test_runner: expose expectFailure message

This change exposes the expectFailure message in the test runner and
adds edge cases for expectFailure ambiguity.
PR-URL: #61563Fixes: #61570
Reviewed-By: Jordan Harband <ljharb@gmail.com>
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: Jacob Smith <jacob@frende.me>

Author: Han5991

doc/api/test.md

@@ -265,11 +265,12 @@ added:
  - v25.5.0
 -->
 
-This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
-in order to "pass"; a test/test-case that does not throw, fails.
+This flips the pass/fail reporting for a specific test or suite: a flagged test
+case must throw in order to pass, and a flagged test case that does not throw
+fails.
 
-In the following, `doTheThing()`returns _currently_`false` (`false` does not equal `true`, causing
-`strictEqual` to throw, so the test-case passes).
+In each of the following, `doTheThing()`fails to return `true`, but since the
+tests are flagged `expectFailure`, they pass.
 
 ```js
 it.expectFailure('should do the thing', () => {
@@ -279,6 +280,50 @@ it.expectFailure('should do the thing', () => {
 it('should do the thing', { expectFailure:true }, () => {
 assert.strictEqual(doTheThing(), true);
 });
+
+it('should do the thing', { expectFailure:'feature not implemented' }, () => {
+assert.strictEqual(doTheThing(), true);
+});
+```
+
+If the value of `expectFailure` is a
+[<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp) |
+[<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function) |
+[<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) |
+[<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error),
+the tests will pass only if they throw a matching value.
+See [`assert.throws`][] for how each value type is handled.
+
+Each of the following tests fails _despite_ being flagged `expectFailure`
+because the failure does not match the specific **expected** failure.
+
+```js
+it('fails because regex does not match', {
+ expectFailure:/expected message/,
+}, () => {
+thrownewError('different message');
+});
+
+it('fails because object matcher does not match', {
+ expectFailure: { code:'ERR_EXPECTED' },
+}, () => {
+consterr=newError('boom');
+err.code='ERR_ACTUAL';
+throw err;
+});
+```
+
+To supply both a reason and specific error for `expectFailure`, use `{ label, match }`.
+
+```js
+it('should fail with specific error and reason', {
+ expectFailure: {
+ label:'reason for failure',
+ match:/error message/,
+ },
+}, () => {
+assert.strictEqual(doTheThing(), true);
+});
 ```
 
 `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
@@ -1684,6 +1729,18 @@ changes:
  thread. If `false`, only one test runs at a time.
  If unspecified, subtests inherit this value from their parent.
 **Default:**`false`.
+*`expectFailure` {boolean|string|RegExp|Function|Object|Error} If truthy, the
+ test is expected to fail. If a non-empty string is provided, that string is displayed
+ in the test results as the reason why the test is expected to fail. If a
+[<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp),
+[<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function),
+[<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object), or
+[<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error)
+ is provided directly (without wrapping in `{ match: … }`), the test passes
+ only if the thrown error matches, following the behavior of
+[`assert.throws`][]. To provide both a reason and validation, pass an object
+ with `label` (string) and `match` (RegExp, Function, Object, or Error).
+**Default:**`false`.
 *`only` {boolean} If truthy, and the test context is configured to run
 `only` tests, then this test will be run. Otherwise, the test is skipped.
 **Default:**`false`.
@@ -4181,6 +4238,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`NODE_V8_COVERAGE`]: cli.md#node_v8_coveragedir
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`assert.throws`]: assert.md#assertthrowsfn-error-message
 [`context.diagnostic`]: #contextdiagnosticmessage
 [`context.skip`]: #contextskipmessage
 [`context.todo`]: #contexttodomessage

lib/internal/test_runner/reporter/tap.js

@@ -87,7 +87,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
 }elseif(todo!==undefined){
 line+=` # TODO${typeoftodo==='string'&&todo.length ? ` ${tapEscape(todo)}` : ''}`;
 }elseif(expectFailure!==undefined){
-line+=' # EXPECTED FAILURE';
+line+=` # EXPECTED FAILURE${typeofexpectFailure==='string' ? ` ${tapEscape(expectFailure)}` : ''}`;
 }
 
 line+='\n';

lib/internal/test_runner/test.js

@@ -1,5 +1,6 @@
 'use strict';
 const{
+ ArrayPrototypeEvery,
  ArrayPrototypePush,
  ArrayPrototypePushApply,
  ArrayPrototypeShift,
@@ -13,6 +14,7 @@ const {
  MathMax,
  Number,
  NumberPrototypeToFixed,
+ ObjectKeys,
  ObjectSeal,
  Promise,
  PromisePrototypeThen,
@@ -40,6 +42,7 @@ const {
  AbortError,
 codes: {
 ERR_INVALID_ARG_TYPE,
+ERR_INVALID_ARG_VALUE,
 ERR_TEST_FAILURE,
 },
 }=require('internal/errors');
@@ -56,7 +59,8 @@ const {
 once: runOnce,
  setOwnProperty,
 }=require('internal/util');
-const{ isPromise }=require('internal/util/types');
+constassert=require('assert');
+const{ isPromise, isRegExp }=require('internal/util/types');
 const{
  validateAbortSignal,
  validateFunction,
@@ -492,6 +496,39 @@ class SuiteContext {
 }
 }
 
+functionparseExpectFailure(expectFailure){
+if(expectFailure===undefined||expectFailure===false){
+returnfalse;
+}
+
+if(typeofexpectFailure==='string'){
+return{__proto__: null,label: expectFailure,match: undefined};
+}
+
+if(typeofexpectFailure==='function'||isRegExp(expectFailure)){
+return{__proto__: null,label: undefined,match: expectFailure};
+}
+
+if(typeofexpectFailure!=='object'){
+return{__proto__: null,label: undefined,match: undefined};
+}
+
+constkeys=ObjectKeys(expectFailure);
+if(keys.length===0){
+thrownewERR_INVALID_ARG_VALUE('options.expectFailure',expectFailure,'must not be an empty object');
+}
+
+if(ArrayPrototypeEvery(keys,(k)=>k==='match'||k==='label')){
+return{
+__proto__: null,
+label: expectFailure.label,
+match: expectFailure.match,
+};
+}
+
+return{__proto__: null,label: undefined,match: expectFailure};
+}
+
 classTestextendsAsyncResource{
 reportedType='test';
 abortController;
@@ -641,7 +678,7 @@ class Test extends AsyncResource {
 this.plan=null;
 this.expectedAssertions=plan;
 this.cancelled=false;
-this.expectFailure=expectFailure!==undefined&&expectFailure!==false;
+this.expectFailure=parseExpectFailure(expectFailure)||this.parent?.expectFailure;
 this.skipped=skip!==undefined&&skip!==false;
 this.isTodo=(todo!==undefined&&todo!==false)||this.parent?.isTodo;
 this.startTime=null;
@@ -955,7 +992,30 @@ class Test extends AsyncResource {
 return;
 }
 
-if(this.expectFailure===true){
+if(this.expectFailure){
+if(typeofthis.expectFailure==='object'&&
+this.expectFailure.match!==undefined){
+const{match: validation}=this.expectFailure;
+try{
+consterrorToCheck=(
+err?.code==='ERR_TEST_FAILURE'&&
+err?.failureType===kTestCodeFailure&&
+err.cause
+) ?
+err.cause :
+err;
+// eslint-disable-next-line no-restricted-syntax
+assert.throws(()=>{throwerrorToCheck;},validation);
+}catch(e){
+this.passed=false;
+this.error=newERR_TEST_FAILURE(
+'The test failed, but the error did not match the expected validation',
+kTestCodeFailure,
+);
+this.error.cause=e;
+return;
+}
+}
 this.passed=true;
 }else{
 this.passed=false;
@@ -965,7 +1025,7 @@ class Test extends AsyncResource {
 }
 
 pass(){
-if(this.error==null&&this.expectFailure===true&&!this.skipped){
+if(this.error==null&&this.expectFailure&&!this.skipped){
 this.passed=false;
 this.error=newERR_TEST_FAILURE(
 'test was expected to fail but passed',
@@ -977,6 +1037,20 @@ class Test extends AsyncResource {
 return;
 }
 
+if(this.skipped||this.isTodo){
+this.passed=true;
+return;
+}
+
+if(this.expectFailure){
+this.passed=false;
+this.error=newERR_TEST_FAILURE(
+'Test passed but was expected to fail',
+kTestCodeFailure,
+);
+return;
+}
+
 this.passed=true;
 }
 
@@ -1366,7 +1440,10 @@ class Test extends AsyncResource {
 }elseif(this.isTodo){
 directive=this.reporter.getTodo(this.message);
 }elseif(this.expectFailure){
-directive=this.reporter.getXFail(this.expectFailure);// TODO(@JakobJingleheimer): support specifying failure
+constmessage=typeofthis.expectFailure==='object' ?
+this.expectFailure.label :
+this.expectFailure;
+directive=this.reporter.getXFail(message);
 }
 
 if(this.reportedType){

test/parallel/test-runner-expect-error-but-pass.js

@@ -8,10 +8,9 @@ if (!process.env.NODE_TEST_CONTEXT) {
 
 stream.on('test:pass',common.mustNotCall());
 stream.on('test:fail',common.mustCall((event)=>{
-assert.strictEqual(event.expectFailure,true);
 assert.strictEqual(event.details.error.code,'ERR_TEST_FAILURE');
 assert.strictEqual(event.details.error.failureType,'expectedFailure');
-assert.strictEqual(event.details.error.cause,'test was expected to fail but passed');
+assert.strictEqual(event.details.error.message,'test was expected to fail but passed');
 },1));
 }else{
 test('passing test',{expectFailure: true},()=>{});