test: update WPT runner

PR-URL: #43455
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

Author: panva

test/common/wpt.js

@@ -5,6 +5,7 @@ const fixtures = require('../common/fixtures');
 constfs=require('fs');
 constfsPromises=fs.promises;
 constpath=require('path');
+constevents=require('events');
 const{ inspect }=require('util');
 const{ Worker }=require('worker_threads');
 
@@ -152,21 +153,30 @@ class WPTTestSpec {
 this.filename=filename;
 
 this.requires=newSet();
-this.failReasons=[];
+this.failedTests=[];
+this.flakyTests=[];
 this.skipReasons=[];
 for(constitemofrules){
 if(item.requires.length){
 for(constreqofitem.requires){
 this.requires.add(req);
 }
 }
-if(item.fail){
-this.failReasons.push(item.fail);
+if(Array.isArray(item.fail?.expected)){
+this.failedTests.push(...item.fail.expected);
+}
+if(Array.isArray(item.fail?.flaky)){
+this.failedTests.push(...item.fail.flaky);
+this.flakyTests.push(...item.fail.flaky);
 }
 if(item.skip){
 this.skipReasons.push(item.skip);
 }
 }
+
+this.failedTests=[...newSet(this.failedTests)];
+this.flakyTests=[...newSet(this.flakyTests)];
+this.skipReasons=[...newSet(this.skipReasons)];
 }
 
 getRelativePath(){
@@ -368,7 +378,7 @@ class WPTRunner {
 
 // TODO(joyeecheung): work with the upstream to port more tests in .html
 // to .js.
-runJsTests(){
+asyncrunJsTests(){
 letqueue=[];
 
 // If the tests are run as `node test/wpt/test-something.js subset.any.js`,
@@ -459,6 +469,8 @@ class WPTRunner {
 );
 this.inProgress.delete(testFileName);
 });
+
+awaitevents.once(worker,'exit').catch(()=>{});
 }
 
 process.on('exit',()=>{
@@ -469,34 +481,72 @@ class WPTRunner {
 }
 }
 inspect.defaultOptions.depth=Infinity;
-console.log(this.results);
+// Sorts the rules to have consistent output
+console.log(JSON.stringify(Object.keys(this.results).sort().reduce(
+(obj,key)=>{
+obj[key]=this.results[key];
+returnobj;
+},
+{}
+),null,2));
 
 constfailures=[];
 letexpectedFailures=0;
 letskipped=0;
-for(constkeyofObject.keys(this.results)){
-constitem=this.results[key];
-if(item.fail&&item.fail.unexpected){
+for(const[key,item]ofObject.entries(this.results)){
+if(item.fail?.unexpected){
 failures.push(key);
 }
-if(item.fail&&item.fail.expected){
+if(item.fail?.expected){
 expectedFailures++;
 }
 if(item.skip){
 skipped++;
 }
 }
+
+constunexpectedPasses=[];
+for(const[key,specMap]ofthis.specMap){
+// File has no expected failures
+if(!specMap.failedTests.length){
+continue;
+}
+
+// File was (maybe even conditionally) skipped
+if(this.results[key]?.skip){
+continue;
+}
+
+// Full check: every expected to fail test is present
+if(specMap.failedTests.some((expectedToFail)=>{
+if(specMap.flakyTests.includes(expectedToFail)){
+returnfalse;
+}
+returnthis.results[key]?.fail?.expected?.includes(expectedToFail)!==true;
+})){
+unexpectedPasses.push(key);
+continue;
+}
+}
+
 constran=total-skipped;
 constpassed=ran-expectedFailures-failures.length;
 console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
 `${passed} passed, ${expectedFailures} expected failures,`,
-`${failures.length} unexpected failures`);
+`${failures.length} unexpected failures,`,
+`${unexpectedPasses.length} unexpected passes`);
 if(failures.length>0){
 constfile=path.join('test','wpt','status',`${this.path}.json`);
 thrownewError(
 `Found ${failures.length} unexpected failures. `+
 `Consider updating ${file} for these files:\n${failures.join('\n')}`);
 }
+if(unexpectedPasses.length>0){
+constfile=path.join('test','wpt','status',`${this.path}.json`);
+thrownewError(
+`Found ${unexpectedPasses.length} unexpected passes. `+
+`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
+}
 });
 }
 
@@ -577,8 +627,9 @@ class WPTRunner {
 if(!result[item.status][key]){
 result[item.status][key]=[];
 }
-if(result[item.status][key].indexOf(item.reason)===-1){
-result[item.status][key].push(item.reason);
+consthasName=result[item.status][key].includes(item.name);
+if(!hasName){
+result[item.status][key].push(item.name);
 }
 }
 }
@@ -589,10 +640,10 @@ class WPTRunner {
 
 fail(filename,test,status){
 constspec=this.specMap.get(filename);
-constexpected=!!(spec.failReasons.length);
+constexpected=spec.failedTests.includes(test.name);
 if(expected){
 console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
-console.log(spec.failReasons.join('; '));
+console.log(test.message||status);
 }else{
 console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
 }
@@ -604,6 +655,7 @@ class WPTRunner {
 ` ${require.main.filename}${filename}`;
 console.log(`Command: ${command}\n`);
 this.addTestResult(filename,{
+name: test.name,
  expected,
 status: kFail,
 reason: test.message||status

test/wpt/README.md

@@ -91,7 +91,11 @@ add this to `test/wpt/status/url.json`:
 
 ```json
 "url-searchparams.any.js": {
-"fail": "explain why the test fails, ideally with links"
+"fail": {
+"expected": [
+"test name in the WPT test case, e.g. second argument passed to test()"
+ ]
+ }
  }
 ```
 
@@ -155,8 +159,17 @@ expected failures.
  // Optional: the test will be skipped with the reason printed
  "skip": "explain why we cannot run a test that's supposed to pass",
 
- // Optional: the test will be skipped with the reason printed
- "fail": "explain why we the test is expected to fail"
+ // Optional: failing tests
+ "fail": {
+ "note": "You may leave an optional arbitrary note e.g. with TODOs",
+ "expected": [
+ "test name in the WPT test case, e.g. second argument passed to test()",
+ "another test name"
+ ],
+ "flaky": [
+ "flaky test name"
+ ]
+ }
  }
 }
 ```

test/wpt/status/FileAPI/blob.json

@@ -1,14 +1,47 @@
 {
-"Blob-constructor.any.js": {
-"skip": "Depends on File API"
- },
 "Blob-constructor-dom.window.js": {
 "skip": "Depends on DOM API"
  },
-"Blob-slice.any.js": {
-"skip": "Depends on File API"
+"Blob-constructor.any.js": {
+"fail": {
+"note": "Depends on File API",
+"expected": [
+"A plain object with @@iterator should be treated as a sequence for the blobParts argument.",
+"A plain object with @@iterator and a length property should be treated as a sequence for the blobParts argument.",
+"A String object should be treated as a sequence for the blobParts argument.",
+"A Uint8Array object should be treated as a sequence for the blobParts argument.",
+"Getters and value conversions should happen in order until an exception is thrown.",
+"Changes to the blobParts array should be reflected in the returned Blob (pop).",
+"Changes to the blobParts array should be reflected in the returned Blob (unshift).",
+"ToString should be called on elements of the blobParts array.",
+"ArrayBuffer elements of the blobParts array should be supported.",
+"Passing typed arrays as elements of the blobParts array should work.",
+"Passing a Float64Array as element of the blobParts array should work.",
+"Array with two blobs",
+"Array with two buffers",
+"Array with two bufferviews",
+"Array with mixed types",
+"options properties should be accessed in lexicographic order.",
+"Arguments should be evaluated from left to right.",
+"Passing null (index 0) for options should use the defaults.",
+"Passing null (index 0) for options should use the defaults (with newlines).",
+"Passing undefined (index 1) for options should use the defaults.",
+"Passing undefined (index 1) for options should use the defaults (with newlines).",
+"Passing object \"[object Object]\" (index 2) for options should use the defaults.",
+"Passing object \"[object Object]\" (index 2) for options should use the defaults (with newlines).",
+"Passing object \"[object Object]\" (index 3) for options should use the defaults.",
+"Passing object \"[object Object]\" (index 3) for options should use the defaults (with newlines).",
+"Passing object \"/regex/\" (index 4) for options should use the defaults.",
+"Passing object \"/regex/\" (index 4) for options should use the defaults (with newlines).",
+"Passing function \"function() {}\" (index 5) for options should use the defaults.",
+"Passing function \"function() {}\" (index 5) for options should use the defaults (with newlines)."
+ ]
+ }
  },
 "Blob-in-worker.worker.js": {
 "skip": "Depends on Web Workers API"
+ },
+"Blob-slice.any.js": {
+"skip": "Depends on File API"
  }
 }