test: report error wpt test results

When a wpt test file is exited for uncaught error, its result should be
recorded in the `wptreport.json` and uploaded to wpt.fyi.
For instance, `html/webappapis/timers/evil-spec-example.any.js` is
exited for uncaught error in Node.js but it shows as "MISSING" at
https://wpt.fyi/results/html/webappapis/timers?label=master&label=experimental&product=chrome&product=node.js&aligned.
PR-URL: #50429
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: legendecas

test/common/wpt.js

@@ -58,46 +58,66 @@ function codeUnitStr(char) {
 return'U+'+char.charCodeAt(0).toString(16);
 }
 
+classReportResult{
+constructor(name){
+this.test=name;
+this.status='OK';
+this.subtests=[];
+}
+
+addSubtest(name,status,message){
+constsubtest={
+ status,
+// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
+name: sanitizeUnpairedSurrogates(name),
+};
+if(message){
+// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
+subtest.message=sanitizeUnpairedSurrogates(message);
+}
+this.subtests.push(subtest);
+returnsubtest;
+}
+
+finish(status){
+this.status=status??'OK';
+}
+}
+
+// Generates a report that can be uploaded to wpt.fyi.
+// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
+// for more details.
 classWPTReport{
 constructor(path){
 this.filename=`report-${path.replaceAll('/','-')}.json`;
-this.results=[];
+/** @type {Map<string, ReportResult>} */
+this.results=newMap();
 this.time_start=Date.now();
 }
 
-addResult(name,status){
-constresult={
-test: name,
- status,
-subtests: [],
-addSubtest(name,status,message){
-constsubtest={
- status,
-// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
-name: sanitizeUnpairedSurrogates(name),
-};
-if(message){
-// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
-subtest.message=sanitizeUnpairedSurrogates(message);
-}
-this.subtests.push(subtest);
-returnsubtest;
-},
-};
-this.results.push(result);
+/**
+ * Get or create a ReportResult for a test spec.
+ * @param {WPTTestSpec} spec
+ */
+getResult(spec){
+constname=`/${spec.getRelativePath()}${spec.variant}`;
+if(this.results.has(name)){
+returnthis.results.get(name);
+}
+constresult=newReportResult(name);
+this.results.set(name,result);
 returnresult;
 }
 
 write(){
 this.time_end=Date.now();
-this.results=this.results.filter((result)=>{
-returnresult.status==='SKIP'||result.subtests.length!==0;
-}).map((result)=>{
-consturl=newURL(result.test,'http://wpt');
-url.pathname=url.pathname.replace(/\.js$/,'.html');
-result.test=url.href.slice(url.origin.length);
-returnresult;
-});
+constresults=Array.from(this.results.values())
+.map((result)=>{
+consturl=newURL(result.test,'http://wpt');
+url.pathname=url.pathname.replace(/\.js$/,'.html');
+result.test=url.href.slice(url.origin.length);
+returnresult;
+});
 
 /**
  * Return required and some optional properties
@@ -110,7 +130,12 @@ class WPTReport {
 os: getOs(),
 };
 
-fs.writeFileSync(`out/wpt/${this.filename}`,JSON.stringify(this));
+fs.writeFileSync(`out/wpt/${this.filename}`,JSON.stringify({
+time_start: this.time_start,
+time_end: this.time_end,
+run_info: this.run_info,
+results: results,
+}));
 }
 }
 
@@ -642,14 +667,13 @@ class WPTRunner {
 this.inProgress.add(spec);
 this.workers.set(spec,worker);
 
-letreportResult;
+constreportResult=this.report?.getResult(spec);
 worker.on('message',(message)=>{
 switch(message.type){
 case'result':
-reportResult||=this.report?.addResult(`/${relativePath}${spec.variant}`,'OK');
 returnthis.resultCallback(spec,message.result,reportResult);
 case'completion':
-returnthis.completionCallback(spec,message.status);
+returnthis.completionCallback(spec,message.status,reportResult);
 default:
 thrownewError(`Unexpected message from worker: ${message.type}`);
 }
@@ -661,6 +685,8 @@ class WPTRunner {
 // This can happen normally, for example in timers tests.
 return;
 }
+// Generate a subtest failure for visibility.
+// No need to record this synthetic failure with wpt.fyi.
 this.fail(
 spec,
 {
@@ -671,6 +697,8 @@ class WPTRunner {
 },
 kUncaught,
 );
+// Mark the whole test as failed in wpt.fyi report.
+reportResult?.finish('ERROR');
 this.inProgress.delete(spec);
 });
 
@@ -680,7 +708,11 @@ class WPTRunner {
 
 process.on('exit',()=>{
 for(constspecofthis.inProgress){
+// No need to record this synthetic failure with wpt.fyi.
 this.fail(spec,{name: 'Incomplete'},kIncomplete);
+// Mark the whole test as failed in wpt.fyi report.
+constreportResult=this.report?.getResult(spec);
+reportResult?.finish('ERROR');
 }
 inspect.defaultOptions.depth=Infinity;
 // Sorts the rules to have consistent output
@@ -780,6 +812,7 @@ class WPTRunner {
  * in one test file).
  * @param {WPTTestSpec} spec
  * @param {Test} test The Test object returned by WPT harness
+ * @param {ReportResult} reportResult The report result object
  */
 resultCallback(spec,test,reportResult){
 conststatus=this.getTestStatus(test.status);
@@ -794,13 +827,19 @@ class WPTRunner {
  * Report the status of each WPT test (one per file)
  * @param {WPTTestSpec} spec
  * @param {object} harnessStatus - The status object returned by WPT harness.
+ * @param {ReportResult} reportResult The report result object
  */
-completionCallback(spec,harnessStatus){
+completionCallback(spec,harnessStatus,reportResult){
 conststatus=this.getTestStatus(harnessStatus.status);
 
 // Treat it like a test case failure
 if(status===kTimeout){
+// No need to record this synthetic failure with wpt.fyi.
 this.fail(spec,{name: 'WPT testharness timeout'},kTimeout);
+// Mark the whole test as TIMEOUT in wpt.fyi report.
+reportResult?.finish('TIMEOUT');
+}else{
+reportResult?.finish();
 }
 this.inProgress.delete(spec);
 // Always force termination of the worker. Some tests allocate resources