benchmark: add --analyze mode to compare.js

Add an --analyze flag that performs statistical analysis directly
after benchmarks complete, eliminating the need for R and compare.R.
When --analyze is specified, compare.js collects the rate data during
the run and prints a statistical summary table instead of CSV output.
The table matches the format of compare.R: improvement percentage,
significance stars (* p<0.05, ** p<0.01, *** p<0.001), and confidence
intervals at three risk levels.
Also adds a --max-regression N option that causes the compare.js to
exit with 1 (error) when the `--new` is N% slower. Useful for CI
use to detect regressions.
Uses the histogram API's welchTest() and cohensD() methods introduced
in the previous commit. Benchmark rates are scaled to integers for
HdrHistogram recording; the --scale option (default 1000) controls
the multiplier for precision.
Usage:
node benchmark/compare.js --old ./node-old --new ./node-new \
--analyze url
Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode/Opus
PR-URL: #65416
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>

Author: jasnell

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}

benchmark/_benchmark_progress.js

@@ -25,9 +25,10 @@ function getTime(diff) {
 // A run is an item in the job queue: { binary, filename, iter }
 // A config is an item in the subqueue: { binary, filename, iter, configs }
 classBenchmarkProgress{
-constructor(queue,benchmarks){
+constructor(queue,benchmarks,options={}){
 this.queue=queue;// Scheduled runs.
 this.benchmarks=benchmarks;// Filenames of scheduled benchmarks.
+this.analyze=!!options.analyze;// stdout is not piped, but unused.
 this.completedRuns=0;// Number of completed runs.
 this.scheduledRuns=queue.length;// Number of scheduled runs.
 // Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
 }
 
 updateProgress(){
-if(!process.stderr.isTTY||process.stdout.isTTY){
+// Progress renders on stderr when stdout is piped (not a TTY).
+// In --analyze mode, stdout is the terminal but is unused during
+// the run, so treat it the same as piped.
+if(!process.stderr.isTTY||(process.stdout.isTTY&&!this.analyze)){
 return;
 }
 readline.clearLine(process.stderr);

benchmark/compare.js

@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
- example 'compare.R'.
+ example 'compare.R'. Use --analyze to perform statistical analysis
+ directly without R.
 
  --new ./new-node-binary new node binary (required)
  --old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  repeated)
  --set variable=value set benchmark variable (can be repeated)
  --no-progress don't show benchmark progress indicator
+ --analyze perform statistical analysis after benchmarks
+ complete (Welch's t-test, effect size) instead
+ of printing csv output
+ --scale 1000 rate-to-integer multiplier for histogram
+ precision when using --analyze (default: 1000)
+ --max-regression N exit with code 1 if any statistically
+ significant regression exceeds N% (implies
+ --analyze)
 
  Examples:
  --set CPUSET=0 Runs benchmarks on CPU core 0.
  --set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
 
  Note: The CPUSET format should match the specifications of the 'taskset' command
-`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress']});
+`,{arrayArgs: ['set','filter','exclude'],boolArgs: ['no-progress','analyze']});
 
 if(!cli.optional.new||!cli.optional.old){
 cli.abort(cli.usage);
 }
 
 constbinaries=['old','new'];
 construns=cli.optional.runs ? parseInt(cli.optional.runs,10) : 30;
+constmaxRegression=cli.optional['max-regression'] ?
+parseFloat(cli.optional['max-regression']) :
+0;
+constanalyze=!!cli.optional.analyze||maxRegression>0;
+constscale=cli.optional.scale ? parseInt(cli.optional.scale,10) : 1000;
 constbenchmarks=cli.benchmarks();
 
 if(benchmarks.length===0){
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
 return;
 }
 
+// When --analyze is set, collect results for statistical analysis.
+constresults=analyze ? newMap() : null;
+
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
 // Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header
-console.log('"binary","filename","configuration","rate","time"');
+// Print csv header (unless analyzing inline).
+if(!analyze){
+console.log('"binary","filename","configuration","rate","time"');
+}
 
 constkStartOfQueue=0;
 
 constshowProgress=!cli.optional['no-progress'];
 letprogress;
 if(showProgress){
-progress=newBenchmarkProgress(queue,benchmarks);
+progress=newBenchmarkProgress(queue,benchmarks,{ analyze });
 progress.startQueue(kStartOfQueue);
 }
 
@@ -99,11 +118,20 @@ if (showProgress) {
 conf+=` ${key}=${inspect(data.conf[key])}`;
 }
 conf=conf.slice(1);
-// Escape quotes (") for correct csv formatting
-conf=conf.replace(/"/g,'""');
 
-console.log(`"${job.binary}","${job.filename}","${conf}",`+
-`${data.rate},${data.time}`);
+if(analyze){
+// Collect results for post-run analysis.
+constname=`${job.filename}${conf}`;
+if(!results.has(name)){
+results.set(name,{old: [],new: []});
+}
+results.get(name)[job.binary].push(data.rate);
+}else{
+// Escape quotes (") for correct csv formatting
+conf=conf.replace(/"/g,'""');
+console.log(`"${job.binary}","${job.filename}","${conf}",`+
+`${data.rate},${data.time}`);
+}
 if(showProgress){
 // One item in the subqueue has been completed.
 progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
 // If there are more benchmarks execute the next
 if(i+1<queue.length){
 recursive(i+1);
+}elseif(analyze){
+printAnalysis(results,scale,maxRegression);
 }
 });
 })(kStartOfQueue);
+
+functionprintAnalysis(results,scale,maxRegression){
+const{ createHistogram }=require('node:perf_hooks');
+
+// Build per-benchmark histograms and run statistical tests.
+constrows=[];
+letmaxNameLen=0;
+
+letskipped=0;
+
+for(const[name,{old: oldRates,new: newRates}]ofresults){
+if(oldRates.length<2||newRates.length<2){
+skipped++;
+continue;
+}
+
+consthOld=createHistogram({figures: 3});
+consthNew=createHistogram({figures: 3});
+
+for(constrofoldRates)hOld.record(Math.max(1,Math.round(r*scale)));
+for(constrofnewRates)hNew.record(Math.max(1,Math.round(r*scale)));
+
+constoldMean=oldRates.reduce((a,b)=>a+b,0)/oldRates.length;
+constnewMean=newRates.reduce((a,b)=>a+b,0)/newRates.length;
+constimprovement=((newMean-oldMean)/oldMean)*100;
+
+// Query the three confidence levels. The p-value and t-statistic
+// are the same regardless of the confidence level, so we extract
+// them from the first result.
+constw95=hOld.welchTest(hNew,{confidence: 0.95});
+constw99=hOld.welchTest(hNew,{confidence: 0.99});
+constw999=hOld.welchTest(hNew,{confidence: 0.999});
+
+// Significance stars matching compare.R convention.
+letstars='';
+if(w95.pValue<0.001)stars='***';
+elseif(w95.pValue<0.01)stars=' **';
+elseif(w95.pValue<0.05)stars=' *';
+
+// Confidence intervals expressed as percentage of the old mean.
+constciPct=(w)=>{
+consthalf=
+(w.confidenceInterval.upper-w.confidenceInterval.lower)/2;
+return(half/(oldMean*scale))*100;
+};
+
+rows.push({
+ name,
+ stars,
+ improvement,
+ci95: ciPct(w95),
+ci99: ciPct(w99),
+ci999: ciPct(w999),
+pValue: w95.pValue,
+});
+
+if(name.length>maxNameLen)maxNameLen=name.length;
+}
+
+// Print header.
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+constrpad=(s,n)=>' '.repeat(Math.max(0,n-s.length))+s;
+
+console.log(`${pad('',maxNameLen)} confidence`+
+` improvement accuracy (*) (**) (***)`);
+
+for(constrowofrows){
+constimp=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
+console.log(
+`${pad(row.name,maxNameLen)}${pad(row.stars,10)}`+
+` ${rpad(imp,11)}`+
+` ±${row.ci95.toFixed(2)}%`+
+` ±${row.ci99.toFixed(2)}%`+
+` ±${row.ci999.toFixed(2)}%`,
+);
+}
+
+if(skipped>0){
+console.log('');
+console.log(
+`Note: ${skipped} configuration${skipped===1 ? ' was' : 's were'}`+
+` skipped because Welch's t-test requires at least 2 samples per`+
+` binary. Use --runs 2 or higher.`,
+);
+}
+
+// --- Bar chart visualization ---
+printChart(rows,maxNameLen);
+
+console.log('');
+console.log(
+`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n`+
+`Use --scale to adjust precision if needed.\n`,
+);
+console.log(
+`Be aware that when doing many comparisons the risk of a false-positive\n`+
+`result increases. In this case, there are ${rows.length} comparisons, `+
+`you can thus\nexpect the following amount of false-positive results:\n`+
+` ${(rows.length*0.05).toFixed(2)} false positives, when considering `+
+`a 5% risk acceptance (*, **, ***),\n`+
+` ${(rows.length*0.01).toFixed(2)} false positives, when considering `+
+`a 1% risk acceptance (**, ***),\n`+
+` ${(rows.length*0.001).toFixed(2)} false positives, when considering `+
+`a 0.1% risk acceptance (***)`,
+);
+
+// Gate: exit with error if any significant regression exceeds the limit.
+if(maxRegression>0){
+constfailures=rows.filter(
+(r)=>r.stars.trim()!==''&&r.improvement<-maxRegression,
+);
+if(failures.length>0){
+console.log('');
+console.log(
+`FAIL: ${failures.length} benchmark${failures.length===1 ? '' : 's'}`+
+` showed a statistically significant regression exceeding`+
+` ${maxRegression}%:`,
+);
+for(constfoffailures){
+console.log(` ${f.name}${f.improvement.toFixed(2)}%`);
+}
+process.exitCode=1;
+}
+}
+}
+
+functionprintChart(rows,maxNameLen){
+if(rows.length===0)return;
+
+// Determine the chart scale from the data. The bar region covers
+// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
+constbarWidth=40;
+consthalfWidth=barWidth/2;
+letmaxAbs=0;
+for(constrowofrows){
+constextent=Math.abs(row.improvement)+row.ci95;
+if(extent>maxAbs)maxAbs=extent;
+}
+if(maxAbs===0)maxAbs=1;
+
+constpad=(s,n)=>s+' '.repeat(Math.max(0,n-s.length));
+
+// Scale axis labels.
+constaxisLeft=`-${maxAbs.toFixed(1)}%`;
+constaxisRight=`+${maxAbs.toFixed(1)}%`;
+constaxisCenter='0%';
+
+// Print axis header.
+constlabelPad=maxNameLen+5;
+constleftLabel=' '.repeat(labelPad)+
+axisLeft+
+' '.repeat(Math.max(0,halfWidth-axisLeft.length-Math.floor(axisCenter.length/2)))+
+axisCenter+
+' '.repeat(Math.max(0,halfWidth-Math.ceil(axisCenter.length/2)-axisRight.length))+
+axisRight;
+console.log('');
+console.log(leftLabel);
+
+for(constrowofrows){
+constimp=row.improvement;
+constci=row.ci95;
+
+// Position of the improvement value in the bar region [0, barWidth].
+constcenter=halfWidth;
+constimpPos=center+(imp/maxAbs)*halfWidth;
+
+// CI extent in bar positions.
+constciLeft=center+((imp-ci)/maxAbs)*halfWidth;
+constciRight=center+((imp+ci)/maxAbs)*halfWidth;
+
+// Build the bar character by character.
+constchars=[];
+for(letx=0;x<barWidth;x++){
+constpos=x+0.5;// Center of this character cell.
+if(x===Math.floor(center)){
+chars.push('|');
+}elseif((imp>=0&&pos>center&&pos<=impPos)||
+(imp<0&&pos<center&&pos>=impPos)){
+chars.push(row.stars ? '\u2588' : '\u2593');// solid or dark shade
+}elseif(pos>=ciLeft&&pos<=ciRight){
+chars.push('\u2591');// Light shade for CI region
+}else{
+chars.push(' ');
+}
+}
+
+constlabel=`${row.improvement>=0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
+constsig=row.stars.trim();
+console.log(`${pad(row.name,maxNameLen)}${chars.join('')}${label}${sig}`);
+}
+}