benchmark: fix calibrate-n option handling

Fix broken parsing for --cv-threshold and --max-increases so the
value after the equals sign is used.
Use the configured CV threshold when checking overall stability and
report the configured thresholds in calibrate-n output. Also handle
invalid numeric values for --runs and --cv-threshold consistently
with the other numeric options.
Signed-off-by: Luan Muniz <luan@luanmuniz.com.br>
PR-URL: #64146
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>

Author: luanmuniz

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });

benchmark/calibrate-n.js

@@ -8,7 +8,9 @@ const { styleText } = require('node:util');
 constDEFAULT_RUNS=30;// Number of runs for each n value
 constCV_THRESHOLD=0.05;// 5% coefficient of variation threshold
 constMAX_N_INCREASE=6;// Maximum number of times to increase n (10**6)
+constMAX_CV_THRESHOLD=0.10;// 10% coefficient of variation threshold for individual configurations
 constINCREASE_FACTOR=10;// Factor by which to increase n
+constSTART_N=10;// Starting n value (10 iterations)
 
 constargs=process.argv.slice(2);
 if(args.length===0){
@@ -19,7 +21,7 @@ Options:
  --runs=N Number of runs for each n value (default: ${DEFAULT_RUNS})
  --cv-threshold=N Target coefficient of variation threshold (default: ${CV_THRESHOLD})
  --max-increases=N Maximum number of n increases to try (default: ${MAX_N_INCREASE})
- --start-n=N Initial n value to start with (default: autodetect)
+ --start-n=N Initial n value to start with (default: ${START_N})
  --increase=N Factor by which to increase n (default: ${INCREASE_FACTOR})
 
 Example:
@@ -34,25 +36,33 @@ let benchmarkPath;
 letruns=DEFAULT_RUNS;
 letcvThreshold=CV_THRESHOLD;
 letmaxIncreases=MAX_N_INCREASE;
-letstartN=10;
+letstartN=START_N;
 letincreaseFactor=INCREASE_FACTOR;
 
 for(constargofargs){
 if(arg.startsWith('--runs=')){
 runs=parseInt(arg.substring(7),10);
+if(isNaN(runs)){
+console.error(`Error: Invalid value for --runs. Using default: ${DEFAULT_RUNS}`);
+runs=DEFAULT_RUNS;
+}
 }elseif(arg.startsWith('--cv-threshold=')){
-cvThreshold=parseFloat(arg.substring(14));
+cvThreshold=parseFloat(arg.substring(15));
+if(isNaN(cvThreshold)){
+console.error(`Error: Invalid value for --cv-threshold. Using default: ${CV_THRESHOLD}`);
+cvThreshold=CV_THRESHOLD;
+}
 }elseif(arg.startsWith('--max-increases=')){
-maxIncreases=parseInt(arg.substring(15),10);
+maxIncreases=parseInt(arg.substring(16),10);
 if(isNaN(maxIncreases)){
 console.error(`Error: Invalid value for --max-increases. Using default: ${MAX_N_INCREASE}`);
 maxIncreases=MAX_N_INCREASE;
 }
 }elseif(arg.startsWith('--start-n=')){
 startN=parseInt(arg.substring(10),10);
 if(isNaN(startN)){
-console.error(`Error: Invalid value for --start-n. Using default: 10`);
-startN=10;
+console.error(`Error: Invalid value for --start-n. Using default: ${START_N}`);
+startN=START_N;
 }
 }elseif(arg.startsWith('--increase=')){
 increaseFactor=parseInt(arg.substring(11),10);
@@ -125,6 +135,7 @@ async function main(n = startN) {
 letbestN=n;
 letbestCV=Infinity;
 letbestGroupStats=null;
+constcvThresholdPercentage=(cvThreshold*100).toFixed(2);
 
 console.log(`
 --------------------------------------------------------
@@ -136,12 +147,12 @@ that produces consistent benchmark results without wasting time.
 How it works:
 1. Run the benchmark multiple times with a specific n value
 2. Group results by configuration
-3. If overall CV is above 5% or any configuration has CV above 10%, increase n and try again
+3. If overall CV is above ${cvThresholdPercentage}% or any configuration has CV above ${MAX_CV_THRESHOLD*100}%, increase n and try again
 
 Configuration:
 - Starting n: ${n.toLocaleString()} iterations
 - Runs per n value: ${runs}
-- Target CV threshold: ${cvThreshold*100}% (lower CV = more stable results)
+- Target CV threshold: ${cvThresholdPercentage}% (lower CV = more stable results)
 - Max increases: ${maxIncreases}
 - Increase factor: ${increaseFactor}x`);
 
@@ -195,23 +206,23 @@ Configuration:
 
 if(groupStats.length>0){
 // Check if any configuration has CV > 10% (too unstable)
-consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>0.10);
+consttooUnstableConfigs=groupStats.filter((g)=>g.stats.cv>MAX_CV_THRESHOLD);
 
 constavgCV=groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length;
 console.log(`\nOverall average CV: ${(avgCV*100).toFixed(2)}%`);
 
-constisOverallStable=avgCV<CV_THRESHOLD;
+constisOverallStable=avgCV<cvThreshold;
 consthasVeryUnstableConfigs=tooUnstableConfigs.length>0;
 
-// Check if overall CV is below CV_THRESHOLD and no configuration has CV > 10%
+// Check if overall CV is below cvThreshold and no configuration has CV > MAX_CV_THRESHOLD
 if(isOverallStable&&!hasVeryUnstableConfigs){
-console.log(styleText(['bold','green'],` ✓ Overall CV is below 5% and no configuration has CV above 10%`));
+console.log(styleText(['bold','green'],` ✓ Overall CV is below ${cvThresholdPercentage}% and no configuration has CV above ${MAX_CV_THRESHOLD*100}%`));
 }else{
 if(!isOverallStable){
-console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above 5%`));
+console.log(styleText(['bold','red'],` ✗ Overall CV (${(avgCV*100).toFixed(2)}%) is above ${cvThresholdPercentage}%`));
 }
 if(hasVeryUnstableConfigs){
-console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above 10%`));
+console.log(styleText(['bold','red'],` ✗ ${tooUnstableConfigs.length} configuration(s) have CV above ${MAX_CV_THRESHOLD*100}%`));
 }
 }
 
@@ -226,7 +237,7 @@ Configuration:
 bestGroupStats.push({
 conf: group.conf,
 stats: stats,
-isStable: stats.cv<=0.10,
+isStable: stats.cv<=MAX_CV_THRESHOLD,
 });
 }
 }
@@ -237,15 +248,15 @@ Configuration:
 }
 
 // Check if we've reached acceptable stability based on new criteria
-// 1. Overall CV should be below CV_THRESHOLD
-// 2. No configuration should have a CV greater than 10%
+// 1. Overall CV should be below cvThreshold
+// 2. No configuration should have a CV greater than MAX_CV_THRESHOLD
 constavgCV=groupStats.length>0 ?
 groupStats.reduce((sum,g)=>sum+g.stats.cv,0)/groupStats.length : Infinity;
-consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>0.10);
-constisOverallStable=avgCV<CV_THRESHOLD;
+consthasUnstableConfig=groupStats.some((g)=>g.stats.cv>MAX_CV_THRESHOLD);
+constisOverallStable=avgCV<cvThreshold;
 
 if(isOverallStable&&!hasUnstableConfig){
-console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < 5% and no configuration has CV > 10%)`);
+console.log(`\n✓ Found optimal n=${n} (Overall CV=${(avgCV*100).toFixed(2)}% < ${cvThresholdPercentage}% and no configuration has CV > ${MAX_CV_THRESHOLD*100}%)`);
 console.log('\nFinal CV for each configuration:');
 groupStats.forEach((g)=>{
 console.log(` ${JSON.stringify(groupedResults[g.confKey].conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
@@ -271,7 +282,7 @@ Configuration:
 if(g.conf){
 console.log(` ${JSON.stringify(g.conf)}: ${(g.stats.cv*100).toFixed(2)}%`);
 if(g.stats.cv>cvThreshold){
-console.log(` ⚠️ This configuration is above the target threshold of ${cvThreshold*100}%`);
+console.log(` ⚠️ This configuration is above the target threshold of ${cvThresholdPercentage}%`);
 }
 }
 });