ci: run pr benchmark against main

Author: danielroe

.github/workflows/cli-bench.yml

@@ -12,7 +12,8 @@ concurrency:
 permissions: {}
 
 jobs:
-# Runs the cheap suites against the published @nuxt/cli: startup timings
+# Runs the cheap suites against a baseline built from the PR's merge base:
+# startup timings
 # (interleaved, so the deltas survive runner noise even when the absolute
 # numbers do not), module counts and install footprint. The dev, restart
 # and build suites need real fixtures and minutes of wall time; run those
@@ -26,6 +27,9 @@ jobs:
 with:
 persist-credentials: false
 
+ - name: Fetch merge base
+run: git fetch --no-tags --depth=1 origin ${{ github.event.pull_request.base.sha }}
+
  - run: npm i -g --force corepack && corepack enable
 
  - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
@@ -39,7 +43,7 @@ jobs:
 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
  - name: 📊 Run benchmark suites
-run: node --experimental-strip-types bench/run.ts --suite startup --suite modules --suite footprint --out bench-report/report.md
+run: node --experimental-strip-types bench/run.ts --baseline ref:${{ github.event.pull_request.base.sha }} --suite startup --suite modules --suite footprint --out bench-report/report.md
 
  - name: ⏫ Upload report
 uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7

bench/README.md

@@ -20,7 +20,7 @@ Useful flags:
 
 | Flag | Default | Meaning |
 | --- | --- | --- |
-|`--baseline`|`latest`| Any npm spec for `@nuxt/cli` to treat as the "before" |
+|`--baseline`|`latest`| Any npm spec for `@nuxt/cli` to treat as the "before", or `ref:<git-ref>` to build the baseline from a commit of this repo (CI uses the PR's base sha)|
 |`--suite`| all | Repeatable. One of `startup`, `modules`, `dev`, `restart`, `build`, `footprint`|
 |`--fixture`|`playground`, `large`| Repeatable. Which fixture to run the project-level suites against |
 |`--workdir`|`~/.cache/nuxt-cli-bench`| Where isolated installs and fixtures live |

bench/lib/targets.ts

@@ -22,7 +22,10 @@ export interface Fixture {
 dir: string
 }
 
-/** The published `@nuxt/cli` version to compare `packages/nuxt-cli` against. */
+/**
+ * The baseline to compare `packages/nuxt-cli` against: an npm dist-tag or
+ * version, or `ref:<git-ref>` to build the package from a commit of this repo.
+ */
 exportconstDEFAULT_BASELINE='latest'
 
 /** Short name for table headers, e.g. `baseline v3.37.0`. */
@@ -33,19 +36,22 @@ export function shortLabel(target: Target): string {
 exportfunctionprepareTargets(workdir: string,baselineSpec: string=DEFAULT_BASELINE): Target[]{
 mkdirSync(workdir,{recursive: true})
 
-consttarball=packHead(workdir)
+consttarball=pack(join(repoRoot,'packages/nuxt-cli'),join(workdir,'pack-head'))
+constbaselineInstallSpec=resolveBaselineSpec(workdir,baselineSpec)
 
 consttargets: Target[]=[
 {id: 'baseline',label: `baseline (@nuxt/cli@${baselineSpec})`,dir: join(workdir,'baseline'),bin: '',version: '',spec: baselineSpec},
 {id: 'head',label: 'head (local main)',dir: join(workdir,'head'),bin: '',version: '',spec: `file:${tarball}`},
 ]
 
+constinstallSpecs: Record<string,string>={baseline: baselineInstallSpec,head: `file:${tarball}`}
+
 for(consttargetoftargets){
 mkdirSync(target.dir,{recursive: true})
 writeFileSync(join(target.dir,'package.json'),`${JSON.stringify({
 name: `nuxt-cli-bench-${target.id}`,
 private: true,
-dependencies: {'@nuxt/cli': target.spec},
+dependencies: {'@nuxt/cli': installSpecs[target.id]},
 },null,2)}\n`)
 npm(['install','--no-audit','--no-fund'],target.dir)
 target.bin=join(target.dir,'node_modules/@nuxt/cli/bin/nuxi.mjs')
@@ -55,12 +61,29 @@ export function prepareTargets(workdir: string, baselineSpec: string = DEFAULT_B
 returntargets
 }
 
-functionpackHead(workdir: string): string{
-constpackageDir=join(repoRoot,'packages/nuxt-cli')
+functionresolveBaselineSpec(workdir: string,spec: string): string{
+if(!spec.startsWith('ref:')){
+returnspec
+}
+constref=spec.slice('ref:'.length)
+constsha=execFileSync('git',['rev-parse',`${ref}^{commit}`],{cwd: repoRoot,encoding: 'utf8'}).trim()
+constdir=join(workdir,`baseline-src-${sha.slice(0,12)}`)
+if(!existsSync(join(dir,'package.json'))){
+rmSync(dir,{recursive: true,force: true})
+mkdirSync(dir,{recursive: true})
+constarchive=execFileSync('git',['archive',sha],{cwd: repoRoot,maxBuffer: 1024**3})
+execFileSync('tar',['-x','-C',dir],{input: archive})
+execFileSync('pnpm',['install'],{cwd: dir,stdio: 'ignore'})
+}
+return`file:${pack(join(dir,'packages/nuxt-cli'),join(workdir,'pack-baseline'))}`
+}
+
+functionpack(packageDir: string,destination: string): string{
+mkdirSync(destination,{recursive: true})
 execFileSync('pnpm',['build'],{cwd: packageDir,stdio: 'ignore'})
-constoutput=execFileSync('npm',['pack','--json','--ignore-scripts','--pack-destination',workdir],{cwd: packageDir,encoding: 'utf8'})
+constoutput=execFileSync('npm',['pack','--json','--ignore-scripts','--pack-destination',destination],{cwd: packageDir,encoding: 'utf8'})
 const[entry]=JSON.parse(output)as{filename: string}[]
-returnjoin(workdir,entry!.filename)
+returnjoin(destination,entry!.filename)
 }
 
 exportfunctionnpm(args: string[],cwd: string): string{