feat(tasks): benchmark NodeJS parser (#2770)

Add NodeJS parser to benchmarks.

Previous attempt #2724 did not work due CodSpeed producing very
inaccurate results (CodSpeedHQ/action#96).

This version runs the actual benchmarks without CodSpeed's
instrumentation. Then another faux-benchmark runs within Codspeed's
instrumented action and just performs meaningless calculations in a loop
for as long as is required to take same amount of time as the original
uninstrumented benchmarks took.

It's unfortunate that we therefore don't get flame graphs on CodSpeed,
but this seems to be the best we can do for now.

.github/workflows/benchmark.yml

@@ -6,6 +6,8 @@ on:
     types: [opened, synchronize]
     paths:
       - '**/*.rs'
+      - 'napi/parser/**/*.js'
+      - 'napi/parser/**/*.mjs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
       - 'tasks/benchmark/codspeed/*.mjs'
@@ -15,6 +17,8 @@ on:
       - bench-*
     paths:
       - '**/*.rs'
+      - 'napi/parser/**/*.js'
+      - 'napi/parser/**/*.mjs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
       - 'tasks/benchmark/codspeed/*.mjs'
@@ -31,7 +35,7 @@ jobs:
       matrix:
         # Run each benchmark in own job.
         # Linter benchmark is by far the slowest, so split each fixture into own job.
-        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap]
+        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, parser_napi]
         include:
           - component: linter
             fixture: 0
@@ -53,7 +57,7 @@ jobs:
       - name: Install Rust Toolchain
         uses: ./.github/actions/rustup
         with:
-          shared-key: 'benchmark'
+          shared-key: ${{ matrix.component == 'parser_napi' && 'benchmark_napi' || 'benchmark' }}
           save-cache: ${{ github.ref_name == 'main' }}
 
       - name: Install codspeed
@@ -77,12 +81,31 @@ jobs:
           pnpm install
           node capture.mjs &
 
-      - name: Build Benchmark
+      # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
+      # so instead run real benchmark without CodSpeed's instrumentation and save the results.
+      # Then "Run benchmark" step below runs a loop of some simple Rust code the number
+      # of times required to take same amount of time as the real benchmark took.
+      # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
+      - name: Build NAPI Benchmark
+        if: ${{ matrix.component == 'parser_napi'}}
+        working-directory: ./napi/parser
+        run: |
+          corepack enable
+          pnpm install
+          pnpm build
+
+      - name: Run NAPI Benchmark
+        if: ${{ matrix.component == 'parser_napi'}}
+        working-directory: ./napi/parser
+        run: node parse.bench.mjs
+
+      - name: Build benchmark
         env:
           RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
         shell: bash
         run: |
-          cargo build --release -p oxc_benchmark --features codspeed --bench ${{ matrix.component }}
+          cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} \
+            --features ${{ matrix.component == 'parser_napi' && 'codspeed_napi' || 'codspeed'}}
           mkdir -p target/codspeed/oxc_benchmark/
           mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
           rm -rf target/codspeed/oxc_benchmark/*.d

napi/parser/package.json

@@ -7,11 +7,10 @@
     "bench": "vitest bench"
   },
   "devDependencies": {
-    "@codspeed/vitest-plugin": "^3.1.0",
     "@napi-rs/cli": "^2.18.0",
     "es-module-lexer": "^1.4.1",
     "flatbuffers": "^23.5.26",
-    "vitest": "^1.3.1"
+    "tinybench": "^2.6.0"
   },
   "engines": {
     "node": ">=14.*"

napi/parser/parse.bench.mjs

@@ -1,10 +1,11 @@
 import {fileURLToPath} from 'url';
 import {join as pathJoin} from 'path';
 import {readFile, writeFile} from 'fs/promises';
-import assert from 'assert';
-import {bench} from 'vitest';
+import {Bench} from 'tinybench';
 import {parseSync} from './index.js';
 
+const IS_CI = !!process.env.CI;
+
 const urls = [
     // TypeScript syntax (2.81MB)
     'https://raw.githubusercontent.com/microsoft/TypeScript/v5.3.3/src/compiler/checker.ts',
@@ -28,9 +29,9 @@ const files = await Promise.all(urls.map(async (url) => {
     let code;
     try {
         code = await readFile(path, 'utf8');
-        console.log('Found cached file:', filename);
+        if (IS_CI) console.log('Found cached file:', filename);
     } catch {
-        console.log('Downloading:', filename);
+        if (IS_CI) console.log('Downloading:', filename);
         const res = await fetch(url);
         code = await res.text();
         await writeFile(path, code);
@@ -39,10 +40,27 @@ const files = await Promise.all(urls.map(async (url) => {
     return {filename, code};
 }));
 
+const bench = new Bench();
+
 for (const {filename, code} of files) {
-    bench(`parser(napi)[${filename}]`, () => {
+    bench.add(`parser_napi[${filename}]`, () => {
         const res = parseSync(code, {sourceFilename: filename});
-        assert(res.errors.length === 0);
         JSON.parse(res.program);
     });
 }
+
+console.log('Warming up');
+await bench.warmup();
+console.log('Running benchmarks');
+await bench.run();
+console.table(bench.table());
+
+// If running on CI, save results to file
+if (IS_CI) {
+    const dataDir = process.env.DATA_DIR;
+    const results = bench.tasks.map(task => ({
+        filename: task.name.match(/\[(.+)\]$/)[1],
+        duration: task.result.period / 1000, // In seconds
+    }));
+    await writeFile(pathJoin(dataDir, 'results.json'), JSON.stringify(results));
+}