[tests] Third-party performance test suite that runs Octane, JetStream, and Web Tooling Benchmark

.github/workflows/ci.yml

@@ -118,15 +118,18 @@ jobs:
       - name: Run integration tests on nightly without alloc_error feature enabled
         run: cargo brimstone-test --release --no-default-features --features nightly -- --reindex --ignore-unimplemented
 
-  build-benchmarks:
+  build-perf:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
 
-      - name: Build benchmarks
+      - name: Build performance test suite
+        run: cargo build -p brimstone_perf
+
+      - name: Build benchmarks tests
         run: cargo bench --no-run
 
   build-fuzzer:

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "tests",
   "tests/fuzz",
   "tests/harness",
+  "tests/perf",
 ]
 resolver = "2"
 

src/benches/README.md

@@ -1,6 +1,6 @@
 # Benchmarks
 
-Brimstone's performance testing is found in this directory.
+Brimstone's first party performance microbenchmark testing is found in this directory.
 
 ## Installation
 

tests/perf/.gitignore

@@ -0,0 +1 @@
+vendor

tests/perf/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "brimstone_perf"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[[bin]]
+name = "bs-perf"
+path = "src/main.rs"
+
+[dependencies]
+clap = { workspace = true, features = ["derive"] }
+regex.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+
+[lints]
+workspace = true

tests/perf/README.md

@@ -0,0 +1,58 @@
+# Performance suite
+
+Runs standard third-party JavaScript performance suites against brimstone.
+
+Supports [Octane](https://github.com/chromium/octane), [JetStream](https://github.com/WebKit/JetStream), and [Web Tooling Benchmark](https://github.com/v8/web-tooling-benchmark/).
+
+## Installation
+
+Suites are vendored into `vendor/` with `./install.sh`. Requires `git`, and `npm` for the Web Tooling Benchmark build.
+
+## Run
+
+```
+# Build bs in release mode and run Octane:
+cargo run -p brimstone_perf -- --suite octane
+
+# Run only specific benchmarks (case-insensitive substring match, repeatable):
+cargo run -p brimstone_perf -- --suite octane --bench richards --bench splay
+
+# Structured JSON, written to a file:
+cargo run -p brimstone_perf -- --suite octane --format json --out octane.json
+```
+
+Useful flags: `--bs-path <path>` (use an existing binary instead of building),
+`--vendor-dir <dir>`, `--format pretty|json`, `--out <file>`, `--flamegraph [<file>]`.
+
+## Profiling a run (flamegraph)
+
+Pass `--flamegraph` to profile the `bs` process for a run and write a flamegraph SVG,
+using the [`flamegraph`](https://github.com/flamegraph-rs/flamegraph) CLI.
+
+```
+# Initial setup: make sure flamegraph is installed
+cargo install flamegraph
+
+# Run a single benchmark with profiling and write flamegraph output
+cargo run --release -p brimstone_perf -- --suite octane --bench raytrace --flamegraph
+```
+
+Notes:
+
+- **Flamegraph arguments.** `--flamegraph-arg=<arg>` forwards any raw `flamegraph` argument
+- **Output file.** `--flamegraph=<file>` writes the output to a paricular file. Defaults to
+  `flamegraph.svg` if no file name is provided.
+- **Build with symbols.** When `--flamegraph` is set and the harness builds `bs` itself, it
+  builds release *with debug info* (`CARGO_PROFILE_RELEASE_DEBUG=true`) so frames are named.
+  If you supply your own `--bs-path`, build it with debug symbols yourself.
+
+## How it works
+
+The harness runs `bs` as a subprocess and prints results to stdout. `bs` is run with:
+
+```
+bs --expose-test-shell-compat <suite files...> <driver> [-- <forwarded cli args>]
+```
+
+`--expose-test-shell-compat` installs the shell host functions that benchmark suites
+expect. Everything after `--` is exposed as `globalThis.arguments`.

tests/perf/install.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Installs the 3p performance suites into ./vendor.
+
+set -e
+
+CURRENT_DIR=$(cd "$(dirname "$0")" && pwd)
+VENDOR_DIR="$CURRENT_DIR/vendor"
+
+# Pinned commits for each 3p suite
+OCTANE_COMMIT=570ad1ccfe86e3eecba0636c8f932ac08edec517
+JETSTREAM_COMMIT=b7babdf323e64e69bd2f6c376189c15825f5c73a
+WEB_TOOLING_COMMIT=4a12828c6a1eed02a70c011bd080445dd319a05f
+
+OCTANE_DIR="$VENDOR_DIR/octane"
+JETSTREAM_DIR="$VENDOR_DIR/jetstream"
+WEB_TOOLING_BENCHMARK_DIR="$VENDOR_DIR/web-tooling-benchmark"
+
+mkdir -p "$VENDOR_DIR"
+
+# Shallow-fetch a single pinned commit: clone_pinned <dir> <url> <commit>.
+clone_pinned() {
+  rm -rf "$1" &&
+  git init -q "$1" &&
+  git -C "$1" remote add origin "$2" &&
+  git -C "$1" fetch -q --depth 1 origin "$3" &&
+  git -C "$1" checkout -q FETCH_HEAD
+}
+
+if [ ! -d "$OCTANE_DIR/.git" ]; then
+  echo "==> Installing Octane"
+  clone_pinned "$OCTANE_DIR" https://github.com/chromium/octane "$OCTANE_COMMIT"
+else
+  echo "==> Octane already installed"
+fi
+
+if [ ! -d "$JETSTREAM_DIR/.git" ]; then
+  echo "==> Installing JetStream"
+  clone_pinned "$JETSTREAM_DIR" https://github.com/WebKit/JetStream "$JETSTREAM_COMMIT"
+else
+  echo "==> JetStream already installed"
+fi
+
+WEB_TOOLING_BENCHMARKS="acorn babel babel-minify babylon buble chai coffeescript espree \
+  esprima jshint lebab postcss prepack prettier source-map terser typescript uglify-js"
+
+if [ ! -f "$WEB_TOOLING_BENCHMARK_DIR/dist/cli-acorn.js" ]; then
+  echo "==> Installing Web Tooling Benchmark"
+  if [ ! -d "$WEB_TOOLING_BENCHMARK_DIR/.git" ]; then
+    clone_pinned "$WEB_TOOLING_BENCHMARK_DIR" https://github.com/v8/web-tooling-benchmark "$WEB_TOOLING_COMMIT"
+  fi
+  (
+    cd "$WEB_TOOLING_BENCHMARK_DIR"
+    [ -d node_modules ] || npm ci
+
+    # Build separate bundles for each individual benchmark so they can be run independently.
+    for bench in $WEB_TOOLING_BENCHMARKS; do
+      echo "    building standalone bundle for $bench"
+      npm run build -- --env.only "$bench"
+      mv dist/cli.js "dist/cli-$bench.js"
+    done
+  )
+else
+  echo "==> Web Tooling Benchmark already installed"
+fi
+
+echo "Done. Vendored suites are in $VENDOR_DIR"

tests/perf/shims/octane.js

@@ -0,0 +1,12 @@
+// Print easily parseable results from a particular run of Octane
+BenchmarkSuite.RunSuites({
+  NotifyResult: function (name, result) {
+    print("RESULT " + name + " " + result);
+  },
+  NotifyError: function (name, error) {
+    print("ERROR " + name + " " + error);
+  },
+  NotifyScore: function (score) {
+    print("SCORE " + score);
+  },
+});