feat(benchmarks): add canonical benchmark page (#125)

* feat(benchmarks): add canonical benchmark page

* docs(benchmarks): add comparison context

* docs(benchmarks): add external comparison table

* style(theme): improve accent text contrast

Author: jack-arturo

.github/workflows/benchmarks-update.yml

@@ -0,0 +1,100 @@
+name: Update Benchmarks
+
+on:
+  repository_dispatch:
+    types: [benchmarks-update]
+  workflow_dispatch:
+    inputs:
+      source_sha:
+        description: "automem commit SHA, branch, or tag to sync"
+        required: false
+      reason:
+        description: "Why benchmark data needs updating"
+        required: false
+
+jobs:
+  update-benchmarks:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Resolve source ref
+        id: resolve
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+        run: |
+          SOURCE_REPO="verygoodplugins/automem"
+          SOURCE_REF="${{ github.event.client_payload.source_sha || github.event.inputs.source_sha || 'main' }}"
+          REASON="${{ github.event.client_payload.reason || github.event.inputs.reason || 'Benchmark update' }}"
+          FULL_SHA=$(gh api "repos/$SOURCE_REPO/commits/$SOURCE_REF" -q '.sha')
+          SHORT_SHA="${FULL_SHA:0:7}"
+
+          echo "source_repo=$SOURCE_REPO" >> "$GITHUB_OUTPUT"
+          echo "source_sha=$FULL_SHA" >> "$GITHUB_OUTPUT"
+          echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT"
+          echo "reason=$REASON" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout automem source
+        uses: actions/checkout@v6
+        with:
+          repository: ${{ steps.resolve.outputs.source_repo }}
+          ref: ${{ steps.resolve.outputs.source_sha }}
+          path: .source-repo
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Sync benchmark data
+        run: npm run sync-benchmarks -- --source .source-repo
+
+      - name: Build
+        run: npm run build
+        env:
+          NODE_OPTIONS: "--max-old-space-size=8192"
+
+      - name: Detect changes
+        id: diff
+        run: |
+          if git diff --quiet -- src/data/benchmarks.json; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create benchmark update PR
+        if: steps.diff.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+          SHORT_SHA: ${{ steps.resolve.outputs.short_sha }}
+          FULL_SHA: ${{ steps.resolve.outputs.source_sha }}
+          REASON: ${{ steps.resolve.outputs.reason }}
+        run: |
+          BRANCH="docs/benchmarks-$SHORT_SHA"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add src/data/benchmarks.json
+          git commit -m "docs(benchmarks): update data from automem@$SHORT_SHA"
+          git push --set-upstream origin "$BRANCH"
+          gh pr create \
+            --title "docs(benchmarks): update benchmark data from automem@$SHORT_SHA" \
+            --body "## Summary
+
+- Syncs benchmark data from verygoodplugins/automem@$FULL_SHA
+- Source trigger: $REASON
+- Verified with npm run build
+
+## Breaking changes
+
+None." \
+            --base main \
+            --head "$BRANCH"

.gitignore

@@ -36,7 +36,7 @@ img/*.jpeg
 .dev.vars
 
 # EmDash local data
-data/
+/data/
 uploads/
 
 # TypeScript

package.json

@@ -8,6 +8,9 @@
     "start": "astro dev",
     "build": "rm -rf dist .astro node_modules/.astro .wrangler/deploy && node scripts/build-pages.mjs",
     "preview": "astro preview",
+    "sync-benchmarks": "node scripts/sync-benchmarks.mjs",
+    "test": "node --test tests/*.test.mjs",
+    "test:benchmarks": "node --test tests/benchmarks-lib.test.mjs tests/sync-benchmarks.test.mjs",
     "astro": "astro",
     "check-links": "node scripts/check-links.js",
     "postinstall": "patch-package"

scripts/check-links.js

@@ -9,41 +9,96 @@
 import { LinkChecker } from 'linkinator';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
+import http from 'http';
 import net from 'net';
 import fs from 'fs';
-import { spawn } from 'child_process';
+import { spawn, spawnSync } from 'child_process';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const distDir = join(__dirname, '../dist');
 
-// Check if dist exists
-if (!fs.existsSync(distDir)) {
-  console.error('❌ dist/ directory not found. Run `npm run build` first.');
-  process.exit(1);
+export function getServeDir(projectRoot = join(__dirname, '..')) {
+  const distDir = join(projectRoot, 'dist');
+  const cloudflareClientDir = join(distDir, 'client');
+
+  if (
+    fs.existsSync(cloudflareClientDir) &&
+    fs.existsSync(join(cloudflareClientDir, 'index.html'))
+  ) {
+    return cloudflareClientDir;
+  }
+
+  return distDir;
+}
+
+export function getServeCommand(serveDir, port) {
+  if (fs.existsSync(join(serveDir, '_worker.js'))) {
+    return {
+      label: 'Cloudflare Pages worker',
+      command: 'npx',
+      args: ['wrangler', 'pages', 'dev', serveDir, '--port', String(port)],
+    };
+  }
+
+  return {
+    label: 'static files',
+    command: 'npx',
+    args: ['serve', serveDir, '-l', String(port)],
+  };
 }
 
 // Find an available port
 function findPort(startPort = 3456) {
   return new Promise((resolve) => {
     const server = net.createServer();
-    server.listen(startPort, () => {
+    server.listen(startPort, '127.0.0.1', () => {
       server.close(() => resolve(startPort));
     });
     server.on('error', () => resolve(findPort(startPort + 1)));
   });
 }
 
+function killProcessesOnPort(port) {
+  const result = spawnSync('lsof', [`-tiTCP:${port}`, '-sTCP:LISTEN'], {
+    encoding: 'utf8',
+  });
+
+  if (result.error || result.status !== 0 || !result.stdout?.trim()) {
+    return;
+  }
+
+  for (const pid of result.stdout.trim().split(/\s+/)) {
+    try {
+      process.kill(Number(pid), 'SIGKILL');
+    } catch {
+      // Ignore cleanup errors; the primary process may have exited already.
+    }
+  }
+}
+
 // Wait for server to be ready
-async function waitForServer(port, maxAttempts = 20) {
+async function waitForServer(port, maxAttempts = 60) {
   for (let i = 0; i < maxAttempts; i++) {
     try {
       await new Promise((resolve, reject) => {
-        const socket = new net.Socket();
-        socket.setTimeout(500);
-        socket.on('connect', () => { socket.destroy(); resolve(); });
-        socket.on('error', reject);
-        socket.on('timeout', () => { socket.destroy(); reject(); });
-        socket.connect(port, 'localhost');
+        const request = http.get({
+          hostname: 'localhost',
+          port,
+          path: '/',
+          timeout: 1000,
+        }, (response) => {
+          response.resume();
+          if (response.statusCode && response.statusCode < 500) {
+            resolve();
+          } else {
+            reject(new Error(`HTTP ${response.statusCode}`));
+          }
+        });
+
+        request.on('error', reject);
+        request.on('timeout', () => {
+          request.destroy();
+          reject(new Error('HTTP readiness timed out'));
+        });
       });
       return true;
     } catch {
@@ -54,14 +109,26 @@ async function waitForServer(port, maxAttempts = 20) {
 }
 
 async function main() {
+  const projectRoot = join(__dirname, '..');
+  const serveDir = getServeDir(projectRoot);
+
+  // Check if dist exists
+  if (!fs.existsSync(serveDir)) {
+    console.error('❌ dist/ directory not found. Run `npm run build` first.');
+    process.exit(1);
+  }
+
   const port = await findPort();
+  const serveCommand = getServeCommand(serveDir, port);
   console.log(`🚀 Starting server on port ${port}...`);
+  console.log(`📁 Serving ${serveDir} via ${serveCommand.label}`);
 
-  // Start serve in the background (no -s flag so 404s are real 404s)
-  const server = spawn('npx', ['serve', 'dist', '-l', String(port)], {
-    cwd: join(__dirname, '..'),
+  // Start the built site in the background. Cloudflare adapter builds need the
+  // Pages worker for dynamic routes; static builds can use serve directly.
+  const server = spawn(serveCommand.command, serveCommand.args, {
+    cwd: projectRoot,
     stdio: ['ignore', 'pipe', 'pipe'],
-    detached: true
+    detached: false
   });
 
   let serverPid = server.pid;
@@ -72,6 +139,12 @@ async function main() {
     spawnError = err;
     console.error('❌ Failed to spawn server:', err);
   });
+  server.stdout?.on('data', (chunk) => {
+    if (process.env.CHECK_LINKS_DEBUG) process.stdout.write(chunk);
+  });
+  server.stderr?.on('data', (chunk) => {
+    if (process.env.CHECK_LINKS_DEBUG) process.stderr.write(chunk);
+  });
 
   try {
     // Check if spawn failed before waiting
@@ -103,6 +176,8 @@ async function main() {
         'railway.com',
         'api.pirsch.io', 
         'echodash.com',  // Bot protection returns 403
+        'github.com',    // GitHub edit/source links often rate-limit CI
+        'automem.ai',    // Canonical/OG URLs point at production before deploy
         'localhost:4321', // Dev server references
         'localhost:4322',
         'localhost:4323',
@@ -128,15 +203,18 @@ async function main() {
     // Only kill if we have a valid PID
     if (serverPid && typeof serverPid === 'number') {
       try { 
-        process.kill(-serverPid); 
+        server.kill('SIGTERM');
       } catch (killErr) {
         // Ignore kill errors in cleanup
       }
     }
+    killProcessesOnPort(port);
   }
 }
 
-main().catch((err) => {
-  console.error('Error:', err);
-  process.exit(1);
-});
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  main().catch((err) => {
+    console.error('Error:', err);
+    process.exit(1);
+  });
+}

scripts/file-doc-map.json

@@ -75,6 +75,9 @@
     "railway.json": ["deployment/railway"],
     "railway.toml": ["deployment/railway"],
     "README.md": ["overview", "getting-started/introduction"],
+    "benchmarks/EXPERIMENT_LOG.md": ["site:/benchmarks", "overview", "core-concepts/hybrid-search", "research"],
+    "benchmarks/publication/**": ["site:/benchmarks", "overview", "core-concepts/hybrid-search", "research"],
+    "docs/BENCHMARK_JUDGE_POLICY.md": ["site:/benchmarks", "research", "development/testing"],
 
     "scripts/health_monitor.py": ["operations/health"],
     "scripts/backup_automem.py": ["operations/backup"],