fix(api): structured 404 envelope for unknown routes + docs drift guard

.github/workflows/deploy-keeperhub.yaml

@@ -199,6 +199,91 @@ jobs:
           CF_ACCESS_CLIENT_ID: ${{ secrets.TO_CF_ACCESS_CLIENT_ID }}
           CF_ACCESS_CLIENT_SECRET: ${{ secrets.TO_CF_ACCESS_CLIENT_SECRET }}
 
+      - name: Probe documented API endpoints
+        # Walks specs/api-coverage.json (committed by the PR-time
+        # check:api-docs guard) and sends a GET to every documented GET
+        # endpoint on the deployed env. We deliberately use GET (not
+        # HEAD): many Next.js App Router handlers and any middleware
+        # that branches on request.method do not implement HEAD, so a
+        # 405 from a healthy GET handler would look like drift.
+        #
+        # Redirects are NOT followed (no curl -L, on purpose): curl
+        # resends custom -H headers - including X-API-Key and the CF
+        # Access secret - across redirects, even cross-host, which would
+        # leak those credentials if BASE_URL ever 30x'd off-origin.
+        #
+        # 404 is treated as drift ONLY for static (parameter-free) paths.
+        # A path-param route (e.g. /api/workflows/{workflowId}) probed
+        # with a fixture id legitimately returns 404 "resource not found"
+        # while the route itself exists and works - indistinguishable from
+        # a missing route file, so we cannot gate a release on it. For
+        # param routes only 5xx and connection failures (000) count as
+        # drift; any 2xx/4xx proves the route is reachable. Static routes
+        # must resolve, so a static 404 is real drift.
+        #
+        # Warn-only on the first deploy after merge so we can see what
+        # fires before it gates a release - flip continue-on-error to
+        # false once a clean run is recorded.
+        continue-on-error: true
+        run: |
+          if [ ! -f specs/api-coverage.json ]; then
+            echo "::error::specs/api-coverage.json not found; PR-time check:api-docs should have produced this"
+            exit 1
+          fi
+          HEADERS=(-H "X-API-Key: ${TEST_API_KEY}")
+          if [[ -n "$CF_ACCESS_CLIENT_ID" && -n "$CF_ACCESS_CLIENT_SECRET" ]]; then
+            HEADERS+=(-H "CF-Access-Client-Id: ${CF_ACCESS_CLIENT_ID}" -H "CF-Access-Client-Secret: ${CF_ACCESS_CLIENT_SECRET}")
+          fi
+          FIXTURE="probe-fixture"
+          FAILED=0
+          REACHED=0
+          while IFS=$'\t' read -r METHOD PATH_TEMPLATE; do
+            [ "$METHOD" = "GET" ] || continue
+            URL_PATH=$(printf '%s' "$PATH_TEMPLATE" | sed -E "s/\{[^}]+\}/${FIXTURE}/g")
+            STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+              "${HEADERS[@]}" \
+              "${{ env.BASE_URL }}${URL_PATH}" || echo "000")
+            HAS_PARAM=0
+            case "$PATH_TEMPLATE" in
+              *"{"*) HAS_PARAM=1 ;;
+            esac
+            case "$STATUS" in
+              200|400|401|403)
+                REACHED=$((REACHED+1))
+                ;;
+              404)
+                if [ "$HAS_PARAM" -eq 1 ]; then
+                  REACHED=$((REACHED+1))
+                  echo "GET ${PATH_TEMPLATE} -> 404 (param route; resource-not-found for a fixture id is expected, route assumed present)"
+                else
+                  FAILED=$((FAILED+1))
+                  echo "::warning::endpoint drift: GET ${PATH_TEMPLATE} returned 404 on ${{ env.BASE_URL }} (static route should resolve)"
+                fi
+                ;;
+              *)
+                FAILED=$((FAILED+1))
+                echo "::warning::endpoint drift: GET ${PATH_TEMPLATE} returned ${STATUS} on ${{ env.BASE_URL }}"
+                ;;
+            esac
+          done < <(jq -r '.endpoints[] | [.method, .path] | @tsv' specs/api-coverage.json)
+          TOTAL=$((REACHED + FAILED))
+          echo "Reached ${REACHED} documented GET endpoints; ${FAILED} drift; ${TOTAL} probed."
+          # Schema sanity guard: if 0 endpoints were probed at all, the
+          # artifact shape changed (renamed field, empty array, jq filter
+          # silently producing nothing) and the probe is meaningless.
+          # We fail rather than greenlight a deploy with no signal.
+          if [ "$TOTAL" -eq 0 ]; then
+            echo "::error::api-coverage probe walked zero endpoints; specs/api-coverage.json may have changed shape"
+            exit 1
+          fi
+          if [ "$FAILED" -gt 0 ]; then
+            exit 1
+          fi
+        env:
+          CF_ACCESS_CLIENT_ID: ${{ secrets.TO_CF_ACCESS_CLIENT_ID }}
+          CF_ACCESS_CLIENT_SECRET: ${{ secrets.TO_CF_ACCESS_CLIENT_SECRET }}
+          TEST_API_KEY: ${{ env.TEST_API_KEY }}
+
       - name: Run Playwright tests against deployed environment
         run: pnpm exec playwright test --grep-invert "happy-paths"
         env:

.github/workflows/pr-checks.yml

@@ -111,6 +111,26 @@ jobs:
             exit 1
           fi
 
+      - name: Check public API docs match route files
+        # Parses every documented (METHOD /api/...) entry in docs/api/*.md
+        # and asserts the corresponding app/api/<path>/route.ts exists and
+        # exports a handler for that method. Also writes
+        # specs/api-coverage.json which the post-deploy live HEAD check
+        # reads. Hackathon teams reported endpoints that 404 in prod or
+        # return shapes that diverge from docs; this guard blocks new drift
+        # at PR time.
+        run: pnpm check:api-docs
+
+      - name: Detect uncommitted api-coverage drift
+        # specs/api-coverage.json is committed and read by the post-deploy
+        # live HEAD check. If the script regenerated it during this job we
+        # need the contributor to commit the regenerated file.
+        run: |
+          if ! git diff --exit-code specs/api-coverage.json; then
+            echo "::error::specs/api-coverage.json is stale. Run 'pnpm check:api-docs' locally and commit the result."
+            exit 1
+          fi
+
   typecheck:
     runs-on: ubuntu-latest
     steps:

app/api/[...slug]/route.ts

@@ -0,0 +1,81 @@
+/**
+ * Catch-all 404 handler for unmatched /api/* paths.
+ *
+ * Next.js's default 404 returns an HTML page, which makes a wrong URL
+ * look identical to a 401 when a builder is probing an API. This handler
+ * returns the canonical {error, detail, request_id} envelope so unknown
+ * routes are unambiguous and machine-parseable.
+ *
+ * Precedence: more specific route segments win over [...slug] in the
+ * Next.js App Router, so existing catch-alls at deeper paths
+ * (app/api/auth/[...all], app/api/execute/[...slug]) keep their own
+ * behavior. This file only fires when no other route matches.
+ *
+ * Cache-Control: no-store so a transient prod misconfig (route file not
+ * shipped, missing env var, etc.) does not get cached at the edge as a
+ * permanent 404.
+ */
+import { type NextRequest, NextResponse } from "next/server";
+import { ApiErrorCodes, apiError } from "@/lib/errors/api-envelope";
+import { ErrorCategory, logUserError } from "@/lib/logging";
+
+function notFoundResponse(request: NextRequest) {
+  const { pathname } = new URL(request.url);
+  // Emit a structured Loki line so frequently-probed unknown routes stay
+  // diagnosable. This is a public surface that bots hammer with random
+  // paths; a miss is a client error, not a system fault, so it must not
+  // page on-call or burn a Sentry event per request. logUserError with no
+  // error argument logs to console/Loki and bumps a bounded Prometheus
+  // counter but skips captureException entirely. The unbounded path lives
+  // in the message (extractContext keeps only the "[api-catch-all]"
+  // prefix as the metric context), never in a metric label, so Prometheus
+  // cardinality stays flat.
+  logUserError(
+    ErrorCategory.VALIDATION,
+    `[api-catch-all] unknown route ${request.method} ${pathname}`,
+    undefined,
+    { method: request.method }
+  );
+  return apiError({
+    status: 404,
+    code: ApiErrorCodes.NOT_FOUND,
+    detail: `Route ${request.method} ${pathname} not found`,
+    requestHeaders: request.headers,
+    headers: { "Cache-Control": "no-store" },
+  });
+}
+
+export function GET(request: NextRequest) {
+  return notFoundResponse(request);
+}
+
+export function POST(request: NextRequest) {
+  return notFoundResponse(request);
+}
+
+export function PUT(request: NextRequest) {
+  return notFoundResponse(request);
+}
+
+export function PATCH(request: NextRequest) {
+  return notFoundResponse(request);
+}
+
+export function DELETE(request: NextRequest) {
+  return notFoundResponse(request);
+}
+
+export function HEAD(request: NextRequest) {
+  // HEAD must not carry a response body (RFC 9110 9.3.2). Reuse the GET
+  // 404 so the status, correlation id, and cache headers stay identical,
+  // then return a body-less response with the same headers.
+  const response = notFoundResponse(request);
+  return new NextResponse(null, {
+    status: response.status,
+    headers: response.headers,
+  });
+}
+
+export function OPTIONS(request: NextRequest) {
+  return notFoundResponse(request);
+}

docs/api/chains.md

@@ -23,35 +23,60 @@ Returns all supported blockchain networks.
 
 ### Response
 
+Returns a bare JSON array of chain objects. The response is not wrapped in a `data` envelope.
+
 ```json
-{
-  "data": [
-    {
-      "id": "chain_1",
-      "chainId": 1,
-      "name": "Ethereum Mainnet",
-      "symbol": "ETH",
-      "chainType": "evm",
-      "defaultPrimaryRpc": "https://...",
-      "defaultFallbackRpc": "https://...",
-      "explorerUrl": "https://etherscan.io",
-      "explorerApiUrl": "https://api.etherscan.io",
-      "isTestnet": false,
-      "isEnabled": true
-    },
-    {
-      "id": "chain_2",
-      "chainId": 11155111,
-      "name": "Sepolia",
-      "symbol": "ETH",
-      "chainType": "evm",
-      "isTestnet": true,
-      "isEnabled": true
-    }
-  ]
-}
+[
+  {
+    "id": "chain_1",
+    "chainId": 1,
+    "name": "Ethereum Mainnet",
+    "symbol": "ETH",
+    "chainType": "evm",
+    "explorerUrl": "https://etherscan.io",
+    "explorerAddressPath": "/address/",
+    "explorerApiUrl": "https://api.etherscan.io",
+    "explorerApiType": "etherscan",
+    "isTestnet": false,
+    "isEnabled": true,
+    "usePrivateMempoolRpc": false
+  },
+  {
+    "id": "chain_2",
+    "chainId": 11155111,
+    "name": "Sepolia",
+    "symbol": "ETH",
+    "chainType": "evm",
+    "explorerUrl": "https://sepolia.etherscan.io",
+    "explorerAddressPath": "/address/",
+    "explorerApiUrl": "https://api-sepolia.etherscan.io",
+    "explorerApiType": "etherscan",
+    "isTestnet": true,
+    "isEnabled": true,
+    "usePrivateMempoolRpc": false
+  }
+]
 ```
 
+### Response Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `id` | string | Internal chain identifier |
+| `chainId` | number | Numeric EVM chain ID (or Solana network ID) |
+| `name` | string | Human-readable chain name |
+| `symbol` | string | Native token symbol |
+| `chainType` | string | `evm` or `solana` (see below) |
+| `explorerUrl` | string \| null | Block explorer base URL |
+| `explorerAddressPath` | string \| null | Path segment appended to `explorerUrl` for address links |
+| `explorerApiUrl` | string \| null | Explorer API base URL for ABI / verification lookups |
+| `explorerApiType` | string \| null | Explorer API family (e.g. `etherscan`, `blockscout`) |
+| `isTestnet` | boolean | Whether this chain is a testnet |
+| `isEnabled` | boolean | Whether the chain is currently available for workflow execution |
+| `usePrivateMempoolRpc` | boolean | Whether KeeperHub routes transactions through a private mempool (Flashbots Protect) by default |
+
+RPC endpoint URLs (`defaultPrimaryRpc`, `defaultFallbackRpc`) are not returned by this endpoint. They may embed provider API keys and are read server-side only; client code should use the user-configurable RPC preferences API instead.
+
 ### Chain Types
 
 | Type | Description |
@@ -114,10 +139,3 @@ Fetches the ABI for a verified contract from the block explorer. The `{chainId}`
 }
 ```
 
-## Alternative ABI Fetch
-
-```http
-GET /api/web3/fetch-abi?address={address}&chainId={chainId}
-```
-
-Alternative endpoint for fetching contract ABIs.

docs/api/user.md

@@ -77,46 +77,57 @@ GET /api/user/rpc-preferences
 
 ### Response
 
+Returns two arrays. `preferences` lists the user's saved overrides; `resolved` lists the effective RPC config for every chain after merging overrides with platform defaults. `source` is `"user"` when a preference is in effect, `"default"` otherwise.
+
 ```json
 {
-  "data": [
+  "preferences": [
+    {
+      "id": "pref_abc123",
+      "chainId": 1,
+      "primaryRpcUrl": "https://custom-rpc.example.com",
+      "fallbackRpcUrl": "https://fallback.example.com",
+      "createdAt": "2026-05-01T12:00:00.000Z",
+      "updatedAt": "2026-05-01T12:00:00.000Z"
+    }
+  ],
+  "resolved": [
     {
       "chainId": 1,
-      "primaryRpc": "https://custom-rpc.example.com",
-      "fallbackRpc": "https://fallback.example.com"
+      "chainName": "Ethereum Mainnet",
+      "primaryRpcUrl": "https://custom-rpc.example.com",
+      "fallbackRpcUrl": "https://fallback.example.com",
+      "primaryWssUrl": null,
+      "fallbackWssUrl": null,
+      "source": "user"
     }
   ]
 }
 ```
 
-### Set RPC Preferences
-
-```http
-POST /api/user/rpc-preferences
-```
-
-### Request Body
-
-```json
-{
-  "chainId": 1,
-  "primaryRpc": "https://custom-rpc.example.com",
-  "fallbackRpc": "https://fallback.example.com"
-}
-```
-
 ### Get Chain RPC Preference
 
 ```http
 GET /api/user/rpc-preferences/{chainId}
 ```
 
-### Update Chain RPC Preference
+### Set or Update Chain RPC Preference
 
 ```http
 PUT /api/user/rpc-preferences/{chainId}
 ```
 
+#### Request Body
+
+```json
+{
+  "primaryRpcUrl": "https://custom-rpc.example.com",
+  "fallbackRpcUrl": "https://fallback.example.com"
+}
+```
+
+`fallbackRpcUrl` is optional. To clear an existing preference, use the DELETE endpoint below instead of sending an empty body.
+
 ### Delete Chain RPC Preference
 
 ```http