ci(e2e): retry live vitest failures

test/e2e-scenario/docs/README.md

@@ -76,8 +76,17 @@ npx vitest run --project e2e-vitest-support --silent=false --reporter=default
 # Opt-in live Vitest scenarios
 npm run build:cli
 NEMOCLAW_RUN_E2E_SCENARIOS=1 npx vitest run --project e2e-scenarios-live --silent=false --reporter=default
+
+# Force two retries locally (three total attempts) for external-service flakes
+NEMOCLAW_RUN_E2E_SCENARIOS=1 NEMOCLAW_E2E_RETRIES=2 npx vitest run --project e2e-scenarios-live
 ```
 
+Live Vitest E2E projects retry failed tests automatically in CI. The default is
+2 retries after the first failure (3 total attempts). Local opt-in runs default
+to no full-test retry; set `NEMOCLAW_E2E_RETRIES=<count>` to override either
+local or CI behavior. Overrides are capped at 5 retries so a typo cannot create
+unbounded credentialed live infrastructure attempts.
+
 The retired `--emit-matrix`, direct `--scenarios` execution, and `--plan-only`
 paths must not be reintroduced.
 

test/e2e-scenario/support-tests/e2e-live-project-config.test.ts

@@ -9,12 +9,14 @@ import {
   shouldRunLiveE2EScenarios,
 } from "../fixtures/live-project-gate.ts";
 import config from "../../../vitest.config.ts";
+import { resolveE2ERetryCount } from "../../helpers/e2e-retries.ts";
 import { readYaml, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts";
 
 interface ProjectConfig {
   test?: {
     name?: string;
     include?: string[];
+    retry?: number;
   };
 }
 
@@ -88,6 +90,31 @@ describe("gated E2E Vitest projects", () => {
     expect(shouldRunBranchValidationE2E({ NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" })).toBe(true);
   });
 
+  it("configures automatic retries only for live E2E Vitest projects", () => {
+    const expectedRetries = resolveE2ERetryCount();
+
+    expect(projectConfig("cli").test?.retry).toBeUndefined();
+    expect(projectConfig("e2e-vitest-support").test?.retry).toBeUndefined();
+    expect(projectConfig("e2e-scenarios-live").test?.retry).toBe(expectedRetries);
+    expect(projectConfig("e2e-branch-validation").test?.retry).toBe(expectedRetries);
+  });
+
+  it("defaults live E2E retries to CI only and supports explicit overrides", () => {
+    expect(resolveE2ERetryCount({})).toBe(0);
+    expect(resolveE2ERetryCount({ CI: "0" })).toBe(0);
+    expect(resolveE2ERetryCount({ CI: "1" })).toBe(2);
+    expect(resolveE2ERetryCount({ CI: "true" })).toBe(2);
+    expect(resolveE2ERetryCount({ GITHUB_ACTIONS: "true" })).toBe(2);
+    expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "0" })).toBe(0);
+    expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "3" })).toBe(3);
+    expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "5" })).toBe(5);
+    expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "6" })).toBe(5);
+    expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "999999" })).toBe(5);
+    expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "-1" })).toBe(2);
+    expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "1.5" })).toBe(2);
+    expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "invalid" })).toBe(2);
+  });
+
   it("sets the branch-validation sentinel in the reusable workflow Vitest step", () => {
     const workflow = readYaml<BranchValidationWorkflow>(
       ".github/workflows/e2e-branch-validation.yaml",

test/helpers/e2e-retries.ts

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+type Environment = Record<string, string | undefined>;
+
+const DEFAULT_CI_E2E_RETRIES = 2;
+const DEFAULT_LOCAL_E2E_RETRIES = 0;
+const MAX_E2E_RETRIES = 5;
+
+export function resolveE2ERetryCount(env: Environment = process.env): number {
+  const override = env.NEMOCLAW_E2E_RETRIES?.trim();
+  if (override && /^[0-9]+$/.test(override)) {
+    return Math.min(Number.parseInt(override, 10), MAX_E2E_RETRIES);
+  }
+
+  const envIsCi = env.GITHUB_ACTIONS === "true" || env.CI === "true" || env.CI === "1";
+  return envIsCi ? DEFAULT_CI_E2E_RETRIES : DEFAULT_LOCAL_E2E_RETRIES;
+}

vitest.config.ts

@@ -8,6 +8,7 @@ import {
   shouldRunInstallerIntegration,
   shouldRunLiveE2EScenarios,
 } from "./test/e2e-scenario/fixtures/live-project-gate.ts";
+import { resolveE2ERetryCount } from "./test/helpers/e2e-retries";
 import { testTimeout } from "./test/helpers/timeouts";
 
 const isGithubActions = process.env.GITHUB_ACTIONS === "true";
@@ -16,6 +17,7 @@ const LIVE_E2E_PROJECT_TIMEOUT_MS = 30 * 60 * 1000;
 const runInstallerIntegration = shouldRunInstallerIntegration();
 const runLiveE2EScenarios = shouldRunLiveE2EScenarios();
 const runBranchValidationE2E = shouldRunBranchValidationE2E();
+const e2eRetryCount = resolveE2ERetryCount();
 
 export default defineConfig({
   test: {
@@ -82,6 +84,10 @@ export default defineConfig({
         test: {
           name: "e2e-scenarios-live",
           testTimeout: testTimeout(LIVE_E2E_PROJECT_TIMEOUT_MS),
+          // Vitest counts retries after the initial failure. In CI the default
+          // value of 2 gives live E2Es up to three total attempts while keeping
+          // local opt-in runs single-shot unless NEMOCLAW_E2E_RETRIES is set.
+          retry: e2eRetryCount,
           include: runLiveE2EScenarios ? ["test/e2e-scenario/live/**/*.test.ts"] : [],
           // Live scenario tests are opt-in because they install, onboard, and
           // mutate real NemoClaw/OpenShell state. Run explicitly with:
@@ -91,6 +97,7 @@ export default defineConfig({
       {
         test: {
           name: "e2e-branch-validation",
+          retry: e2eRetryCount,
           include: runBranchValidationE2E ? ["test/e2e/brev-e2e.test.ts"] : [],
           // Branch validation E2E: rsyncs the branch over a Brev instance
           // provisioned from the published NemoClaw launchable image and