diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index b01ac687fc..762573f032 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -76,8 +76,17 @@ npx vitest run --project e2e-vitest-support --silent=false --reporter=default # Opt-in live Vitest scenarios npm run build:cli NEMOCLAW_RUN_E2E_SCENARIOS=1 npx vitest run --project e2e-scenarios-live --silent=false --reporter=default + +# Force two retries locally (three total attempts) for external-service flakes +NEMOCLAW_RUN_E2E_SCENARIOS=1 NEMOCLAW_E2E_RETRIES=2 npx vitest run --project e2e-scenarios-live ``` +Live Vitest E2E projects retry failed tests automatically in CI. The default is +2 retries after the first failure (3 total attempts). Local opt-in runs default +to no full-test retry; set `NEMOCLAW_E2E_RETRIES=` to override either +local or CI behavior. Overrides are capped at 5 retries so a typo cannot create +unbounded credentialed live infrastructure attempts. + The retired `--emit-matrix`, direct `--scenarios` execution, and `--plan-only` paths must not be reintroduced. diff --git a/test/e2e-scenario/support-tests/e2e-live-project-config.test.ts b/test/e2e-scenario/support-tests/e2e-live-project-config.test.ts index 0a969176ce..0c9dc5acd8 100644 --- a/test/e2e-scenario/support-tests/e2e-live-project-config.test.ts +++ b/test/e2e-scenario/support-tests/e2e-live-project-config.test.ts @@ -9,12 +9,14 @@ import { shouldRunLiveE2EScenarios, } from "../fixtures/live-project-gate.ts"; import config from "../../../vitest.config.ts"; +import { resolveE2ERetryCount } from "../../helpers/e2e-retries.ts"; import { readYaml, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; interface ProjectConfig { test?: { name?: string; include?: string[]; + retry?: number; }; } @@ -88,6 +90,31 @@ describe("gated E2E Vitest projects", () => { expect(shouldRunBranchValidationE2E({ NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" })).toBe(true); }); + it("configures automatic retries only for live E2E Vitest projects", () => { + const expectedRetries = resolveE2ERetryCount(); + + expect(projectConfig("cli").test?.retry).toBeUndefined(); + expect(projectConfig("e2e-vitest-support").test?.retry).toBeUndefined(); + expect(projectConfig("e2e-scenarios-live").test?.retry).toBe(expectedRetries); + expect(projectConfig("e2e-branch-validation").test?.retry).toBe(expectedRetries); + }); + + it("defaults live E2E retries to CI only and supports explicit overrides", () => { + expect(resolveE2ERetryCount({})).toBe(0); + expect(resolveE2ERetryCount({ CI: "0" })).toBe(0); + expect(resolveE2ERetryCount({ CI: "1" })).toBe(2); + expect(resolveE2ERetryCount({ CI: "true" })).toBe(2); + expect(resolveE2ERetryCount({ GITHUB_ACTIONS: "true" })).toBe(2); + expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "0" })).toBe(0); + expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "3" })).toBe(3); + expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "5" })).toBe(5); + expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "6" })).toBe(5); + expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "999999" })).toBe(5); + expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "-1" })).toBe(2); + expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "1.5" })).toBe(2); + expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "invalid" })).toBe(2); + }); + it("sets the branch-validation sentinel in the reusable workflow Vitest step", () => { const workflow = readYaml( ".github/workflows/e2e-branch-validation.yaml", diff --git a/test/helpers/e2e-retries.ts b/test/helpers/e2e-retries.ts new file mode 100644 index 0000000000..12f3984c89 --- /dev/null +++ b/test/helpers/e2e-retries.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type Environment = Record; + +const DEFAULT_CI_E2E_RETRIES = 2; +const DEFAULT_LOCAL_E2E_RETRIES = 0; +const MAX_E2E_RETRIES = 5; + +export function resolveE2ERetryCount(env: Environment = process.env): number { + const override = env.NEMOCLAW_E2E_RETRIES?.trim(); + if (override && /^[0-9]+$/.test(override)) { + return Math.min(Number.parseInt(override, 10), MAX_E2E_RETRIES); + } + + const envIsCi = env.GITHUB_ACTIONS === "true" || env.CI === "true" || env.CI === "1"; + return envIsCi ? DEFAULT_CI_E2E_RETRIES : DEFAULT_LOCAL_E2E_RETRIES; +} diff --git a/vitest.config.ts b/vitest.config.ts index 80ddf2f6a5..e463c496fd 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,6 +8,7 @@ import { shouldRunInstallerIntegration, shouldRunLiveE2EScenarios, } from "./test/e2e-scenario/fixtures/live-project-gate.ts"; +import { resolveE2ERetryCount } from "./test/helpers/e2e-retries"; import { testTimeout } from "./test/helpers/timeouts"; const isGithubActions = process.env.GITHUB_ACTIONS === "true"; @@ -16,6 +17,7 @@ const LIVE_E2E_PROJECT_TIMEOUT_MS = 30 * 60 * 1000; const runInstallerIntegration = shouldRunInstallerIntegration(); const runLiveE2EScenarios = shouldRunLiveE2EScenarios(); const runBranchValidationE2E = shouldRunBranchValidationE2E(); +const e2eRetryCount = resolveE2ERetryCount(); export default defineConfig({ test: { @@ -82,6 +84,10 @@ export default defineConfig({ test: { name: "e2e-scenarios-live", testTimeout: testTimeout(LIVE_E2E_PROJECT_TIMEOUT_MS), + // Vitest counts retries after the initial failure. In CI the default + // value of 2 gives live E2Es up to three total attempts while keeping + // local opt-in runs single-shot unless NEMOCLAW_E2E_RETRIES is set. + retry: e2eRetryCount, include: runLiveE2EScenarios ? ["test/e2e-scenario/live/**/*.test.ts"] : [], // Live scenario tests are opt-in because they install, onboard, and // mutate real NemoClaw/OpenShell state. Run explicitly with: @@ -91,6 +97,7 @@ export default defineConfig({ { test: { name: "e2e-branch-validation", + retry: e2eRetryCount, include: runBranchValidationE2E ? ["test/e2e/brev-e2e.test.ts"] : [], // Branch validation E2E: rsyncs the branch over a Brev instance // provisioned from the published NemoClaw launchable image and