test(repo): fix live e2e reboot flake and cut cancellations (#7016)

## TL;DR

fixes the live e2e ssl reboot flake and stops develop pushes from
cancelling a running live suite

- `ssl-enforcement update` reboots the database and `get` can report it
applied before the reboot ends, so the reboot landed in the next file
and killed `migration up` with `57P01`. the test now waits for a new
postmaster after the toggle and the restore, and `queryLiveDb` no longer
crashes when a restart drops an idle client

- merge queue bursts land develop pushes closer together than one live
run, so each push cancelled the running suite and turned its commit red.
pushes now share a lane that never cancels a running suite

## ref

- spotted in: [migration
up](https://github.com/supabase/cli/actions/runs/37284888772/job/111681321268)
[cancelled
run](https://github.com/supabase/cli/actions/runs/37296027587)

Author: 7ttp

.github/MAINTAINERS.md

@@ -87,7 +87,7 @@ from **Issues → Labels** if it is missing.
 
 [`Live E2E`](./workflows/live-e2e.yml) exercises managed staging after every push
 to `develop`, daily at 06:23 UTC, and on manual dispatch. New `develop` pushes
-cancel superseded push runs; nightly and manual runs execute independently.
+replace only a queued push run; nightly and manual runs execute independently.
 Nightly runs do not depend on a new beta version: they also detect staging
 changes between CLI releases.
 

.github/workflows/live-e2e-suite.yml

@@ -12,11 +12,6 @@ on:
         required: false
         type: string
         default: live-e2e
-      cancel_in_progress:
-        description: Cancel an older run in the same group
-        required: false
-        type: boolean
-        default: false
     secrets:
       SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN:
         required: true
@@ -33,7 +28,7 @@ jobs:
     timeout-minutes: 30
     concurrency:
       group: ${{ inputs.concurrency_group }}
-      cancel-in-progress: ${{ inputs.cancel_in_progress }}
+      cancel-in-progress: false
     env:
       SUPABASE_LIVE_API_URL: https://api.supabase.green
       SUPABASE_LIVE_PROJECT_NAME: supabase-cli-live

.github/workflows/live-e2e.yml

@@ -17,10 +17,9 @@ jobs:
     uses: ./.github/workflows/live-e2e-suite.yml
     with:
       ref: ${{ github.sha }}
-      # Pushes to develop share a branch group and cancel superseded pushes.
+      # Pushes to develop share a branch group that never cancels a running suite.
       # Manual, scheduled, and stable-gate runs use independent groups because
       # they each own separate temporary projects and may run concurrently.
       concurrency_group: live-e2e-${{ github.event_name == 'push' && github.ref || github.run_id }}
-      cancel_in_progress: ${{ github.event_name == 'push' }}
     secrets:
       SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN: ${{ secrets.SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN }}

apps/cli/src/commands/ssl-enforcement/update/update.live.test.ts

@@ -4,6 +4,7 @@ import { expect } from "vitest";
 import {
   experimentalProjectLiveFlags,
   type LiveFixtures,
+  queryLiveDb,
   requireLiveSuccess,
   test,
   throwWithCleanup,
@@ -15,10 +16,11 @@ type LiveRun = Awaited<ReturnType<LiveFixtures["cli"]>>;
 // Bound the polled gets and the restore so one hung subprocess cannot exhaust
 // the live testTimeout and leave the shared project with a flipped posture.
 const POLL_ATTEMPT_EXIT_TIMEOUT_MS = 20_000;
+const DB_PROBE_TIMEOUT_MS = 10_000;
 const RESTORE_EXIT_TIMEOUT_MS = 60_000;
 const PROOF_INTERVAL_MS = 2_000;
 const PROOF_TIMEOUT_MS = 60_000;
-// Fits the worst case (bounded capture/toggle + two proofs + restore) with headroom.
+// Fits the worst case (bounded capture/toggle + proofs and reboot waits + restore) with headroom.
 const LIVE_TIMEOUT_MS = 600_000;
 
 const SslEnforcementPosture = Schema.Struct({
@@ -108,6 +110,54 @@ function expectApplied(
   );
 }
 
+function postmasterStart(dbUrl: string, label: string) {
+  return Effect.tryPromise({
+    try: () =>
+      queryLiveDb<{ started: string }>(
+        dbUrl,
+        "select pg_postmaster_start_time()::text as started",
+        undefined,
+        DB_PROBE_TIMEOUT_MS,
+      ),
+    catch: (error) =>
+      new SslEnforcementLiveError({
+        message: `${label}: ${error instanceof Error ? error.message : String(error)}`,
+        cause: error,
+      }),
+  }).pipe(
+    Effect.timeoutOrElse({
+      duration: DB_PROBE_TIMEOUT_MS,
+      orElse: () =>
+        Effect.fail(
+          new SslEnforcementLiveError({
+            message: `${label}: no answer within ${DB_PROBE_TIMEOUT_MS}ms`,
+          }),
+        ),
+    }),
+    Effect.flatMap(([row]) =>
+      row === undefined
+        ? Effect.fail(
+            new SslEnforcementLiveError({ message: `${label}: no postmaster start time` }),
+          )
+        : Effect.succeed(row.started),
+    ),
+  );
+}
+
+// Each posture change reboots the database, possibly after get already reports it applied.
+function awaitReboot(dbUrl: string, previous: string, label: string) {
+  return postmasterStart(dbUrl, label).pipe(
+    Effect.filterOrFail(
+      (started) => started !== previous,
+      () =>
+        new SslEnforcementLiveError({
+          message: `${label}: the postmaster started at ${previous} is still running`,
+        }),
+    ),
+    Effect.retry(proofSchedule),
+  );
+}
+
 // Not wired to the test `signal`: an interrupt SIGKILLs an in-flight restore
 // mid-request (the run's scope release kills the process group), so letting the
 // bounded restore run out is strictly safer.
@@ -125,6 +175,10 @@ test(
           RESTORE_EXIT_TIMEOUT_MS,
         );
         const posture = captured.currentConfig.database;
+        const initialStart = yield* postmasterStart(
+          project.dbUrl,
+          "postmaster start capture for ssl-enforcement update",
+        );
 
         const toggle = Effect.gen(function* () {
           const updated = yield* cliEffect(
@@ -146,7 +200,13 @@ test(
             !posture,
             "ssl-enforcement get proof for ssl-enforcement update",
           );
+          return yield* awaitReboot(
+            project.dbUrl,
+            initialStart,
+            "database reboot after ssl-enforcement update",
+          );
         });
+        const toggleExit = yield* Effect.exit(toggle);
 
         const restore = Effect.gen(function* () {
           const restored = yield* cliEffect(
@@ -160,10 +220,16 @@ test(
             posture,
             "ssl-enforcement get proof of the restored posture for ssl-enforcement update",
           );
+          if (Exit.isSuccess(toggleExit)) {
+            yield* awaitReboot(
+              project.dbUrl,
+              toggleExit.value,
+              "database reboot after the ssl-enforcement update restore",
+            );
+          }
         });
 
         // The restore runs whatever the toggle did; neither failure hides the other.
-        const toggleExit = yield* Effect.exit(toggle);
         const restoreExit = yield* Effect.exit(restore);
         return {
           toggleError: Exit.isFailure(toggleExit) ? Cause.squash(toggleExit.cause) : undefined,

apps/cli/tests/helpers/live.ts

@@ -278,9 +278,16 @@ export async function queryLiveDb<T extends Record<string, unknown>>(
   dbUrl: string,
   query: string,
   values?: ReadonlyArray<unknown>,
+  connectionTimeoutMillis?: number,
 ): Promise<T[]> {
   // Verified TLS against the Supabase CA, so the query works whatever the project's SSL enforcement.
-  const client = new pg.Client({ connectionString: dbUrl, ssl: { ca: rootCaBundle() } });
+  const client = new pg.Client({
+    connectionString: dbUrl,
+    ssl: { ca: rootCaBundle() },
+    connectionTimeoutMillis,
+  });
+  // A restart that drops an idle or closing client surfaces only as an `error` event.
+  client.on("error", () => {});
   await client.connect();
   try {
     const result = await client.query(query, values === undefined ? undefined : [...values]);