feat(stack): persist service logs, ship them to Analytics, and add stack logs history

apps/cli/docs/stack-commands.md

@@ -16,7 +16,7 @@ and password-reconciliation connection uses that administrative role, so a nativ
 | `supabase stack prepare` | Download artifacts without starting services.                                     |
 | `supabase stack start`   | Create or resume the project's stack.                                             |
 | `supabase stack status`  | Show identity, readiness, and drift, or export connection variables with `--env`. |
-| `supabase stack logs`    | Stream live stack logs.                                                           |
+| `supabase stack logs`    | Print retained stack logs; `-f` streams new lines.                                |
 | `supabase stack restart` | Restart an existing stack using its saved effective configuration.                |
 | `supabase stack stop`    | Stop a stack while retaining its data.                                            |
 
@@ -160,7 +160,7 @@ Other values are rejected. The override is applied before reading the project co
 
 `supabase services` follows the same backend selection. In stack mode it lists image versions and
 canonical `ghcr.io/supabase/cli/...` names from the installed CLI's artifact catalog, including
-Mailpit and Vector. PostgreSQL uses the selected major version (15 or 17); invalid configuration
+Mailpit. PostgreSQL uses the selected major version (15 or 17); invalid configuration
 or an unsupported PostgreSQL major warns with the cause and uses catalog defaults. This inventory describes the
 CLI catalog, not running containers, downloaded images, or service health. The Docker and native
 stack runtimes use the same catalog versions, though a running stack launched by another CLI
@@ -237,18 +237,25 @@ lint transaction (always rolled back). It does not launch a client binary.
 
 ## Reading stack logs
 
-`supabase stack logs` streams live stdout/stderr from composition members without
-starting an owner or service. Select `--stack <name>` or `--stack-id <id>`;
-`--service <kind-or-instance-id>` can include standalone services too. The command
-requires a reachable owner and streams until interrupted. Ctrl-C leaves services
-running. There is no retained history, `--tail`, or `--follow` flag.
-
-Text uses `<timestamp> <service>/<instance-id>/<stream>: <line>` and strips terminal
-control sequences. For automation use `--output-format stream-json`: each
-`log-entry` contains `timestamp`, `service`, `instance_id`, `stream`, `line`, and
-`source: "live"`. Finite JSON output is not supported. Delivery is best effort;
-stdout/stderr and different services may interleave. Missing stacks, unavailable
-owners, and unmatched services fail with status 1; interruption exits 130.
+`supabase stack logs` prints the retained stdout/stderr of composition members and
+exits; `-f/--follow` then streams new lines until interrupted. Select `--stack <name>`
+or `--stack-id <id>`; the repeatable `--service <kind-or-instance-id>` can include
+standalone services too. History is read from the persisted log files, so it works
+while the stack is stopped; `--follow` requires a running owner and fails before
+printing anything without one. Neither mode starts an owner or service, and Ctrl-C
+leaves services running.
+
+`--tail N` (default 200) keeps the newest lines across the selected services and
+`--since` takes a duration (`10m`, `1h30m`), an ISO-8601 time, or `start` for each
+service's latest launch. Text uses `<service> | <HH:MM:SS.mmm> <line>` with aligned
+labels, shows launches and lost output as dim separators, strips terminal control
+sequences, and notes on stderr when the tail hides older lines. For automation use
+`--output-format stream-json`: each `log-entry` contains `timestamp`, `service`,
+`instance_id`, `stream`, `line`, and `source` (`history` or `live`), and markers are
+`log-marker` events with `kind` and, for lost output, `count`. `--output-format json`
+prints the history as one array and does not accept `--follow`. Missing stacks,
+unmatched services, and `--follow` without an owner fail with status 1; interruption
+exits 130.
 
 ## Data and configuration
 

apps/cli/src/command-internal/colors.ts

@@ -77,3 +77,18 @@ export function green(text: string, stream: ColorStream = process.stderr): strin
 export function gray(text: string, stream: ColorStream = process.stderr): string {
   return supportsColor(stream) ? styleText("gray", text, { validateStream: false }) : text;
 }
+
+/** Renders in magenta. */
+export function magenta(text: string, stream: ColorStream = process.stderr): string {
+  return supportsColor(stream) ? styleText("magenta", text, { validateStream: false }) : text;
+}
+
+/** Renders in blue. */
+export function blue(text: string, stream: ColorStream = process.stderr): string {
+  return supportsColor(stream) ? styleText("blue", text, { validateStream: false }) : text;
+}
+
+/** Renders dimmed. */
+export function dim(text: string, stream: ColorStream = process.stderr): string {
+  return supportsColor(stream) ? styleText("dim", text, { validateStream: false }) : text;
+}

apps/cli/src/command-internal/db-config.integration.test.ts

@@ -465,7 +465,7 @@ describe("dbConfigResolver (db-url under the stack backend)", () => {
         registered: true,
       }),
       followStatus: Stream.empty,
-      logs: Stream.empty,
+      readLogs: () => Stream.empty,
       credentials: () => unused,
       saveSnapshot: () => unused,
       restoreSnapshot: () => unused,

apps/cli/src/command-internal/db-target-flags.ts

@@ -139,6 +139,7 @@ export const VALUE_CONSUMING_LONG_FLAGS = new Set([
   // experimental stack flags
   "capability",
   "service",
+  "since",
   "stack",
   "stack-id",
   "preparation",

apps/cli/src/command-internal/stack-config.ts

@@ -228,12 +228,6 @@ const ANALYTICS_PORT: PortSetting = {
   key: "port",
   configPath: "analytics.port",
 };
-const ANALYTICS_VECTOR_PORT: PortSetting = {
-  envVar: "SUPABASE_ANALYTICS_VECTOR_PORT",
-  section: "analytics",
-  key: "vector_port",
-  configPath: "analytics.vector_port",
-};
 const EDGE_RUNTIME_INSPECTOR_PORT: PortSetting = {
   envVar: "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT",
   section: "edge_runtime",
@@ -268,7 +262,6 @@ const endpointSettingsByServiceEndpoint: Readonly<Record<string, StackEndpointSe
   "database.sql": DB_PORT,
   "pooler.sql": DB_POOLER_PORT,
   "analytics.http": ANALYTICS_PORT,
-  "vector.http": ANALYTICS_VECTOR_PORT,
   "studio.http": STUDIO_PORT,
   "mail.http": LOCAL_SMTP_PORT,
   "mail.smtp": LOCAL_SMTP_SMTP_PORT,
@@ -659,17 +652,6 @@ const resolveEffectiveCliConfig = (
       env,
     ),
     port: resolvedPort("SUPABASE_ANALYTICS_PORT", analytics.port, "analytics.port", env),
-    ...(analytics.vector_port === undefined &&
-    envOverride("SUPABASE_ANALYTICS_VECTOR_PORT", undefined, env) === undefined
-      ? {}
-      : {
-          vector_port: resolvedPort(
-            "SUPABASE_ANALYTICS_VECTOR_PORT",
-            analytics.vector_port ?? 0,
-            "analytics.vector_port",
-            env,
-          ),
-        }),
     backend: envOverrideAnalyticsBackend(analytics.backend, env),
     gcp_project_id: envOverride("SUPABASE_ANALYTICS_GCP_PROJECT_ID", analytics.gcp_project_id, env),
     gcp_project_number: envOverride(
@@ -1375,24 +1357,6 @@ export const loadStackConfig = Effect.fn("StackConfig.load")(
                   } satisfies ServiceCreationType,
                 ]
               : []),
-            ...(validatedConfig.analytics.enabled
-              ? [
-                  {
-                    service: "vector" as const,
-                    config: { apiKey: "api-key" },
-                    endpoints: {
-                      http: endpoint(
-                        resolvePort(
-                          ANALYTICS_VECTOR_PORT,
-                          document,
-                          validatedConfig.analytics.vector_port ?? 0,
-                          context.projectEnvValues,
-                        ),
-                      ),
-                    },
-                  } satisfies ServiceCreationType,
-                ]
-              : []),
             ...(validatedConfig.storage.image_transformation?.enabled === true
               ? [
                   {

apps/cli/src/command-internal/stack-log-events.ts

@@ -0,0 +1,37 @@
+import type { LogRecord, StackLogRecord } from "@supabase/stack/effect";
+import type { StreamEvent } from "../shared/output/types.ts";
+
+/** Whether a record was read as history or followed live. */
+export type LogSource = "history" | "live";
+
+/** The `log-entry` or `log-marker` event of a record. */
+export const logEvent = (record: StackLogRecord, source: LogSource): StreamEvent => {
+  const subject = { service: record.service, instance_id: record.instanceId };
+  if (record.kind === "stdout" || record.kind === "stderr")
+    return {
+      type: "log-entry",
+      timestamp: record.timestamp,
+      source,
+      ...subject,
+      stream: record.kind,
+      line: record.text ?? "",
+    };
+  return {
+    type: "log-marker",
+    timestamp: record.timestamp,
+    source,
+    ...subject,
+    kind: record.kind,
+    ...(record.stream === undefined ? {} : { stream: record.stream }),
+    ...(record.count === undefined ? {} : { count: record.count }),
+  };
+};
+
+/** The text of a launch or lost marker, as `stack logs` prints it. */
+export const markerText = (record: LogRecord) => {
+  if (record.kind === "launch")
+    return record.launchId === undefined ? "--- launch ---" : `--- launch ${record.launchId} ---`;
+  if (record.count === undefined) return "--- older records were removed by retention ---";
+  const unit = record.count === 1 ? "chunk" : "chunks";
+  return `--- ${record.count} ${record.stream ?? "output"} ${unit} lost ---`;
+};

apps/cli/src/commands/db/dump/dump.integration.test.ts

@@ -124,7 +124,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => {
       registered: true,
     }),
     followStatus: Stream.empty,
-    logs: Stream.empty,
+    readLogs: () => Stream.empty,
     credentials: () =>
       Effect.succeed({ databaseUrl: "postgresql://postgres:secret@127.0.0.1:54322/postgres" }),
     saveSnapshot: () => Effect.die("unused"),

apps/cli/src/commands/db/reset/reset.integration.test.ts

@@ -565,7 +565,7 @@ const stackService = (
     prepare: Effect.void,
     status: observation,
     followStatus: Stream.empty,
-    logs: Stream.empty,
+    readLogs: () => Stream.empty,
     credentials: () => Effect.succeed({}),
   } satisfies Omit<ServiceInstance, "service">;
   switch (creation.service) {

apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts

@@ -112,7 +112,7 @@ function generateStackApi(workdir: string) {
     saveSnapshot: unusedStackFn,
     restoreSnapshot: unusedStackFn,
     resetData: unusedStack,
-    logs: Stream.empty,
+    readLogs: () => Stream.empty,
     followStatus: Stream.empty,
     status: Effect.succeed({
       id: "primary",

apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts

@@ -114,7 +114,7 @@ function syncStackApi(workdir: string, port: number) {
     saveSnapshot: unusedSyncFn,
     restoreSnapshot: unusedSyncFn,
     resetData: unusedSync,
-    logs: Stream.empty,
+    readLogs: () => Stream.empty,
     followStatus: Stream.empty,
     status: Effect.succeed({
       id: "primary",

apps/cli/src/commands/db/start/start.integration.test.ts

@@ -1633,7 +1633,7 @@ describe("db start stack backend", () => {
       registered: true,
     })),
     followStatus: Stream.empty,
-    logs: Stream.empty,
+    readLogs: () => Stream.empty,
     credentials: () =>
       Effect.sync(() => {
         state.credentialsCalled = true;

apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md

@@ -1,42 +1,80 @@
 # `supabase stack logs`
 
-Streams live stdout/stderr from the selected saved stack. The experimental
-feature flag controls command registration. It connects to an existing owner;
-it never launches an owner or starts/stops a service.
+Prints the retained service logs of the selected saved stack and exits; with
+`-f/--follow` it then streams new lines until interrupted. The experimental
+feature flag controls command registration. It never launches an owner or
+starts/stops a service.
 
 ## Selection and files
 
 Select the current project/branch/name, `--stack <name>`, or `--stack-id <id>`.
 The selectors are mutually exclusive. By default, only composition members are
-included. `--service <kind-or-instance-id>` can also select standalone instances.
-An unavailable owner, missing stack, or unmatched service fails with status 1.
+included. `--service <kind-or-instance-id>` is repeatable and can also select
+standalone instances; a value that matches no saved instance fails with status 1.
+A missing stack fails with status 1.
 
-Reads saved definitions under `<SUPABASE_HOME or ~/.supabase>/stacks/<id>/`.
-Discovery ensures the registry directory exists with mode 0700 and probes local
-owners. Shared routing/settings may read project config and profiles. No project
-files, service configuration, artifacts, or data are changed.
+Reads saved definitions under `<SUPABASE_HOME or ~/.supabase>/stacks/<id>/` and
+the persisted segments under
+`<SUPABASE_HOME or ~/.supabase>/stacks/<id>/logs/<service>/<instance-id>/<generation>.log`
+directly, so history is readable while the owner is down. Discovery ensures the
+registry directory exists with mode 0700 and probes local owners. Shared
+routing/settings may read project config and profiles. No project files,
+service configuration, artifacts, logs, or data are changed.
 
-## Output and cancellation
+## History, follow, and flags
 
-This is a live-only stream with no retained history, cursor, or `--tail` option.
-It continues until interrupted or the selected streams close. `--follow` is
-unnecessary and is not accepted. Legacy `-o/--output` and finite JSON output are
-rejected; use text or `--output-format stream-json`.
+`--tail N` (default 200, 0 prints none) keeps the newest N output lines across
+the selected services, with the `launch` and `lost` markers between them.
+`--since` accepts a duration before now (`30s`, `10m`, `1h30m`, `2d`), an
+ISO-8601 time, or `start`, which keeps the records of each instance's current
+launch and later ones, in history and while following. The current launch is the
+launch id saved in the stack definition, which keeps increasing across owner
+restarts, or else the highest launch record in history (all records when
+retention removed that launch record). Records are
+ordered by timestamp, service, instance, and file position. Timestamps are the
+owner's clock at each line's first byte. Lines end at `\n`, `\r\n`, or a lone
+`\r`, so carriage-return progress updates print as separate lines. History is
+read as a stream holding at most N lines per instance, while every matching line
+is counted for the footer.
 
-Text writes `<timestamp> <service>/<instance-id>/<stream>: <line>`. Terminal
-control sequences are stripped from text. Stream JSON emits `log-entry` events
-with `timestamp`, `service`, `instance_id`, `stream`, `line`, and `source: "live"`.
-Lines preserve their content in machine output. UTF-8 and line fragments are
-assembled separately for each instance and stdout/stderr channel. Timestamps
-reflect receipt by the CLI. Delivery is best effort: slow subscribers can lose entries. Ordering across
-stdout/stderr channels and different services is not guaranteed.
+`-f/--follow` requires a reachable owner: without one it fails with status 1
+before printing anything and suggests running without `--follow`. It prints the
+history, then streams each instance's new records through the owner from the
+last record read, so no record is repeated or skipped between history and live
+output. With `--tail 0` no history is read and the owner starts each follow at
+its current end. A selected instance the running owner does not serve is named
+in a warning and not followed; when it serves none of them, the command fails
+with status 1 after printing the history. The owner's log shipping to Analytics reads the persisted logs
+separately and does not affect this command.
 
-Interrupting the command cancels its subscriptions, exits with status 130, and
-leaves the owner and services running. Successful stream completion exits 0;
-selection, connection, or stream failures exit 1.
+## Output
+
+Text writes `<service> | <HH:MM:SS.mmm> <line>` with the service labels padded
+to one width; a service kind selected more than once is labelled
+`<service>:<first 8 characters of the instance id>`. Times are local. `launch`
+and `lost` records are dim separator lines (`--- launch 2 ---`,
+`--- 3 stdout chunks lost ---`, `--- older records were removed by retention ---`).
+Labels are coloured and markers dimmed only when stdout is a colour-capable
+terminal. Terminal control sequences are stripped from text. When a non-zero
+tail hides older lines, stderr gets
+`showing last 200 of 5,234 lines, use --tail/--since`; an empty history without
+`--follow` prints `No retained log lines for the selected services.` on stderr.
+
+Stream JSON emits a `log-entry` event per output line with `timestamp`,
+`service`, `instance_id`, `stream` (`stdout` or `stderr`), `line`, and `source`
+(`history` or `live`), and a `log-marker` event per marker with `timestamp`,
+`service`, `instance_id`, `kind` (`launch` or `lost`), `source`, and, for chunks
+lost before they were written, `stream` and `count`. A `lost` marker without
+`count` reports segments removed by retention. `--output-format json` prints
+one array of the same objects; it cannot be combined with `--follow`. Lines
+preserve their content in machine output. Legacy `-o/--output` is rejected.
+
+Interrupting a follow cancels its subscriptions, exits with status 130, and
+leaves the owner and services running. History reads and completed follows exit
+0; selection, owner, or read failures exit 1.
 
 ## Telemetry
 
-Standard command telemetry is retained; log contents are not custom telemetry
-properties. Telemetry flushes on every exit to
+Standard command telemetry is retained, with `-f` reported as `follow`; log
+contents are not custom telemetry properties. Telemetry flushes on every exit to
 `<SUPABASE_HOME or ~/.supabase>/telemetry.json`.