diff --git a/packages/loader/container-loader/api-report/container-loader.legacy.alpha.api.md b/packages/loader/container-loader/api-report/container-loader.legacy.alpha.api.md index f48239247c93..5c81bbceb2ae 100644 --- a/packages/loader/container-loader/api-report/container-loader.legacy.alpha.api.md +++ b/packages/loader/container-loader/api-report/container-loader.legacy.alpha.api.md @@ -7,6 +7,9 @@ // @alpha @legacy export function asLegacyAlpha(base: IContainer): ContainerAlpha; +// @alpha @legacy +export function captureFullContainerState(input: ICaptureFullContainerStateProps): Promise; + // @public export enum ConnectionState { CatchingUp = 1, @@ -44,6 +47,14 @@ export interface IBaseProtocolHandler { snapshot(): IQuorumSnapshot; } +// @alpha @legacy +export interface ICaptureFullContainerStateProps { + readonly documentServiceFactory: IDocumentServiceFactory; + readonly logger?: ITelemetryBaseLogger | undefined; + readonly request: IRequest; + readonly urlResolver: IUrlResolver; +} + // @beta @deprecated @legacy (undocumented) export interface ICodeDetailsLoader extends Partial { load(source: IFluidCodeDetails): Promise; diff --git a/packages/loader/container-loader/src/captureReferencedContents.ts b/packages/loader/container-loader/src/captureReferencedContents.ts new file mode 100644 index 000000000000..a433ed606893 --- /dev/null +++ b/packages/loader/container-loader/src/captureReferencedContents.ts @@ -0,0 +1,446 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { bufferToString } from "@fluid-internal/client-utils"; +import type { + IDocumentStorageService, + ISequencedDocumentMessage, + ISnapshot, + ISnapshotTree, +} from "@fluidframework/driver-definitions/internal"; +import { readAndParse } from "@fluidframework/driver-utils/internal"; + +import type { + IBase64BlobContents, + ISerializableBlobContents, +} from "./containerStorageAdapter.js"; + +/** + * Wire-format constants this module needs to walk and filter snapshots. + * Authoritative definitions live in `container-runtime` and + * `runtime-definitions`; the values are duplicated here to avoid a + * loader → runtime layering dependency. A contract test in + * `packages/test/local-server-tests` asserts these match the authoritative + * values; do not change them in isolation. + * + * Authoritative sources: + * - `blobsTreeName`, `redirectTableBlobName`: `packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts` + * - `blobManagerBasePath`: `packages/runtime/container-runtime/src/blobManager/blobManager.ts` + * - `gcTreeKey`, `gcBlobPrefix`, `gcTombstoneBlobKey`, `gcDeletedBlobKey`: `packages/runtime/runtime-definitions/src/garbageCollectionDefinitions.ts` + * + * @internal + */ +export const wireFormatConstants = { + blobsTreeName: ".blobs", + redirectTableBlobName: ".redirectTable", + blobManagerBasePath: "_blobs", + gcTreeKey: "gc", + gcBlobPrefix: "__gc", + gcTombstoneBlobKey: "__tombstones", + gcDeletedBlobKey: "__deletedNodes", +} as const; + +const { + blobsTreeName, + redirectTableBlobName, + blobManagerBasePath, + gcTreeKey, + gcBlobPrefix, + gcTombstoneBlobKey, + gcDeletedBlobKey, +} = wireFormatConstants; + +interface IGcNodeData { + outboundRoutes: string[]; + unreferencedTimestampMs?: number; +} + +interface IGcState { + gcNodes: { [id: string]: IGcNodeData }; +} + +/** + * The parsed subset of the `gc` subtree that drives reachability decisions. + */ +export interface IGcSnapshotData { + gcState: IGcState | undefined; + tombstones: string[] | undefined; + deletedNodes: string[] | undefined; +} + +/** Reader that returns a blob's contents for a given storage id. */ +type BlobReader = (id: string) => Promise; + +/** + * Upper bound on concurrent `readBlob` calls. Driver/service back-pressure is + * real for large documents, and unbounded `Promise.all` can trigger throttling + * or spike memory. The value is a pragmatic middle ground — high enough to + * keep a typical driver's request pipeline full, low enough to avoid storms. + */ +const maxReadConcurrency = 32; + +/** + * Runs `fn` over `items` with at most `limit` promises in flight. Preserves + * input order on output (not that any caller depends on it today). + * + * Exported for unit tests; not part of the package public API. + * + * @internal + */ +export async function mapWithConcurrency( + items: readonly T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results: R[] = Array.from({ length: items.length }); + let cursor = 0; + const workerCount = Math.min(limit, items.length); + const workers = Array.from({ length: workerCount }, async () => { + while (cursor < items.length) { + const index = cursor++; + const item = items[index]; + if (item !== undefined) { + results[index] = await fn(item); + } + } + }); + await Promise.all(workers); + return results; +} + +/** + * Parses the `gc` subtree of a base snapshot. Returns `undefined` if the + * snapshot has no GC tree (GC disabled or pre-GC document). + */ +export async function parseGcSnapshotData( + baseSnapshot: ISnapshotTree, + storage: Pick, +): Promise { + const gcSnapshotTree: ISnapshotTree | undefined = baseSnapshot.trees[gcTreeKey]; + if (gcSnapshotTree === undefined) { + return undefined; + } + let gcState: IGcState | undefined; + let tombstones: string[] | undefined; + let deletedNodes: string[] | undefined; + for (const [key, blobId] of Object.entries(gcSnapshotTree.blobs)) { + if (key === gcDeletedBlobKey) { + deletedNodes = await readAndParse(storage, blobId); + } else if (key === gcTombstoneBlobKey) { + tombstones = await readAndParse(storage, blobId); + } else if (key.startsWith(gcBlobPrefix)) { + const partial = await readAndParse(storage, blobId); + if (gcState === undefined) { + gcState = { gcNodes: { ...partial.gcNodes } }; + } else { + for (const [nodeId, nodeData] of Object.entries(partial.gcNodes)) { + gcState.gcNodes[nodeId] ??= nodeData; + } + } + } + } + return { gcState, tombstones, deletedNodes }; +} + +/** + * Walks a snapshot and inlines the contents of every blob reachable without + * crossing an `unreferenced` subtree boundary. Subtrees flagged + * `unreferenced: true` are skipped entirely — the summarizer sets that flag + * from GC state, so honouring it filters out dead subtrees without a + * separate GC-path traversal. + * + * The root-level `.blobs` subtree is special-cased: only its `.redirectTable` + * blob is read, because attachment blob contents are captured separately via + * {@link captureReferencedAttachmentBlobs}. + */ +export async function readReferencedSnapshotBlobs( + snapshot: ISnapshot | ISnapshotTree, + storage: Pick, +): Promise { + const { tree, read } = toTreeAndReader(snapshot, storage); + const ids = new Set(); + collectReferencedBlobIds(tree, true, ids); + const blobs: ISerializableBlobContents = {}; + await mapWithConcurrency([...ids], maxReadConcurrency, async (id) => { + const data = await read(id); + blobs[id] = bufferToString(data, "utf8"); + }); + return blobs; +} + +/** + * Synchronously walks the snapshot tree and gathers the set of blob ids that + * should be inlined. Subtrees flagged `unreferenced: true` are skipped + * entirely. The root-level `.blobs` subtree is special-cased: only its + * `.redirectTable` id is collected, because attachment blob contents are + * captured separately via {@link captureReferencedAttachmentBlobs}. + */ +function collectReferencedBlobIds( + tree: ISnapshotTree, + isRoot: boolean, + ids: Set, +): void { + if (tree.unreferenced === true) { + return; + } + for (const blobId of Object.values(tree.blobs)) { + ids.add(blobId); + } + for (const [key, subTree] of Object.entries(tree.trees)) { + if (isRoot && key === blobsTreeName) { + const tableBlobId = subTree.blobs[redirectTableBlobName]; + if (tableBlobId !== undefined) { + ids.add(tableBlobId); + } + } else { + collectReferencedBlobIds(subTree, false, ids); + } + } +} + +function toTreeAndReader( + snapshot: ISnapshot | ISnapshotTree, + storage: Pick, +): { tree: ISnapshotTree; read: BlobReader } { + if ("snapshotTree" in snapshot) { + const blobContents = snapshot.blobContents; + return { + tree: snapshot.snapshotTree, + read: async (id) => blobContents.get(id) ?? storage.readBlob(id), + }; + } + return { tree: snapshot, read: async (id) => storage.readBlob(id) }; +} + +/** + * Fetches attachment blob contents from a snapshot, filtered by GC + * reachability. Blobs GC has explicitly marked unreferenced, tombstoned, or + * deleted are skipped. Blobs absent from the GC graph are kept — GC state + * lags behind recent attachments and dropping them would lose live data. + * If `gcData` is `undefined`, every attachment blob is returned. + * + * The returned map is keyed by attachment blob storage id. Values are the + * raw bytes encoded as **base64** strings — attachment blobs may carry + * arbitrary binary payloads (images, encrypted data, etc.) and a + * UTF-8 round-trip would silently corrupt non-UTF-8 byte sequences with + * replacement characters. The runtime's own pending-blob serializer uses + * base64 for the same reason. This diverges from the structural-blob path + * in {@link readReferencedSnapshotBlobs}, which encodes UTF-8 because those + * blobs are JSON or other text the runtime authored. Callers must keep the + * two encodings on separate fields of the pending state so the load side + * can decode each correctly. + */ +export async function captureReferencedAttachmentBlobs( + baseSnapshot: ISnapshotTree, + storage: Pick, + gcData: IGcSnapshotData | undefined, +): Promise { + const blobsTree: ISnapshotTree | undefined = baseSnapshot.trees[blobsTreeName]; + if (blobsTree === undefined) { + return {}; + } + const localIdToStorageId = await readRedirectTable(blobsTree, storage); + if (localIdToStorageId.size === 0) { + return {}; + } + + const unreferencedLocalIds = + gcData === undefined ? undefined : collectUnreferencedBlobLocalIds(gcData); + + const storageIdsToFetch = new Set(); + for (const [localId, storageId] of localIdToStorageId) { + if (unreferencedLocalIds?.has(localId) !== true) { + storageIdsToFetch.add(storageId); + } + } + + const contents: IBase64BlobContents = {}; + await mapWithConcurrency([...storageIdsToFetch], maxReadConcurrency, async (storageId) => { + const buffer = await storage.readBlob(storageId); + contents[storageId] = bufferToString(buffer, "base64"); + }); + return contents; +} + +/** + * Reconstructs the BlobManager's redirect table from a `.blobs` subtree. + * Mirrors `toRedirectTable` in blobManagerSnapSum.ts. + */ +async function readRedirectTable( + blobsTree: ISnapshotTree, + storage: Pick, +): Promise> { + const redirectTable = new Map(); + const tableBlobId: string | undefined = blobsTree.blobs[redirectTableBlobName]; + if (tableBlobId !== undefined) { + const entries = await readAndParse<[string, string][]>(storage, tableBlobId); + for (const [localId, storageId] of entries) { + redirectTable.set(localId, storageId); + } + } + for (const [key, storageId] of Object.entries(blobsTree.blobs)) { + if (key !== redirectTableBlobName) { + // Identity mapping: storage ids referenced directly in handles (legacy). + redirectTable.set(storageId, storageId); + } + } + return redirectTable; +} + +/** + * Collects the set of blob localIds that GC has explicitly marked as + * unreferenced (via `unreferencedTimestampMs` on a gc node), tombstoned, or + * deleted. Tombstones and deletedNodes are applied regardless of whether + * `gcState` is present — they are authoritative on their own and must not + * be silently dropped when gc state is absent but tombstone/deleted lists + * exist. + */ +function collectUnreferencedBlobLocalIds(gcData: IGcSnapshotData): Set { + const blobPathPrefix = `/${blobManagerBasePath}/`; + const unreferenced = new Set(); + if (gcData.gcState !== undefined) { + for (const [nodePath, nodeData] of Object.entries(gcData.gcState.gcNodes)) { + if ( + nodePath.startsWith(blobPathPrefix) && + nodeData.unreferencedTimestampMs !== undefined + ) { + unreferenced.add(nodePath.slice(blobPathPrefix.length)); + } + } + } + for (const nodePath of [...(gcData.tombstones ?? []), ...(gcData.deletedNodes ?? [])]) { + if (nodePath.startsWith(blobPathPrefix)) { + unreferenced.add(nodePath.slice(blobPathPrefix.length)); + } + } + return unreferenced; +} + +/** + * A blob reference extracted from a `BlobAttach` op. `localId` is the + * `BlobManager` GC identity for the blob; `storageId` is the id used for + * `IDocumentStorageService.readBlob`. + * + * @internal + */ +export interface IBlobAttachReference { + readonly localId: string; + readonly storageId: string; +} + +interface IBlobAttachLikeMetadata { + readonly localId: string; + readonly blobId: string; +} + +function isBlobAttachLikeMetadata(metadata: unknown): metadata is IBlobAttachLikeMetadata { + if (typeof metadata !== "object" || metadata === null) { + return false; + } + const candidate = metadata as { localId?: unknown; blobId?: unknown }; + return typeof candidate.localId === "string" && typeof candidate.blobId === "string"; +} + +/** + * Extracts every `BlobAttach` reference an op carries. Returns an empty array + * for non-blobAttach ops. + * + * This is the single place in the loader that interprets the BlobAttach + * wire format. Capture and load-side reasoning about ops should call into + * this function rather than reading `op.metadata` directly, so a future + * protocol change touches one site. + * + * BlobAttach ops carry `(localId, storageId)` directly on + * `ISequencedDocumentMessage.metadata` and are not grouped — the container + * runtime routes them through a separate `outbox.submitBlobAttach` lane, + * and `OpGroupingManager.groupBatch` asserts (0x5dd) that no op carrying + * non-batch metadata enters a grouped batch. If either guarantee changes, + * extend this function rather than each call site. + * + * @internal + */ +export function extractBlobAttachReferences( + op: Pick, +): IBlobAttachReference[] { + if (!isBlobAttachLikeMetadata(op.metadata)) { + return []; + } + return [{ localId: op.metadata.localId, storageId: op.metadata.blobId }]; +} + +/** + * Set of attachment-blob localIds that GC has marked unreferenced, + * tombstoned, or deleted in the base snapshot. `undefined` if `gcData` + * is `undefined` (GC disabled / pre-GC document). + * + * @internal + */ +export function unreferencedAttachmentBlobLocalIds( + gcData: IGcSnapshotData | undefined, +): Set | undefined { + return gcData === undefined ? undefined : collectUnreferencedBlobLocalIds(gcData); +} + +/** + * Inline attachment blob contents for the given `(localId, storageId)` + * references. Skips entries already present in `existing` (de-dupe with + * the snapshot path) and entries whose `localId` is in + * `unreferencedLocalIds`. Returns only the freshly-read entries; the + * caller merges them into the existing map. + * + * @internal + */ +export async function inlineAttachmentBlobsByReference( + references: readonly IBlobAttachReference[], + storage: Pick, + unreferencedLocalIds: ReadonlySet | undefined, + existing: Readonly, +): Promise { + const storageIdsToFetch = new Set(); + for (const { localId, storageId } of references) { + if (unreferencedLocalIds?.has(localId) === true) { + continue; + } + if (existing[storageId] !== undefined) { + continue; + } + storageIdsToFetch.add(storageId); + } + const added: IBase64BlobContents = {}; + if (storageIdsToFetch.size === 0) { + return added; + } + await mapWithConcurrency([...storageIdsToFetch], maxReadConcurrency, async (storageId) => { + const buffer = await storage.readBlob(storageId); + added[storageId] = bufferToString(buffer, "base64"); + }); + return added; +} + +/** + * Returns true if any referenced subtree of `baseSnapshot` declares a + * `groupId` — the snapshot-tree wire field that carries the runtime's + * loading-group identifier. Subtrees flagged `unreferenced` are skipped — + * a dead subtree's `groupId` would not be loaded by the runtime either. + * + * `captureFullContainerState` does not yet support loading groups: prefetching + * per-group snapshots adds a code path that has no end-to-end coverage and no + * known production consumer. Callers use this to fail fast with a `UsageError` + * rather than silently producing a pending state that omits group data. + */ +export function snapshotHasLoadingGroups(baseSnapshot: ISnapshotTree): boolean { + if (baseSnapshot.unreferenced === true) { + return false; + } + if (baseSnapshot.groupId !== undefined) { + return true; + } + for (const child of Object.values(baseSnapshot.trees)) { + if (snapshotHasLoadingGroups(child)) { + return true; + } + } + return false; +} diff --git a/packages/loader/container-loader/src/containerStorageAdapter.ts b/packages/loader/container-loader/src/containerStorageAdapter.ts index 869aa5d71f76..54d8dc50506c 100644 --- a/packages/loader/container-loader/src/containerStorageAdapter.ts +++ b/packages/loader/container-loader/src/containerStorageAdapter.ts @@ -36,13 +36,32 @@ import type { import { convertSnapshotInfoToSnapshot } from "./utils.js"; /** - * Stringified blobs from a summary/snapshot tree. + * Stringified blobs from a summary/snapshot tree, keyed by blob id. + * Values are **UTF-8-encoded** — this is the right encoding for JSON or + * other text the runtime authors and consumes through this map. For + * arbitrary binary payloads (e.g. attachment blob contents), use + * {@link IBase64BlobContents} instead; a UTF-8 round-trip silently + * corrupts non-UTF-8 byte sequences with replacement characters. * @internal */ export interface ISerializableBlobContents { [id: string]: string; } +/** + * Stringified blobs inlined in a summary/snapshot tree, keyed by blob id. + * Values are **base64-encoded** raw bytes. Used for attachment-blob + * payloads, which may carry arbitrary binary data (images, encrypted + * blobs, etc.). Mirrors the encoding used by the runtime's own + * pending-blob serializer in `BlobManager`. Structurally identical to + * {@link ISerializableBlobContents}; the two types exist to keep the + * encoding contract visible at every call site. + * @internal + */ +export interface IBase64BlobContents { + [id: string]: string; +} + /** * This class wraps the actual storage and make sure no wrong apis are called according to * container attach state. diff --git a/packages/loader/container-loader/src/createAndLoadContainerUtils.ts b/packages/loader/container-loader/src/createAndLoadContainerUtils.ts index 6cb0705bef58..7bca03aac831 100644 --- a/packages/loader/container-loader/src/createAndLoadContainerUtils.ts +++ b/packages/loader/container-loader/src/createAndLoadContainerUtils.ts @@ -19,11 +19,16 @@ import type { import type { IClientDetails } from "@fluidframework/driver-definitions"; import type { IDocumentServiceFactory, + ISequencedDocumentMessage, + ISnapshot, + ISnapshotTree, IUrlResolver, } from "@fluidframework/driver-definitions/internal"; -import { DriverHeader } from "@fluidframework/driver-definitions/internal"; +import { DriverHeader, FetchSource } from "@fluidframework/driver-definitions/internal"; +import { getSnapshotTree } from "@fluidframework/driver-utils/internal"; import { GenericError, + UsageError, normalizeError, createChildMonitoringContext, mixinMonitoringContext, @@ -33,16 +38,28 @@ import { } from "@fluidframework/telemetry-utils/internal"; import { v4 as uuid } from "uuid"; +import { + captureReferencedAttachmentBlobs, + extractBlobAttachReferences, + inlineAttachmentBlobsByReference, + parseGcSnapshotData, + readReferencedSnapshotBlobs, + snapshotHasLoadingGroups, + unreferencedAttachmentBlobLocalIds, + type IBlobAttachReference, +} from "./captureReferencedContents.js"; import { DebugLogger } from "./debugLogger.js"; import { createFrozenDocumentServiceFactory } from "./frozenServices.js"; import { Loader } from "./loader.js"; import { pkgVersion } from "./packageVersion.js"; import type { ProtocolHandlerBuilder } from "./protocol.js"; +import type { IPendingContainerState } from "./serializedStateManager.js"; import type { LoadSummarizerSummaryResult, OnDemandSummaryResults, SummarizeOnDemandResults, } from "./summarizerResultTypes.js"; +import { getDocumentAttributes } from "./utils.js"; interface OnDemandSummarizeResultsPromises { readonly summarySubmitted: Promise; @@ -245,6 +262,185 @@ export async function loadFrozenContainerFromPendingState( }); } +/** + * Properties for {@link captureFullContainerState}. + * @legacy @alpha + */ +export interface ICaptureFullContainerStateProps { + /** + * The url resolver used to resolve the request into a Fluid resolved url. + */ + readonly urlResolver: IUrlResolver; + /** + * The document service factory used to construct the driver services + * against which the state is captured. + */ + readonly documentServiceFactory: IDocumentServiceFactory; + /** + * The request identifying the container whose state is to be captured. + */ + readonly request: IRequest; + /** + * Optional logger for driver-side telemetry. + */ + readonly logger?: ITelemetryBaseLogger | undefined; +} + +/** + * Captures the current state of an attached container using only driver-level + * services, without instantiating a runtime or loading a full container. The + * returned string is a serialized pending container state in the same wire + * format produced by a live container's pending-state serialization, and can + * be handed to {@link loadExistingContainer} as `pendingLocalState`. + * + * The output is a self-contained view of the container's referenced graph: + * the latest snapshot, inlined contents of every blob reachable through + * referenced subtrees, inlined contents of every referenced attachment blob + * keyed by storage id, and all ops with sequence numbers after the base + * snapshot's sequence number (as read from its attributes blob). + * + * Reachability respects GC. Snapshot subtrees flagged `unreferenced: true` + * are skipped (their contents are not inlined). Attachment blobs that GC has + * marked unreferenced, tombstoned, or deleted are skipped. When the snapshot + * has no GC tree (GC disabled or pre-GC document), no filtering is applied. + * + * Blob reads on load hit the `ContainerStorageAdapter` cache populated from + * the captured `snapshotBlobs` map, so a frozen loader can serve the full + * referenced graph without a live storage service. + * + * `pendingRuntimeState` is `undefined` — no runtime is instantiated — so the + * output cannot carry DDS-level in-flight changes. It is intended for state + * relay, inspection, and durable-state snapshot use cases. + * + * Containers that declare loading groups are not yet supported: the function + * throws `UsageError` if any referenced subtree carries a `groupId`. Group + * snapshots would need a separate prefetch + serialization path; until there + * is a known consumer and end-to-end coverage, the capture refuses rather + * than silently producing pending state that omits group data. + * + * Note: if a new snapshot lands between the snapshot fetch and the ops fetch, + * the returned state may not reflect the very latest snapshot, but remains + * internally consistent: ops are anchored to the snapshot that was captured. + * + * No `mixinMonitoringContext` / `configProvider` is wired here, deliberately + * diverging from the sibling entry points in this file. The function reads + * no feature flags and instantiates no runtime, so there is nothing for a + * monitoring context to gate or attribute. If a future change introduces + * config-gated behavior or runtime-attributed telemetry, add the wiring + * back together with that change. + * @legacy @alpha + */ +export async function captureFullContainerState({ + urlResolver, + documentServiceFactory, + request, + logger, +}: ICaptureFullContainerStateProps): Promise { + const resolvedUrl = await urlResolver.resolve(request); + if (resolvedUrl === undefined) { + throw new UsageError("Failed to resolve request to a Fluid URL"); + } + + const documentService = await documentServiceFactory.createDocumentService( + resolvedUrl, + logger, + ); + try { + const storage = await documentService.connectToStorage(); + + const versions = await storage.getVersions( + // `null` signals "latest" + // eslint-disable-next-line unicorn/no-null + null, + 1, + "captureFullContainerState", + FetchSource.noCache, + ); + const version = versions[0]; + const snapshot: ISnapshot | ISnapshotTree | undefined = + storage.getSnapshot === undefined + ? ((await storage.getSnapshotTree(version, "captureFullContainerState")) ?? undefined) + : await storage.getSnapshot({ + cacheSnapshot: false, + versionId: version?.id, + scenarioName: "captureFullContainerState", + }); + if (snapshot === undefined) { + throw new GenericError("Failed to fetch snapshot for captureFullContainerState"); + } + + const baseSnapshot = getSnapshotTree(snapshot); + if (snapshotHasLoadingGroups(baseSnapshot)) { + throw new UsageError( + "captureFullContainerState does not yet support containers with loading groups", + ); + } + const attributes = await getDocumentAttributes(storage, baseSnapshot); + const gcData = await parseGcSnapshotData(baseSnapshot, storage); + // Structural snapshot blobs (JSON/text the runtime authored) are + // UTF-8-encoded; attachment blobs may carry arbitrary binary bytes + // and are base64-encoded. Keep them on separate fields of the + // pending state so the load side can apply the matching decoder + // without ambiguity. See IPendingContainerState.attachmentBlobContents. + const [snapshotBlobs, attachmentBlobContents] = await Promise.all([ + readReferencedSnapshotBlobs(snapshot, storage), // utf8 encoded + captureReferencedAttachmentBlobs(baseSnapshot, storage, gcData), // base64 encoded + ]); + + const deltaStorage = await documentService.connectToDeltaStorage(); + const opsStream = deltaStorage.fetchMessages( + attributes.sequenceNumber + 1, + undefined, + undefined, + false, + "captureFullContainerState", + ); + const savedOps: ISequencedDocumentMessage[] = []; + const postSnapshotBlobReferences: IBlobAttachReference[] = []; + let opsResult = await opsStream.read(); + while (!opsResult.done) { + for (const op of opsResult.value) { + savedOps.push(op); + // Blobs uploaded after the base snapshot are not in its + // `.blobs` redirect table, so `captureReferencedAttachmentBlobs` + // did not see them. The wire-format BlobAttach op carries + // `(localId, storageId)` in its metadata; collect those here so + // we can backfill the bytes before sealing the artifact. + const refs = extractBlobAttachReferences(op); + if (refs.length > 0) { + postSnapshotBlobReferences.push(...refs); + } + } + opsResult = await opsStream.read(); + } + + if (postSnapshotBlobReferences.length > 0) { + const added = await inlineAttachmentBlobsByReference( + postSnapshotBlobReferences, + storage, + unreferencedAttachmentBlobLocalIds(gcData), + attachmentBlobContents, + ); + Object.assign(attachmentBlobContents, added); + } + + const pendingState: IPendingContainerState = { + attached: true, + baseSnapshot, + snapshotBlobs, + attachmentBlobContents: + Object.keys(attachmentBlobContents).length === 0 ? undefined : attachmentBlobContents, + loadedGroupIdSnapshots: undefined, + pendingRuntimeState: undefined, + savedOps, + url: resolvedUrl.url, + }; + return JSON.stringify(pendingState); + } finally { + documentService.dispose(); + } +} + /** * Loads a summarizer container with the required headers, triggers an on-demand summary, and then closes it. * Returns success/failure and an optional error for host-side handling. diff --git a/packages/loader/container-loader/src/index.ts b/packages/loader/container-loader/src/index.ts index 391e3796d938..916ba3aec6e9 100644 --- a/packages/loader/container-loader/src/index.ts +++ b/packages/loader/container-loader/src/index.ts @@ -7,11 +7,13 @@ export { ConnectionState } from "./connectionState.js"; export { type ContainerAlpha, waitContainerToCatchUp, asLegacyAlpha } from "./container.js"; export { createFrozenDocumentServiceFactory } from "./frozenServices.js"; export { + captureFullContainerState, createDetachedContainer, loadExistingContainer, rehydrateDetachedContainer, loadFrozenContainerFromPendingState, loadSummarizerContainerAndMakeSummary, + type ICaptureFullContainerStateProps, type ICreateAndLoadContainerProps, type ICreateDetachedContainerProps, type ILoadExistingContainerProps, @@ -55,3 +57,8 @@ export type { QuorumProposalsSnapshot, } from "./protocol/index.js"; export { PendingLocalStateStore } from "./pendingLocalStateStore.js"; +export { + extractBlobAttachReferences, + wireFormatConstants, + type IBlobAttachReference, +} from "./captureReferencedContents.js"; diff --git a/packages/loader/container-loader/src/pendingLocalStateStore.ts b/packages/loader/container-loader/src/pendingLocalStateStore.ts index 15f866e03d34..f6f43d785d81 100644 --- a/packages/loader/container-loader/src/pendingLocalStateStore.ts +++ b/packages/loader/container-loader/src/pendingLocalStateStore.ts @@ -45,6 +45,7 @@ export class PendingLocalStateStore { readonly #pendingStates = new Map(); readonly #savedOps: Record = {}; readonly #blobs: Record = {}; + readonly #attachmentBlobs: Record = {}; readonly #loadingGroups: Record = {}; /** @@ -92,7 +93,8 @@ export class PendingLocalStateStore { */ set(key: TKey, pendingLocalState: string): this { const state = getAttachedContainerStateFromSerializedContainer(pendingLocalState); - const { savedOps, snapshotBlobs, loadedGroupIdSnapshots, url } = state; + const { savedOps, snapshotBlobs, attachmentBlobContents, loadedGroupIdSnapshots, url } = + state; // Normalize URL by removing trailing slash for comparison const normalizedUrl = url.replace(/\/$/, ""); @@ -108,6 +110,11 @@ export class PendingLocalStateStore { for (const [id, blob] of Object.entries(snapshotBlobs)) { snapshotBlobs[id] = this.#blobs[id] ??= blob; } + if (attachmentBlobContents !== undefined) { + for (const [id, blob] of Object.entries(attachmentBlobContents)) { + attachmentBlobContents[id] = this.#attachmentBlobs[id] ??= blob; + } + } if (loadedGroupIdSnapshots !== undefined) { for (const [id, lg] of Object.entries(loadedGroupIdSnapshots)) { if ( diff --git a/packages/loader/container-loader/src/serializedStateManager.ts b/packages/loader/container-loader/src/serializedStateManager.ts index e79d4e75710b..5a43e1bab980 100644 --- a/packages/loader/container-loader/src/serializedStateManager.ts +++ b/packages/loader/container-loader/src/serializedStateManager.ts @@ -33,6 +33,7 @@ import { import { getBlobContentsFromTree, type ContainerStorageAdapter, + type IBase64BlobContents, type ISerializableBlobContents, } from "./containerStorageAdapter.js"; import { SnapshotRefresher } from "./snapshotRefresher.js"; @@ -83,6 +84,21 @@ export interface IPendingContainerState extends SnapshotWithBlobs { * Any group snapshots (aka delay-loaded) we've downloaded from the service for this container */ loadedGroupIdSnapshots?: Record; + /** + * Attachment blob contents inlined by storage id, encoded as base64. + * + * Carried separately from {@link SnapshotWithBlobs.snapshotBlobs} because + * attachment blobs may contain arbitrary binary payloads, and the + * UTF-8 encoding used for `snapshotBlobs` (which holds JSON/text the + * runtime authors) would corrupt non-UTF-8 byte sequences with + * replacement characters. Populated by `captureFullContainerState`; the + * live container's pending-state path leaves this `undefined` because + * it does not inline attachment blob contents. + * + * On load, entries are decoded from base64 and merged into the same + * blob cache that `snapshotBlobs` populates. + */ + attachmentBlobContents?: IBase64BlobContents; /** * All ops since base snapshot sequence number up to the latest op * seen when the container was closed. Used to apply stashed (saved pending) @@ -265,11 +281,22 @@ export class SerializedStateManager implements IDisposable { } return { snapshot, version, attributes }; } else { - const { baseSnapshot, snapshotBlobs, savedOps } = pendingLocalState; + const { baseSnapshot, snapshotBlobs, attachmentBlobContents, savedOps } = + pendingLocalState; const blobContents = new Map(); + // Structural snapshot blobs (snapshot trees, `.attributes`, `.redirectTable`) + // are JSON/text the runtime authored, so UTF-8 round-trip is lossless. for (const [id, value] of Object.entries(snapshotBlobs)) { blobContents.set(id, stringToBuffer(value, "utf8")); } + // Attachment blobs are base64-encoded — see IPendingContainerState + // docs. Decoded after structural blobs because storage-id collisions + // between the two namespaces should resolve to the binary form. + if (attachmentBlobContents !== undefined) { + for (const [id, value] of Object.entries(attachmentBlobContents)) { + blobContents.set(id, stringToBuffer(value, "base64")); + } + } this.storageAdapter.cacheSnapshotBlobs(blobContents); const attributes = await getDocumentAttributes(this.storageAdapter, baseSnapshot); diff --git a/packages/loader/container-loader/src/test/captureReferencedContents.spec.ts b/packages/loader/container-loader/src/test/captureReferencedContents.spec.ts new file mode 100644 index 000000000000..1e5babaa9536 --- /dev/null +++ b/packages/loader/container-loader/src/test/captureReferencedContents.spec.ts @@ -0,0 +1,727 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils"; +import type { + IDocumentStorageService, + ISnapshot, + ISnapshotTree, +} from "@fluidframework/driver-definitions/internal"; + +import { + captureReferencedAttachmentBlobs, + extractBlobAttachReferences, + inlineAttachmentBlobsByReference, + mapWithConcurrency, + parseGcSnapshotData, + readReferencedSnapshotBlobs, + snapshotHasLoadingGroups, + unreferencedAttachmentBlobLocalIds, + type IBlobAttachReference, + type IGcSnapshotData, +} from "../captureReferencedContents.js"; + +/** Minimal storage shim whose readBlob is backed by an id → string map. */ +function mockStorage( + blobs: Record, +): Pick { + return { + readBlob: async (id) => { + const content: string | undefined = blobs[id]; + assert(content !== undefined, `Test storage missing blob ${id}`); + return stringToBuffer(content, "utf8"); + }, + }; +} + +function tree(partial: Partial): ISnapshotTree { + return { blobs: {}, trees: {}, ...partial }; +} + +/** + * Encodes the same UTF-8 bytes the test storage shim returns for `content`, + * matching the base64 output `captureReferencedAttachmentBlobs` produces. + */ +const toB64 = (content: string): string => + bufferToString(stringToBuffer(content, "utf8"), "base64"); + +describe("captureReferencedContents", () => { + describe("readReferencedSnapshotBlobs", () => { + it("inlines every blob in a fully-referenced tree", async () => { + const snapshot = tree({ + blobs: { attributes: "a" }, + trees: { + ".channels": tree({ + trees: { + ds1: tree({ + blobs: { ".component": "b" }, + trees: { + root: tree({ blobs: { header: "c" } }), + }, + }), + }, + }), + }, + }); + const storage = mockStorage({ a: "A", b: "B", c: "C" }); + const result = await readReferencedSnapshotBlobs(snapshot, storage); + assert.deepStrictEqual(result, { a: "A", b: "B", c: "C" }); + }); + + it("skips subtrees flagged unreferenced", async () => { + const snapshot = tree({ + trees: { + live: tree({ blobs: { live: "kept" } }), + dead: tree({ + unreferenced: true, + blobs: { dead: "skipped" }, + trees: { nested: tree({ blobs: { nested: "skipped-too" } }) }, + }), + }, + }); + const storage = mockStorage({ kept: "KEPT" }); + // Missing entries would throw in the storage shim, asserting we never read them. + const result = await readReferencedSnapshotBlobs(snapshot, storage); + assert.deepStrictEqual(result, { kept: "KEPT" }); + }); + + it("special-cases root .blobs: reads only the redirect table", async () => { + const snapshot = tree({ + trees: { + ".blobs": tree({ + blobs: { + ".redirectTable": "rt", + "attachment-storage-id": "attachment-storage-id", + }, + }), + }, + }); + const storage = mockStorage({ rt: "RT" }); + const result = await readReferencedSnapshotBlobs(snapshot, storage); + assert.deepStrictEqual( + result, + { rt: "RT" }, + "attachment blob contents must not be read via the general walker", + ); + }); + + it("prefers ISnapshot.blobContents over storage when given an ISnapshot", async () => { + const snapshotTree = tree({ blobs: { x: "content-id" } }); + const snapshot: ISnapshot = { + snapshotTree, + blobContents: new Map([["content-id", stringToBuffer("IN-MEMORY", "utf8")]]), + ops: [], + sequenceNumber: 10, + latestSequenceNumber: undefined, + snapshotFormatV: 1, + }; + // storage has a different value — if it's consulted, the test fails. + const storage = mockStorage({ "content-id": "FROM-STORAGE" }); + const result = await readReferencedSnapshotBlobs(snapshot, storage); + assert.deepStrictEqual(result, { "content-id": "IN-MEMORY" }); + }); + }); + + describe("parseGcSnapshotData", () => { + it("returns undefined when the snapshot has no gc tree", async () => { + const result = await parseGcSnapshotData(tree({}), mockStorage({})); + assert.strictEqual(result, undefined); + }); + + it("parses gc state, tombstones, and deleted nodes from their blob keys", async () => { + const snapshot = tree({ + trees: { + gc: tree({ + blobs: { + __gc_root: "gcblob", + __tombstones: "tsblob", + __deletedNodes: "delblob", + }, + }), + }, + }); + const storage = mockStorage({ + gcblob: JSON.stringify({ + gcNodes: { "/a": { outboundRoutes: [], unreferencedTimestampMs: 1 } }, + }), + tsblob: JSON.stringify(["/b"]), + delblob: JSON.stringify(["/c"]), + }); + const result = await parseGcSnapshotData(snapshot, storage); + assert(result !== undefined); + assert.deepStrictEqual(result.tombstones, ["/b"]); + assert.deepStrictEqual(result.deletedNodes, ["/c"]); + assert.strictEqual( + result.gcState?.gcNodes["/a"]?.unreferencedTimestampMs, + 1, + "gc state merged from __gc-prefixed blobs", + ); + }); + }); + + describe("captureReferencedAttachmentBlobs", () => { + function attachmentsOnly( + table: [string, string][], + blobBytes: Record, + ): { + snapshot: ISnapshotTree; + storage: ReturnType; + } { + const blobs: Record = { + rt: JSON.stringify(table), + ...blobBytes, + }; + const snapshot = tree({ + trees: { + ".blobs": tree({ blobs: { ".redirectTable": "rt" } }), + }, + }); + return { snapshot, storage: mockStorage(blobs) }; + } + + it("returns {} when there is no .blobs subtree", async () => { + const result = await captureReferencedAttachmentBlobs( + tree({}), + mockStorage({}), + undefined, + ); + assert.deepStrictEqual(result, {}); + }); + + it("includes every attachment blob when gc data is undefined", async () => { + const { snapshot, storage } = attachmentsOnly( + [ + ["l1", "s1"], + ["l2", "s2"], + ], + { s1: "S1", s2: "S2" }, + ); + const result = await captureReferencedAttachmentBlobs(snapshot, storage, undefined); + assert.deepStrictEqual(result, { s1: toB64("S1"), s2: toB64("S2") }); + }); + + it("skips blobs marked unreferenced in gc state", async () => { + const { snapshot, storage } = attachmentsOnly( + [ + ["keep", "keep-storage"], + ["drop", "drop-storage"], + ], + { "keep-storage": "K", "drop-storage": "must-not-read" }, + ); + const gcData: IGcSnapshotData = { + gcState: { + gcNodes: { + "/_blobs/drop": { outboundRoutes: [], unreferencedTimestampMs: 123 }, + "/_blobs/keep": { outboundRoutes: [] }, + }, + }, + tombstones: undefined, + deletedNodes: undefined, + }; + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + assert.deepStrictEqual(result, { "keep-storage": toB64("K") }); + }); + + it("skips blobs listed in tombstones or deletedNodes", async () => { + const { snapshot, storage } = attachmentsOnly( + [ + ["tomb", "tomb-storage"], + ["del", "del-storage"], + ["ok", "ok-storage"], + ], + { "tomb-storage": "x", "del-storage": "y", "ok-storage": "OK" }, + ); + const gcData: IGcSnapshotData = { + gcState: { gcNodes: {} }, + tombstones: ["/_blobs/tomb"], + deletedNodes: ["/_blobs/del"], + }; + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + assert.deepStrictEqual(result, { "ok-storage": toB64("OK") }); + }); + + it("still applies tombstones and deletedNodes when gcState is undefined", async () => { + const { snapshot, storage } = attachmentsOnly( + [ + ["tomb", "tomb-storage"], + ["del", "del-storage"], + ["ok", "ok-storage"], + ], + { "tomb-storage": "x", "del-storage": "y", "ok-storage": "OK" }, + ); + const gcData: IGcSnapshotData = { + gcState: undefined, + tombstones: ["/_blobs/tomb"], + deletedNodes: ["/_blobs/del"], + }; + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + assert.deepStrictEqual( + result, + { "ok-storage": toB64("OK") }, + "tombstones and deletedNodes are authoritative even without gcState", + ); + }); + + it("keeps blobs that are absent from the gc graph (gc lag tolerance)", async () => { + const { snapshot, storage } = attachmentsOnly([["recent", "recent-storage"]], { + "recent-storage": "R", + }); + const gcData: IGcSnapshotData = { + gcState: { gcNodes: {} }, // empty graph: the blob isn't listed at all + tombstones: undefined, + deletedNodes: undefined, + }; + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + assert.deepStrictEqual( + result, + { "recent-storage": toB64("R") }, + "blobs absent from the GC graph must be kept", + ); + }); + + it("returns legacy identity-mapped blobs from .blobs (no .redirectTable entry)", async () => { + // Pre-redirect-table format: `.blobs` listed attachment storage ids + // directly under their own keys, so the redirect table entry is the + // identity mapping `(storageId, storageId)`. readRedirectTable + // reconstructs that mapping and captureReferencedAttachmentBlobs + // must then read those blobs. + const snapshot = tree({ + trees: { + ".blobs": tree({ + blobs: { "legacy-storage-id": "legacy-storage-id" }, + }), + }, + }); + const storage = mockStorage({ "legacy-storage-id": "LEGACY" }); + const result = await captureReferencedAttachmentBlobs(snapshot, storage, undefined); + assert.deepStrictEqual(result, { "legacy-storage-id": toB64("LEGACY") }); + }); + + it("returns legacy identity-mapped blobs alongside redirect-table entries", async () => { + // Mixed-format `.blobs`: some entries under a `.redirectTable` blob, + // others as direct storage-id keys. Both must surface, and a + // `.redirectTable` keyed entry must not also be treated as a legacy + // identity-mapped entry. + const snapshot = tree({ + trees: { + ".blobs": tree({ + blobs: { + ".redirectTable": "rt", + "legacy-storage-id": "legacy-storage-id", + }, + }), + }, + }); + const storage = mockStorage({ + rt: JSON.stringify([["modern-local", "modern-storage"]]), + "modern-storage": "MODERN", + "legacy-storage-id": "LEGACY", + }); + const result = await captureReferencedAttachmentBlobs(snapshot, storage, undefined); + assert.deepStrictEqual(result, { + "modern-storage": toB64("MODERN"), + "legacy-storage-id": toB64("LEGACY"), + }); + }); + + it("ignores gc nodes for non-attachment-blob paths", async () => { + // gcState may contain unreferenced/tombstoned nodes for data stores, + // channels, etc. Those paths must not be confused with blob localIds + // — the attachment filter only looks at /_blobs/ paths. + const { snapshot, storage } = attachmentsOnly([["live", "live-storage"]], { + "live-storage": "LIVE", + }); + const gcData: IGcSnapshotData = { + gcState: { + gcNodes: { + "/dataStores/some-id": { + outboundRoutes: [], + unreferencedTimestampMs: 123, + }, + }, + }, + tombstones: ["/dataStores/another"], + deletedNodes: ["/channels/x"], + }; + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + assert.deepStrictEqual( + result, + { "live-storage": toB64("LIVE") }, + "non-blob gc paths must not influence attachment filtering", + ); + }); + + it("integrates with parseGcSnapshotData on a snapshot that carries a real gc subtree", async () => { + // End-to-end through both helpers: build a snapshot whose gc subtree + // blobs encode unreferenced + tombstoned + deleted simultaneously, + // run parseGcSnapshotData on it, then feed that into the attachment + // filter. Verifies the full GC-driven exclusion path that + // captureFullContainerState relies on, not just the helpers in + // isolation. + const snapshot = tree({ + trees: { + ".blobs": tree({ blobs: { ".redirectTable": "rt" } }), + gc: tree({ + blobs: { + __gc_root: "gc-blob", + __tombstones: "ts-blob", + __deletedNodes: "del-blob", + }, + }), + }, + }); + const storage = mockStorage({ + rt: JSON.stringify([ + ["live", "live-storage"], + ["unref", "unref-storage"], + ["tomb", "tomb-storage"], + ["del", "del-storage"], + ]), + "gc-blob": JSON.stringify({ + gcNodes: { + "/_blobs/live": { outboundRoutes: [] }, + "/_blobs/unref": { + outboundRoutes: [], + unreferencedTimestampMs: 1700000000000, + }, + }, + }), + "ts-blob": JSON.stringify(["/_blobs/tomb"]), + "del-blob": JSON.stringify(["/_blobs/del"]), + "live-storage": "LIVE", + "unref-storage": "must-not-read", + "tomb-storage": "must-not-read", + "del-storage": "must-not-read", + }); + + const gcData = await parseGcSnapshotData(snapshot, storage); + assert(gcData !== undefined, "snapshot has a gc subtree, so gcData must parse"); + const result = await captureReferencedAttachmentBlobs(snapshot, storage, gcData); + + assert.deepStrictEqual( + result, + { "live-storage": toB64("LIVE") }, + "only the live blob survives all three GC mechanisms", + ); + }); + }); + + describe("snapshotHasLoadingGroups", () => { + it("returns false for a snapshot with no groupIds anywhere", () => { + const snapshot = tree({ + trees: { + a: tree({ trees: { nested: tree({}) } }), + b: tree({}), + }, + }); + assert.strictEqual(snapshotHasLoadingGroups(snapshot), false); + }); + + it("returns true for a groupId on a top-level subtree", () => { + const snapshot = tree({ + trees: { a: tree({ groupId: "g1" }) }, + }); + assert.strictEqual(snapshotHasLoadingGroups(snapshot), true); + }); + + it("returns true for a groupId on a deeply nested subtree", () => { + const snapshot = tree({ + trees: { + a: tree({ + trees: { + fine: tree({}), + deep: tree({ + trees: { deeper: tree({ groupId: "g1" }) }, + }), + }, + }), + }, + }); + assert.strictEqual(snapshotHasLoadingGroups(snapshot), true); + }); + + it("ignores groupIds inside unreferenced subtrees", () => { + const snapshot = tree({ + trees: { + dead: tree({ unreferenced: true, groupId: "dead-group" }), + }, + }); + assert.strictEqual( + snapshotHasLoadingGroups(snapshot), + false, + "unreferenced subtrees would not be loaded by the runtime, so their groupIds don't count", + ); + }); + + it("returns false when the entire snapshot is unreferenced", () => { + const snapshot = tree({ unreferenced: true, groupId: "g1" }); + assert.strictEqual(snapshotHasLoadingGroups(snapshot), false); + }); + }); + + describe("extractBlobAttachReferences", () => { + it("extracts (localId, storageId) from BlobAttach metadata", () => { + const result = extractBlobAttachReferences({ + metadata: { localId: "L", blobId: "S" }, + }); + assert.deepStrictEqual(result, [{ localId: "L", storageId: "S" }]); + }); + + it("returns [] when metadata is undefined", () => { + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: undefined }), []); + }); + + it("returns [] when metadata is null", () => { + // eslint-disable-next-line unicorn/no-null + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: null }), []); + }); + + it("returns [] when metadata is not an object", () => { + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: "string" }), []); + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: 42 }), []); + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: true }), []); + }); + + it("returns [] when localId is missing", () => { + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: { blobId: "S" } }), []); + }); + + it("returns [] when blobId is missing", () => { + assert.deepStrictEqual(extractBlobAttachReferences({ metadata: { localId: "L" } }), []); + }); + + it("returns [] when localId is not a string", () => { + assert.deepStrictEqual( + extractBlobAttachReferences({ metadata: { localId: 1, blobId: "S" } }), + [], + ); + }); + + it("returns [] when blobId is not a string", () => { + assert.deepStrictEqual( + extractBlobAttachReferences({ metadata: { localId: "L", blobId: 2 } }), + [], + ); + }); + + it("tolerates extra fields on metadata", () => { + const result = extractBlobAttachReferences({ + metadata: { localId: "L", blobId: "S", batchId: "b", extra: 99 }, + }); + assert.deepStrictEqual(result, [{ localId: "L", storageId: "S" }]); + }); + }); + + describe("unreferencedAttachmentBlobLocalIds", () => { + it("returns undefined when gcData is undefined", () => { + assert.strictEqual(unreferencedAttachmentBlobLocalIds(undefined), undefined); + }); + + it("returns an empty set when gcData has no unreferenced/tombstoned/deleted blobs", () => { + const result = unreferencedAttachmentBlobLocalIds({ + gcState: { + gcNodes: { "/_blobs/live": { outboundRoutes: [] } }, + }, + tombstones: undefined, + deletedNodes: undefined, + }); + assert.deepStrictEqual([...(result ?? [])], []); + }); + + it("collects localIds from gcState, tombstones, and deletedNodes", () => { + const result = unreferencedAttachmentBlobLocalIds({ + gcState: { + gcNodes: { + "/_blobs/unref": { + outboundRoutes: [], + unreferencedTimestampMs: 1, + }, + "/_blobs/live": { outboundRoutes: [] }, + }, + }, + tombstones: ["/_blobs/tomb"], + deletedNodes: ["/_blobs/del"], + }); + assert.deepStrictEqual([...(result ?? [])].sort(), ["del", "tomb", "unref"]); + }); + + it("ignores non-/_blobs/ paths in all three sources", () => { + const result = unreferencedAttachmentBlobLocalIds({ + gcState: { + gcNodes: { + "/dataStores/x": { + outboundRoutes: [], + unreferencedTimestampMs: 1, + }, + }, + }, + tombstones: ["/dataStores/y"], + deletedNodes: ["/channels/z"], + }); + assert.deepStrictEqual([...(result ?? [])], []); + }); + }); + + describe("inlineAttachmentBlobsByReference", () => { + it("returns {} for empty references", async () => { + const storage = mockStorage({}); + const result = await inlineAttachmentBlobsByReference([], storage, undefined, {}); + assert.deepStrictEqual(result, {}); + }); + + it("reads each unique storageId once and base64-encodes", async () => { + const refs: IBlobAttachReference[] = [ + { localId: "l1", storageId: "s1" }, + { localId: "l2", storageId: "s2" }, + ]; + const storage = mockStorage({ s1: "S1", s2: "S2" }); + const result = await inlineAttachmentBlobsByReference(refs, storage, undefined, {}); + assert.deepStrictEqual(result, { s1: toB64("S1"), s2: toB64("S2") }); + }); + + it("collapses multiple references that share a storageId to a single read", async () => { + let reads = 0; + const storage: Pick = { + readBlob: async (id) => { + reads++; + return stringToBuffer(`bytes-${id}`, "utf8"); + }, + }; + const refs: IBlobAttachReference[] = [ + { localId: "a", storageId: "shared" }, + { localId: "b", storageId: "shared" }, + { localId: "c", storageId: "shared" }, + ]; + const result = await inlineAttachmentBlobsByReference(refs, storage, undefined, {}); + assert.deepStrictEqual(result, { shared: toB64("bytes-shared") }); + assert.strictEqual(reads, 1, "shared storageId must be read at most once"); + }); + + it("skips references whose localId is in unreferencedLocalIds", async () => { + const refs: IBlobAttachReference[] = [ + { localId: "keep", storageId: "keep-s" }, + { localId: "drop", storageId: "drop-s" }, + ]; + // drop-s is intentionally absent from storage — touching it would throw. + const storage = mockStorage({ "keep-s": "K" }); + const result = await inlineAttachmentBlobsByReference( + refs, + storage, + new Set(["drop"]), + {}, + ); + assert.deepStrictEqual(result, { "keep-s": toB64("K") }); + }); + + it("skips references whose storageId is already in existing", async () => { + const refs: IBlobAttachReference[] = [ + { localId: "old", storageId: "old-s" }, + { localId: "new", storageId: "new-s" }, + ]; + // old-s is intentionally absent from storage — touching it would throw. + const storage = mockStorage({ "new-s": "N" }); + const existing = { "old-s": toB64("PRE") }; + const result = await inlineAttachmentBlobsByReference( + refs, + storage, + undefined, + existing, + ); + assert.deepStrictEqual( + result, + { "new-s": toB64("N") }, + "only freshly-read entries are returned; caller merges with existing", + ); + }); + + it("returns {} when every reference is filtered out", async () => { + const refs: IBlobAttachReference[] = [ + { localId: "drop", storageId: "drop-s" }, + { localId: "dup", storageId: "dup-s" }, + ]; + const storage = mockStorage({}); + const result = await inlineAttachmentBlobsByReference(refs, storage, new Set(["drop"]), { + "dup-s": toB64("X"), + }); + assert.deepStrictEqual(result, {}); + }); + }); + + describe("mapWithConcurrency", () => { + it("returns [] for empty input", async () => { + const calls: number[] = []; + const result = await mapWithConcurrency([], 4, async (x) => { + calls.push(x); + return x; + }); + assert.deepStrictEqual(result, []); + assert.deepStrictEqual(calls, []); + }); + + it("preserves input order on output", async () => { + // Reverse the natural completion order: earlier indices wait longer, + // so an order-by-completion implementation would visibly fail. + const items = [0, 1, 2, 3, 4]; + const result = await mapWithConcurrency(items, 8, async (x) => { + await new Promise((resolve) => setTimeout(resolve, (items.length - x) * 2)); + return x * 10; + }); + assert.deepStrictEqual(result, [0, 10, 20, 30, 40]); + }); + + it("processes every item exactly once", async () => { + const seen = new Set(); + const items = Array.from({ length: 25 }, (_, i) => i); + await mapWithConcurrency(items, 4, async (x) => { + assert(!seen.has(x), `item ${x} processed twice`); + seen.add(x); + return x; + }); + assert.strictEqual(seen.size, items.length); + }); + + it("never exceeds the configured concurrency limit", async () => { + let inFlight = 0; + let peak = 0; + const limit = 3; + const items = Array.from({ length: 20 }, (_, i) => i); + await mapWithConcurrency(items, limit, async () => { + inFlight++; + peak = Math.max(peak, inFlight); + // Yield so other workers can ramp up before this one finishes. + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight--; + }); + assert(peak <= limit, `peak concurrency ${peak} exceeded limit ${limit}`); + assert(peak >= 2, `expected concurrency > 1, got peak ${peak}`); + }); + + it("caps worker count at items.length when limit > items.length", async () => { + let peak = 0; + let inFlight = 0; + await mapWithConcurrency([1, 2], 100, async () => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 2)); + inFlight--; + }); + assert(peak <= 2, `peak concurrency ${peak} exceeded item count`); + }); + + it("propagates errors from fn", async () => { + await assert.rejects( + mapWithConcurrency([1, 2, 3], 2, async (x) => { + if (x === 2) { + throw new Error("boom"); + } + return x; + }), + /boom/, + ); + }); + }); +}); diff --git a/packages/runtime/container-runtime/src/blobManager/blobManager.ts b/packages/runtime/container-runtime/src/blobManager/blobManager.ts index 634d8a8c5782..4918875848b0 100644 --- a/packages/runtime/container-runtime/src/blobManager/blobManager.ts +++ b/packages/runtime/container-runtime/src/blobManager/blobManager.ts @@ -229,6 +229,9 @@ interface IBlobManagerInternalEvents { const createAbortError = (): LoggingError => new LoggingError("uploadBlob aborted"); +/** + * @internal + */ export const blobManagerBasePath = "_blobs"; export class BlobManager { diff --git a/packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts b/packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts index a98d59fab3dd..ca9a7b1e4777 100644 --- a/packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts +++ b/packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts @@ -18,6 +18,9 @@ export interface IBlobManagerLoadInfo { redirectTable?: [string, string][]; } +/** + * @internal + */ export const redirectTableBlobName = ".redirectTable"; /** diff --git a/packages/runtime/container-runtime/src/index.ts b/packages/runtime/container-runtime/src/index.ts index d5b27604c3ea..a5c982f1db2e 100644 --- a/packages/runtime/container-runtime/src/index.ts +++ b/packages/runtime/container-runtime/src/index.ts @@ -30,6 +30,11 @@ export type { } from "./messageTypes.js"; export { ContainerMessageType } from "./messageTypes.js"; export type { IBlobManagerLoadInfo } from "./blobManager/index.js"; +export { + blobManagerBasePath, + blobsTreeName, + redirectTableBlobName, +} from "./blobManager/index.js"; export type { IDataStoreAliasMessage } from "./dataStore.js"; export { FluidDataStoreRegistry } from "./dataStoreRegistry.js"; export { diff --git a/packages/test/local-server-tests/src/test/captureFullContainerState.spec.ts b/packages/test/local-server-tests/src/test/captureFullContainerState.spec.ts new file mode 100644 index 000000000000..6ecba677edc0 --- /dev/null +++ b/packages/test/local-server-tests/src/test/captureFullContainerState.spec.ts @@ -0,0 +1,513 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "assert"; + +import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils"; +import { + captureFullContainerState, + createDetachedContainer, + extractBlobAttachReferences, + loadFrozenContainerFromPendingState, + asLegacyAlpha, + type ContainerAlpha, +} from "@fluidframework/container-loader/internal"; +import type { FluidObject } from "@fluidframework/core-interfaces/internal"; +import type { + IDocumentService, + IDocumentServiceFactory, + IDocumentStorageService, +} from "@fluidframework/driver-definitions/internal"; +import type { + LocalDocumentServiceFactory, + LocalResolver, +} from "@fluidframework/local-driver/internal"; +import { SharedMap, type ISharedMap } from "@fluidframework/map/internal"; +import { isFluidHandle, toFluidHandleInternal } from "@fluidframework/runtime-utils/internal"; +import { LocalDeltaConnectionServer } from "@fluidframework/server-local-server"; +import { + timeoutPromise, + type ITestFluidObject, + type LocalCodeLoader, + type TestFluidObject, +} from "@fluidframework/test-utils/internal"; + +import { createLoader } from "../utils.js"; + +const toComparableArray = (map: ISharedMap): [string, unknown][] => + [...map.entries()].map(([key, value]) => [ + key, + isFluidHandle(value) ? toFluidHandleInternal(value).absolutePath : value, + ]); + +/** + * Wraps a document service factory so that any `readBlob` call on the + * resulting storage throws. Frozen-load delegates `readBlob` to the inner + * storage when one is provided, which masks the case where the artifact + * itself is not self-contained. Routing through this wrapper turns "blob + * fell through to live storage" into a hard failure that the test can catch. + */ +function makeFactoryWithFailingReadBlob( + inner: IDocumentServiceFactory, +): IDocumentServiceFactory { + const wrapService = (svc: IDocumentService): IDocumentService => + new Proxy(svc, { + get: (target, prop, receiver) => { + if (prop === "connectToStorage") { + return async (): Promise => { + const innerStorage = await target.connectToStorage(); + return new Proxy(innerStorage, { + get: (storageTarget, storageProp, storageReceiver) => { + if (storageProp === "readBlob") { + return async (_id: string): Promise => { + throw new Error( + "readBlob hit live storage — captured artifact was not self-contained", + ); + }; + } + return Reflect.get(storageTarget, storageProp, storageReceiver) as unknown; + }, + }); + }; + } + return Reflect.get(target, prop, receiver) as unknown; + }, + }); + + return { + createContainer: async (...args) => wrapService(await inner.createContainer(...args)), + createDocumentService: async (...args) => + wrapService(await inner.createDocumentService(...args)), + }; +} + +const initialize = async (): Promise<{ + container: ContainerAlpha; + testFluidObject: ITestFluidObject; + urlResolver: LocalResolver; + codeLoader: LocalCodeLoader; + documentServiceFactory: LocalDocumentServiceFactory; +}> => { + const deltaConnectionServer = LocalDeltaConnectionServer.create(); + const { urlResolver, codeDetails, codeLoader, loaderProps, documentServiceFactory } = + createLoader({ deltaConnectionServer }); + + const container = asLegacyAlpha( + await createDetachedContainer({ codeDetails, ...loaderProps }), + ); + const entryPoint: FluidObject = (await container.getEntryPoint()) ?? {}; + assert( + entryPoint.ITestFluidObject !== undefined, + "Expected entrypoint to be a valid TestFluidObject", + ); + return { + container, + testFluidObject: entryPoint.ITestFluidObject, + urlResolver, + codeLoader, + documentServiceFactory, + }; +}; + +describe("captureFullContainerState", () => { + it("captures state that can rehydrate a frozen container with matching data", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + for (let i = 0; i < 5; i++) { + testFluidObject.root.set(`detached-${i}`, i); + } + await container.attach(urlResolver.createCreateNewRequest("test")); + for (let i = 0; i < 5; i++) { + testFluidObject.root.set(`attached-${i}`, i); + } + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + + const parsed = JSON.parse(pendingLocalState) as { + attached: boolean; + pendingRuntimeState: unknown; + url: string; + savedOps: unknown[]; + baseSnapshot: unknown; + snapshotBlobs: Record; + }; + assert.strictEqual(parsed.attached, true, "captured state should be marked attached"); + assert.strictEqual( + parsed.pendingRuntimeState, + undefined, + "pendingRuntimeState must be undefined for driver-only capture", + ); + assert( + typeof parsed.url === "string" && parsed.url.length > 0, + "captured state should include the resolved container url", + ); + assert(parsed.baseSnapshot !== undefined, "captured state should include a base snapshot"); + assert( + Object.keys(parsed.snapshotBlobs).length > 0, + "captured state should inline snapshot blobs", + ); + + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + + assert.deepEqual( + toComparableArray(frozenEntryPoint.ITestFluidObject.root), + toComparableArray(testFluidObject.root), + "frozen container should reflect state at time of capture", + ); + }); + + it("includes ops posted after the snapshot in savedOps", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + await container.attach(urlResolver.createCreateNewRequest("test")); + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + // Make changes after attach so there are ops beyond the base snapshot. + for (let i = 0; i < 10; i++) { + testFluidObject.root.set(`post-snapshot-${i}`, i); + } + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + const parsed = JSON.parse(pendingLocalState) as { + savedOps: { sequenceNumber: number }[]; + }; + assert( + parsed.savedOps.length > 0, + "savedOps should contain the ops posted after the snapshot", + ); + for (let i = 1; i < parsed.savedOps.length; i++) { + assert( + parsed.savedOps[i].sequenceNumber > parsed.savedOps[i - 1].sequenceNumber, + "savedOps should be ordered by ascending sequence number", + ); + } + + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + for (let i = 0; i < 10; i++) { + assert.strictEqual( + frozenEntryPoint.ITestFluidObject.root.get(`post-snapshot-${i}`), + i, + `frozen container should replay op for post-snapshot-${i}`, + ); + } + }); + + it("captures DDS and blob references written before capture", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + await container.attach(urlResolver.createCreateNewRequest("test")); + + const nested = SharedMap.create(testFluidObject.runtime); + nested.set("nestedKey", "nestedValue"); + testFluidObject.root.set("nestedMapId", nested.handle); + + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + const retrieved = (await frozenEntryPoint.ITestFluidObject.root + .get("nestedMapId") + .get()) as ISharedMap; + assert(retrieved !== undefined, "Expected to retrieve nested SharedMap from frozen state"); + assert.strictEqual(retrieved.get("nestedKey"), "nestedValue"); + }); + + it("inlines attachment blob contents so reads don't go back to storage", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + // Upload before attach so the attach summary carries the blob in the + // `.blobs` subtree. Local tests run without a summarizer, so this is + // the only way to get attachment blobs into a fetched snapshot. + const blobPayload = "attachment-blob-payload"; + const blobHandle = await testFluidObject.runtime.uploadBlob( + stringToBuffer(blobPayload, "utf8"), + ); + testFluidObject.root.set("blobHandle", blobHandle); + + await container.attach(urlResolver.createCreateNewRequest("test")); + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + const parsed = JSON.parse(pendingLocalState) as { + snapshotBlobs: Record; + attachmentBlobContents?: Record; + }; + assert( + parsed.attachmentBlobContents !== undefined, + "Expected captured state to populate attachmentBlobContents for attachment blobs", + ); + const inlinedPayloads = Object.values(parsed.attachmentBlobContents).map((v) => + bufferToString(stringToBuffer(v, "base64"), "utf8"), + ); + assert( + inlinedPayloads.includes(blobPayload), + "Expected captured state to inline attachment blob contents by storage ID", + ); + + // Round-trip: the frozen container reads the blob through the cached + // attachmentBlobContents entry (base64-decoded into the blob cache on + // load), confirming the inlined copy is used on rehydrate. + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + const retrievedBlob = await frozenEntryPoint.ITestFluidObject.root.get("blobHandle").get(); + assert(retrievedBlob !== undefined, "Expected blob handle to resolve in frozen container"); + assert.strictEqual(bufferToString(retrievedBlob, "utf8"), blobPayload); + }); + + it("round-trips non-UTF-8 attachment blob bytes byte-exactly", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + // Deliberately invalid UTF-8: 0xff/0xfe/0xc0 are not valid lead bytes + // in any UTF-8 sequence, and a UTF-8 round-trip would replace them + // with U+FFFD, losing the original bytes irrecoverably. Base64 is the + // established encoding for binary attachment blobs in this codebase. + const binaryPayload = new Uint8Array([0xff, 0xfe, 0x00, 0x80, 0xc0]); + const blobHandle = await testFluidObject.runtime.uploadBlob(binaryPayload.buffer); + testFluidObject.root.set("binaryBlobHandle", blobHandle); + + await container.attach(urlResolver.createCreateNewRequest("test")); + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + + // First, prove the captured pending state itself encodes the binary + // bytes losslessly. local-server's FrozenDocumentService delegates + // readBlob through to live storage, which masks corruption in the + // pending state during rehydration; assert directly against the + // captured payload so this test fails if attachment-blob encoding + // regresses to UTF-8. + const parsed = JSON.parse(pendingLocalState) as { + attachmentBlobContents?: Record; + }; + assert( + parsed.attachmentBlobContents !== undefined, + "attachment blobs must be captured into attachmentBlobContents (not snapshotBlobs)", + ); + const capturedDecoded = Object.values(parsed.attachmentBlobContents).map( + (v) => new Uint8Array(stringToBuffer(v, "base64")), + ); + const matched = capturedDecoded.find( + (bytes) => + bytes.length === binaryPayload.length && bytes.every((b, i) => b === binaryPayload[i]), + ); + assert( + matched !== undefined, + `captured attachmentBlobContents must contain the original bytes; got ${capturedDecoded + .map((b) => `[${[...b].join(",")}]`) + .join(", ")}`, + ); + + // Then verify end-to-end through rehydration that the frozen + // container resolves the handle to the same bytes. + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + const retrievedBlob = await frozenEntryPoint.ITestFluidObject.root + .get("binaryBlobHandle") + .get(); + assert(retrievedBlob !== undefined, "Expected blob handle to resolve in frozen container"); + assert.deepStrictEqual( + new Uint8Array(retrievedBlob), + binaryPayload, + "Non-UTF-8 attachment blob bytes must round-trip byte-exactly through capture/rehydrate", + ); + }); + + it("inlines blobs uploaded after the base snapshot via blobAttach replay", async () => { + const { container, testFluidObject, urlResolver, codeLoader, documentServiceFactory } = + await initialize(); + + // Take the container live first. The attach summary becomes the base + // snapshot; any blob uploaded after this point reaches the captured + // artifact only as a BlobAttach op in the tail, not via the snapshot's + // `.blobs` redirect table. + await container.attach(urlResolver.createCreateNewRequest("test")); + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const blobPayload = "post-snapshot-attachment-payload"; + const blobHandle = await testFluidObject.runtime.uploadBlob( + stringToBuffer(blobPayload, "utf8"), + ); + testFluidObject.root.set("postSnapshotBlobHandle", blobHandle); + if (container.isDirty) { + await timeoutPromise((resolve) => container.once("saved", () => resolve())); + } + + const url = await container.getAbsoluteUrl(""); + assert(url !== undefined, "Expected container to provide a valid absolute URL"); + + const pendingLocalState = await captureFullContainerState({ + urlResolver, + documentServiceFactory, + request: { url }, + }); + + // Sanity: the savedOps tail must contain at least one blobAttach op, + // otherwise the test does not actually exercise the post-snapshot path. + // Route through the public extractor so this prerequisite stays correct + // if BlobAttach ops are ever wrapped in a groupedBatch. + const parsed = JSON.parse(pendingLocalState) as { + savedOps: { metadata?: unknown; contents: unknown }[]; + attachmentBlobContents?: Record; + }; + const totalBlobAttachReferences = parsed.savedOps.reduce( + (count, op) => count + extractBlobAttachReferences(op).length, + 0, + ); + assert( + totalBlobAttachReferences > 0, + "test prerequisite: savedOps must carry at least one blobAttach reference", + ); + assert( + parsed.attachmentBlobContents !== undefined, + "Expected attachmentBlobContents to be populated for the post-snapshot blob", + ); + const inlinedPayloads = Object.values(parsed.attachmentBlobContents).map((v) => + bufferToString(stringToBuffer(v, "base64"), "utf8"), + ); + assert( + inlinedPayloads.includes(blobPayload), + `Expected captured artifact to inline the post-snapshot blob; got [${inlinedPayloads.join(", ")}]`, + ); + + // Round-trip with a factory whose live storage throws on readBlob. + // If the captured artifact is genuinely self-contained, the handle + // resolves from the cached attachment bytes and live storage is + // never consulted. + const noLiveStorageFactory = makeFactoryWithFailingReadBlob(documentServiceFactory); + const frozenContainer = await loadFrozenContainerFromPendingState({ + codeLoader, + documentServiceFactory: noLiveStorageFactory, + urlResolver, + request: { url }, + pendingLocalState, + }); + const frozenEntryPoint: FluidObject = + await frozenContainer.getEntryPoint(); + assert( + frozenEntryPoint.ITestFluidObject !== undefined, + "Expected frozen container entrypoint to be a valid TestFluidObject", + ); + const retrievedBlob = await frozenEntryPoint.ITestFluidObject.root + .get("postSnapshotBlobHandle") + .get(); + assert(retrievedBlob !== undefined, "Expected blob handle to resolve in frozen container"); + assert.strictEqual(bufferToString(retrievedBlob, "utf8"), blobPayload); + }); +}); diff --git a/packages/test/local-server-tests/src/test/wireFormatConstants.spec.ts b/packages/test/local-server-tests/src/test/wireFormatConstants.spec.ts new file mode 100644 index 000000000000..406ff97a8c94 --- /dev/null +++ b/packages/test/local-server-tests/src/test/wireFormatConstants.spec.ts @@ -0,0 +1,42 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +import { wireFormatConstants } from "@fluidframework/container-loader/internal"; +import { + blobManagerBasePath, + blobsTreeName, + redirectTableBlobName, +} from "@fluidframework/container-runtime/internal"; +import { + gcBlobPrefix, + gcDeletedBlobKey, + gcTombstoneBlobKey, + gcTreeKey, +} from "@fluidframework/runtime-definitions/internal"; + +/** + * `container-loader` duplicates a handful of wire-format constants because the + * authoritative definitions live in `container-runtime` and + * `runtime-definitions`, which the loader cannot depend on. This contract test + * imports both copies and asserts they match + * + * Ideally these never change, if they do great care will be needed + * to preserve the correctness of the container-loader code that uses them. + */ +describe("wireFormatConstants contract", () => { + it("matches container-runtime and runtime-definitions values", () => { + assert.deepStrictEqual(wireFormatConstants, { + blobsTreeName, + redirectTableBlobName, + blobManagerBasePath, + gcTreeKey, + gcBlobPrefix, + gcTombstoneBlobKey, + gcDeletedBlobKey, + }); + }); +});