feat(tree): internals for field batch format with specialized node shapes

packages/dds/tree/src/feature-libraries/chunked-forest/codec/chunkDecoding.ts

@@ -40,14 +40,17 @@ import {
 import type { IncrementalDecoder } from "./codecs.js";
 import {
 	type EncodedAnyShape,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 	type EncodedChunkShapeV2,
+	type EncodedChunkShapeVTextExperimental,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedFieldBatchV2,
+	type EncodedFieldShape,
 	type EncodedIncrementalChunkShape,
 	type EncodedInlineArrayShape,
 	type EncodedNestedArrayShape,
 	type EncodedNodeShape,
+	type EncodedSpecializedNodeShape,
 	type EncodedValueShape,
 	SpecialField,
 	supportsIncrementalEncoding,
@@ -76,9 +79,109 @@ export function decode(
 	);
 }
 
+/**
+ * Walks a chain of specialized node shapes (`f`) back to the concrete `c` node shape they are
+ * ultimately based on.
+ */
+function resolveToNodeShape(
+	shapeIndex: number,
+	context: DecoderContext<EncodedChunkShape>,
+	visited: Set<number> = new Set(),
+): EncodedNodeShape {
+	assert(!visited.has(shapeIndex), "cyclic specialized node shape chain");
+	visited.add(shapeIndex);
+	const encoded = context.shapes[shapeIndex];
+	assert(encoded !== undefined, "base shape index out of bounds");
+	if (encoded.c !== undefined) {
+		return encoded.c;
+	}
+	assert(
+		"f" in encoded && encoded.f !== undefined,
+		"specialized node shape base must resolve to a node shape",
+	);
+	return applySpecialization(
+		resolveToNodeShape(encoded.f.base, context, visited),
+		encoded.f,
+		context,
+	);
+}
+
+/**
+ * Produces a merged {@link EncodedNodeShape} by overlaying `spec` onto `base`.
+ *
+ * `type` is always inherited from `base`. `value` and `extraFields` are inherited from `base`
+ * unless `spec` includes them as own properties (regardless of value), in which case `spec`'s
+ * value wins.
+ *
+ * For `fields`, override entries whose key matches a key in `base.fields` replace that entry's
+ * shape index in place; override entries with keys not present in `base` are appended at the end
+ * in `spec.fields` order. Field order from `base` is preserved.
+ */
+function applySpecialization(
+	base: EncodedNodeShape,
+	spec: EncodedSpecializedNodeShape,
+	context: DecoderContext<EncodedChunkShape>,
+): EncodedNodeShape {
+	const overrides = new Map<FieldKey, number>();
+	for (const [keyEncoded, shapeIndex] of spec.fields ?? []) {
+		const key = context.identifier<FieldKey>(keyEncoded);
+		assert(!overrides.has(key), "duplicate field key in specialized node shape");
+		overrides.set(key, shapeIndex);
+	}
+
+	const mergedFields: EncodedFieldShape[] = [];
+	const overriddenKeys = new Set<FieldKey>();
+	const baseKeys = new Set<FieldKey>();
+	for (const [keyEncoded, shapeIndex] of base.fields ?? []) {
+		const key = context.identifier<FieldKey>(keyEncoded);
+		assert(!baseKeys.has(key), "duplicate field key in base node shape");
+		baseKeys.add(key);
+		const overrideShape = overrides.get(key);
+		if (overrideShape === undefined) {
+			mergedFields.push([keyEncoded, shapeIndex]);
+		} else {
+			overriddenKeys.add(key);
+			mergedFields.push([keyEncoded, overrideShape]);
+		}
+	}
+	for (const [keyEncoded, shapeIndex] of spec.fields ?? []) {
+		if (!overriddenKeys.has(context.identifier<FieldKey>(keyEncoded))) {
+			mergedFields.push([keyEncoded, shapeIndex]);
+		}
+	}
+
+	return {
+		type: base.type,
+		value: "value" in spec ? spec.value : base.value,
+		fields: mergedFields.length > 0 ? mergedFields : undefined,
+		extraFields: "extraFields" in spec ? spec.extraFields : base.extraFields,
+	};
+}
+
+/**
+ * Decoder for {@link EncodedSpecializedNodeShape}s.
+ * Merges the specialization's field overrides with the resolved base node shape, then delegates
+ * to a {@link NodeDecoder} built from the merged shape.
+ */
+export class SpecializedNodeDecoder implements ChunkDecoder {
+	private readonly inner: NodeDecoder;
+	public constructor(
+		shape: EncodedSpecializedNodeShape,
+		context: DecoderContext<EncodedChunkShape>,
+	) {
+		this.inner = new NodeDecoder(
+			applySpecialization(resolveToNodeShape(shape.base, context), shape, context),
+			context,
+		);
+	}
+	public decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {
+		return this.inner.decode(decoders, stream);
+	}
+}
+
 const decoderLibrary = new DiscriminatedUnionDispatcher<
-	EncodedChunkShapeV1OrV2,
-	[context: DecoderContext<EncodedChunkShapeV1OrV2>],
+	EncodedChunkShapeVTextExperimental,
+	[context: DecoderContext<EncodedChunkShape>],
 	ChunkDecoder
 >({
 	a(shape: EncodedNestedArrayShape, context): ChunkDecoder {
@@ -99,6 +202,9 @@ const decoderLibrary = new DiscriminatedUnionDispatcher<
 	): ChunkDecoder {
 		return new IncrementalChunkDecoder(context);
 	},
+	f(shape: EncodedSpecializedNodeShape, context): ChunkDecoder {
+		return new SpecializedNodeDecoder(shape, context);
+	},
 });
 
 /**
@@ -300,7 +406,7 @@ type BasicFieldDecoder = (
  * Get a decoder for fields of a provided (via `shape` and `context`).
  */
 function fieldDecoder(
-	context: DecoderContext<EncodedChunkShapeV1OrV2>,
+	context: DecoderContext<EncodedChunkShape>,
 	key: FieldKey,
 	shape: number,
 ): BasicFieldDecoder {
@@ -319,7 +425,7 @@ export class NodeDecoder implements ChunkDecoder {
 	private readonly fieldDecoders: readonly BasicFieldDecoder[];
 	public constructor(
 		private readonly shape: EncodedNodeShape,
-		private readonly context: DecoderContext<EncodedChunkShapeV1OrV2>,
+		private readonly context: DecoderContext<EncodedChunkShape>,
 	) {
 		this.type = shape.type === undefined ? undefined : context.identifier(shape.type);
 

packages/dds/tree/src/feature-libraries/chunked-forest/codec/compressedEncode.ts

@@ -31,7 +31,7 @@ import type { FieldBatch } from "./fieldBatch.js";
 import {
 	type EncodedAnyShape,
 	type EncodedChunkShapeV1,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 	type EncodedChunkShapeV2,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedNestedArrayShape,
@@ -63,8 +63,8 @@ export function compressedEncode(
 	return updateShapesAndIdentifiersEncoding(context.version, batchBuffer);
 }
 
-export type BufferFormat = BufferFormatGeneric<EncodedChunkShapeV1OrV2>;
-export type Shape = ShapeGeneric<EncodedChunkShapeV1OrV2>;
+export type BufferFormat = BufferFormatGeneric<EncodedChunkShape>;
+export type Shape = ShapeGeneric<EncodedChunkShape>;
 
 /**
  * Like {@link FieldEncoder}, except data will be prefixed with the key.
@@ -166,7 +166,7 @@ export function asNodesEncoder(encoder: NodeEncoder): NodesEncoder {
 /**
  * Encodes a chunk with {@link EncodedAnyShape} by prefixing the data with its shape.
  */
-export class AnyShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
+export class AnyShape extends ShapeGeneric<EncodedChunkShape> {
 	private constructor() {
 		super();
 	}
@@ -271,7 +271,7 @@ export const anyFieldEncoder: FieldEncoder = {
  * which is an easy way to keep all the related code together without extra objects.
  */
 export class InlineArrayEncoder
-	extends ShapeGeneric<EncodedChunkShapeV1OrV2>
+	extends ShapeGeneric<EncodedChunkShape>
 	implements NodesEncoder, FieldEncoder
 {
 	public static readonly empty: InlineArrayEncoder = new InlineArrayEncoder(0, {
@@ -355,7 +355,7 @@ export class InlineArrayEncoder
 /**
  * Encodes the shape for a nested array as {@link EncodedNestedArrayShape} shape.
  */
-export class NestedArrayShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
+export class NestedArrayShape extends ShapeGeneric<EncodedChunkShape> {
 	/**
 	 * @param innerShape - The shape of each item in this nested array.
 	 */
@@ -366,7 +366,7 @@ export class NestedArrayShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
 	public encodeShape(
 		identifiers: DeduplicationTable<string>,
 		shapes: DeduplicationTable<Shape>,
-	): EncodedChunkShapeV1OrV2 {
+	): EncodedChunkShape {
 		const shape: EncodedNestedArrayShape =
 			shapes.valueToIndex.get(this.innerShape) ??
 			fail(0xb4f /* index for shape not found in table */);

packages/dds/tree/src/feature-libraries/chunked-forest/codec/format/formatV2.ts

@@ -16,16 +16,20 @@ import { shapesV1 } from "./formatV1.js";
 export type EncodedIncrementalChunkShape = Static<typeof EncodedIncrementalChunkShape>;
 export const EncodedIncrementalChunkShape = Type.Literal(0);
 
+/**
+ * The chunk shapes supported by the V2 format.
+ * @remarks
+ * See {@link EncodedChunkShapeV2}.
+ */
+export const shapesV2 = {
+	...shapesV1,
+	e: Type.Optional(EncodedIncrementalChunkShape),
+} as const;
+
 /**
  * V2 extension of {@link EncodedChunkShapeV1}.
  * @remarks
  * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
  */
 export type EncodedChunkShapeV2 = Static<typeof EncodedChunkShapeV2>;
-export const EncodedChunkShapeV2 = Type.Object(
-	{
-		...shapesV1,
-		e: Type.Optional(EncodedIncrementalChunkShape),
-	},
-	unionOptions,
-);
+export const EncodedChunkShapeV2 = Type.Object(shapesV2, unionOptions);

packages/dds/tree/src/feature-libraries/chunked-forest/codec/format/formatVText.ts

@@ -0,0 +1,74 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { type Static, Type } from "@sinclair/typebox";
+
+import { unionOptions } from "../../../../codec/index.js";
+
+import { ShapeIndex } from "./formatGeneric.js";
+import { EncodedFieldShape, EncodedValueShape } from "./formatV1.js";
+import { shapesV2 } from "./formatV2.js";
+
+/**
+ * A node shape that derives from another node shape by overlaying property-level overrides.
+ *
+ * @remarks
+ * Compresses runs of node shapes that differ only in a few properties: a base node shape
+ * defines the structural skeleton, and the specialization narrows specific properties.
+ *
+ * For example, a base `FormatNode` with a variable-boolean `bold` field can be specialized
+ * to a shape that pins `bold` to a constant `true` — every node decoded with the
+ * specialization contributes zero stream tokens for `bold`.
+ *
+ * Merge rules (see {@link applySpecialization}): `type` is always inherited from the resolved
+ * base. `fields`, `value`, and `extraFields` are inherited unless the specialization sets them
+ * as own properties — to inherit, the property must be omitted; setting it explicitly (even to
+ * `false` or `undefined`) is treated as an override.
+ */
+export type EncodedSpecializedNodeShape = Static<typeof EncodedSpecializedNodeShape>;
+export const EncodedSpecializedNodeShape = Type.Object(
+	{
+		/**
+		 * Index into the enclosing batch's shapes array of the shape this specializes.
+		 * Must resolve to either an {@link EncodedNodeShape} or another
+		 * `EncodedSpecializedNodeShape`; chains are followed transitively until a node shape
+		 * is reached. This restriction is enforced at runtime, not by the schema.
+		 */
+		base: ShapeIndex,
+		/**
+		 * Field-level overrides applied to the resolved base's `fields`. Entries whose key
+		 * matches a base field replace that field's shape index in place; entries with new
+		 * keys are appended after all base fields, in the order given here. Base field order
+		 * is preserved — this is the stream consumption order at decode time, so encoders
+		 * must serialize per-field tokens in the merged order, not in this list's order.
+		 */
+		fields: Type.Optional(Type.Array(EncodedFieldShape)),
+		/**
+		 * If present, replaces the resolved base's value shape.
+		 */
+		value: Type.Optional(EncodedValueShape),
+		/**
+		 * If present, replaces the resolved base's extraFields shape.
+		 */
+		extraFields: Type.Optional(ShapeIndex),
+	},
+	{ additionalProperties: false },
+);
+
+/**
+ * Experimental extension of {@link EncodedChunkShapeV2}.
+ * @remarks
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export type EncodedChunkShapeVTextExperimental = Static<
+	typeof EncodedChunkShapeVTextExperimental
+>;
+export const EncodedChunkShapeVTextExperimental = Type.Object(
+	{
+		...shapesV2,
+		f: Type.Optional(EncodedSpecializedNodeShape),
+	},
+	unionOptions,
+);

packages/dds/tree/src/feature-libraries/chunked-forest/codec/format/index.ts

@@ -16,14 +16,19 @@ export {
 	SpecialField,
 } from "./formatV1.js";
 export { EncodedIncrementalChunkShape, EncodedChunkShapeV2 } from "./formatV2.js";
+export {
+	EncodedChunkShapeVTextExperimental,
+	EncodedSpecializedNodeShape,
+} from "./formatVText.js";
 export {
 	FieldBatchFormatVersion,
 	EncodedFieldBatchV1,
 	EncodedFieldBatchV2,
+	EncodedFieldBatchVTextExperimental,
 	supportsIncrementalEncoding,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedFieldBatchV1AndV2,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 } from "./versions.js";
 export type {
 	ShapeIndex,

packages/dds/tree/src/feature-libraries/chunked-forest/codec/format/versions.ts

@@ -10,6 +10,7 @@ import { strictEnum, type Values } from "../../../../util/index.js";
 import { EncodedFieldBatchGeneric } from "./formatGeneric.js";
 import { EncodedChunkShapeV1 } from "./formatV1.js";
 import { EncodedChunkShapeV2 } from "./formatV2.js";
+import { EncodedChunkShapeVTextExperimental } from "./formatVText.js";
 
 /**
  * The format version for the field batch.
@@ -29,6 +30,10 @@ export const FieldBatchFormatVersion = strictEnum("FieldBatchFormatVersion", {
 	 * {@link EncodedIncrementalChunkShape} was added in this version.
 	 */
 	v2: 2,
+	/**
+	 * Experimental codec with optimizations for text.
+	 */
+	vTextExperimental: "text",
 });
 
 /**
@@ -68,6 +73,17 @@ export const EncodedFieldBatchV2 = EncodedFieldBatchGeneric(
 	EncodedChunkShapeV2,
 );
 
+/**
+ * Encoded {@link FieldBatch} using the experimental text optimized format.
+ */
+export type EncodedFieldBatchVTextExperimental = Static<
+	typeof EncodedFieldBatchVTextExperimental
+>;
+export const EncodedFieldBatchVTextExperimental = EncodedFieldBatchGeneric(
+	FieldBatchFormatVersion.vTextExperimental,
+	EncodedChunkShapeVTextExperimental,
+);
+
 /**
  * Encoded {@link FieldBatch}, which might use V2 features, but might also have been from a V1 encoder.
  * @remarks
@@ -88,9 +104,14 @@ export type EncodedFieldBatchV1OrV2 = EncodedFieldBatchV1 | EncodedFieldBatchV2;
 export type EncodedFieldBatchV1AndV2 = EncodedFieldBatchV1 & EncodedFieldBatchV2;
 
 /**
- * Encoded chunk shape, which might use V2 features, but might also have been from a V1 encoder.
+ * An encoded chunk shape from any known {@link FieldBatchFormatVersion}.
+ *
  * @remarks
- * Type wise, equivalent to V2, as that is a superset of V1.
- * Used instead of just V2 for clarity.
+ * Use this when working with chunk shapes uniformly across versions — for example, in the
+ * shared decoder dispatcher and in encoder shape base classes. New format versions should
+ * add their chunk shape variant to this union.
  */
-export type EncodedChunkShapeV1OrV2 = EncodedChunkShapeV1 | EncodedChunkShapeV2;
+export type EncodedChunkShape =
+	| EncodedChunkShapeV1
+	| EncodedChunkShapeV2
+	| EncodedChunkShapeVTextExperimental;