From a6683e183e3d53d16386ab8788415b2ab6b00a5f Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sun, 12 Apr 2026 11:03:23 +0800 Subject: [PATCH 01/12] Add headless YAOS CLI --- package-lock.json | 66 ++- package.json | 3 + packages/cli/README.md | 55 ++ packages/cli/package.json | 23 + packages/cli/src/config.ts | 217 ++++++++ packages/cli/src/index.ts | 168 ++++++ packages/cli/src/nodeDiskMirror.ts | 804 +++++++++++++++++++++++++++++ packages/cli/src/nodeVaultSync.ts | 187 +++++++ packages/cli/src/shims.d.ts | 2 + packages/cli/tests/config.test.ts | 119 +++++ packages/cli/tsconfig.json | 13 + src/sync/exclude.ts | 16 +- src/sync/snapshotClient.ts | 8 +- src/sync/vaultSync.ts | 69 ++- src/utils/normalizeVaultPath.ts | 7 + 15 files changed, 1713 insertions(+), 44 deletions(-) create mode 100644 packages/cli/README.md create mode 100644 packages/cli/package.json create mode 100644 packages/cli/src/config.ts create mode 100644 packages/cli/src/index.ts create mode 100644 packages/cli/src/nodeDiskMirror.ts create mode 100644 packages/cli/src/nodeVaultSync.ts create mode 100644 packages/cli/src/shims.d.ts create mode 100644 packages/cli/tests/config.test.ts create mode 100644 packages/cli/tsconfig.json create mode 100644 src/utils/normalizeVaultPath.ts diff --git a/package-lock.json b/package-lock.json index c1d6eb2..6ce41cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,16 @@ { "name": "yaos", - "version": "1.5.0", + "version": "1.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "yaos", - "version": "1.5.0", + "version": "1.5.1", "license": "0-BSD", + "workspaces": [ + "packages/*" + ], "dependencies": { "fast-diff": "^1.3.0", "fflate": "^0.8.2", @@ -1029,6 +1032,10 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@yaos/cli": { + "resolved": "packages/cli", + "link": true + }, "node_modules/acorn": { "version": "8.15.0", "dev": true, @@ -1087,7 +1094,6 @@ }, "node_modules/argparse": { "version": "2.0.1", - "dev": true, "license": "Python-2.0" }, "node_modules/array-buffer-byte-length": { @@ -1357,6 +1363,21 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/cliui": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", @@ -1382,6 +1403,15 @@ "version": "1.1.4", "license": "MIT" }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/concat-map": { "version": "0.0.1", "dev": true, @@ -3159,7 +3189,6 @@ }, "node_modules/js-yaml": { "version": "4.1.1", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -3762,6 +3791,19 @@ "dev": true, "license": "MIT" }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "dev": true, @@ -4907,6 +4949,22 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "packages/cli": { + "name": "@yaos/cli", + "version": "0.0.0", + "dependencies": { + "chokidar": "^4.0.3", + "commander": "^14.0.0" + }, + "bin": { + "yaos-cli": "dist/index.js" + }, + "devDependencies": { + "@types/node": "^16.11.6", + "esbuild": "0.25.5", + "typescript": "^5.8.3" + } } } } diff --git a/package.json b/package.json index a196dac..79a0853 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "A zero-terminal, real-time sync engine powered by your own Cloudflare Worker.", "main": "main.js", "type": "module", + "workspaces": [ + "packages/*" + ], "scripts": { "dev": "node esbuild.config.mjs", "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production", diff --git a/packages/cli/README.md b/packages/cli/README.md new file mode 100644 index 0000000..4ca3b0d --- /dev/null +++ b/packages/cli/README.md @@ -0,0 +1,55 @@ +# YAOS CLI + +Headless YAOS client for mirroring a Markdown vault directory to the YAOS CRDT room. + +## Commands + +```bash +yaos-cli daemon --host --token --vault-id --dir +yaos-cli sync --host --token --vault-id --dir +yaos-cli status --host --token --vault-id +``` + +- `daemon` performs startup reconciliation, starts the filesystem watcher, and stays running. +- `sync` performs one reconciliation pass and exits. +- `status` connects to YAOS and prints current connection/cache state as JSON. + +## Configuration precedence + +1. CLI flags +2. Environment variables +3. `~/.config/yaos/cli.json` (or `$XDG_CONFIG_HOME/yaos/cli.json`) + +## Environment variables + +- `YAOS_HOST` +- `YAOS_TOKEN` +- `YAOS_VAULT_ID` +- `YAOS_DIR` +- `YAOS_DEVICE_NAME` +- `YAOS_DEBUG` +- `YAOS_EXCLUDE_PATTERNS` +- `YAOS_MAX_FILE_SIZE_KB` +- `YAOS_EXTERNAL_EDIT_POLICY` +- `YAOS_FRONTMATTER_GUARD` +- `YAOS_CONFIG_DIR` + +## Config file example + +```json +{ + "host": "https://sync.example.com", + "token": "...", + "vaultId": "vault-123", + "dir": "/srv/vault", + "deviceName": "n100-headless", + "debug": false, + "excludePatterns": "templates/,scratch/", + "maxFileSizeKB": 2048, + "externalEditPolicy": "always", + "frontmatterGuardEnabled": true, + "configDir": ".obsidian" +} +``` + +Use file mode `0600` if you store the token in this file. diff --git a/packages/cli/package.json b/packages/cli/package.json new file mode 100644 index 0000000..3c45626 --- /dev/null +++ b/packages/cli/package.json @@ -0,0 +1,23 @@ +{ + "name": "@yaos/cli", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "yaos-cli": "dist/index.cjs" + }, + "scripts": { + "build": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs", + "typecheck": "tsc --noEmit -p tsconfig.json", + "test": "node --import jiti/register --test tests/*.ts" + }, + "dependencies": { + "chokidar": "^4.0.3", + "commander": "^14.0.0" + }, + "devDependencies": { + "@types/node": "^16.11.6", + "esbuild": "0.25.5", + "typescript": "^5.8.3" + } +} diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts new file mode 100644 index 0000000..0c7ed59 --- /dev/null +++ b/packages/cli/src/config.ts @@ -0,0 +1,217 @@ +import { promises as fs } from "node:fs"; +import * as os from "node:os"; +import * as nodePath from "node:path"; +import type { ExternalEditPolicy } from "../../../src/settings"; + +const DEFAULT_CONFIG_DIR = ".obsidian"; + +export interface CliCommandOptions { + host?: string; + token?: string; + vaultId?: string; + dir?: string; + deviceName?: string; + debug?: boolean; + excludePatterns?: string; + maxFileSizeKB?: number; + externalEditPolicy?: ExternalEditPolicy; + frontmatterGuardEnabled?: boolean; + configDir?: string; +} + +export interface CliFileConfig { + host?: string; + token?: string; + vaultId?: string; + dir?: string; + deviceName?: string; + debug?: boolean; + excludePatterns?: string; + maxFileSizeKB?: number; + externalEditPolicy?: ExternalEditPolicy; + frontmatterGuardEnabled?: boolean; + configDir?: string; +} + +export interface ResolvedCliConfig { + host?: string; + token?: string; + vaultId?: string; + dir?: string; + deviceName: string; + debug: boolean; + excludePatterns: string; + maxFileSizeKB: number; + externalEditPolicy: ExternalEditPolicy; + frontmatterGuardEnabled: boolean; + configDir: string; + configPath: string; +} + +export interface RuntimeCliConfig extends ResolvedCliConfig { + host: string; + token: string; + vaultId: string; + dir: string; +} + +const DEFAULTS = { + deviceName: os.hostname(), + debug: false, + excludePatterns: "", + maxFileSizeKB: 2048, + externalEditPolicy: "always" as ExternalEditPolicy, + frontmatterGuardEnabled: true, + configDir: DEFAULT_CONFIG_DIR, +}; + +export function getDefaultConfigPath(): string { + const baseDir = process.env.XDG_CONFIG_HOME + ? nodePath.resolve(process.env.XDG_CONFIG_HOME) + : nodePath.join(os.homedir(), ".config"); + return nodePath.join(baseDir, "yaos", "cli.json"); +} + +export async function resolveCliConfig(options: CliCommandOptions): Promise { + const configPath = getDefaultConfigPath(); + const fileConfig = await readConfigFile(configPath); + const envConfig = readEnvConfig(process.env); + + return { + ...DEFAULTS, + ...fileConfig, + ...envConfig, + ...pickDefined(options), + configPath, + }; +} + +export function requireRuntimeConfig( + config: ResolvedCliConfig, + requirements: { requireDir: boolean }, +): RuntimeCliConfig { + const missing = [ + config.host ? null : "host", + config.token ? null : "token", + config.vaultId ? null : "vaultId", + requirements.requireDir && !config.dir ? "dir" : null, + ].filter((value): value is string => value !== null); + + if (missing.length > 0) { + throw new Error(`Missing required configuration: ${missing.join(", ")}`); + } + + return { + ...config, + host: config.host!, + token: config.token!, + vaultId: config.vaultId!, + dir: config.dir!, + }; +} + +async function readConfigFile(configPath: string): Promise { + try { + const raw = await fs.readFile(configPath, "utf8"); + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("config file must contain a JSON object"); + } + return sanitizeFileConfig(parsed as Record); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return {}; + } + throw new Error(`Failed to load ${configPath}: ${(error as Error).message}`); + } +} + +function sanitizeFileConfig(value: Record): CliFileConfig { + const config: CliFileConfig = {}; + assignString(config, "host", value.host); + assignString(config, "token", value.token); + assignString(config, "vaultId", value.vaultId); + assignString(config, "dir", value.dir); + assignString(config, "deviceName", value.deviceName); + assignBoolean(config, "debug", value.debug); + assignString(config, "excludePatterns", value.excludePatterns); + assignNumber(config, "maxFileSizeKB", value.maxFileSizeKB); + assignExternalEditPolicy(config, value.externalEditPolicy); + assignBoolean(config, "frontmatterGuardEnabled", value.frontmatterGuardEnabled); + assignString(config, "configDir", value.configDir); + return config; +} + +function readEnvConfig(env: NodeJS.ProcessEnv): CliFileConfig { + const config: CliFileConfig = {}; + assignString(config, "host", env.YAOS_HOST); + assignString(config, "token", env.YAOS_TOKEN); + assignString(config, "vaultId", env.YAOS_VAULT_ID); + assignString(config, "dir", env.YAOS_DIR); + assignString(config, "deviceName", env.YAOS_DEVICE_NAME); + assignBoolean(config, "debug", parseBooleanEnv(env.YAOS_DEBUG)); + assignString(config, "excludePatterns", env.YAOS_EXCLUDE_PATTERNS); + assignNumber(config, "maxFileSizeKB", parseNumberEnv(env.YAOS_MAX_FILE_SIZE_KB)); + assignExternalEditPolicy(config, env.YAOS_EXTERNAL_EDIT_POLICY); + assignBoolean(config, "frontmatterGuardEnabled", parseBooleanEnv(env.YAOS_FRONTMATTER_GUARD)); + assignString(config, "configDir", env.YAOS_CONFIG_DIR); + return config; +} + +function pickDefined(value: T): Partial { + return Object.fromEntries( + Object.entries(value as Record).filter(([, entry]) => entry !== undefined), + ) as Partial; +} + +function assignString( + target: CliFileConfig, + key: K, + value: unknown, +): void { + if (typeof value === "string" && value.length > 0) { + target[key] = value as CliFileConfig[K]; + } +} + +function assignBoolean( + target: CliFileConfig, + key: K, + value: unknown, +): void { + if (typeof value === "boolean") { + target[key] = value as CliFileConfig[K]; + } +} + +function assignNumber( + target: CliFileConfig, + key: K, + value: unknown, +): void { + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + target[key] = value as CliFileConfig[K]; + } +} + +function assignExternalEditPolicy( + target: CliFileConfig, + value: unknown, +): void { + if (value === "always" || value === "closed-only" || value === "never") { + target.externalEditPolicy = value; + } +} + +function parseBooleanEnv(value: string | undefined): boolean | undefined { + if (value == null) return undefined; + if (value === "1" || value.toLowerCase() === "true") return true; + if (value === "0" || value.toLowerCase() === "false") return false; + return undefined; +} + +function parseNumberEnv(value: string | undefined): number | undefined { + if (value == null || value.length === 0) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts new file mode 100644 index 0000000..ccd7f9e --- /dev/null +++ b/packages/cli/src/index.ts @@ -0,0 +1,168 @@ +#!/usr/bin/env node + +import { Command, InvalidOptionArgumentError, Option } from "commander"; +import type { CliCommandOptions, ResolvedCliConfig } from "./config"; +import { requireRuntimeConfig, resolveCliConfig } from "./config"; +import { createNodeVaultSync, HeadlessYaosClient } from "./nodeVaultSync"; + +const program = new Command(); + +program + .name("yaos-cli") + .description("Headless YAOS client for filesystem-backed Markdown vaults") + .showHelpAfterError(); + +addCommonOptions( + program + .command("daemon") + .description("Run the headless YAOS client and keep watching the vault") + .action(async (options: CliCommandOptions) => { + const resolved = await resolveCliConfig(options); + const runtime = requireRuntimeConfig(resolved, { requireDir: true }); + const client = new HeadlessYaosClient(runtime); + const startup = await client.startup({ watch: true }); + console.log(JSON.stringify({ + mode: "daemon", + config: summarizeConfig(runtime), + startup, + status: client.getStatus(), + }, null, 2)); + await waitForShutdown(client); + }), + { includeDir: true }, +); + +addCommonOptions( + program + .command("sync") + .description("Perform one reconciliation pass and exit") + .action(async (options: CliCommandOptions) => { + const resolved = await resolveCliConfig(options); + const runtime = requireRuntimeConfig(resolved, { requireDir: true }); + const client = new HeadlessYaosClient(runtime); + try { + const startup = await client.startup({ watch: false }); + console.log(JSON.stringify({ + mode: "sync", + config: summarizeConfig(runtime), + startup, + status: client.getStatus(), + }, null, 2)); + } finally { + await client.stop(); + } + }), + { includeDir: true }, +); + +addCommonOptions( + program + .command("status") + .description("Show current connection and local-cache status") + .action(async (options: CliCommandOptions) => { + const resolved = await resolveCliConfig(options); + const vaultSync = createNodeVaultSync(resolved); + try { + const localLoaded = await vaultSync.waitForLocalPersistence(); + const providerSynced = await vaultSync.waitForProviderSync(); + console.log(JSON.stringify({ + mode: "status", + config: summarizeConfig(resolved), + localLoaded, + providerSynced, + connected: vaultSync.connected, + localReady: vaultSync.localReady, + connectionGeneration: vaultSync.connectionGeneration, + storedSchemaVersion: vaultSync.storedSchemaVersion, + safeReconcileMode: vaultSync.getSafeReconcileMode(), + fatalAuthError: vaultSync.fatalAuthError, + fatalAuthCode: vaultSync.fatalAuthCode, + activeMarkdownPaths: vaultSync.getActiveMarkdownPaths().length, + }, null, 2)); + } finally { + vaultSync.destroy(); + } + }), + { includeDir: false }, +); + +void program.parseAsync(process.argv); + +function addCommonOptions( + command: T, + options: { includeDir: boolean }, +): T { + command + .option("--host ", "YAOS server URL") + .option("--token ", "YAOS sync token") + .option("--vault-id ", "YAOS vault ID") + .option("--device-name ", "Device name reported to YAOS") + .option("--debug", "Enable verbose logging") + .option("--exclude-patterns ", "Comma-separated path prefixes to exclude") + .addOption( + new Option("--max-file-size-kb ", "Maximum markdown file size to sync") + .argParser(parsePositiveInteger), + ) + .addOption( + new Option("--external-edit-policy ", "How to treat local filesystem edits") + .choices(["always", "closed-only", "never"]), + ); + + if (options.includeDir) { + command.option("--dir ", "Vault directory to mirror"); + } + + return command; +} + +function parsePositiveInteger(value: string): number { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new InvalidOptionArgumentError(`Expected a positive integer, received ${value}`); + } + return parsed; +} + +function summarizeConfig(config: ResolvedCliConfig): Record { + return { + host: config.host ?? null, + vaultId: config.vaultId ?? null, + dir: config.dir ?? null, + deviceName: config.deviceName, + debug: config.debug, + excludePatterns: config.excludePatterns, + maxFileSizeKB: config.maxFileSizeKB, + externalEditPolicy: config.externalEditPolicy, + frontmatterGuardEnabled: config.frontmatterGuardEnabled, + configDir: config.configDir, + configPath: config.configPath, + }; +} + +async function waitForShutdown(client: HeadlessYaosClient): Promise { + await new Promise((resolve, reject) => { + const finish = async () => { + cleanup(); + try { + await client.stop(); + resolve(); + } catch (error) { + reject(error); + } + }; + + const onSigint = () => { + void finish(); + }; + const onSigterm = () => { + void finish(); + }; + const cleanup = () => { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + }; + + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + }); +} diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts new file mode 100644 index 0000000..ccbd9b6 --- /dev/null +++ b/packages/cli/src/nodeDiskMirror.ts @@ -0,0 +1,804 @@ +import chokidar, { type FSWatcher } from "chokidar"; +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import type { Dirent, Stats } from "node:fs"; +import * as nodePath from "node:path"; +import * as Y from "yjs"; +import type { ExternalEditPolicy } from "../../../src/settings"; +import { applyDiffToYText } from "../../../src/sync/diff"; +import { isExcluded } from "../../../src/sync/exclude"; +import { + isFrontmatterBlocked, + validateFrontmatterTransition, +} from "../../../src/sync/frontmatterGuard"; +import type { ReconcileMode, ReconcileResult, VaultSync } from "../../../src/sync/vaultSync"; +import { isMarkdownSyncable, ORIGIN_SEED } from "../../../src/types"; +import { normalizeVaultPath } from "../../../src/utils/normalizeVaultPath"; + +const WRITE_DEBOUNCE_MS = 300; +const WRITE_DEBOUNCE_BURST_MS = 1_000; +const WRITE_BURST_THRESHOLD = 20; +const MARKDOWN_DIRTY_SETTLE_MS = 350; +const SUPPRESS_MS = 500; +const MAX_CONCURRENT_WRITES = 5; +const WATCHER_STABILITY_MS = 200; +const WATCHER_POLL_MS = 50; + +const LOCAL_STRING_ORIGINS = new Set([ + ORIGIN_SEED, + "disk-sync", +]); + +type DirtyReason = "create" | "modify"; + +interface SuppressionEntry { + kind: "write" | "delete"; + expiresAt: number; + expectedBytes?: number; + expectedHash?: string; +} + +interface ScannedDiskState { + contents: Map; + presentPaths: Set; +} + +interface DirtyFile { + path: string; + reason: DirtyReason; + content: string; + stats: Stats | null; +} + +export interface NodeDiskMirrorOptions { + rootDir: string; + deviceName: string; + debug: boolean; + excludePatterns: string[]; + maxFileSizeKB: number; + externalEditPolicy: ExternalEditPolicy; + frontmatterGuardEnabled: boolean; + configDir?: string; +} + +export interface NodeDiskMirrorDebugSnapshot { + watcherReady: boolean; + dirtyCount: number; + deletedCount: number; + queuedWrites: string[]; + suppressedCount: number; +} + +function isLocalOrigin(origin: unknown, provider: unknown): boolean { + if (origin === provider) return false; + if (typeof origin === "string") return LOCAL_STRING_ORIGINS.has(origin); + if (origin == null) return true; + return true; +} + +export class NodeDiskMirror { + private readonly rootDir: string; + private readonly configDir: string; + private readonly maxFileSize: number; + private watcher: FSWatcher | null = null; + private watcherReady = false; + private mapObserverCleanups: Array<() => void> = []; + private dirtyMarkdownPaths = new Map(); + private deletedMarkdownPaths = new Set(); + private markdownDrainPromise: Promise | null = null; + private markdownDrainTimer: ReturnType | null = null; + private lastMarkdownDirtyAt = 0; + private suppressedPaths = new Map(); + private writeQueue = new Set(); + private forcedWritePaths = new Set(); + private debounceTimers = new Map>(); + private writeDrainPromise: Promise | null = null; + private pathWriteLocks = new Map>(); + + constructor( + private readonly vaultSync: VaultSync, + private readonly options: NodeDiskMirrorOptions, + ) { + this.rootDir = nodePath.resolve(options.rootDir); + this.configDir = options.configDir ?? ".obsidian"; + this.maxFileSize = options.maxFileSizeKB * 1024; + } + + startMapObservers(): void { + if (this.mapObserverCleanups.length > 0) return; + + const metaObserver = (event: Y.YMapEvent) => { + if (isLocalOrigin(event.transaction.origin, this.vaultSync.provider)) { + return; + } + event.changes.keys.forEach((change, fileId) => { + const oldMeta = change.oldValue as import("../../../src/types").FileMeta | undefined; + const newMeta = this.vaultSync.meta.get(fileId); + const oldPath = typeof oldMeta?.path === "string" ? normalizeVaultPath(oldMeta.path) : null; + const newPath = typeof newMeta?.path === "string" ? normalizeVaultPath(newMeta.path) : null; + const wasDeleted = this.vaultSync.isFileMetaDeleted(oldMeta); + const isDeleted = this.vaultSync.isFileMetaDeleted(newMeta); + + if (newPath && isDeleted && !wasDeleted) { + void this.handleRemoteDelete(newPath); + return; + } + + if (newPath && !isDeleted && wasDeleted) { + this.scheduleWrite(newPath); + return; + } + + if (oldPath && newPath && oldPath !== newPath && !isDeleted) { + void this.handleRemoteRename(oldPath, newPath); + return; + } + + if ((change.action === "add" || change.action === "update") && newPath && !isDeleted) { + this.scheduleWrite(newPath); + } + }); + }; + + this.vaultSync.meta.observe(metaObserver); + this.mapObserverCleanups.push(() => this.vaultSync.meta.unobserve(metaObserver)); + + const afterTxnHandler = (txn: Y.Transaction) => { + if (isLocalOrigin(txn.origin, this.vaultSync.provider)) return; + + for (const [changedType] of txn.changed) { + if (!(changedType instanceof Y.Text)) continue; + const fileId = this.vaultSync.getFileIdForText(changedType); + if (!fileId) continue; + const meta = this.vaultSync.meta.get(fileId); + if (!meta || this.vaultSync.isFileMetaDeleted(meta)) continue; + this.scheduleWrite(meta.path); + } + }; + + this.vaultSync.ydoc.on("afterTransaction", afterTxnHandler); + this.mapObserverCleanups.push(() => this.vaultSync.ydoc.off("afterTransaction", afterTxnHandler)); + } + + async reconcileFromDisk(mode: ReconcileMode): Promise { + const disk = await this.scanMarkdownFiles(); + const result = this.vaultSync.reconcileVault( + disk.contents, + disk.presentPaths, + mode, + this.options.deviceName, + ); + + for (const path of result.createdOnDisk) { + this.queueImmediateWrite(path, `reconcile-create:${mode}`, true); + } + for (const path of result.updatedOnDisk) { + this.queueImmediateWrite(path, `reconcile-update:${mode}`, true); + } + await this.kickWriteDrain(); + return result; + } + + async startWatching(): Promise { + if (this.watcher) return; + + const watcher = chokidar.watch(".", { + cwd: this.rootDir, + persistent: true, + ignoreInitial: true, + alwaysStat: true, + awaitWriteFinish: { + stabilityThreshold: WATCHER_STABILITY_MS, + pollInterval: WATCHER_POLL_MS, + }, + ignored: (rawPath, stats) => this.shouldIgnoreWatchPath(rawPath, stats ?? null), + }); + + watcher + .on("add", (rawPath, stats) => this.onDiskAdd(rawPath, stats ?? null)) + .on("change", (rawPath, stats) => this.onDiskChange(rawPath, stats ?? null)) + .on("unlink", (rawPath) => this.onDiskDelete(rawPath)) + .on("error", (error) => { + console.error("[yaos-cli] chokidar watcher error:", error); + }); + + this.watcher = watcher; + await new Promise((resolve, reject) => { + watcher.once("ready", () => { + this.watcherReady = true; + resolve(); + }); + watcher.once("error", reject); + }); + } + + async stop(): Promise { + if (this.markdownDrainTimer) { + clearTimeout(this.markdownDrainTimer); + this.markdownDrainTimer = null; + } + for (const timer of this.debounceTimers.values()) { + clearTimeout(timer); + } + this.debounceTimers.clear(); + await this.markdownDrainPromise; + await this.writeDrainPromise; + if (this.watcher) { + await this.watcher.close(); + this.watcher = null; + } + this.watcherReady = false; + for (const cleanup of this.mapObserverCleanups) { + cleanup(); + } + this.mapObserverCleanups = []; + this.dirtyMarkdownPaths.clear(); + this.deletedMarkdownPaths.clear(); + this.writeQueue.clear(); + this.forcedWritePaths.clear(); + this.suppressedPaths.clear(); + this.pathWriteLocks.clear(); + } + + getDebugSnapshot(): NodeDiskMirrorDebugSnapshot { + return { + watcherReady: this.watcherReady, + dirtyCount: this.dirtyMarkdownPaths.size, + deletedCount: this.deletedMarkdownPaths.size, + queuedWrites: Array.from(this.writeQueue), + suppressedCount: this.suppressedPaths.size, + }; + } + + private onDiskAdd(rawPath: string, stats: Stats | null): void { + const path = this.normalizeEventPath(rawPath); + if (!path || this.shouldIgnoreNormalizedPath(path, stats)) return; + this.markMarkdownDirty(path, "create"); + } + + private onDiskChange(rawPath: string, stats: Stats | null): void { + const path = this.normalizeEventPath(rawPath); + if (!path || this.shouldIgnoreNormalizedPath(path, stats)) return; + this.markMarkdownDirty(path, "modify"); + } + + private onDiskDelete(rawPath: string): void { + const path = this.normalizeEventPath(rawPath); + if (!path || !this.isMarkdownPathSyncable(path)) return; + this.deletedMarkdownPaths.add(path); + this.dirtyMarkdownPaths.delete(path); + this.lastMarkdownDirtyAt = Date.now(); + this.scheduleMarkdownDrain(); + } + + private markMarkdownDirty(path: string, reason: DirtyReason): void { + const previous = this.dirtyMarkdownPaths.get(path); + if (previous !== "create") { + this.dirtyMarkdownPaths.set(path, reason); + } + this.deletedMarkdownPaths.delete(path); + this.lastMarkdownDirtyAt = Date.now(); + this.scheduleMarkdownDrain(); + } + + private scheduleMarkdownDrain(): void { + if (this.markdownDrainTimer) { + clearTimeout(this.markdownDrainTimer); + } + + this.markdownDrainTimer = setTimeout(() => { + this.markdownDrainTimer = null; + const sinceLastDirty = Date.now() - this.lastMarkdownDirtyAt; + if (sinceLastDirty < MARKDOWN_DIRTY_SETTLE_MS) { + this.scheduleMarkdownDrain(); + return; + } + void this.kickMarkdownDrain(); + }, MARKDOWN_DIRTY_SETTLE_MS); + } + + private kickMarkdownDrain(): Promise { + if (this.markdownDrainPromise) return this.markdownDrainPromise; + this.markdownDrainPromise = this.drainDirtyMarkdownPaths().finally(() => { + this.markdownDrainPromise = null; + if (this.dirtyMarkdownPaths.size > 0 || this.deletedMarkdownPaths.size > 0) { + this.scheduleMarkdownDrain(); + } + }); + return this.markdownDrainPromise; + } + + private async drainDirtyMarkdownPaths(): Promise { + if (this.dirtyMarkdownPaths.size === 0 && this.deletedMarkdownPaths.size === 0) return; + + const batchDirty = Array.from(this.dirtyMarkdownPaths.entries()); + const batchDeletes = Array.from(this.deletedMarkdownPaths); + this.dirtyMarkdownPaths.clear(); + this.deletedMarkdownPaths.clear(); + + const survivingDeletes = new Set(); + for (const path of batchDeletes) { + if (!this.consumeDeleteSuppression(path)) { + survivingDeletes.add(path); + } + } + + const dirtyFiles: DirtyFile[] = []; + for (const [path, reason] of batchDirty) { + const current = await this.readDirtyFile(path, reason); + if (!current) continue; + const suppressed = reason === "create" + ? await this.shouldSuppressWriteEvent(path, "create", current.stats) + : await this.shouldSuppressWriteEvent(path, "modify", current.stats); + if (suppressed) continue; + dirtyFiles.push(current); + } + + const renamePairs = this.inferRenamePairs( + dirtyFiles.filter((entry) => entry.reason === "create"), + survivingDeletes, + ); + + for (const [oldPath, newPath] of renamePairs) { + this.vaultSync.queueRename(oldPath, newPath); + survivingDeletes.delete(oldPath); + } + + for (const path of survivingDeletes) { + this.vaultSync.handleDelete(path, this.options.deviceName); + } + + for (const dirtyFile of dirtyFiles) { + if (dirtyFile.reason === "create" && this.vaultSync.isPendingRenameTarget(dirtyFile.path)) { + continue; + } + await this.syncFileFromDisk(dirtyFile.path, dirtyFile.content); + } + } + + private inferRenamePairs( + creates: DirtyFile[], + deletes: Set, + ): Map { + const renames = new Map(); + for (const create of creates) { + const exactBasenameMatches = this.findRenameCandidates(create, deletes, true); + const candidates = exactBasenameMatches.length === 1 + ? exactBasenameMatches + : this.findRenameCandidates(create, deletes, false); + if (candidates.length !== 1) continue; + const oldPath = candidates[0]; + if (!oldPath) continue; + renames.set(oldPath, create.path); + deletes.delete(oldPath); + } + return renames; + } + + private findRenameCandidates( + create: DirtyFile, + deletes: Set, + requireSameBasename: boolean, + ): string[] { + const matches: string[] = []; + const newBasename = nodePath.posix.basename(create.path); + for (const oldPath of deletes) { + if (requireSameBasename && nodePath.posix.basename(oldPath) !== newBasename) { + continue; + } + const oldText = this.vaultSync.getTextForPath(oldPath); + if (!oldText) continue; + if (oldText.toJSON() !== create.content) continue; + matches.push(oldPath); + } + return matches; + } + + private async readDirtyFile(path: string, reason: DirtyReason): Promise { + const absolutePath = this.toAbsolutePath(path); + try { + const [stats, content] = await Promise.all([ + fs.stat(absolutePath), + fs.readFile(absolutePath, "utf8"), + ]); + if (this.maxFileSize > 0 && content.length > this.maxFileSize) { + this.log( + `syncFileFromDisk: skipping "${path}" (${Math.round(content.length / 1024)} KB exceeds limit)`, + ); + return null; + } + return { path, reason, content, stats }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + console.error(`[yaos-cli] failed reading dirty file "${path}":`, error); + } + return null; + } + } + + private async syncFileFromDisk(path: string, content: string): Promise { + if (!this.isMarkdownPathSyncable(path)) return; + if (this.options.externalEditPolicy === "never") { + this.log(`syncFileFromDisk: skipping "${path}" (external edit policy: never)`); + return; + } + + const existingText = this.vaultSync.getTextForPath(path); + if (existingText) { + const crdtContent = existingText.toJSON(); + if (crdtContent === content) return; + if (this.shouldBlockFrontmatterIngest(path, crdtContent, content, "disk-to-crdt")) { + return; + } + applyDiffToYText(existingText, crdtContent, content, "disk-sync"); + return; + } + + if (this.shouldBlockFrontmatterIngest(path, null, content, "disk-to-crdt-seed")) { + return; + } + + this.vaultSync.ensureFile(path, content, this.options.deviceName); + } + + private shouldBlockFrontmatterIngest( + path: string, + previousContent: string | null, + nextContent: string, + reason: string, + ): boolean { + if (!this.options.frontmatterGuardEnabled) return false; + const validation = validateFrontmatterTransition(previousContent, nextContent); + if (!isFrontmatterBlocked(validation)) return false; + this.log( + `frontmatter ingest blocked for "${path}" ` + + `(${validation.reasons.join(", ") || validation.risk}) [${reason}]`, + ); + return true; + } + + private scheduleWrite(path: string): void { + path = normalizeVaultPath(path); + const existing = this.debounceTimers.get(path); + if (existing) clearTimeout(existing); + const delay = this.writeQueue.size >= WRITE_BURST_THRESHOLD + ? WRITE_DEBOUNCE_BURST_MS + : WRITE_DEBOUNCE_MS; + this.debounceTimers.set( + path, + setTimeout(() => { + this.debounceTimers.delete(path); + this.writeQueue.add(path); + void this.kickWriteDrain(); + }, delay), + ); + } + + private queueImmediateWrite(path: string, reason: string, force = false): void { + path = normalizeVaultPath(path); + if (force) { + this.forcedWritePaths.add(path); + } + this.writeQueue.add(path); + this.log(`queueImmediateWrite: "${path}" (${reason}${force ? ", forced" : ""})`); + void this.kickWriteDrain(); + } + + private kickWriteDrain(): Promise { + if (this.writeDrainPromise) return this.writeDrainPromise; + this.writeDrainPromise = this.drainWriteQueue().finally(() => { + this.writeDrainPromise = null; + }); + return this.writeDrainPromise; + } + + private async drainWriteQueue(): Promise { + while (this.writeQueue.size > 0) { + if (this.writeQueue.size > WRITE_BURST_THRESHOLD) { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + + const batch: string[] = []; + for (const path of this.writeQueue) { + batch.push(path); + if (batch.length >= MAX_CONCURRENT_WRITES) break; + } + for (const path of batch) { + this.writeQueue.delete(path); + } + + await Promise.all( + batch.map((path) => { + const force = this.forcedWritePaths.delete(path); + return this.flushWrite(path, force); + }), + ); + } + } + + private async flushWrite(path: string, force = false): Promise { + path = normalizeVaultPath(path); + return this.runPathWriteLocked(path, () => this.flushWriteUnlocked(path, force)); + } + + private async flushWriteUnlocked(path: string, _force: boolean): Promise { + const ytext = this.vaultSync.getTextForPath(path); + if (!ytext) return; + const content = ytext.toJSON(); + const absolutePath = this.toAbsolutePath(path); + const currentContent = await this.readFileIfExists(absolutePath); + if (currentContent === content) return; + if (this.shouldBlockFrontmatterWrite(path, currentContent, content)) return; + + await fs.mkdir(nodePath.dirname(absolutePath), { recursive: true }); + await this.suppressWrite(path, content); + await fs.writeFile(absolutePath, content, "utf8"); + } + + private shouldBlockFrontmatterWrite( + path: string, + previousContent: string | null, + nextContent: string, + ): boolean { + if (!this.options.frontmatterGuardEnabled) return false; + const validation = validateFrontmatterTransition(previousContent, nextContent); + if (!isFrontmatterBlocked(validation)) return false; + this.log( + `frontmatter write blocked for "${path}" ` + + `(${validation.reasons.join(", ") || validation.risk})`, + ); + return true; + } + + private async handleRemoteDelete(path: string): Promise { + path = normalizeVaultPath(path); + this.deletedMarkdownPaths.delete(path); + this.dirtyMarkdownPaths.delete(path); + this.writeQueue.delete(path); + this.forcedWritePaths.delete(path); + const timer = this.debounceTimers.get(path); + if (timer) { + clearTimeout(timer); + this.debounceTimers.delete(path); + } + + this.suppressDelete(path); + const absolutePath = this.toAbsolutePath(path); + try { + await fs.rm(absolutePath, { force: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + console.error(`[yaos-cli] remote delete failed for "${path}":`, error); + } + } + } + + private async handleRemoteRename(oldPath: string, newPath: string): Promise { + oldPath = normalizeVaultPath(oldPath); + newPath = normalizeVaultPath(newPath); + if (oldPath === newPath) return; + + this.deletedMarkdownPaths.delete(oldPath); + this.dirtyMarkdownPaths.delete(oldPath); + this.dirtyMarkdownPaths.delete(newPath); + this.writeQueue.delete(oldPath); + this.forcedWritePaths.delete(oldPath); + const oldTimer = this.debounceTimers.get(oldPath); + if (oldTimer) { + clearTimeout(oldTimer); + this.debounceTimers.delete(oldPath); + } + + const newContent = this.vaultSync.getTextForPath(newPath)?.toJSON() ?? this.vaultSync.getTextForPath(oldPath)?.toJSON() ?? null; + if (newContent != null) { + await this.suppressWrite(newPath, newContent); + } + this.suppressDelete(oldPath); + + const oldAbsolutePath = this.toAbsolutePath(oldPath); + const newAbsolutePath = this.toAbsolutePath(newPath); + try { + await fs.mkdir(nodePath.dirname(newAbsolutePath), { recursive: true }); + await fs.rename(oldAbsolutePath, newAbsolutePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + this.log(`remote rename fell back to write for "${oldPath}" -> "${newPath}"`); + } + } + this.queueImmediateWrite(newPath, "remote-rename", true); + } + + private consumeDeleteSuppression(path: string): boolean { + path = normalizeVaultPath(path); + const entry = this.getActiveSuppression(path); + if (!entry) return false; + this.suppressedPaths.delete(path); + return entry.kind === "delete"; + } + + private async shouldSuppressWriteEvent( + path: string, + event: "modify" | "create", + stats: Stats | null, + ): Promise { + path = normalizeVaultPath(path); + const entry = this.getActiveSuppression(path); + if (!entry) return false; + if (entry.kind !== "write") { + this.suppressedPaths.delete(path); + return false; + } + + if ( + stats + && typeof entry.expectedBytes === "number" + && stats.size !== entry.expectedBytes + ) { + this.suppressedPaths.delete(path); + this.log( + `suppression: "${path}" ${event} size mismatch ` + + `(expected=${entry.expectedBytes}, observed=${stats.size})`, + ); + return false; + } + + try { + const content = await fs.readFile(this.toAbsolutePath(path), "utf8"); + const fingerprint = this.fingerprintContent(content); + if ( + fingerprint.bytes === entry.expectedBytes + && fingerprint.hash === entry.expectedHash + ) { + this.suppressedPaths.delete(path); + return true; + } + } catch { + // Fall through and let normal sync handle it. + } + + this.suppressedPaths.delete(path); + return false; + } + + private getActiveSuppression(path: string): SuppressionEntry | null { + path = normalizeVaultPath(path); + const entry = this.suppressedPaths.get(path); + if (!entry) return null; + if (Date.now() < entry.expiresAt) { + return entry; + } + this.suppressedPaths.delete(path); + return null; + } + + private async suppressWrite(path: string, content: string): Promise { + const fingerprint = this.fingerprintContent(content); + this.suppressedPaths.set(normalizeVaultPath(path), { + kind: "write", + expiresAt: Date.now() + SUPPRESS_MS, + expectedBytes: fingerprint.bytes, + expectedHash: fingerprint.hash, + }); + } + + private suppressDelete(path: string): void { + this.suppressedPaths.set(normalizeVaultPath(path), { + kind: "delete", + expiresAt: Date.now() + SUPPRESS_MS, + }); + } + + private fingerprintContent(content: string): { bytes: number; hash: string } { + const bytes = Buffer.byteLength(content, "utf8"); + const hash = createHash("sha256").update(content, "utf8").digest("hex"); + return { bytes, hash }; + } + + private runPathWriteLocked(path: string, work: () => Promise): Promise { + const previous = this.pathWriteLocks.get(path) ?? Promise.resolve(); + const next = previous.catch(() => undefined).then(work); + let tracked: Promise; + tracked = next.finally(() => { + if (this.pathWriteLocks.get(path) === tracked) { + this.pathWriteLocks.delete(path); + } + }); + this.pathWriteLocks.set(path, tracked); + return tracked; + } + + private async scanMarkdownFiles(): Promise { + const contents = new Map(); + const presentPaths = new Set(); + await this.scanDirectory("", contents, presentPaths); + return { contents, presentPaths }; + } + + private async scanDirectory( + relativeDir: string, + contents: Map, + presentPaths: Set, + ): Promise { + const absoluteDir = relativeDir + ? this.toAbsolutePath(relativeDir) + : this.rootDir; + let entries: Dirent[]; + try { + entries = await fs.readdir(absoluteDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + + for (const entry of entries) { + const relativePath = normalizeVaultPath( + relativeDir ? `${relativeDir}/${entry.name}` : entry.name, + ); + const absolutePath = this.toAbsolutePath(relativePath); + const stats = await fs.lstat(absolutePath); + if (stats.isDirectory()) { + if (isExcluded(`${relativePath}/`, this.options.excludePatterns, this.configDir)) { + continue; + } + await this.scanDirectory(relativePath, contents, presentPaths); + continue; + } + if (!stats.isFile()) continue; + if (!this.isMarkdownPathSyncable(relativePath)) continue; + presentPaths.add(relativePath); + const content = await fs.readFile(absolutePath, "utf8"); + if (this.maxFileSize > 0 && content.length > this.maxFileSize) { + continue; + } + contents.set(relativePath, content); + } + } + + private shouldIgnoreWatchPath(rawPath: string, stats: Stats | null): boolean { + const path = this.normalizeEventPath(rawPath); + if (!path) return false; + return this.shouldIgnoreNormalizedPath(path, stats); + } + + private shouldIgnoreNormalizedPath(path: string, stats: Stats | null): boolean { + if (stats?.isDirectory()) { + return isExcluded(`${path}/`, this.options.excludePatterns, this.configDir); + } + return !this.isMarkdownPathSyncable(path); + } + + private isMarkdownPathSyncable(path: string): boolean { + return isMarkdownSyncable(path, this.options.excludePatterns, this.configDir); + } + + private normalizeEventPath(rawPath: string): string | null { + const normalized = normalizeVaultPath(rawPath); + return normalized.length > 0 ? normalized : null; + } + + private toAbsolutePath(vaultPath: string): string { + const parts = normalizeVaultPath(vaultPath) + .split("/") + .filter((segment) => segment.length > 0); + return nodePath.join(this.rootDir, ...parts); + } + + private async readFileIfExists(absolutePath: string): Promise { + try { + return await fs.readFile(absolutePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } + } + + private log(message: string): void { + if (this.options.debug) { + console.debug(`[yaos-cli:disk] ${message}`); + } + } +} diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts new file mode 100644 index 0000000..933fe19 --- /dev/null +++ b/packages/cli/src/nodeVaultSync.ts @@ -0,0 +1,187 @@ +import type { VaultSyncSettings } from "../../../src/settings"; +import { + VaultSync, + type ReconcileMode, + type ReconcileResult, + type VaultSyncOptions, + type VaultSyncPersistence, +} from "../../../src/sync/vaultSync"; +import { NodeDiskMirror } from "./nodeDiskMirror"; +import type { ResolvedCliConfig, RuntimeCliConfig } from "./config"; + +const NOOP_INDEXEDDB_ERROR = new Error("IndexedDB is unavailable in the headless Node runtime"); + +export interface HeadlessStartupResult { + localLoaded: boolean; + providerSynced: boolean; + mode: ReconcileMode; + reconcileResult: ReconcileResult; +} + +export function createNodeVaultSync( + config: ResolvedCliConfig, + options?: Omit, +): VaultSync { + return new VaultSync(toVaultSyncSettings(config), { + ...options, + persistenceFactory: () => createNoopPersistence(), + logPersistenceOpenError: false, + }); +} + +export class HeadlessYaosClient { + readonly vaultSync: VaultSync; + readonly diskMirror: NodeDiskMirror; + private reconcileInFlight = false; + private reconcilePending = false; + private awaitingFirstProviderSyncAfterStartup = false; + private lastReconciledGeneration = 0; + private reconnectionHandlerInstalled = false; + private stopped = false; + + constructor(private readonly config: RuntimeCliConfig) { + this.vaultSync = createNodeVaultSync(config); + this.diskMirror = new NodeDiskMirror(this.vaultSync, { + rootDir: config.dir, + deviceName: config.deviceName, + debug: config.debug, + excludePatterns: config.excludePatterns + .split(",") + .map((value) => value.trim()) + .filter((value) => value.length > 0), + maxFileSizeKB: config.maxFileSizeKB, + externalEditPolicy: config.externalEditPolicy, + frontmatterGuardEnabled: config.frontmatterGuardEnabled, + configDir: config.configDir, + }); + } + + async startup(options: { watch: boolean }): Promise { + const localLoaded = await this.vaultSync.waitForLocalPersistence(); + const providerSynced = await this.vaultSync.waitForProviderSync(); + if (this.vaultSync.fatalAuthError) { + throw new Error(formatFatalAuthError(this.vaultSync)); + } + + const mode = this.vaultSync.getSafeReconcileMode(); + const reconcileResult = await this.diskMirror.reconcileFromDisk(mode); + this.lastReconciledGeneration = this.vaultSync.connectionGeneration; + this.awaitingFirstProviderSyncAfterStartup = !providerSynced; + + if (options.watch) { + this.diskMirror.startMapObservers(); + this.installReconnectionHandler(); + await this.diskMirror.startWatching(); + } + + return { + localLoaded, + providerSynced, + mode, + reconcileResult, + }; + } + + async stop(): Promise { + this.stopped = true; + await this.diskMirror.stop(); + this.vaultSync.destroy(); + } + + getStatus(): Record { + return { + host: this.config.host, + vaultId: this.config.vaultId, + deviceName: this.config.deviceName, + connected: this.vaultSync.connected, + localReady: this.vaultSync.localReady, + connectionGeneration: this.vaultSync.connectionGeneration, + storedSchemaVersion: this.vaultSync.storedSchemaVersion, + safeReconcileMode: this.vaultSync.getSafeReconcileMode(), + fatalAuthError: this.vaultSync.fatalAuthError, + fatalAuthCode: this.vaultSync.fatalAuthCode, + diskMirror: this.diskMirror.getDebugSnapshot(), + }; + } + + private installReconnectionHandler(): void { + if (this.reconnectionHandlerInstalled) return; + this.reconnectionHandlerInstalled = true; + this.vaultSync.onProviderSync((generation) => { + if (this.stopped) return; + if (this.awaitingFirstProviderSyncAfterStartup) { + this.awaitingFirstProviderSyncAfterStartup = false; + if (this.reconcileInFlight) { + this.reconcilePending = true; + return; + } + void this.runReconnectReconciliation(generation); + return; + } + if (generation <= this.lastReconciledGeneration) { + return; + } + if (this.reconcileInFlight) { + this.reconcilePending = true; + return; + } + void this.runReconnectReconciliation(generation); + }); + } + + private async runReconnectReconciliation(generation: number): Promise { + if (this.stopped) return; + this.reconcileInFlight = true; + try { + await this.diskMirror.reconcileFromDisk("authoritative"); + this.lastReconciledGeneration = generation; + this.awaitingFirstProviderSyncAfterStartup = false; + } finally { + this.reconcileInFlight = false; + if (!this.reconcilePending || this.stopped) return; + this.reconcilePending = false; + if (this.vaultSync.connectionGeneration > this.lastReconciledGeneration) { + void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); + } + } + } +} + +function toVaultSyncSettings(config: ResolvedCliConfig): VaultSyncSettings { + return { + host: config.host ?? "", + token: config.token ?? "", + vaultId: config.vaultId ?? "", + deviceName: config.deviceName, + debug: config.debug, + frontmatterGuardEnabled: config.frontmatterGuardEnabled, + excludePatterns: config.excludePatterns, + maxFileSizeKB: config.maxFileSizeKB, + externalEditPolicy: config.externalEditPolicy, + enableAttachmentSync: false, + attachmentSyncExplicitlyConfigured: true, + maxAttachmentSizeKB: 0, + attachmentConcurrency: 1, + showRemoteCursors: false, + updateRepoUrl: "", + updateRepoBranch: "main", + }; +} + +function createNoopPersistence(): VaultSyncPersistence { + const db = Promise.reject(NOOP_INDEXEDDB_ERROR); + db.catch(() => undefined); + return { + once() { + // Headless v1 intentionally skips local CRDT persistence. + }, + destroy() { + return; + }, + _db: db, + }; +} + +function formatFatalAuthError(vaultSync: VaultSync): string { + return `Provider rejected the connection (${vaultSync.fatalAuthCode ?? "unknown"})`; +} diff --git a/packages/cli/src/shims.d.ts b/packages/cli/src/shims.d.ts new file mode 100644 index 0000000..b1b9798 --- /dev/null +++ b/packages/cli/src/shims.d.ts @@ -0,0 +1,2 @@ +declare module "js-yaml"; +declare module "qrcode"; diff --git a/packages/cli/tests/config.test.ts b/packages/cli/tests/config.test.ts new file mode 100644 index 0000000..15c7df6 --- /dev/null +++ b/packages/cli/tests/config.test.ts @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import * as os from "node:os"; +import * as nodePath from "node:path"; +import test from "node:test"; +import { requireRuntimeConfig, resolveCliConfig } from "../src/config"; + +const ENV_KEYS = [ + "XDG_CONFIG_HOME", + "YAOS_HOST", + "YAOS_TOKEN", + "YAOS_VAULT_ID", + "YAOS_DIR", + "YAOS_DEVICE_NAME", + "YAOS_DEBUG", + "YAOS_EXCLUDE_PATTERNS", + "YAOS_MAX_FILE_SIZE_KB", + "YAOS_EXTERNAL_EDIT_POLICY", + "YAOS_FRONTMATTER_GUARD", + "YAOS_CONFIG_DIR", +] as const; + +test("resolveCliConfig applies CLI > env > file precedence", async () => { + const originalEnv = snapshotEnv(); + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-config-")); + try { + process.env.XDG_CONFIG_HOME = tempRoot; + const configDir = nodePath.join(tempRoot, "yaos"); + await mkdir(configDir, { recursive: true }); + await writeFile( + nodePath.join(configDir, "cli.json"), + JSON.stringify({ + host: "https://file.example", + token: "file-token", + vaultId: "file-vault", + dir: "/file/vault", + deviceName: "file-device", + debug: false, + excludePatterns: "from-file/", + maxFileSizeKB: 111, + externalEditPolicy: "never", + frontmatterGuardEnabled: false, + configDir: ".obsidian-file", + }), + "utf8", + ); + + process.env.YAOS_HOST = "https://env.example"; + process.env.YAOS_TOKEN = "env-token"; + process.env.YAOS_VAULT_ID = "env-vault"; + process.env.YAOS_DIR = "/env/vault"; + process.env.YAOS_DEVICE_NAME = "env-device"; + process.env.YAOS_DEBUG = "true"; + process.env.YAOS_EXCLUDE_PATTERNS = "from-env/"; + process.env.YAOS_MAX_FILE_SIZE_KB = "222"; + process.env.YAOS_EXTERNAL_EDIT_POLICY = "closed-only"; + process.env.YAOS_FRONTMATTER_GUARD = "true"; + process.env.YAOS_CONFIG_DIR = ".obsidian-env"; + + const resolved = await resolveCliConfig({ + host: "https://flag.example", + token: "flag-token", + vaultId: "flag-vault", + dir: "/flag/vault", + deviceName: "flag-device", + debug: false, + excludePatterns: "from-flag/", + maxFileSizeKB: 333, + externalEditPolicy: "always", + configDir: ".obsidian-flag", + }); + + assert.equal(resolved.host, "https://flag.example"); + assert.equal(resolved.token, "flag-token"); + assert.equal(resolved.vaultId, "flag-vault"); + assert.equal(resolved.dir, "/flag/vault"); + assert.equal(resolved.deviceName, "flag-device"); + assert.equal(resolved.debug, false); + assert.equal(resolved.excludePatterns, "from-flag/"); + assert.equal(resolved.maxFileSizeKB, 333); + assert.equal(resolved.externalEditPolicy, "always"); + assert.equal(resolved.frontmatterGuardEnabled, true); + assert.equal(resolved.configDir, ".obsidian-flag"); + } finally { + restoreEnv(originalEnv); + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +test("requireRuntimeConfig rejects missing required fields", () => { + assert.throws( + () => requireRuntimeConfig({ + deviceName: "device", + debug: false, + excludePatterns: "", + maxFileSizeKB: 2048, + externalEditPolicy: "always", + frontmatterGuardEnabled: true, + configDir: ".obsidian", + configPath: "/tmp/cli.json", + }, { requireDir: true }), + /Missing required configuration: host, token, vaultId, dir/, + ); +}); + +function snapshotEnv(): Record { + return Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])); +} + +function restoreEnv(snapshot: Record): void { + for (const key of ENV_KEYS) { + const value = snapshot[key]; + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json new file mode 100644 index 0000000..7bb0264 --- /dev/null +++ b/packages/cli/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "outDir": "dist", + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/src/sync/exclude.ts b/src/sync/exclude.ts index c392b24..06fc8bd 100644 --- a/src/sync/exclude.ts +++ b/src/sync/exclude.ts @@ -1,14 +1,8 @@ -/** Paths that are always excluded, regardless of user settings. */ -function normalizePrefix(path: string): string { - return path - .replace(/\\/g, "/") - .replace(/\/{2,}/g, "/") - .replace(/^\.\//, "") - .replace(/^\/+/, ""); -} +import { normalizeVaultPath } from "../utils/normalizeVaultPath"; +/** Paths that are always excluded, regardless of user settings. */ function alwaysExcludedPrefixes(configDir: string): string[] { - const normalizedConfigDir = normalizePrefix(configDir).replace(/\/$/, ""); + const normalizedConfigDir = normalizeVaultPath(configDir).replace(/\/$/, ""); return [ `${normalizedConfigDir}/`, ".trash/", @@ -26,12 +20,12 @@ function alwaysExcludedPrefixes(configDir: string): string[] { * @returns true if the path matches any exclude pattern */ export function isExcluded(path: string, patterns: string[], configDir: string): boolean { - const normalizedPath = normalizePrefix(path); + const normalizedPath = normalizeVaultPath(path); for (const prefix of alwaysExcludedPrefixes(configDir)) { if (normalizedPath.startsWith(prefix)) return true; } for (const prefix of patterns) { - if (normalizedPath.startsWith(normalizePrefix(prefix))) return true; + if (normalizedPath.startsWith(normalizeVaultPath(prefix))) return true; } return false; } diff --git a/src/sync/snapshotClient.ts b/src/sync/snapshotClient.ts index f3a977a..0dbaf2b 100644 --- a/src/sync/snapshotClient.ts +++ b/src/sync/snapshotClient.ts @@ -16,6 +16,7 @@ import type { FileMeta, BlobRef } from "../types"; import { appendTraceParams, type TraceHttpContext } from "../debug/trace"; import { obsidianRequest } from "../utils/http"; import { yTextToString } from "../utils/format"; +import { normalizeVaultPath } from "../utils/normalizeVaultPath"; // ------------------------------------------------------------------- // Types (mirrors server SnapshotIndex) @@ -64,13 +65,6 @@ export interface SnapshotDiff { blobsChanged: Array<{ path: string; snapshotHash: string; currentHash: string }>; } -function normalizeVaultPath(path: string): string { - return path - .replace(/\\/g, "/") - .replace(/\/{2,}/g, "/") - .replace(/^\.\//, "") - .replace(/^\/+/, ""); -} function getStoredSchemaVersion(doc: Y.Doc): number | null { const stored = doc.getMap("sys").get("schemaVersion"); diff --git a/src/sync/vaultSync.ts b/src/sync/vaultSync.ts index 37d0d0d..f75066f 100644 --- a/src/sync/vaultSync.ts +++ b/src/sync/vaultSync.ts @@ -1,12 +1,12 @@ import * as Y from "yjs"; import YSyncProvider from "y-partyserver/provider"; import { IndexeddbPersistence } from "y-indexeddb"; -import { normalizePath } from "obsidian"; import { type FileMeta, type BlobRef, type BlobMeta, type BlobTombstone, ORIGIN_SEED } from "../types"; import type { VaultSyncSettings } from "../settings"; import type { TraceHttpContext, TraceRecord } from "../debug/trace"; import { randomBase64Url } from "../utils/base64url"; import { formatUnknown } from "../utils/format"; +import { normalizeVaultPath } from "../utils/normalizeVaultPath"; /** Current schema version. Stored in sys.schemaVersion. */ export const SCHEMA_VERSION = 2; @@ -27,6 +27,33 @@ const MAX_BACKOFF_TIME_MS = 30_000; /** Debounce window for batching rename events (folder renames). */ const RENAME_BATCH_MS = 50; +/** Persistence adapter contract shared by browser and headless clients. */ +export interface VaultSyncPersistenceDatabase { + addEventListener( + type: "error", + listener: (event: { target: { error?: unknown } | null }) => void, + ): void; +} + +export interface VaultSyncPersistence { + once(event: "synced", listener: () => void): void; + destroy(): Promise | void; + _db: Promise; +} + +export type VaultSyncPersistenceFactory = (name: string, doc: Y.Doc) => VaultSyncPersistence; + +export interface VaultSyncOptions { + traceContext?: TraceHttpContext; + trace?: TraceRecord; + persistenceFactory?: VaultSyncPersistenceFactory; + logPersistenceOpenError?: boolean; +} + +function createIndexedDbPersistence(name: string, doc: Y.Doc): VaultSyncPersistence { + return new IndexeddbPersistence(name, doc) as unknown as VaultSyncPersistence; +} + /** Reconciliation mode determines what operations are safe. */ export type ReconcileMode = "conservative" | "authoritative"; type FatalAuthCode = "unauthorized" | "server_misconfigured" | "unclaimed" | "update_required"; @@ -37,7 +64,6 @@ interface FatalAuthMessage { roomSchemaVersion: number | null; reason: string | null; } - const FATAL_AUTH_CODES = new Set([ "unauthorized", "server_misconfigured", @@ -102,7 +128,7 @@ interface IndexedDbErrorDetails { export class VaultSync { readonly ydoc: Y.Doc; readonly provider: YSyncProvider; - readonly persistence: IndexeddbPersistence; + readonly persistence: VaultSyncPersistence; readonly pathToId: Y.Map; readonly idToText: Y.Map; @@ -165,10 +191,7 @@ export class VaultSync { constructor( settings: VaultSyncSettings, - options?: { - traceContext?: TraceHttpContext; - trace?: TraceRecord; - }, + options?: VaultSyncOptions, ) { this.debug = settings.debug; this._device = settings.deviceName || undefined; @@ -194,19 +217,22 @@ export class VaultSync { this.log(`IndexedDB database: ${idbName}`); // Start both persistence and provider in parallel. - this.persistence = new IndexeddbPersistence(idbName, this.ydoc); + const persistenceFactory = options?.persistenceFactory ?? createIndexedDbPersistence; + this.persistence = persistenceFactory(idbName, this.ydoc); + const persistenceDb = this.persistence._db; // Catch IndexedDB open/write failures (unavailable, quota, permissions). // y-indexeddb's internal _db promise rejects if IDB can't open. // We also listen for unhandled IDB transaction errors. - (this.persistence as unknown as { _db: Promise })._db - .catch((err: unknown) => { - this.captureIndexedDbError(err, "open"); + persistenceDb.catch((err: unknown) => { + this.captureIndexedDbError(err, "open"); + if (options?.logPersistenceOpenError !== false) { console.error("[yaos] IndexedDB failed to open:", err); - }); + } + }); - (this.persistence as unknown as { _db: Promise })._db - .then((db: IDBDatabase) => { + persistenceDb + .then((db) => { db.addEventListener("error", (event) => { const target = event.target as { error?: unknown } | null; this.captureIndexedDbError( @@ -307,13 +333,12 @@ export class VaultSync { }); // Also resolve (false) if IDB errors out after we started waiting - (this.persistence as unknown as { _db: Promise })._db - .catch(() => { - clearTimeout(timeout); - this.captureIndexedDbError(new Error("IndexedDB failed during waitForLocalPersistence"), "wait"); - this.log("IndexedDB errored during wait — proceeding without cache"); - resolve(false); - }); + this.persistence._db.catch(() => { + clearTimeout(timeout); + this.captureIndexedDbError(new Error("IndexedDB failed during waitForLocalPersistence"), "wait"); + this.log("IndexedDB errored during wait — proceeding without cache"); + resolve(false); + }); }); } @@ -426,7 +451,7 @@ export class VaultSync { /** Normalize a vault-relative path for consistent CRDT keys. */ private normPath(path: string): string { - return normalizePath(path); + return normalizeVaultPath(path); } isFileMetaDeleted(meta: FileMeta | undefined): boolean { diff --git a/src/utils/normalizeVaultPath.ts b/src/utils/normalizeVaultPath.ts new file mode 100644 index 0000000..cbab4b7 --- /dev/null +++ b/src/utils/normalizeVaultPath.ts @@ -0,0 +1,7 @@ +export function normalizeVaultPath(path: string): string { + return path + .replace(/\\/g, "/") + .replace(/\/{2,}/g, "/") + .replace(/^(\.\/)+/, "") + .replace(/^\/+/, ""); +} From b2f1471ea2948444f98f9e575a7633e585f70b84 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sun, 12 Apr 2026 13:33:56 +0800 Subject: [PATCH 02/12] Add WebSocket polyfill (ws) for Node.js runtime --- package-lock.json | 17 ++++++++++++++--- packages/cli/package.json | 4 +++- packages/cli/src/nodeVaultSync.ts | 3 +++ src/sync/vaultSync.ts | 2 ++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ce41cf..b30dcd5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -801,6 +801,16 @@ "@types/estree": "*" } }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.35.1", "dev": true, @@ -4709,7 +4719,6 @@ "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -4955,13 +4964,15 @@ "version": "0.0.0", "dependencies": { "chokidar": "^4.0.3", - "commander": "^14.0.0" + "commander": "^14.0.0", + "ws": "^8.18.2" }, "bin": { - "yaos-cli": "dist/index.js" + "yaos-cli": "dist/index.cjs" }, "devDependencies": { "@types/node": "^16.11.6", + "@types/ws": "^8.18.0", "esbuild": "0.25.5", "typescript": "^5.8.3" } diff --git a/packages/cli/package.json b/packages/cli/package.json index 3c45626..c4d0a09 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -13,10 +13,12 @@ }, "dependencies": { "chokidar": "^4.0.3", - "commander": "^14.0.0" + "commander": "^14.0.0", + "ws": "^8.18.2" }, "devDependencies": { "@types/node": "^16.11.6", + "@types/ws": "^8.18.0", "esbuild": "0.25.5", "typescript": "^5.8.3" } diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index 933fe19..8e33f27 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -1,3 +1,5 @@ +import WebSocket from "ws"; + import type { VaultSyncSettings } from "../../../src/settings"; import { VaultSync, @@ -24,6 +26,7 @@ export function createNodeVaultSync( ): VaultSync { return new VaultSync(toVaultSyncSettings(config), { ...options, +webSocketPolyfill: WebSocket as unknown as typeof globalThis.WebSocket, persistenceFactory: () => createNoopPersistence(), logPersistenceOpenError: false, }); diff --git a/src/sync/vaultSync.ts b/src/sync/vaultSync.ts index f75066f..27dec42 100644 --- a/src/sync/vaultSync.ts +++ b/src/sync/vaultSync.ts @@ -48,6 +48,7 @@ export interface VaultSyncOptions { trace?: TraceRecord; persistenceFactory?: VaultSyncPersistenceFactory; logPersistenceOpenError?: boolean; + webSocketPolyfill?: typeof WebSocket; } function createIndexedDbPersistence(name: string, doc: Y.Doc): VaultSyncPersistence { @@ -261,6 +262,7 @@ export class VaultSync { params, connect: true, maxBackoffTime: MAX_BACKOFF_TIME_MS, + WebSocketPolyfill: options?.webSocketPolyfill, }); // Track connection generations for reconnect detection From 0bb4cb43c592b61c4189fc672c154bb5effed01b Mon Sep 17 00:00:00 2001 From: enieuwy Date: Mon, 13 Apr 2026 09:31:47 +0800 Subject: [PATCH 03/12] Fix chokidar watcher: handle null-stats and array-wrap ignored callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented the filesystem watcher from working: 1. shouldIgnoreNormalizedPath treated null-stats paths as files. When chokidar calls _isIgnored for the root directory without stats, the path was checked against isMarkdownSyncable (returns false for directory names not ending in .md), causing the entire tree to be pruned. Fix: when stats are null, only ignore paths that are definitively non-markdown files (have an extension but not .md). 2. Chokidar's internal _isIgnored calls .map() on the ignored option. When ignored is a bare function, .map() throws TypeError (functions don't have .map), which is silently caught — the watcher starts but watches nothing. Wrapping in an array preserves the function through normalizeIgnored's type check. Also upgraded chokidar from 4.0.3 to 5.0.0. --- package-lock.json | 20 ++++++++++---------- packages/cli/package.json | 2 +- packages/cli/src/nodeDiskMirror.ts | 11 ++++++++++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index b30dcd5..dfadaeb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1374,15 +1374,15 @@ } }, "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "license": "MIT", "dependencies": { - "readdirp": "^4.0.1" + "readdirp": "^5.0.0" }, "engines": { - "node": ">= 14.16.0" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -3802,12 +3802,12 @@ "license": "MIT" }, "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", "license": "MIT", "engines": { - "node": ">= 14.18.0" + "node": ">= 20.19.0" }, "funding": { "type": "individual", @@ -4963,7 +4963,7 @@ "name": "@yaos/cli", "version": "0.0.0", "dependencies": { - "chokidar": "^4.0.3", + "chokidar": "^5.0.0", "commander": "^14.0.0", "ws": "^8.18.2" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index c4d0a09..0fb0c67 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -12,7 +12,7 @@ "test": "node --import jiti/register --test tests/*.ts" }, "dependencies": { - "chokidar": "^4.0.3", + "chokidar": "^5.0.0", "commander": "^14.0.0", "ws": "^8.18.2" }, diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index ccbd9b6..4651d86 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -191,7 +191,7 @@ export class NodeDiskMirror { stabilityThreshold: WATCHER_STABILITY_MS, pollInterval: WATCHER_POLL_MS, }, - ignored: (rawPath, stats) => this.shouldIgnoreWatchPath(rawPath, stats ?? null), + ignored: [(rawPath, stats) => this.shouldIgnoreWatchPath(rawPath, stats ?? null)], }); watcher @@ -766,6 +766,15 @@ export class NodeDiskMirror { if (stats?.isDirectory()) { return isExcluded(`${path}/`, this.options.excludePatterns, this.configDir); } + // When stats are unavailable, only ignore if we can definitively determine + // the path is not a syncable markdown file. If there's no extension match + // but stats are missing, it might be a directory — don't prune it. + if (stats === null) { + // Cannot determine if directory; only ignore known non-markdown files + // that have an extension (i.e., are definitely files). + if (path.includes(".") && !path.endsWith(".md")) return true; + return false; + } return !this.isMarkdownPathSyncable(path); } From 1fe0cf1fdb733ecf4016e117b1dfed56989494b8 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Mon, 13 Apr 2026 12:28:43 +0800 Subject: [PATCH 04/12] fix(cli): code review fixes for headless CLI P1: Add trailing slash removal to normalizeVaultPath to match Obsidian normalizePath() semantics, preventing CRDT key mismatches between plugin and headless clients. P1: Add requireRuntimeConfig() to status command so host/token/vaultId are validated before creating a sync client. P1: Move startMapObservers() before reconcileFromDisk() so remote edits arriving during startup disk scan are immediately mirrored. Safe because observers filter local origins (ORIGIN_SEED). P2: Resolve relative vault dir to absolute path in resolveCliConfig so YAOS_DIR/config values don't depend on process working directory. P2: Reject invalid externalEditPolicy values instead of silently defaulting to the most permissive policy (always). P2: Change createNodeVaultSync to accept RuntimeCliConfig instead of ResolvedCliConfig, making the type system enforce that connection settings are validated before sync client construction. Also includes prior uncommitted fixes: Promise.allSettled for batch writes, fs.rm cleanup on rename fallback, ENOENT handling during vault walk, early reconnection handler installation, reconcileInFlight flag, commander exitOverride, and prepare/dev scripts. --- packages/cli/package.json | 2 ++ packages/cli/src/config.ts | 20 +++++++++++--- packages/cli/src/index.ts | 16 ++++++++--- packages/cli/src/nodeDiskMirror.ts | 27 ++++++++++++++++--- packages/cli/src/nodeVaultSync.ts | 43 ++++++++++++++++++++++++------ src/utils/normalizeVaultPath.ts | 3 ++- 6 files changed, 92 insertions(+), 19 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 0fb0c67..544e028 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -7,6 +7,8 @@ "yaos-cli": "dist/index.cjs" }, "scripts": { + "dev": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --watch", + "prepare": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs", "build": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs", "typecheck": "tsc --noEmit -p tsconfig.json", "test": "node --import jiti/register --test tests/*.ts" diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 0c7ed59..73c2180 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -77,15 +77,19 @@ export async function resolveCliConfig(options: CliCommandOptions): Promise( } } +const VALID_EXTERNAL_EDIT_POLICIES = ["always", "closed-only", "never"] as const; + function assignExternalEditPolicy( target: CliFileConfig, value: unknown, ): void { - if (value === "always" || value === "closed-only" || value === "never") { - target.externalEditPolicy = value; + if (value == null) return; + if (typeof value === "string" && VALID_EXTERNAL_EDIT_POLICIES.includes(value as ExternalEditPolicy)) { + target.externalEditPolicy = value as ExternalEditPolicy; + return; } + const display = typeof value === "string" ? `"${value}"` : typeof value; + throw new Error( + `Invalid externalEditPolicy: ${display}. Must be one of: ${VALID_EXTERNAL_EDIT_POLICIES.join(", ")}`, + ); } function parseBooleanEnv(value: string | undefined): boolean | undefined { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ccd7f9e..bd08ae1 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -10,7 +10,8 @@ const program = new Command(); program .name("yaos-cli") .description("Headless YAOS client for filesystem-backed Markdown vaults") - .showHelpAfterError(); + .showHelpAfterError() + .exitOverride(); addCommonOptions( program @@ -61,7 +62,8 @@ addCommonOptions( .description("Show current connection and local-cache status") .action(async (options: CliCommandOptions) => { const resolved = await resolveCliConfig(options); - const vaultSync = createNodeVaultSync(resolved); + const runtime = requireRuntimeConfig(resolved, { requireDir: false }); + const vaultSync = createNodeVaultSync(runtime); try { const localLoaded = await vaultSync.waitForLocalPersistence(); const providerSynced = await vaultSync.waitForProviderSync(); @@ -86,7 +88,15 @@ addCommonOptions( { includeDir: false }, ); -void program.parseAsync(process.argv); +program.parseAsync(process.argv).catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + // CommanderError carries its own exitCode (0 for --help, 1 for errors). + const exitCode = (error as { exitCode?: number })?.exitCode ?? 1; + if (exitCode !== 0) { + process.stderr.write(`error: ${message}\n`); + } + process.exit(exitCode); +}); function addCommonOptions( command: T, diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index 4651d86..669d6f1 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -507,12 +507,18 @@ export class NodeDiskMirror { this.writeQueue.delete(path); } - await Promise.all( + const results = await Promise.allSettled( batch.map((path) => { const force = this.forcedWritePaths.delete(path); return this.flushWrite(path, force); }), ); + for (let i = 0; i < results.length; i++) { + const result = results[i]; + if (result != null && result.status === "rejected") { + console.error(`[yaos-cli] write failed for "${batch[i]}":`, result.reason); + } + } } } @@ -604,6 +610,9 @@ export class NodeDiskMirror { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { this.log(`remote rename fell back to write for "${oldPath}" -> "${newPath}"`); } + // Remove the old file to prevent stale content from being + // seeded back into the CRDT on the next reconciliation pass. + await fs.rm(oldAbsolutePath, { force: true }).catch(() => undefined); } this.queueImmediateWrite(newPath, "remote-rename", true); } @@ -737,7 +746,13 @@ export class NodeDiskMirror { relativeDir ? `${relativeDir}/${entry.name}` : entry.name, ); const absolutePath = this.toAbsolutePath(relativePath); - const stats = await fs.lstat(absolutePath); + let stats: Stats; + try { + stats = await fs.lstat(absolutePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } if (stats.isDirectory()) { if (isExcluded(`${relativePath}/`, this.options.excludePatterns, this.configDir)) { continue; @@ -748,7 +763,13 @@ export class NodeDiskMirror { if (!stats.isFile()) continue; if (!this.isMarkdownPathSyncable(relativePath)) continue; presentPaths.add(relativePath); - const content = await fs.readFile(absolutePath, "utf8"); + let content: string; + try { + content = await fs.readFile(absolutePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } if (this.maxFileSize > 0 && content.length > this.maxFileSize) { continue; } diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index 8e33f27..e0858ef 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -21,7 +21,7 @@ export interface HeadlessStartupResult { } export function createNodeVaultSync( - config: ResolvedCliConfig, + config: RuntimeCliConfig, options?: Omit, ): VaultSync { return new VaultSync(toVaultSyncSettings(config), { @@ -60,6 +60,13 @@ export class HeadlessYaosClient { } async startup(options: { watch: boolean }): Promise { + if (options.watch) { + // Install reconnection handler before any async waits so that + // late provider sync events are never missed between timeouts + // and handler installation. + this.installReconnectionHandler(); + } + const localLoaded = await this.vaultSync.waitForLocalPersistence(); const providerSynced = await this.vaultSync.waitForProviderSync(); if (this.vaultSync.fatalAuthError) { @@ -67,13 +74,33 @@ export class HeadlessYaosClient { } const mode = this.vaultSync.getSafeReconcileMode(); - const reconcileResult = await this.diskMirror.reconcileFromDisk(mode); + + // Start CRDT observers before reconciliation so that remote edits + // arriving while we scan disk are immediately mirrored to disk. + // Safe because observers filter out local origins (ORIGIN_SEED). + if (options.watch) { + this.diskMirror.startMapObservers(); + } + + this.reconcileInFlight = true; + let reconcileResult: ReconcileResult; + try { + reconcileResult = await this.diskMirror.reconcileFromDisk(mode); + } finally { + this.reconcileInFlight = false; + } this.lastReconciledGeneration = this.vaultSync.connectionGeneration; this.awaitingFirstProviderSyncAfterStartup = !providerSynced; + // Drain any deferred reconciliation that was requested during startup. + if (this.reconcilePending && !this.stopped) { + this.reconcilePending = false; + if (this.vaultSync.connectionGeneration > this.lastReconciledGeneration) { + void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); + } + } + if (options.watch) { - this.diskMirror.startMapObservers(); - this.installReconnectionHandler(); await this.diskMirror.startWatching(); } @@ -150,11 +177,11 @@ export class HeadlessYaosClient { } } -function toVaultSyncSettings(config: ResolvedCliConfig): VaultSyncSettings { +function toVaultSyncSettings(config: RuntimeCliConfig): VaultSyncSettings { return { - host: config.host ?? "", - token: config.token ?? "", - vaultId: config.vaultId ?? "", + host: config.host, + token: config.token, + vaultId: config.vaultId, deviceName: config.deviceName, debug: config.debug, frontmatterGuardEnabled: config.frontmatterGuardEnabled, diff --git a/src/utils/normalizeVaultPath.ts b/src/utils/normalizeVaultPath.ts index cbab4b7..1e79e11 100644 --- a/src/utils/normalizeVaultPath.ts +++ b/src/utils/normalizeVaultPath.ts @@ -3,5 +3,6 @@ export function normalizeVaultPath(path: string): string { .replace(/\\/g, "/") .replace(/\/{2,}/g, "/") .replace(/^(\.\/)+/, "") - .replace(/^\/+/, ""); + .replace(/^\/+/, "") + .replace(/\/+$/, ""); } From 7369b93c7efbe32f83a1f18de7b70a88fe3d090b Mon Sep 17 00:00:00 2001 From: enieuwy Date: Tue, 14 Apr 2026 01:01:42 +0800 Subject: [PATCH 05/12] =?UTF-8?q?fix(cli):=20address=20code=20review=20fin?= =?UTF-8?q?dings=20=E2=80=94=20security=20and=20correctness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 fixes: - nodeDiskMirror: reject path traversal via .. segments in toAbsolutePath() - nodeDiskMirror: don't prune directories when chokidar has no stats - exclude: preserve trailing slash in user exclude patterns P2 fixes: - config: expand ~ in vault directory before resolve - cli: use strict regex for positive integer parsing - config: filter empty strings from CLI overrides in pickDefined - nodeVaultSync: gate reconnect callback until startup state initialized - nodeDiskMirror: write new file before deleting old in rename fallback --- packages/cli/src/config.ts | 7 +++++-- packages/cli/src/index.ts | 5 ++--- packages/cli/src/nodeDiskMirror.ts | 31 ++++++++++++++++++++---------- packages/cli/src/nodeVaultSync.ts | 7 +++++++ src/sync/exclude.ts | 4 +++- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 73c2180..ae78f8c 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -87,9 +87,10 @@ export async function resolveCliConfig(options: CliCommandOptions): Promise(value: T): Partial { return Object.fromEntries( - Object.entries(value as Record).filter(([, entry]) => entry !== undefined), + Object.entries(value as Record).filter( + ([, entry]) => entry !== undefined && entry !== "", + ), ) as Partial; } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index bd08ae1..27b299b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -126,11 +126,10 @@ function addCommonOptions( } function parsePositiveInteger(value: string): number { - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { + if (!/^[1-9]\d*$/.test(value)) { throw new InvalidOptionArgumentError(`Expected a positive integer, received ${value}`); } - return parsed; + return Number.parseInt(value, 10); } function summarizeConfig(config: ResolvedCliConfig): Record { diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index 669d6f1..64689b7 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -603,6 +603,7 @@ export class NodeDiskMirror { const oldAbsolutePath = this.toAbsolutePath(oldPath); const newAbsolutePath = this.toAbsolutePath(newPath); + let needsWriteFallback = false; try { await fs.mkdir(nodePath.dirname(newAbsolutePath), { recursive: true }); await fs.rename(oldAbsolutePath, newAbsolutePath); @@ -610,11 +611,19 @@ export class NodeDiskMirror { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { this.log(`remote rename fell back to write for "${oldPath}" -> "${newPath}"`); } - // Remove the old file to prevent stale content from being - // seeded back into the CRDT on the next reconciliation pass. + needsWriteFallback = true; + } + + if (needsWriteFallback) { + // Write the new file first, then delete the old file after the + // write completes. Deleting before the write risks losing both + // files if the write also fails. + this.queueImmediateWrite(newPath, "remote-rename", true); + await this.kickWriteDrain(); await fs.rm(oldAbsolutePath, { force: true }).catch(() => undefined); + } else { + this.queueImmediateWrite(newPath, "remote-rename", true); } - this.queueImmediateWrite(newPath, "remote-rename", true); } private consumeDeleteSuppression(path: string): boolean { @@ -787,13 +796,10 @@ export class NodeDiskMirror { if (stats?.isDirectory()) { return isExcluded(`${path}/`, this.options.excludePatterns, this.configDir); } - // When stats are unavailable, only ignore if we can definitively determine - // the path is not a syncable markdown file. If there's no extension match - // but stats are missing, it might be a directory — don't prune it. if (stats === null) { - // Cannot determine if directory; only ignore known non-markdown files - // that have an extension (i.e., are definitely files). - if (path.includes(".") && !path.endsWith(".md")) return true; + // No stats available yet — don't prune. Chokidar will re-evaluate + // with stats once it has them, so directories like "notes.v2" are + // not incorrectly excluded. return false; } return !this.isMarkdownPathSyncable(path); @@ -812,7 +818,12 @@ export class NodeDiskMirror { const parts = normalizeVaultPath(vaultPath) .split("/") .filter((segment) => segment.length > 0); - return nodePath.join(this.rootDir, ...parts); + const absolute = nodePath.join(this.rootDir, ...parts); + const relative = nodePath.relative(this.rootDir, absolute); + if (relative.startsWith("..") || nodePath.isAbsolute(relative)) { + throw new Error(`Path traversal rejected: "${vaultPath}" resolves outside vault root`); + } + return absolute; } private async readFileIfExists(absolutePath: string): Promise { diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index e0858ef..497d4b3 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -38,6 +38,7 @@ export class HeadlessYaosClient { private reconcileInFlight = false; private reconcilePending = false; private awaitingFirstProviderSyncAfterStartup = false; + private startupInitialized = false; private lastReconciledGeneration = 0; private reconnectionHandlerInstalled = false; private stopped = false; @@ -91,6 +92,7 @@ export class HeadlessYaosClient { } this.lastReconciledGeneration = this.vaultSync.connectionGeneration; this.awaitingFirstProviderSyncAfterStartup = !providerSynced; + this.startupInitialized = true; // Drain any deferred reconciliation that was requested during startup. if (this.reconcilePending && !this.stopped) { @@ -139,6 +141,11 @@ export class HeadlessYaosClient { this.reconnectionHandlerInstalled = true; this.vaultSync.onProviderSync((generation) => { if (this.stopped) return; + if (!this.startupInitialized) { + // Defer until startup has initialized its state. + this.reconcilePending = true; + return; + } if (this.awaitingFirstProviderSyncAfterStartup) { this.awaitingFirstProviderSyncAfterStartup = false; if (this.reconcileInFlight) { diff --git a/src/sync/exclude.ts b/src/sync/exclude.ts index 06fc8bd..7a97d2c 100644 --- a/src/sync/exclude.ts +++ b/src/sync/exclude.ts @@ -25,7 +25,9 @@ export function isExcluded(path: string, patterns: string[], configDir: string): if (normalizedPath.startsWith(prefix)) return true; } for (const prefix of patterns) { - if (normalizedPath.startsWith(normalizeVaultPath(prefix))) return true; + const normalizedPrefix = normalizeVaultPath(prefix); + const matchPrefix = prefix.endsWith("/") ? normalizedPrefix + "/" : normalizedPrefix; + if (normalizedPath.startsWith(matchPrefix)) return true; } return false; } From eb4215e5a065b6d94522e5d999ed333d6d876bde Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 14:46:35 +0800 Subject: [PATCH 06/12] fix(cli): atomic writes for headless filesystem output --- packages/cli/src/fs.ts | 85 ++++++++++++++++++++++++++++++ packages/cli/src/nodeDiskMirror.ts | 7 +-- packages/cli/tests/fs.test.ts | 62 ++++++++++++++++++++++ 3 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/fs.ts create mode 100644 packages/cli/tests/fs.test.ts diff --git a/packages/cli/src/fs.ts b/packages/cli/src/fs.ts new file mode 100644 index 0000000..eed8234 --- /dev/null +++ b/packages/cli/src/fs.ts @@ -0,0 +1,85 @@ +import { randomBytes } from "node:crypto"; +import { promises as fs } from "node:fs"; +import * as nodePath from "node:path"; + +/** + * Write a file atomically: write to a unique temp file, fsync, then rename. + * A crash mid-write leaves at most an orphan temp file rather than a corrupt + * final file. Rename is atomic on POSIX when source and destination share a + * filesystem, which they do because the temp file is created next to the target. + */ +export async function writeFileAtomic( + absolutePath: string, + content: string | Uint8Array, + options: { mode?: number } = {}, +): Promise { + const dir = nodePath.dirname(absolutePath); + const mode = options.mode ?? await readExistingMode(absolutePath); + const tmpPath = nodePath.join( + dir, + `.yaos-write-${process.pid}.${Date.now()}.${randomBytes(8).toString("hex")}.tmp`, + ); + + let renamed = false; + try { + const fh = await fs.open(tmpPath, "wx", mode); + try { + await fh.writeFile(content); + await fh.datasync(); + } finally { + await fh.close(); + } + + await fs.rename(tmpPath, absolutePath); + renamed = true; + await syncDirectoryBestEffort(dir); + } finally { + if (!renamed) { + await fs.rm(tmpPath, { force: true }); + } + } +} + +export async function ensureDirectoryDurable(dir: string): Promise { + const created = await fs.mkdir(dir, { recursive: true }); + if (!created) return; + await syncCreatedDirectoryParents(created, dir); +} + +async function readExistingMode(absolutePath: string): Promise { + try { + const stats = await fs.stat(absolutePath); + return stats.mode & 0o7777; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +async function syncCreatedDirectoryParents(firstCreated: string, targetDir: string): Promise { + const target = nodePath.resolve(targetDir); + let current = nodePath.resolve(firstCreated); + while (true) { + await syncDirectoryBestEffort(nodePath.dirname(current)); + if (current === target) return; + if (!target.startsWith(current + nodePath.sep)) return; + const nextSegment = target.slice(current.length + 1).split(nodePath.sep)[0]; + if (!nextSegment) return; + current = nodePath.join(current, nextSegment); + } +} + +async function syncDirectoryBestEffort(dir: string): Promise { + let dh: fs.FileHandle | null = null; + try { + dh = await fs.open(dir, "r"); + await dh.sync(); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EINVAL" && code !== "ENOTSUP" && code !== "EPERM" && code !== "EISDIR") { + throw error; + } + } finally { + await dh?.close(); + } +} diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index 64689b7..c055d8e 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -1,6 +1,7 @@ import chokidar, { type FSWatcher } from "chokidar"; import { createHash } from "node:crypto"; import { promises as fs } from "node:fs"; +import { ensureDirectoryDurable, writeFileAtomic } from "./fs"; import type { Dirent, Stats } from "node:fs"; import * as nodePath from "node:path"; import * as Y from "yjs"; @@ -536,9 +537,9 @@ export class NodeDiskMirror { if (currentContent === content) return; if (this.shouldBlockFrontmatterWrite(path, currentContent, content)) return; - await fs.mkdir(nodePath.dirname(absolutePath), { recursive: true }); + await ensureDirectoryDurable(nodePath.dirname(absolutePath)); await this.suppressWrite(path, content); - await fs.writeFile(absolutePath, content, "utf8"); + await writeFileAtomic(absolutePath, content); } private shouldBlockFrontmatterWrite( @@ -605,7 +606,7 @@ export class NodeDiskMirror { const newAbsolutePath = this.toAbsolutePath(newPath); let needsWriteFallback = false; try { - await fs.mkdir(nodePath.dirname(newAbsolutePath), { recursive: true }); + await ensureDirectoryDurable(nodePath.dirname(newAbsolutePath)); await fs.rename(oldAbsolutePath, newAbsolutePath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { diff --git a/packages/cli/tests/fs.test.ts b/packages/cli/tests/fs.test.ts new file mode 100644 index 0000000..be0ddd1 --- /dev/null +++ b/packages/cli/tests/fs.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import * as os from "node:os"; +import * as nodePath from "node:path"; +import test from "node:test"; +import { writeFileAtomic } from "../src/fs"; + +test("writeFileAtomic writes content and leaves no temp files after success", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-atomic-")); + try { + const target = nodePath.join(tempRoot, "note.md"); + await writeFileAtomic(target, "hello\n"); + await writeFileAtomic(target, "goodbye\n"); + + assert.equal(await readFile(target, "utf8"), "goodbye\n"); + assert.deepEqual(await listTempFiles(tempRoot), []); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +test("writeFileAtomic preserves the existing file when the target directory rejects temp creation", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-atomic-")); + try { + const target = nodePath.join(tempRoot, "note.md"); + await writeFile(target, "original\n", "utf8"); + await chmod(tempRoot, 0o500); + try { + await assert.rejects( + () => writeFileAtomic(target, "new\n"), + /EACCES|EPERM/, + ); + } finally { + await chmod(tempRoot, 0o700); + } + assert.equal(await readFile(target, "utf8"), "original\n"); + assert.deepEqual(await listTempFiles(tempRoot), []); + } finally { + await chmod(tempRoot, 0o700).catch(() => undefined); + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +test("writeFileAtomic preserves an existing file mode", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-atomic-")); + try { + const target = nodePath.join(tempRoot, "secret.md"); + await writeFile(target, "secret\n", { encoding: "utf8", mode: 0o600 }); + await chmod(target, 0o600); + + await writeFileAtomic(target, "updated\n"); + + assert.equal(await readFile(target, "utf8"), "updated\n"); + assert.equal((await stat(target)).mode & 0o777, 0o600); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +async function listTempFiles(dir: string): Promise { + return (await readdir(dir)).filter((entry) => entry.includes(".tmp")); +} From d018c78086c2f8387399be12e4f4c06948d9ab04 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 14:46:46 +0800 Subject: [PATCH 07/12] fix(sync): normalize vault paths to unicode nfc --- packages/cli/src/nodeDiskMirror.ts | 40 ++++++++++++++++++- packages/cli/tests/normalizeVaultPath.test.ts | 17 ++++++++ src/utils/normalizeVaultPath.ts | 1 + 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 packages/cli/tests/normalizeVaultPath.test.ts diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index c055d8e..623d944 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -752,10 +752,17 @@ export class NodeDiskMirror { } for (const entry of entries) { - const relativePath = normalizeVaultPath( + const rawRelativePath = this.normalizePathWithoutUnicode( relativeDir ? `${relativeDir}/${entry.name}` : entry.name, ); - const absolutePath = this.toAbsolutePath(relativePath); + const relativePath = normalizeVaultPath(rawRelativePath); + if (rawRelativePath !== relativePath) { + throw new Error( + `Non-NFC filesystem path is unsupported by yaos-cli: "${rawRelativePath}". ` + + `Rename it to "${relativePath}" before syncing.`, + ); + } + const absolutePath = this.toAbsoluteRawPath(rawRelativePath); let stats: Stats; try { stats = await fs.lstat(absolutePath); @@ -811,10 +818,39 @@ export class NodeDiskMirror { } private normalizeEventPath(rawPath: string): string | null { + const rawNormalized = this.normalizePathWithoutUnicode(rawPath); const normalized = normalizeVaultPath(rawPath); + if (rawNormalized !== normalized) { + console.error( + `[yaos-cli] ignoring non-NFC filesystem path "${rawNormalized}"; ` + + `rename it to "${normalized}" before syncing.`, + ); + return null; + } return normalized.length > 0 ? normalized : null; } + private normalizePathWithoutUnicode(path: string): string { + return path + .replace(/\\/g, "/") + .replace(/\/{2,}/g, "/") + .replace(/^(\.\/)+/, "") + .replace(/^\/+/, "") + .replace(/\/+$/, ""); + } + + private toAbsoluteRawPath(vaultPath: string): string { + const parts = this.normalizePathWithoutUnicode(vaultPath) + .split("/") + .filter((segment) => segment.length > 0); + const absolute = nodePath.join(this.rootDir, ...parts); + const relative = nodePath.relative(this.rootDir, absolute); + if (relative.startsWith("..") || nodePath.isAbsolute(relative)) { + throw new Error(`Path traversal rejected: "${vaultPath}" resolves outside vault root`); + } + return absolute; + } + private toAbsolutePath(vaultPath: string): string { const parts = normalizeVaultPath(vaultPath) .split("/") diff --git a/packages/cli/tests/normalizeVaultPath.test.ts b/packages/cli/tests/normalizeVaultPath.test.ts new file mode 100644 index 0000000..2747081 --- /dev/null +++ b/packages/cli/tests/normalizeVaultPath.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { normalizeVaultPath } from "../../../src/utils/normalizeVaultPath"; + +test("normalizeVaultPath preserves existing slash semantics", () => { + assert.equal(normalizeVaultPath("./folder//nested\\note.md/"), "folder/nested/note.md"); + assert.equal(normalizeVaultPath("/folder/note.md"), "folder/note.md"); + assert.equal(normalizeVaultPath("folder/"), "folder"); +}); + +test("normalizeVaultPath normalizes decomposed Unicode filenames to NFC", () => { + const nfc = "caf\u00e9.md"; + const nfd = "cafe\u0301.md"; + assert.notEqual(nfd, nfc); + assert.equal(normalizeVaultPath(nfd), nfc); + assert.equal(normalizeVaultPath(nfd), normalizeVaultPath(nfc)); +}); diff --git a/src/utils/normalizeVaultPath.ts b/src/utils/normalizeVaultPath.ts index 1e79e11..015c942 100644 --- a/src/utils/normalizeVaultPath.ts +++ b/src/utils/normalizeVaultPath.ts @@ -1,5 +1,6 @@ export function normalizeVaultPath(path: string): string { return path + .normalize("NFC") .replace(/\\/g, "/") .replace(/\/{2,}/g, "/") .replace(/^(\.\/)+/, "") From fb6519ad9fe7018ed8afabc404012fc9ce04a2dd Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 14:48:23 +0800 Subject: [PATCH 08/12] feat(cli): persist identity-checked yjs state --- packages/cli/src/index.ts | 8 + packages/cli/src/nodeVaultSync.ts | 87 ++++++++-- packages/cli/src/statePersistence.ts | 174 ++++++++++++++++++++ packages/cli/tests/statePersistence.test.ts | 94 +++++++++++ src/sync/vaultSync.ts | 4 + 5 files changed, 352 insertions(+), 15 deletions(-) create mode 100644 packages/cli/src/statePersistence.ts create mode 100644 packages/cli/tests/statePersistence.test.ts diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 27b299b..a176649 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -41,6 +41,14 @@ addCommonOptions( const resolved = await resolveCliConfig(options); const runtime = requireRuntimeConfig(resolved, { requireDir: true }); const client = new HeadlessYaosClient(runtime); + const stateStatus = client.getStatePersistenceStatus(); + if (!stateStatus.loaded) { + console.warn( + "WARN: No local YAOS state cache found. Downloading full vault history. " + + "Future runs will use '.yaos-state.bin' for delta sync. " + + "For continuous updates, use 'yaos-cli daemon' to prevent rate-limiting.", + ); + } try { const startup = await client.startup({ watch: false }); console.log(JSON.stringify({ diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index 497d4b3..215697d 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -1,5 +1,6 @@ import WebSocket from "ws"; +import * as Y from "yjs"; import type { VaultSyncSettings } from "../../../src/settings"; import { VaultSync, @@ -9,9 +10,17 @@ import { type VaultSyncPersistence, } from "../../../src/sync/vaultSync"; import { NodeDiskMirror } from "./nodeDiskMirror"; -import type { ResolvedCliConfig, RuntimeCliConfig } from "./config"; +import type { RuntimeCliConfig } from "./config"; +import { + loadStateUpdate, + persistStateUpdate, + type LoadedStateUpdate, + type StatePersistenceMetadata, +} from "./statePersistence"; -const NOOP_INDEXEDDB_ERROR = new Error("IndexedDB is unavailable in the headless Node runtime"); +interface CreateNodeVaultSyncOptions extends Omit { + initialStateUpdate?: Uint8Array | null; +} export interface HeadlessStartupResult { localLoaded: boolean; @@ -22,12 +31,13 @@ export interface HeadlessStartupResult { export function createNodeVaultSync( config: RuntimeCliConfig, - options?: Omit, + options?: CreateNodeVaultSyncOptions, ): VaultSync { + const { initialStateUpdate, ...vaultSyncOptions } = options ?? {}; return new VaultSync(toVaultSyncSettings(config), { - ...options, -webSocketPolyfill: WebSocket as unknown as typeof globalThis.WebSocket, - persistenceFactory: () => createNoopPersistence(), + ...vaultSyncOptions, + webSocketPolyfill: WebSocket as unknown as typeof globalThis.WebSocket, + persistenceFactory: (_name, doc) => createHeadlessPersistence(doc, initialStateUpdate), logPersistenceOpenError: false, }); } @@ -35,6 +45,8 @@ webSocketPolyfill: WebSocket as unknown as typeof globalThis.WebSocket, export class HeadlessYaosClient { readonly vaultSync: VaultSync; readonly diskMirror: NodeDiskMirror; + private readonly loadedState: LoadedStateUpdate; + private lastPersistedState: StatePersistenceMetadata | null = null; private reconcileInFlight = false; private reconcilePending = false; private awaitingFirstProviderSyncAfterStartup = false; @@ -44,7 +56,10 @@ export class HeadlessYaosClient { private stopped = false; constructor(private readonly config: RuntimeCliConfig) { - this.vaultSync = createNodeVaultSync(config); + this.loadedState = loadStateUpdate(config.dir, { host: config.host, vaultId: config.vaultId }); + this.vaultSync = createNodeVaultSync(config, { + initialStateUpdate: this.loadedState.update, + }); this.diskMirror = new NodeDiskMirror(this.vaultSync, { rootDir: config.dir, deviceName: config.deviceName, @@ -116,8 +131,14 @@ export class HeadlessYaosClient { async stop(): Promise { this.stopped = true; - await this.diskMirror.stop(); - this.vaultSync.destroy(); + try { + await this.diskMirror.stop(); + if (this.startupInitialized) { + await this.persistState(); + } + } finally { + this.vaultSync.destroy(); + } } getStatus(): Record { @@ -133,9 +154,35 @@ export class HeadlessYaosClient { fatalAuthError: this.vaultSync.fatalAuthError, fatalAuthCode: this.vaultSync.fatalAuthCode, diskMirror: this.diskMirror.getDebugSnapshot(), + statePersistence: this.getStatePersistenceStatus(), + }; + } + + getStatePersistenceStatus(): Record { + return { + loaded: this.loadedState.loaded, + path: this.loadedState.updatePath, + byteLength: this.loadedState.byteLength, + stateVectorHash: this.loadedState.stateVectorHash, + lastPersisted: this.lastPersistedState, }; } + private async persistState(): Promise { + if (!this.canPersistState()) return; + this.lastPersistedState = await persistStateUpdate(this.config.dir, this.vaultSync.ydoc, { + host: this.config.host, + vaultId: this.config.vaultId, + schemaVersion: this.vaultSync.storedSchemaVersion, + activePathCount: this.vaultSync.getActiveMarkdownPaths().length, + }); + } + + private canPersistState(): boolean { + if (this.vaultSync.providerSynced) return true; + return this.loadedState.loaded && this.vaultSync.isInitialized; + } + private installReconnectionHandler(): void { if (this.reconnectionHandlerInstalled) return; this.reconnectionHandlerInstalled = true; @@ -172,6 +219,7 @@ export class HeadlessYaosClient { try { await this.diskMirror.reconcileFromDisk("authoritative"); this.lastReconciledGeneration = generation; + await this.persistState(); this.awaitingFirstProviderSyncAfterStartup = false; } finally { this.reconcileInFlight = false; @@ -205,17 +253,26 @@ function toVaultSyncSettings(config: RuntimeCliConfig): VaultSyncSettings { }; } -function createNoopPersistence(): VaultSyncPersistence { - const db = Promise.reject(NOOP_INDEXEDDB_ERROR); - db.catch(() => undefined); +function createHeadlessPersistence( + doc: Y.Doc, + initialStateUpdate: Uint8Array | null | undefined, +): VaultSyncPersistence { + const loaded = initialStateUpdate != null && initialStateUpdate.byteLength > 0; + if (loaded) { + Y.applyUpdate(doc, initialStateUpdate); + } return { - once() { - // Headless v1 intentionally skips local CRDT persistence. + once(_event, listener) { + queueMicrotask(listener); }, destroy() { return; }, - _db: db, + _db: Promise.resolve({ + addEventListener() { + return; + }, + }), }; } diff --git a/packages/cli/src/statePersistence.ts b/packages/cli/src/statePersistence.ts new file mode 100644 index 0000000..845ba4a --- /dev/null +++ b/packages/cli/src/statePersistence.ts @@ -0,0 +1,174 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import * as nodePath from "node:path"; +import * as Y from "yjs"; +import { writeFileAtomic } from "./fs"; + +export const STATE_UPDATE_FILENAME = ".yaos-state.bin"; +export const STATE_METADATA_FILENAME = ".yaos-state.json"; + +export interface StatePersistencePaths { + updatePath: string; + metadataPath: string; +} + +export interface LoadedStateUpdate { + loaded: boolean; + update: Uint8Array | null; + updatePath: string; + byteLength: number; + stateVectorHash: string | null; +} + +export interface StatePersistenceMetadata { + host: string; + vaultId: string; + schemaVersion: number | null; + activePathCount: number; + syncedAt: string; + stateUpdateBytes: number; + stateVectorHash: string; +} + +export interface StatePersistenceIdentity { + host: string; + vaultId: string; +} + +export function getStatePersistencePaths(rootDir: string): StatePersistencePaths { + return { + updatePath: nodePath.join(rootDir, STATE_UPDATE_FILENAME), + metadataPath: nodePath.join(rootDir, STATE_METADATA_FILENAME), + }; +} + +export function loadStateUpdate(rootDir: string, expectedIdentity?: StatePersistenceIdentity): LoadedStateUpdate { + const { updatePath, metadataPath } = getStatePersistencePaths(rootDir); + let bytes: Buffer; + try { + bytes = readFileSync(updatePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { + loaded: false, + update: null, + updatePath, + byteLength: 0, + stateVectorHash: null, + }; + } + throw error; + } + + if (bytes.byteLength === 0) { + throw new Error(`YAOS state file is empty: ${updatePath}. Delete it to force a full resync.`); + } + + if (expectedIdentity) { + const metadata = loadStateMetadata(metadataPath); + if (metadata.host !== expectedIdentity.host || metadata.vaultId !== expectedIdentity.vaultId) { + throw new Error( + `YAOS state cache identity mismatch for ${updatePath}. ` + + `Expected host=${expectedIdentity.host} vaultId=${expectedIdentity.vaultId}, ` + + `found host=${metadata.host} vaultId=${metadata.vaultId}. ` + + "Delete the state files to force a full resync.", + ); + } + } + + const validationDoc = new Y.Doc(); + try { + Y.applyUpdate(validationDoc, bytes); + } catch (error) { + throw new Error( + `Failed to load YAOS state file ${updatePath}: ${(error as Error).message}. ` + + "Delete it to force a full resync.", + ); + } + + return { + loaded: true, + update: new Uint8Array(bytes), + updatePath, + byteLength: bytes.byteLength, + stateVectorHash: hashStateVector(validationDoc), + }; +} + +export async function persistStateUpdate( + rootDir: string, + doc: Y.Doc, + metadata: Omit & { + syncedAt?: string; + }, +): Promise { + const { updatePath, metadataPath } = getStatePersistencePaths(rootDir); + const update = Y.encodeStateAsUpdate(doc); + const persistedMetadata: StatePersistenceMetadata = { + ...metadata, + syncedAt: metadata.syncedAt ?? new Date().toISOString(), + stateUpdateBytes: update.byteLength, + stateVectorHash: hashStateVector(doc), + }; + + await writeFileAtomic(updatePath, update, { mode: 0o600 }); + await writeFileAtomic( + metadataPath, + JSON.stringify(persistedMetadata, null, 2) + "\n", + { mode: 0o600 }, + ); + + return persistedMetadata; +} + +function loadStateMetadata(metadataPath: string): StatePersistenceMetadata { + let raw: string; + try { + raw = readFileSync(metadataPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error( + `YAOS state metadata is missing: ${metadataPath}. ` + + "Delete the state files to force a full resync.", + ); + } + throw error; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error( + `Failed to parse YAOS state metadata ${metadataPath}: ${(error as Error).message}. ` + + "Delete the state files to force a full resync.", + ); + } + + if (!isStatePersistenceMetadata(parsed)) { + throw new Error( + `Invalid YAOS state metadata: ${metadataPath}. ` + + "Delete the state files to force a full resync.", + ); + } + + return parsed; +} + +function isStatePersistenceMetadata(value: unknown): value is StatePersistenceMetadata { + if (!value || typeof value !== "object") return false; + const record = value as Record; + return ( + typeof record.host === "string" + && typeof record.vaultId === "string" + && (record.schemaVersion === null || typeof record.schemaVersion === "number") + && typeof record.activePathCount === "number" + && typeof record.syncedAt === "string" + && typeof record.stateUpdateBytes === "number" + && typeof record.stateVectorHash === "string" + ); +} + +export function hashStateVector(doc: Y.Doc): string { + return createHash("sha256").update(Y.encodeStateVector(doc)).digest("hex"); +} diff --git a/packages/cli/tests/statePersistence.test.ts b/packages/cli/tests/statePersistence.test.ts new file mode 100644 index 0000000..0a67e49 --- /dev/null +++ b/packages/cli/tests/statePersistence.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import * as os from "node:os"; +import * as nodePath from "node:path"; +import test from "node:test"; +import * as Y from "yjs"; +import { + getStatePersistencePaths, + loadStateUpdate, + persistStateUpdate, +} from "../src/statePersistence"; + +test("persistStateUpdate stores a Yjs update that can seed a fresh doc", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-state-")); + try { + const source = new Y.Doc(); + source.getMap("pathToId").set("cafe.md", "file-1"); + + const metadata = await persistStateUpdate(tempRoot, source, { + host: "https://sync.example", + vaultId: "vault-1", + schemaVersion: 2, + activePathCount: 1, + syncedAt: "2026-05-02T00:00:00.000Z", + }); + + const loaded = loadStateUpdate(tempRoot, { host: "https://sync.example", vaultId: "vault-1" }); + assert.equal(loaded.loaded, true); + assert.equal(loaded.byteLength, metadata.stateUpdateBytes); + assert.equal(loaded.stateVectorHash, metadata.stateVectorHash); + assert.ok(loaded.update); + + const restored = new Y.Doc(); + Y.applyUpdate(restored, loaded.update!); + assert.equal(restored.getMap("pathToId").get("cafe.md"), "file-1"); + + const { metadataPath } = getStatePersistencePaths(tempRoot); + const rawMetadata = JSON.parse(await readFile(metadataPath, "utf8")) as typeof metadata; + assert.equal(rawMetadata.host, "https://sync.example"); + assert.equal(rawMetadata.vaultId, "vault-1"); + assert.equal(rawMetadata.stateVectorHash, metadata.stateVectorHash); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +test("loadStateUpdate reports a missing cache without pretending local state loaded", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-state-")); + try { + const loaded = loadStateUpdate(tempRoot); + assert.equal(loaded.loaded, false); + assert.equal(loaded.update, null); + assert.equal(loaded.byteLength, 0); + assert.equal(loaded.stateVectorHash, null); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); + +test("loadStateUpdate rejects corrupt state instead of silently full-syncing", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-state-")); + try { + const { updatePath } = getStatePersistencePaths(tempRoot); + await writeFile(updatePath, "not a yjs update", "utf8"); + assert.throws( + () => loadStateUpdate(tempRoot), + /Delete it to force a full resync/, + ); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); + + +test("loadStateUpdate rejects state from a different room identity", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-state-")); + try { + const source = new Y.Doc(); + source.getMap("pathToId").set("private.md", "file-1"); + await persistStateUpdate(tempRoot, source, { + host: "https://sync.example", + vaultId: "vault-1", + schemaVersion: 2, + activePathCount: 1, + }); + + assert.throws( + () => loadStateUpdate(tempRoot, { host: "https://sync.example", vaultId: "vault-2" }), + /state cache identity mismatch/, + ); + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +}); \ No newline at end of file diff --git a/src/sync/vaultSync.ts b/src/sync/vaultSync.ts index 8c8bcb8..e5818ac 100644 --- a/src/sync/vaultSync.ts +++ b/src/sync/vaultSync.ts @@ -346,6 +346,10 @@ export class VaultSync { waitForProviderSync(): Promise { if (this._providerSynced) return Promise.resolve(true); + if (this.provider.synced) { + this._providerSynced = true; + return Promise.resolve(true); + } if (this._fatalAuthError) return Promise.resolve(false); return new Promise((resolve) => { From a15740a27dd299de4da4f1606b0706eb16348906 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 14:48:31 +0800 Subject: [PATCH 09/12] fix(cli): use distinct exit code for update_required --- packages/cli/src/errors.ts | 28 ++++++++++++++++++++++++++++ packages/cli/src/index.ts | 10 ++++++++-- packages/cli/src/nodeVaultSync.ts | 3 ++- packages/cli/tests/errors.test.ts | 23 +++++++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/errors.ts create mode 100644 packages/cli/tests/errors.test.ts diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts new file mode 100644 index 0000000..50e5e28 --- /dev/null +++ b/packages/cli/src/errors.ts @@ -0,0 +1,28 @@ +export const CLI_EXIT_CODES = { + success: 0, + failure: 1, + updateRequired: 2, +} as const; + +export type HeadlessFatalAuthCode = "unauthorized" | "server_misconfigured" | "unclaimed" | "update_required"; + +export class HeadlessCliError extends Error { + constructor( + message: string, + readonly fatalAuthCode?: HeadlessFatalAuthCode | null, + ) { + super(message); + this.name = "HeadlessCliError"; + } +} + +export function exitCodeForError(error: unknown): number { + const commanderExitCode = (error as { exitCode?: unknown })?.exitCode; + if (typeof commanderExitCode === "number") { + return commanderExitCode; + } + if (error instanceof HeadlessCliError && error.fatalAuthCode === "update_required") { + return CLI_EXIT_CODES.updateRequired; + } + return CLI_EXIT_CODES.failure; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index a176649..61147de 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -3,6 +3,7 @@ import { Command, InvalidOptionArgumentError, Option } from "commander"; import type { CliCommandOptions, ResolvedCliConfig } from "./config"; import { requireRuntimeConfig, resolveCliConfig } from "./config"; +import { exitCodeForError, HeadlessCliError } from "./errors"; import { createNodeVaultSync, HeadlessYaosClient } from "./nodeVaultSync"; const program = new Command(); @@ -75,6 +76,12 @@ addCommonOptions( try { const localLoaded = await vaultSync.waitForLocalPersistence(); const providerSynced = await vaultSync.waitForProviderSync(); + if (vaultSync.fatalAuthError) { + throw new HeadlessCliError( + `Provider rejected the connection (${vaultSync.fatalAuthCode ?? "unknown"})`, + vaultSync.fatalAuthCode, + ); + } console.log(JSON.stringify({ mode: "status", config: summarizeConfig(resolved), @@ -98,8 +105,7 @@ addCommonOptions( program.parseAsync(process.argv).catch((error: unknown) => { const message = error instanceof Error ? error.message : String(error); - // CommanderError carries its own exitCode (0 for --help, 1 for errors). - const exitCode = (error as { exitCode?: number })?.exitCode ?? 1; + const exitCode = exitCodeForError(error); if (exitCode !== 0) { process.stderr.write(`error: ${message}\n`); } diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index 215697d..db6ae4a 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -9,6 +9,7 @@ import { type VaultSyncOptions, type VaultSyncPersistence, } from "../../../src/sync/vaultSync"; +import { HeadlessCliError } from "./errors"; import { NodeDiskMirror } from "./nodeDiskMirror"; import type { RuntimeCliConfig } from "./config"; import { @@ -86,7 +87,7 @@ export class HeadlessYaosClient { const localLoaded = await this.vaultSync.waitForLocalPersistence(); const providerSynced = await this.vaultSync.waitForProviderSync(); if (this.vaultSync.fatalAuthError) { - throw new Error(formatFatalAuthError(this.vaultSync)); + throw new HeadlessCliError(formatFatalAuthError(this.vaultSync), this.vaultSync.fatalAuthCode); } const mode = this.vaultSync.getSafeReconcileMode(); diff --git a/packages/cli/tests/errors.test.ts b/packages/cli/tests/errors.test.ts new file mode 100644 index 0000000..7e92a87 --- /dev/null +++ b/packages/cli/tests/errors.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { CLI_EXIT_CODES, exitCodeForError, HeadlessCliError } from "../src/errors"; + +test("exitCodeForError preserves commander exit codes", () => { + assert.equal(exitCodeForError({ exitCode: 0 }), CLI_EXIT_CODES.success); + assert.equal(exitCodeForError({ exitCode: 9 }), 9); +}); + +test("exitCodeForError maps update_required to a non-restarting daemon code", () => { + assert.equal(CLI_EXIT_CODES.updateRequired, 2); + assert.notEqual(CLI_EXIT_CODES.updateRequired, CLI_EXIT_CODES.failure); + assert.notEqual(CLI_EXIT_CODES.updateRequired, CLI_EXIT_CODES.success); + assert.equal( + exitCodeForError(new HeadlessCliError("update required", "update_required")), + CLI_EXIT_CODES.updateRequired, + ); +}); + +test("exitCodeForError maps other failures to generic failure", () => { + assert.equal(exitCodeForError(new HeadlessCliError("unauthorized", "unauthorized")), CLI_EXIT_CODES.failure); + assert.equal(exitCodeForError(new Error("boom")), CLI_EXIT_CODES.failure); +}); From 1ad86ee1ed52f73652afa12a027af1a1be406137 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 14:48:34 +0800 Subject: [PATCH 10/12] docs(cli): document headless runtime constraints --- packages/cli/README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/cli/README.md b/packages/cli/README.md index 4ca3b0d..9e66f6f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -14,6 +14,42 @@ yaos-cli status --host --token --vault-id - `sync` performs one reconciliation pass and exits. - `status` connects to YAOS and prints current connection/cache state as JSON. +## State persistence + +`sync` and `daemon` persist local Yjs state in the mirrored vault directory: + +- `.yaos-state.bin` — operational Yjs update cache used on the next startup for delta sync. +- `.yaos-state.json` — human-readable metadata about the last persisted state. + +If `.yaos-state.bin` is missing, `yaos-cli sync` must download the current room state before reconciling. Future runs reuse the cache. +If the file is corrupt, the CLI fails loudly instead of silently pretending the cache loaded; delete the file only when you intentionally want a full resync. + +## Runtime support constraints + +Supported: + +- Linux on a local filesystem. +- One YAOS headless process per vault directory. + +Unsupported: + +- NFS, SMB, FUSE, cloud-drive mounts, or other non-local filesystems. +- Running two `yaos-cli daemon` processes against the same vault directory. +- Running `yaos-cli daemon` against a directory that an Obsidian YAOS plugin instance is also writing through a shared drive. +- Attachment/blob sync. The CLI is markdown-only for now. +- `.obsidian` settings/plugin sync. + +These constraints are correctness boundaries, not performance suggestions. Multiple writers against the same filesystem path can generate colliding file IDs and orphan CRDT state. + +## systemd restart behavior + +If the server reports `update_required`, the CLI exits with status `2`. For systemd services, prevent restart loops with: + +```ini +Restart=always +RestartPreventExitStatus=2 +``` + ## Configuration precedence 1. CLI flags From 40da1bfafb594220e234101435393270b1cbf593 Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 15:48:38 +0800 Subject: [PATCH 11/12] fix(cli): address headless hardening review findings --- packages/cli/README.md | 4 +- packages/cli/src/fs.ts | 28 +++++ packages/cli/src/index.ts | 22 +++- packages/cli/src/nodeDiskMirror.ts | 135 ++++++++++++++++------ packages/cli/src/nodeVaultSync.ts | 47 ++++++-- packages/cli/tests/fs.test.ts | 18 +++ packages/cli/tests/nodeDiskMirror.test.ts | 22 ++++ src/sync/vaultSync.ts | 29 +++++ 8 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 packages/cli/tests/nodeDiskMirror.test.ts diff --git a/packages/cli/README.md b/packages/cli/README.md index 9e66f6f..df67c3b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -21,7 +21,9 @@ yaos-cli status --host --token --vault-id - `.yaos-state.bin` — operational Yjs update cache used on the next startup for delta sync. - `.yaos-state.json` — human-readable metadata about the last persisted state. -If `.yaos-state.bin` is missing, `yaos-cli sync` must download the current room state before reconciling. Future runs reuse the cache. +If `.yaos-state.bin` is missing, `yaos-cli sync` warns because it must fetch room state before it can delta-sync. +If provider sync times out, the CLI continues in conservative mode rather than pretending it has current room state. +Future runs reuse the cache after a synced state is persisted. If the file is corrupt, the CLI fails loudly instead of silently pretending the cache loaded; delete the file only when you intentionally want a full resync. ## Runtime support constraints diff --git a/packages/cli/src/fs.ts b/packages/cli/src/fs.ts index eed8234..157362f 100644 --- a/packages/cli/src/fs.ts +++ b/packages/cli/src/fs.ts @@ -24,6 +24,9 @@ export async function writeFileAtomic( try { const fh = await fs.open(tmpPath, "wx", mode); try { + if (mode !== undefined) { + await fh.chmod(mode); + } await fh.writeFile(content); await fh.datasync(); } finally { @@ -46,6 +49,21 @@ export async function ensureDirectoryDurable(dir: string): Promise { await syncCreatedDirectoryParents(created, dir); } +export async function removeFileDurable(absolutePath: string): Promise { + await fs.rm(absolutePath, { force: true }); + await syncDirectoryIfPresent(nodePath.dirname(absolutePath)); +} + +export async function renameFileDurable(oldAbsolutePath: string, newAbsolutePath: string): Promise { + await fs.rename(oldAbsolutePath, newAbsolutePath); + const oldDir = nodePath.dirname(oldAbsolutePath); + const newDir = nodePath.dirname(newAbsolutePath); + await syncDirectoryBestEffort(oldDir); + if (newDir !== oldDir) { + await syncDirectoryBestEffort(newDir); + } +} + async function readExistingMode(absolutePath: string): Promise { try { const stats = await fs.stat(absolutePath); @@ -69,6 +87,16 @@ async function syncCreatedDirectoryParents(firstCreated: string, targetDir: stri } } +async function syncDirectoryIfPresent(dir: string): Promise { + try { + await syncDirectoryBestEffort(dir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } +} + async function syncDirectoryBestEffort(dir: string): Promise { let dh: fs.FileHandle | null = null; try { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 61147de..8d8aa50 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -163,11 +163,18 @@ function summarizeConfig(config: ResolvedCliConfig): Record { } async function waitForShutdown(client: HeadlessYaosClient): Promise { - await new Promise((resolve, reject) => { + let cleanup = () => undefined; + let stopping = false; + const stopClient = async () => { + if (stopping) return; + stopping = true; + cleanup(); + await client.stop(); + }; + const signalPromise = new Promise((resolve, reject) => { const finish = async () => { - cleanup(); try { - await client.stop(); + await stopClient(); resolve(); } catch (error) { reject(error); @@ -180,7 +187,7 @@ async function waitForShutdown(client: HeadlessYaosClient): Promise { const onSigterm = () => { void finish(); }; - const cleanup = () => { + cleanup = () => { process.off("SIGINT", onSigint); process.off("SIGTERM", onSigterm); }; @@ -188,4 +195,11 @@ async function waitForShutdown(client: HeadlessYaosClient): Promise { process.on("SIGINT", onSigint); process.on("SIGTERM", onSigterm); }); + + const fatalAuthPromise = client.waitForFatalAuth().then(async (error) => { + await stopClient(); + throw error; + }); + + await Promise.race([signalPromise, fatalAuthPromise]); } diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index 623d944..a0949f6 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -1,7 +1,7 @@ import chokidar, { type FSWatcher } from "chokidar"; import { createHash } from "node:crypto"; import { promises as fs } from "node:fs"; -import { ensureDirectoryDurable, writeFileAtomic } from "./fs"; +import { ensureDirectoryDurable, removeFileDurable, renameFileDurable, writeFileAtomic } from "./fs"; import type { Dirent, Stats } from "node:fs"; import * as nodePath from "node:path"; import * as Y from "yjs"; @@ -70,6 +70,16 @@ export interface NodeDiskMirrorDebugSnapshot { suppressedCount: number; } +export function splitSafeVaultPathParts(originalPath: string, normalizedPath: string): string[] { + const parts = normalizedPath + .split("/") + .filter((segment) => segment.length > 0); + if (parts.some((segment) => segment === "." || segment === "..")) { + throw new Error(`Path traversal rejected: "${originalPath}" contains dot segments`); + } + return parts; +} + function isLocalOrigin(origin: unknown, provider: unknown): boolean { if (origin === provider) return false; if (typeof origin === "string") return LOCAL_STRING_ORIGINS.has(origin); @@ -95,6 +105,7 @@ export class NodeDiskMirror { private debounceTimers = new Map>(); private writeDrainPromise: Promise | null = null; private pathWriteLocks = new Map>(); + private rootRealPath: string | null = null; constructor( private readonly vaultSync: VaultSync, @@ -214,16 +225,6 @@ export class NodeDiskMirror { } async stop(): Promise { - if (this.markdownDrainTimer) { - clearTimeout(this.markdownDrainTimer); - this.markdownDrainTimer = null; - } - for (const timer of this.debounceTimers.values()) { - clearTimeout(timer); - } - this.debounceTimers.clear(); - await this.markdownDrainPromise; - await this.writeDrainPromise; if (this.watcher) { await this.watcher.close(); this.watcher = null; @@ -233,6 +234,9 @@ export class NodeDiskMirror { cleanup(); } this.mapObserverCleanups = []; + + await this.flushPendingWorkBeforeStop(); + this.dirtyMarkdownPaths.clear(); this.deletedMarkdownPaths.clear(); this.writeQueue.clear(); @@ -241,6 +245,39 @@ export class NodeDiskMirror { this.pathWriteLocks.clear(); } + private async flushPendingWorkBeforeStop(): Promise { + if (this.markdownDrainTimer) { + clearTimeout(this.markdownDrainTimer); + this.markdownDrainTimer = null; + } + + while ( + this.markdownDrainPromise + || this.dirtyMarkdownPaths.size > 0 + || this.deletedMarkdownPaths.size > 0 + ) { + if (this.markdownDrainPromise) { + await this.markdownDrainPromise; + } else { + await this.kickMarkdownDrain(); + } + } + + for (const [path, timer] of this.debounceTimers.entries()) { + clearTimeout(timer); + this.writeQueue.add(path); + } + this.debounceTimers.clear(); + + while (this.writeDrainPromise || this.writeQueue.size > 0) { + if (this.writeDrainPromise) { + await this.writeDrainPromise; + } else { + await this.kickWriteDrain(); + } + } + } + getDebugSnapshot(): NodeDiskMirrorDebugSnapshot { return { watcherReady: this.watcherReady, @@ -398,16 +435,16 @@ export class NodeDiskMirror { private async readDirtyFile(path: string, reason: DirtyReason): Promise { const absolutePath = this.toAbsolutePath(path); try { - const [stats, content] = await Promise.all([ - fs.stat(absolutePath), - fs.readFile(absolutePath, "utf8"), - ]); - if (this.maxFileSize > 0 && content.length > this.maxFileSize) { + const stats = await fs.lstat(absolutePath); + if (!stats.isFile()) return null; + if (this.maxFileSize > 0 && stats.size > this.maxFileSize) { this.log( - `syncFileFromDisk: skipping "${path}" (${Math.round(content.length / 1024)} KB exceeds limit)`, + `syncFileFromDisk: skipping "${path}" (${Math.round(stats.size / 1024)} KB exceeds limit)`, ); return null; } + await this.assertSafeParentPath(absolutePath, path); + const content = await fs.readFile(absolutePath, "utf8"); return { path, reason, content, stats }; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { @@ -533,11 +570,13 @@ export class NodeDiskMirror { if (!ytext) return; const content = ytext.toJSON(); const absolutePath = this.toAbsolutePath(path); + await this.assertSafeParentPath(absolutePath, path); const currentContent = await this.readFileIfExists(absolutePath); if (currentContent === content) return; if (this.shouldBlockFrontmatterWrite(path, currentContent, content)) return; await ensureDirectoryDurable(nodePath.dirname(absolutePath)); + await this.assertSafeParentPath(absolutePath, path); await this.suppressWrite(path, content); await writeFileAtomic(absolutePath, content); } @@ -572,7 +611,8 @@ export class NodeDiskMirror { this.suppressDelete(path); const absolutePath = this.toAbsolutePath(path); try { - await fs.rm(absolutePath, { force: true }); + await this.assertSafeParentPath(absolutePath, path); + await removeFileDurable(absolutePath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { console.error(`[yaos-cli] remote delete failed for "${path}":`, error); @@ -606,8 +646,10 @@ export class NodeDiskMirror { const newAbsolutePath = this.toAbsolutePath(newPath); let needsWriteFallback = false; try { + await this.assertSafeParentPath(oldAbsolutePath, oldPath); await ensureDirectoryDurable(nodePath.dirname(newAbsolutePath)); - await fs.rename(oldAbsolutePath, newAbsolutePath); + await this.assertSafeParentPath(newAbsolutePath, newPath); + await renameFileDurable(oldAbsolutePath, newAbsolutePath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { this.log(`remote rename fell back to write for "${oldPath}" -> "${newPath}"`); @@ -621,7 +663,7 @@ export class NodeDiskMirror { // files if the write also fails. this.queueImmediateWrite(newPath, "remote-rename", true); await this.kickWriteDrain(); - await fs.rm(oldAbsolutePath, { force: true }).catch(() => undefined); + await removeFileDurable(oldAbsolutePath).catch(() => undefined); } else { this.queueImmediateWrite(newPath, "remote-rename", true); } @@ -780,6 +822,9 @@ export class NodeDiskMirror { if (!stats.isFile()) continue; if (!this.isMarkdownPathSyncable(relativePath)) continue; presentPaths.add(relativePath); + if (this.maxFileSize > 0 && stats.size > this.maxFileSize) { + continue; + } let content: string; try { content = await fs.readFile(absolutePath, "utf8"); @@ -787,9 +832,6 @@ export class NodeDiskMirror { if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; throw error; } - if (this.maxFileSize > 0 && content.length > this.maxFileSize) { - continue; - } contents.set(relativePath, content); } } @@ -840,9 +882,24 @@ export class NodeDiskMirror { } private toAbsoluteRawPath(vaultPath: string): string { - const parts = this.normalizePathWithoutUnicode(vaultPath) - .split("/") - .filter((segment) => segment.length > 0); + return this.toAbsolutePathFromParts(vaultPath, this.safeVaultPathParts( + vaultPath, + this.normalizePathWithoutUnicode(vaultPath), + )); + } + + private toAbsolutePath(vaultPath: string): string { + return this.toAbsolutePathFromParts(vaultPath, this.safeVaultPathParts( + vaultPath, + normalizeVaultPath(vaultPath), + )); + } + + private safeVaultPathParts(originalPath: string, normalizedPath: string): string[] { + return splitSafeVaultPathParts(originalPath, normalizedPath); + } + + private toAbsolutePathFromParts(vaultPath: string, parts: string[]): string { const absolute = nodePath.join(this.rootDir, ...parts); const relative = nodePath.relative(this.rootDir, absolute); if (relative.startsWith("..") || nodePath.isAbsolute(relative)) { @@ -851,16 +908,26 @@ export class NodeDiskMirror { return absolute; } - private toAbsolutePath(vaultPath: string): string { - const parts = normalizeVaultPath(vaultPath) - .split("/") - .filter((segment) => segment.length > 0); - const absolute = nodePath.join(this.rootDir, ...parts); - const relative = nodePath.relative(this.rootDir, absolute); + private async assertSafeParentPath(absolutePath: string, vaultPath: string): Promise { + let parentRealPath: string; + try { + parentRealPath = await fs.realpath(nodePath.dirname(absolutePath)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + const rootRealPath = await this.getRootRealPath(); + const relative = nodePath.relative(rootRealPath, parentRealPath); if (relative.startsWith("..") || nodePath.isAbsolute(relative)) { - throw new Error(`Path traversal rejected: "${vaultPath}" resolves outside vault root`); + throw new Error(`Symlink traversal rejected: "${vaultPath}" resolves outside vault root`); } - return absolute; + } + + private async getRootRealPath(): Promise { + if (this.rootRealPath == null) { + this.rootRealPath = await fs.realpath(this.rootDir); + } + return this.rootRealPath; } private async readFileIfExists(absolutePath: string): Promise { diff --git a/packages/cli/src/nodeVaultSync.ts b/packages/cli/src/nodeVaultSync.ts index db6ae4a..84cdf95 100644 --- a/packages/cli/src/nodeVaultSync.ts +++ b/packages/cli/src/nodeVaultSync.ts @@ -48,6 +48,8 @@ export class HeadlessYaosClient { readonly diskMirror: NodeDiskMirror; private readonly loadedState: LoadedStateUpdate; private lastPersistedState: StatePersistenceMetadata | null = null; + private readonly fatalAuthPromise: Promise; + private removeFatalAuthHandler: () => void = () => undefined; private reconcileInFlight = false; private reconcilePending = false; private awaitingFirstProviderSyncAfterStartup = false; @@ -61,6 +63,14 @@ export class HeadlessYaosClient { this.vaultSync = createNodeVaultSync(config, { initialStateUpdate: this.loadedState.update, }); + this.fatalAuthPromise = new Promise((resolve) => { + this.removeFatalAuthHandler = this.vaultSync.onFatalAuth(() => { + resolve(new HeadlessCliError( + formatFatalAuthError(this.vaultSync), + this.vaultSync.fatalAuthCode, + )); + }); + }); this.diskMirror = new NodeDiskMirror(this.vaultSync, { rootDir: config.dir, deviceName: config.deviceName, @@ -110,13 +120,7 @@ export class HeadlessYaosClient { this.awaitingFirstProviderSyncAfterStartup = !providerSynced; this.startupInitialized = true; - // Drain any deferred reconciliation that was requested during startup. - if (this.reconcilePending && !this.stopped) { - this.reconcilePending = false; - if (this.vaultSync.connectionGeneration > this.lastReconciledGeneration) { - void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); - } - } + this.drainPendingReconciliation(); if (options.watch) { await this.diskMirror.startWatching(); @@ -138,6 +142,7 @@ export class HeadlessYaosClient { await this.persistState(); } } finally { + this.removeFatalAuthHandler(); this.vaultSync.destroy(); } } @@ -169,6 +174,16 @@ export class HeadlessYaosClient { }; } + waitForFatalAuth(): Promise { + if (this.vaultSync.fatalAuthError) { + return Promise.resolve(new HeadlessCliError( + formatFatalAuthError(this.vaultSync), + this.vaultSync.fatalAuthCode, + )); + } + return this.fatalAuthPromise; + } + private async persistState(): Promise { if (!this.canPersistState()) return; this.lastPersistedState = await persistStateUpdate(this.config.dir, this.vaultSync.ydoc, { @@ -224,11 +239,19 @@ export class HeadlessYaosClient { this.awaitingFirstProviderSyncAfterStartup = false; } finally { this.reconcileInFlight = false; - if (!this.reconcilePending || this.stopped) return; - this.reconcilePending = false; - if (this.vaultSync.connectionGeneration > this.lastReconciledGeneration) { - void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); - } + this.drainPendingReconciliation(); + } + } + + private drainPendingReconciliation(): void { + if (!this.reconcilePending || this.stopped) return; + this.reconcilePending = false; + if (this.awaitingFirstProviderSyncAfterStartup) { + void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); + return; + } + if (this.vaultSync.connectionGeneration > this.lastReconciledGeneration) { + void this.runReconnectReconciliation(this.vaultSync.connectionGeneration); } } } diff --git a/packages/cli/tests/fs.test.ts b/packages/cli/tests/fs.test.ts index be0ddd1..efb7bed 100644 --- a/packages/cli/tests/fs.test.ts +++ b/packages/cli/tests/fs.test.ts @@ -57,6 +57,24 @@ test("writeFileAtomic preserves an existing file mode", async () => { } }); +test("writeFileAtomic preserves an existing file mode under a restrictive umask", async () => { + const tempRoot = await mkdtemp(nodePath.join(os.tmpdir(), "yaos-cli-atomic-")); + const previousUmask = process.umask(0o077); + try { + const target = nodePath.join(tempRoot, "shared.md"); + await writeFile(target, "shared\n", "utf8"); + await chmod(target, 0o664); + + await writeFileAtomic(target, "updated\n"); + + assert.equal(await readFile(target, "utf8"), "updated\n"); + assert.equal((await stat(target)).mode & 0o777, 0o664); + } finally { + process.umask(previousUmask); + await rm(tempRoot, { recursive: true, force: true }); + } +}); + async function listTempFiles(dir: string): Promise { return (await readdir(dir)).filter((entry) => entry.includes(".tmp")); } diff --git a/packages/cli/tests/nodeDiskMirror.test.ts b/packages/cli/tests/nodeDiskMirror.test.ts new file mode 100644 index 0000000..a027213 --- /dev/null +++ b/packages/cli/tests/nodeDiskMirror.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { splitSafeVaultPathParts } from "../src/nodeDiskMirror"; +import { normalizeVaultPath } from "../../../src/utils/normalizeVaultPath"; + +test("splitSafeVaultPathParts rejects dot-segment traversal", () => { + assert.throws( + () => splitSafeVaultPathParts("folder/../victim.md", normalizeVaultPath("folder/../victim.md")), + /Path traversal rejected/, + ); + assert.throws( + () => splitSafeVaultPathParts("folder/./victim.md", normalizeVaultPath("folder/./victim.md")), + /Path traversal rejected/, + ); +}); + +test("splitSafeVaultPathParts accepts normalized vault-relative paths", () => { + assert.deepEqual( + splitSafeVaultPathParts("folder/note.md", normalizeVaultPath("folder/note.md")), + ["folder", "note.md"], + ); +}); diff --git a/src/sync/vaultSync.ts b/src/sync/vaultSync.ts index e5818ac..de0d573 100644 --- a/src/sync/vaultSync.ts +++ b/src/sync/vaultSync.ts @@ -161,6 +161,7 @@ export class VaultSync { */ private _connectionGeneration = 0; private _providerSyncWaiters = new Set<(value: boolean) => void>(); + private _fatalAuthWaiters = new Set<() => void>(); /** * True if the server sent an explicit auth error message. @@ -295,6 +296,9 @@ export class VaultSync { } this.provider.disconnect(); this.resolvePendingProviderSyncWaiters(false); + if (firstFatal) { + this.resolvePendingFatalAuthWaiters(); + } }; // y-partyserver emits "__YPS:" control payloads via "custom-message". @@ -397,6 +401,17 @@ export class VaultSync { }); } + onFatalAuth(callback: () => void): () => void { + if (this._fatalAuthError) { + queueMicrotask(callback); + return () => undefined; + } + this._fatalAuthWaiters.add(callback); + return () => { + this._fatalAuthWaiters.delete(callback); + }; + } + // ------------------------------------------------------------------- // Sentinel // ------------------------------------------------------------------- @@ -1479,6 +1494,7 @@ export class VaultSync { this.log("Destroying VaultSync"); if (this._renameTimer) clearTimeout(this._renameTimer); this.clearPendingRenames(); + this._fatalAuthWaiters.clear(); this.provider.destroy(); void this.persistence.destroy(); this.ydoc.destroy(); @@ -1565,6 +1581,19 @@ export class VaultSync { } } + private resolvePendingFatalAuthWaiters(): void { + if (this._fatalAuthWaiters.size === 0) return; + const waiters = Array.from(this._fatalAuthWaiters); + this._fatalAuthWaiters.clear(); + for (const waiter of waiters) { + try { + waiter(); + } catch { + // Ignore waiter errors; each promise handles its own lifecycle. + } + } + } + private classifyIndexedDbError(err: unknown): { kind: IndexedDbErrorKind; name: string | null; From fc81e9b2348fde6cf0b878755ff6e6d8e07f4a2e Mon Sep 17 00:00:00 2001 From: enieuwy Date: Sat, 2 May 2026 15:59:58 +0800 Subject: [PATCH 12/12] fix(cli): close symlink gaps in rename fallback --- packages/cli/src/nodeDiskMirror.ts | 34 +++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/nodeDiskMirror.ts b/packages/cli/src/nodeDiskMirror.ts index a0949f6..67b7018 100644 --- a/packages/cli/src/nodeDiskMirror.ts +++ b/packages/cli/src/nodeDiskMirror.ts @@ -645,8 +645,8 @@ export class NodeDiskMirror { const oldAbsolutePath = this.toAbsolutePath(oldPath); const newAbsolutePath = this.toAbsolutePath(newPath); let needsWriteFallback = false; + await this.assertSafeParentPath(oldAbsolutePath, oldPath); try { - await this.assertSafeParentPath(oldAbsolutePath, oldPath); await ensureDirectoryDurable(nodePath.dirname(newAbsolutePath)); await this.assertSafeParentPath(newAbsolutePath, newPath); await renameFileDurable(oldAbsolutePath, newAbsolutePath); @@ -661,8 +661,7 @@ export class NodeDiskMirror { // Write the new file first, then delete the old file after the // write completes. Deleting before the write risks losing both // files if the write also fails. - this.queueImmediateWrite(newPath, "remote-rename", true); - await this.kickWriteDrain(); + await this.flushWrite(newPath, true); await removeFileDurable(oldAbsolutePath).catch(() => undefined); } else { this.queueImmediateWrite(newPath, "remote-rename", true); @@ -909,22 +908,37 @@ export class NodeDiskMirror { } private async assertSafeParentPath(absolutePath: string, vaultPath: string): Promise { - let parentRealPath: string; - try { - parentRealPath = await fs.realpath(nodePath.dirname(absolutePath)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return; - throw error; - } const rootRealPath = await this.getRootRealPath(); + const parentRealPath = await this.getNearestExistingAncestorRealPath( + nodePath.dirname(absolutePath), + ); const relative = nodePath.relative(rootRealPath, parentRealPath); if (relative.startsWith("..") || nodePath.isAbsolute(relative)) { throw new Error(`Symlink traversal rejected: "${vaultPath}" resolves outside vault root`); } } + private async getNearestExistingAncestorRealPath(absolutePath: string): Promise { + let current = absolutePath; + while (true) { + try { + return await fs.realpath(current); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + const parent = nodePath.dirname(current); + if (parent === current) { + throw error; + } + current = parent; + } + } + } + private async getRootRealPath(): Promise { if (this.rootRealPath == null) { + await ensureDirectoryDurable(this.rootDir); this.rootRealPath = await fs.realpath(this.rootDir); } return this.rootRealPath;