diff --git a/.changeset/curly-olives-jog.md b/.changeset/curly-olives-jog.md new file mode 100644 index 00000000..a845151c --- /dev/null +++ b/.changeset/curly-olives-jog.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.changeset/gate-release-notes.md b/.changeset/gate-release-notes.md new file mode 100644 index 00000000..24bfac20 --- /dev/null +++ b/.changeset/gate-release-notes.md @@ -0,0 +1,4 @@ +--- +--- + +CI: gate PRs targeting main on a properly-filled release-notes-next.md file. diff --git a/.changeset/in-process-mirror-scheduler.md b/.changeset/in-process-mirror-scheduler.md new file mode 100644 index 00000000..125c3c85 --- /dev/null +++ b/.changeset/in-process-mirror-scheduler.md @@ -0,0 +1,6 @@ +--- +"ornn-api": minor +"ornn-web": minor +--- + +In-process GitHub mirror reconcile scheduler with admin-configurable cadence (#437). The k8s `CronJob` is gone — the periodic mirror reconcile now runs inside the `ornn-api` pod via Agenda, multipod-safe via per-fire row locking on a shared MongoDB collection. The schedule is editable on the admin GitHub mirror settings page (preset dropdown + custom cron), interpreted in Singapore time (UTC+8, no DST), defaulting to `0 2 * * *` (daily 2am SGT). Empty schedule disables the scheduled reconcile without affecting publish-time webhooks. Mirror configuration is now unified under `SettingsService` — a one-shot boot migration copies any legacy `platform_settings.githubMirror` values into the new section on first boot. diff --git a/.changeset/mirror-scheduled-run-status.md b/.changeset/mirror-scheduled-run-status.md new file mode 100644 index 00000000..a34f5158 --- /dev/null +++ b/.changeset/mirror-scheduled-run-status.md @@ -0,0 +1,6 @@ +--- +"ornn-api": minor +"ornn-web": minor +--- + +Surface last-run status of the scheduled mirror reconcile (#475). Both the GitHub mirror settings page and the legacy mirror dashboard now show whether the most recent *scheduled* fire succeeded, failed (with the error message), is currently running, or hasn't happened yet — sourced from Agenda's persisted recurring-job doc so the view is consistent across pods and survives restarts. The previous in-process `lastReconcile` block on `GET /admin/mirror/status` is replaced with a new `scheduledRun` block. Manual `Reconcile now` clicks from the dashboard still work but do not appear in this widget; the 409 "already running" guard on the manual reconcile endpoint is unchanged. diff --git a/.changeset/release-body-too-long-fallback.md b/.changeset/release-body-too-long-fallback.md new file mode 100644 index 00000000..b159ca4a --- /dev/null +++ b/.changeset/release-body-too-long-fallback.md @@ -0,0 +1,4 @@ +--- +--- + +CI: harden the `changeset-release` workflow against GitHub's 125 000-char release-body limit. Big releases like v0.6.0 (87 consumed changesets) used to fail the `Create tag + GitHub Release` step; the workflow now falls back to a short body linking to the in-repo CHANGELOGs when the inline body would exceed the cap. diff --git a/.changeset/release-notes-from-repo-file.md b/.changeset/release-notes-from-repo-file.md new file mode 100644 index 00000000..7e689709 --- /dev/null +++ b/.changeset/release-notes-from-repo-file.md @@ -0,0 +1,4 @@ +--- +--- + +CI: release workflow now reads `.github/release-notes-next.md` as the GitHub Release body, with a fall-back to a short link-to-CHANGELOG body when the file isn't curated. Replaces the raw-CHANGELOG dump that blew past GitHub's 125 000-char limit on v0.6.0. diff --git a/.changeset/release-notes-template-and-dated.md b/.changeset/release-notes-template-and-dated.md new file mode 100644 index 00000000..89ad4965 --- /dev/null +++ b/.changeset/release-notes-template-and-dated.md @@ -0,0 +1,4 @@ +--- +--- + +CI: restructure release notes into a fixed `release-notes-template.md` (immutable, holds format and instructions) + per-release `release-notes-.md` files (copied from template, filled in, retained in repo as historical record). CI gate on develop → main PRs now validates the most recent dated file; release workflow reads the same file. Docs in CONTRIBUTING.md + CLAUDE.md updated. diff --git a/.github/release-notes-20260513.md b/.github/release-notes-20260513.md new file mode 100644 index 00000000..889edc9d --- /dev/null +++ b/.github/release-notes-20260513.md @@ -0,0 +1,15 @@ +## Fixed + +- Few technical bugs fixed. + +## New Feature + +- Mirror reconcile now runs in-process; no external Kubernetes CronJob. +- Mirror schedule editable from admin settings — presets or custom cron, Singapore time. +- "Last run" status on mirror settings page and dashboard: succeeded, failed, running, or never. +- Technical enhancement. + +## Changed + +- Mirror dashboard's "Last reconcile" tile is now consistent across pods and survives restarts. +- Technical enhancement. diff --git a/.github/release-notes-template.md b/.github/release-notes-template.md new file mode 100644 index 00000000..4a4a0723 --- /dev/null +++ b/.github/release-notes-template.md @@ -0,0 +1,73 @@ + + +## Fixed + +- (write here) + +## New Feature + +- (write here) + +## Changed + +- (write here) diff --git a/.github/workflows/changeset-release.yml b/.github/workflows/changeset-release.yml index a4cfdbbd..ae7eb019 100644 --- a/.github/workflows/changeset-release.yml +++ b/.github/workflows/changeset-release.yml @@ -161,26 +161,84 @@ jobs: git tag -a "${TAG}" -m "Release ${TAG}" git push origin "${TAG}" - extract_section() { - awk -v v="$1" ' - /^## / { - if (found) { exit } - if ($0 ~ "^## " v "$") { found=1; next } - } - found { print } - ' "$2" - } - API_NOTES=$(extract_section "${VERSION}" ornn-api/CHANGELOG.md) - WEB_NOTES=$(extract_section "${VERSION}" ornn-web/CHANGELOG.md) - - BODY=$(cat <.md` file + # (filename sort, descending). If it exists and the + # author has filled it in (placeholder string + # `(write here)` removed), strip its HTML comment block + # and use the remaining prose as the release body. + # Footer with CHANGELOG links is always appended. + # + # 2. Short body that just links to the per-package + # CHANGELOG sections on the tag. Used when (1) wasn't + # filled in, or when the full inline notes would + # exceed GitHub's 125 000-char body cap (#429 / #430). + # + # `.github/release-notes-template.md` is the immutable + # template the dated file is copied from — never read here. + # The filename-pattern filter (8-digit date) excludes it + # from the dated-file search. + # + # Inline-CHANGELOG body was retired — even when it fits, the + # raw consumed changesets are engineer-speak that doesn't + # belong on a public release page. Authors curate the + # user-facing summary into the dated file before opening + # their develop → main PR, and the gate in + # `.github/workflows/check-release-notes.yml` enforces it + # on PRs to main. + REPO_URL="https://github.com/${GITHUB_REPOSITORY}" + CHANGELOG_FOOTER=$(cat </dev/null \ + | grep -E '/release-notes-[0-9]{8}\.md$' \ + | sort -r \ + | head -1) + + USE_CURATED=false + if [ -n "$RELEASE_NOTES_FILE" ] && [ -f "$RELEASE_NOTES_FILE" ] && ! grep -qF '(write here)' "$RELEASE_NOTES_FILE"; then + USE_CURATED=true + fi + + if $USE_CURATED; then + echo "Using curated release notes from $RELEASE_NOTES_FILE" + # Strip the leading HTML comment block (instructions to the + # author). Everything between `` + # on its own line is dropped. + CURATED=$(awk ' + BEGIN { in_comment = 0 } + /^$/ { if (in_comment) { in_comment = 0; next } } + !in_comment { print } + ' "$RELEASE_NOTES_FILE") + BODY="${CURATED}${CHANGELOG_FOOTER}" + else + echo "No curated dated release-notes file found (or placeholder still present) — using short-body fallback." + BODY=$(cat <.md` file. The release workflow +# falls back to a short link-to-CHANGELOG body when no dated file +# exists or it isn't curated — that fallback is fine for emergencies +# but a poor default. This gate forces the maintainer to create and +# fill in a dated release-notes file BEFORE the release-bump train +# starts, so v always ships with curated release notes on the +# GitHub Releases page. Tracked in #433 / #435. +# +# Files involved: +# - `.github/release-notes-template.md` — immutable template (never +# edited; this gate ignores it and even fails if someone deletes +# it). +# - `.github/release-notes-.md` — one per release, copied +# from the template and filled in. The most recent (by filename +# sort) is what the gate validates. +# +# Required-status check binding (one-time, in the repo Settings UI): +# add `Check release notes` to main's branch-protection list so the +# gate is actually load-bearing. + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, edited] + +jobs: + check-release-notes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Validate release notes file + run: | + set -u + TEMPLATE=".github/release-notes-template.md" + + # ── 0. Template must exist ───────────────────────────────── + # The template is the canonical source for the dated-file + # format. If someone deletes it, the next maintainer has no + # source to copy from. Fail loud. + if [ ! -f "$TEMPLATE" ]; then + echo "::error file=$TEMPLATE::Template file does not exist. Restore $TEMPLATE — it is the canonical source for per-release notes files." + exit 1 + fi + + # ── 1. Find the most recent dated release-notes file ────── + # Pattern: release-notes-YYYYMMDD.md (8 digits). The + # template's filename (release-notes-template.md) does NOT + # match this pattern, so it's naturally excluded. Sort + # descending so the freshest date wins. + LATEST=$(ls -1 .github/release-notes-*.md 2>/dev/null \ + | grep -E '/release-notes-[0-9]{8}\.md$' \ + | sort -r \ + | head -1) + + if [ -z "$LATEST" ]; then + EXPECTED=".github/release-notes-$(date -u +%Y%m%d).md" + echo "::error file=$TEMPLATE::No dated release-notes file found. Copy the template to a dated file before merging to main:" + echo "::error file=$TEMPLATE:: cp $TEMPLATE $EXPECTED" + echo "::error file=$TEMPLATE:: \$EDITOR $EXPECTED # fill in the three sections" + exit 1 + fi + + echo "Validating: $LATEST" + FAILED=0 + + # ── 2. Filename format sanity ────────────────────────────── + # ls already filtered by the regex, but assert again so a + # future ls flag change can't silently let through a + # `release-notes-foo.md` lookalike. + BASENAME=$(basename "$LATEST") + if ! echo "$BASENAME" | grep -qE '^release-notes-[0-9]{8}\.md$'; then + echo "::error file=$LATEST::Filename does not match the expected pattern \`release-notes-YYYYMMDD.md\` (8 digits). Got: $BASENAME" + FAILED=1 + fi + + # ── 3. All three section headings present ────────────────── + for section in "## Fixed" "## New Feature" "## Changed"; do + if ! grep -qF "$section" "$LATEST"; then + echo "::error file=$LATEST::Missing required heading: \`$section\`. Release notes must contain Fixed / New Feature / Changed sections — leave a section with only the technical-bucket bullet (\"Few technical bugs fixed\" / \"Technical enhancement\") if nothing notable happened in that bucket." + FAILED=1 + fi + done + + # ── 4. Placeholder `(write here)` removed ────────────────── + # The template uses `(write here)` for each bullet slot. Any + # remaining occurrence in the dated file means at least one + # section hasn't been filled in. + if grep -qF '(write here)' "$LATEST"; then + echo "::error file=$LATEST::Release notes still contain the \`(write here)\` placeholder. Replace each placeholder with brief user-facing bullets (6-12 words each). Cluster technical-only items into a single \`Few technical bugs fixed\` / \`Technical enhancement\` trailing bullet." + FAILED=1 + fi + + if [ "$FAILED" -ne 0 ]; then + echo "" + echo "Release-notes flow reminder:" + echo " - $TEMPLATE has the formatting rules and the per-file copy command." + echo " - Run \`bun changeset status\` to see what's pending for this release." + echo " - One bullet = one product-level fact, 6-12 words." + echo " - Drop a section entirely if there's genuinely nothing for it." + exit 1 + fi + + echo "✓ Release notes valid — $LATEST has all three sections, no placeholders." diff --git a/CLAUDE.md b/CLAUDE.md index 71c4c29b..5e08b4d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,7 +111,19 @@ Select affected package(s), semver bump level (`patch` / `minor` / `major`), wri Fully automated on the `main` side. No local script to run — developer action is "open a PR, review a PR". `.github/workflows/changeset-release.yml` is a state machine driven by `push: main`. -**Step 1 — Promote `develop` → `main`.** Open a PR `develop → main` and merge it. Regular PR; it carries whatever features + unconsumed `.changeset/*.md` files have piled up on `develop` since the last release. +**Step 0 — Curate release notes.** Before opening the develop → main PR, the maintainer creates a dated release-notes file: + +```bash +cp .github/release-notes-template.md .github/release-notes-$(date -u +%Y%m%d).md +# edit the new file — replace each `(write here)` with user-facing bullets +git add .github/release-notes-*.md && git commit && git push +``` + +The template (`.github/release-notes-template.md`) is immutable — it stays in the repo and supplies the format / instructions for future releases. The dated file (`.github/release-notes-.md`) is per-release; after release it stays in the repo as a historical record. The release workflow reads the most recent dated file at release time and uses it as the GitHub Release body (HTML comment block stripped, CHANGELOG-link footer appended automatically). The CI gate `.github/workflows/check-release-notes.yml` fails the develop → main PR if no dated file exists, if any of `## Fixed` / `## New Feature` / `## Changed` is missing, or if the `(write here)` placeholder is still in the file. + +Each dated file has three sections — see the template's comment block for the formatting rules. The short version: one bullet = one product-level fact, 6–12 words; cluster purely-technical items into a single trailing `Few technical bugs fixed` / `Technical enhancement` bullet per section; no PR / issue refs, no version numbers. + +**Step 1 — Promote `develop` → `main`.** Open a PR `develop → main` and merge it. Regular PR; it carries whatever features + unconsumed `.changeset/*.md` files + the new `release-notes-.md` file have piled up on `develop` since the last release. **Step 2 — Review the bot's release-bump PR.** On the `main` push from Step 1, the workflow sees pending `.changeset/*.md` files, so it: @@ -125,7 +137,7 @@ Review that PR. Merge with **Squash and merge** (keeps history linear; `main` en **Step 3 — Tag + GitHub Release + sync back to `develop`.** On the `main` push from Step 2, the workflow sees no pending changesets + `ornn-api/package.json`'s version has no matching `v` tag, so it: 1. Creates an annotated `v` tag and pushes it. -2. Extracts the `## ` section from each package's `CHANGELOG.md`, builds a combined body, and calls `gh release create`. +2. Finds the most recent `.github/release-notes-.md`, strips its HTML comment block, appends a CHANGELOG-links footer, and calls `gh release create` with that body. If no dated file exists (or it still has the `(write here)` placeholder), falls back to a short body that just links to the in-repo `CHANGELOG.md` files. A length safety check kicks the body back to the short fallback if it exceeds 120 000 chars (GitHub API caps release bodies at 125 000). 3. Creates branch `sync/post-release-v` from `main`. 4. Opens PR `sync/post-release-v → develop` — **auto-approved + auto-merged** by the same workflow via a direct `PUT /repos/.../pulls/:n/merge` API call with `merge_method: merge`. No human action for the sync step; the PR is a deterministic replay of a commit that already passed CI on `main`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8705e6f..4e348e32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,6 +158,34 @@ bun changeset --empty Releases are fully automated via Changesets. Maintainer-driven; contributors don't need to do anything beyond including a changeset on each PR. The flow is documented in [`CLAUDE.md`](CLAUDE.md#versioning--releases). +### Release notes — maintainer task per release + +The auto-generated `CHANGELOG.md` is engineer-speak (PR refs, author thanks, paragraph-long rationales). The public **GitHub Releases page** uses a curated, user-facing summary instead. + +The release-notes flow uses two files: + +- **[`.github/release-notes-template.md`](.github/release-notes-template.md)** — the immutable template. Never edited; provides the format and instructions. +- **`.github/release-notes-.md`** — one per release. Copied from the template, filled in by the maintainer (or their local Claude), and committed to develop before opening the `develop → main` release PR. After release it stays in the repo as a historical record. + +Before opening a `develop → main` release PR: + +```bash +cp .github/release-notes-template.md .github/release-notes-$(date -u +%Y%m%d).md +# edit the new dated file — see the template's comment block for rules +``` + +Each dated file has three sections: + +- **Fixed** — bug fixes the user notices. Cluster technical-only fixes into a single trailing `Few technical bugs fixed` bullet. +- **New Feature** — new features the user notices. Cluster technical-only work into a single trailing `Technical enhancement` bullet. +- **Changed** — changes to existing features. Same `Technical enhancement` cluster for the technical-only bucket. + +One bullet = 6–12 words. Plain prose. No PR / issue refs. The full per-PR detail is linked at the bottom of every release body automatically. + +**CI gate**: [`.github/workflows/check-release-notes.yml`](.github/workflows/check-release-notes.yml) fails the `develop → main` PR if no dated file exists, if it's missing any of the three section headings, or if the `(write here)` placeholder is still present. The PR can't merge until the gate is green. + +**Release workflow** (`changeset-release.yml`) reads the most recent dated file at release time and uses its prose as the GitHub Release body (HTML comment block stripped, CHANGELOG-link footer appended). If no dated file is present, the workflow falls back to a short body that links to the in-repo `CHANGELOG.md` files — release still publishes, just without curated prose. + ## Where to ask questions - Usage / how-to → [Discussions → Q&A](https://github.com/ChronoAIProject/Ornn/discussions/categories/q-a) diff --git a/bun.lock b/bun.lock index fcbeba07..ed5878f1 100644 --- a/bun.lock +++ b/bun.lock @@ -16,8 +16,11 @@ }, "ornn-api": { "name": "ornn-api", - "version": "0.5.0", + "version": "0.6.0", "dependencies": { + "@agendajs/mongo-backend": "^4.0.2", + "agenda": "^6.2.5", + "cron-parser": "^5.5.0", "hono": "^4.12.18", "jszip": "^3.10.1", "mongodb": "^7.0.0", @@ -37,10 +40,11 @@ }, "ornn-web": { "name": "ornn-web", - "version": "0.5.0", + "version": "0.6.0", "dependencies": { "@hookform/resolvers": "^5.2.2", "@tanstack/react-query": "^5.62.0", + "cron-parser": "^5.5.0", "diff": "^9.0.0", "framer-motion": "^12.38.0", "highlight.js": "^11.10.0", @@ -83,7 +87,7 @@ }, "sdk/typescript": { "name": "@chronoai/ornn-sdk", - "version": "0.2.0", + "version": "0.2.1", "devDependencies": { "@types/bun": "latest", "typescript": "^6.0.0", @@ -102,6 +106,8 @@ "packages": { "@adobe/css-tools": ["@adobe/css-tools@4.4.4", "", {}, "sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg=="], + "@agendajs/mongo-backend": ["@agendajs/mongo-backend@4.0.2", "", { "dependencies": { "debug": "^4.4.0" }, "peerDependencies": { "agenda": "6.2.5", "mongodb": "^6.0.0 || ^7.0.0" } }, "sha512-EeDnn6bMYGerpamU/iOFbw9b2uD3Om/Sw/lWqJUjLGo3yntT2SALC1taLs2KzsFHCHSjHt6iOenlhSPc6JpGBg=="], + "@antfu/install-pkg": ["@antfu/install-pkg@1.1.0", "", { "dependencies": { "package-manager-detector": "^1.3.0", "tinyexec": "^1.0.1" } }, "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ=="], "@asamuzakjp/css-color": ["@asamuzakjp/css-color@5.1.11", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@csstools/css-calc": "^3.2.0", "@csstools/css-color-parser": "^4.1.0", "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg=="], @@ -560,6 +566,8 @@ "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + "agenda": ["agenda@6.2.5", "", { "dependencies": { "cron-parser": "^5.0.0", "date.js": "~0.3.3", "debug": "^4.4.0", "human-interval": "~2.0.1", "luxon": "^3.2.1" } }, "sha512-wBTsMsbHwF3Gd65rafkylJTeRTPI6iJg5P4LzKaE0PN6V764UzAh9XRxRI8ytnG7VQoTKois8th8eAOLeb6oOg=="], + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], "ajv": ["ajv@6.15.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw=="], @@ -658,6 +666,8 @@ "cose-base": ["cose-base@1.0.3", "", { "dependencies": { "layout-base": "^1.0.0" } }, "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg=="], + "cron-parser": ["cron-parser@5.5.0", "", { "dependencies": { "luxon": "^3.7.1" } }, "sha512-oML4lKUXxizYswqmxuOCpgFS8BNUJpIu6k/2HVHyaL8Ynnf3wdf9tkns0yRdJLSIjkJ+b0DXHMZEHGpMwjnPww=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="], @@ -742,6 +752,8 @@ "dataloader": ["dataloader@1.4.0", "", {}, "sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw=="], + "date.js": ["date.js@0.3.3", "", { "dependencies": { "debug": "~3.1.0" } }, "sha512-HgigOS3h3k6HnW011nAb43c5xx5rBXk8P2v/WIT9Zv4koIaVXiH2BURguI78VVp+5Qc076T7OR378JViCnZtBw=="], + "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], "dayjs": ["dayjs@1.11.20", "", {}, "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ=="], @@ -914,6 +926,8 @@ "human-id": ["human-id@4.1.3", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-tsYlhAYpjCKa//8rXZ9DqKEawhPoSytweBC2eNvcaDK+57RZLHGqNs3PZTQO6yekLFSuvA6AlnAfrw1uBvtb+Q=="], + "human-interval": ["human-interval@2.0.1", "", { "dependencies": { "numbered": "^1.1.0" } }, "sha512-r4Aotzf+OtKIGQCB3odUowy4GfUDTy3aTWTfLd7ZF2gBCy3XW3v/dJLRefZnOFFnjqs5B1TypvS8WarpBkYUNQ=="], + "i18next": ["i18next@26.1.0", "", { "peerDependencies": { "typescript": "^5 || ^6" }, "optionalPeers": ["typescript"] }, "sha512-dIU6td04DvQuIqVst5S9g0GviTmhZ0DYD4b9ociVGJmuCa5vZ2de/t+Enf4olvj87mF8Y2lwjNQBwC9QZsvzKQ=="], "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], @@ -1042,6 +1056,8 @@ "lru-cache": ["lru-cache@11.3.6", "", {}, "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A=="], + "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], @@ -1182,6 +1198,8 @@ "node-releases": ["node-releases@2.0.44", "", {}, "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ=="], + "numbered": ["numbered@1.1.0", "", {}, "sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g=="], + "obug": ["obug@2.1.1", "", {}, "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ=="], "on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="], @@ -1606,6 +1624,8 @@ "d3-sankey/d3-shape": ["d3-shape@1.3.7", "", { "dependencies": { "d3-path": "1" } }, "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw=="], + "date.js/debug": ["debug@3.1.0", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g=="], + "eslint-plugin-react-hooks/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], @@ -1638,6 +1658,8 @@ "d3-sankey/d3-shape/d3-path": ["d3-path@1.0.9", "", {}, "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg=="], + "date.js/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "find-cache-dir/make-dir/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "mongodb-connection-string-url/whatwg-url/tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="], diff --git a/deployment/ornn-api/mirror-cronjob.yaml b/deployment/ornn-api/mirror-cronjob.yaml deleted file mode 100644 index d5430a90..00000000 --- a/deployment/ornn-api/mirror-cronjob.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: ornn-mirror-reconcile - namespace: ${NAMESPACE} -spec: - # Every hour at :17 — offset from common scrape windows so we don't - # collide with backup / monitoring spikes. - schedule: "17 * * * *" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 5 - jobTemplate: - spec: - backoffLimit: 1 - ttlSecondsAfterFinished: 3600 - template: - metadata: - labels: - app: ornn-api - role: mirror-reconcile - spec: - restartPolicy: Never - containers: - - name: reconcile - image: ${ORNN_API_IMAGE} - imagePullPolicy: ${IMAGE_PULL_POLICY} - # Mirror config (kill switch + repo coords + GitHub App - # credentials) lives in the platform_settings DB row — - # the script reads + self-gates on every run, so when - # disabled or incomplete it logs and exits 0. - # Same env shape as ornn-api/deployment.yaml — kept in - # lockstep so a mis-set var on either side doesn't drift - # only the runtime path. Cross-checked against config.ts - # + infra/url.ts: every key below has a code consumer. - env: - - name: PORT - value: "${PORT}" - - name: LOG_LEVEL - value: "${LOG_LEVEL}" - - name: LOG_PRETTY - value: "${LOG_PRETTY}" - - name: MONGODB_DB - value: "${MONGODB_DB}" - - name: MAX_PACKAGE_SIZE_BYTES - value: "${MAX_PACKAGE_SIZE_BYTES}" - - name: ALLOWED_ORIGINS - value: "${ALLOWED_ORIGINS}" - - name: ORNN_PUBLIC_ORIGIN - value: "${ORNN_PUBLIC_ORIGIN}" - - name: ORNN_URL_ALLOWLIST_CIDR - value: "${ORNN_URL_ALLOWLIST_CIDR}" - - name: AGENTSEAL_PYTHON - value: "${AGENTSEAL_PYTHON}" - - name: AGENTSEAL_SCRIPT - value: "${AGENTSEAL_SCRIPT}" - envFrom: - - secretRef: - name: ornn-api-secret - command: ["bun", "run", "scripts/reconcile-mirror.ts"] - resources: - requests: - cpu: "50m" - memory: "128Mi" - limits: - cpu: "500m" - memory: "512Mi" diff --git a/ornn-api/package.json b/ornn-api/package.json index d026a2a3..5e33ae4e 100644 --- a/ornn-api/package.json +++ b/ornn-api/package.json @@ -13,6 +13,9 @@ "audit:reserved-verbs": "bun run scripts/audit-reserved-verbs.ts" }, "dependencies": { + "@agendajs/mongo-backend": "^4.0.2", + "agenda": "^6.2.5", + "cron-parser": "^5.5.0", "hono": "^4.12.18", "jszip": "^3.10.1", "mongodb": "^7.0.0", diff --git a/ornn-api/scripts/reconcile-mirror.ts b/ornn-api/scripts/reconcile-mirror.ts index 91e5f032..574325ec 100644 --- a/ornn-api/scripts/reconcile-mirror.ts +++ b/ornn-api/scripts/reconcile-mirror.ts @@ -1,27 +1,24 @@ /** * One-shot mirror reconciliation entry point. * - * Wires up the same dependency graph as `bootstrap.ts` (mongo + skill - * repo + skill service + platform settings) but skips the HTTP server - * — runs `MirrorService.reconcileAll()` once and exits with code 0 on - * success / 1 on failure. + * Wires up the dependency graph (mongo + skill repo + skill service + + * SettingsService) and runs `MirrorService.reconcileAll()` once, then + * exits with code 0 on success / 1 on failure. * - * Used by the k8s `CronJob` (every hour) so any state the publish-time - * webhook dropped is caught the next time the cron fires. Mirror - * settings (kill switch + repo coords + GitHub App credentials) live - * in the `platform_settings` Mongo collection and are surfaced through - * the admin UI; this script reads them via `PlatformSettingsService` - * and no-ops cleanly when disabled or incomplete (the same self-gating + * The in-process scheduler in `ornn-api` is the production driver for + * this work — this script remains as a manual debugging shim operators + * can run from a developer box (e.g. to force an immediate reconcile + * without waiting for the schedule). Mirror settings (kill switch + + * repo coords + GitHub App credentials) live in the `platform_settings` + * Mongo collection under the `mirror` section and are surfaced through + * the admin UI; this script reads them via `SettingsServiceImpl` and + * no-ops cleanly when disabled or incomplete (the same self-gating * code path the long-running pod uses). * * Run locally: * MONGODB_URI=... MONGODB_DB=ornn ENCRYPTION_KEY=... \ * bun run scripts/reconcile-mirror.ts * - * NyxID SA credentials and other operator-flippable settings live in - * the `platform_settings` collection — the script reads them through - * `SettingsService` and self-gates when missing. - * * @module scripts/reconcile-mirror */ @@ -34,8 +31,8 @@ import { SkillRepository } from "../src/domains/skills/crud/repository"; import { SkillVersionRepository } from "../src/domains/skills/crud/skillVersionRepository"; import { SkillService } from "../src/domains/skills/crud/service"; import { MirrorService } from "../src/domains/skills/mirror/mirrorService"; -import { PlatformSettingsRepository } from "../src/domains/platform/repository"; -import { PlatformSettingsService } from "../src/domains/platform/service"; +import { SettingsRepository } from "../src/domains/settings/repository"; +import { SettingsServiceImpl } from "../src/domains/settings/service"; async function main(): Promise { const logger = pino({ level: "info" }).child({ service: "reconcile-mirror" }); @@ -47,12 +44,14 @@ async function main(): Promise { const skillVersionRepo = new SkillVersionRepository(mongo.db); await skillVersionRepo.ensureIndexes(); - const platformSettingsRepo = new PlatformSettingsRepository(mongo.db); - const platformSettingsService = new PlatformSettingsService(platformSettingsRepo, { + const settingsRepo = new SettingsRepository(mongo.db); + const settingsService = new SettingsServiceImpl({ + repo: settingsRepo, encryptionKey: config.encryptionKey, }); + const saTokenProvider = new NyxidSaTokenProvider(async () => { - const s = await platformSettingsService.getNyxidIntegration(); + const s = await settingsService.getNyxid(); return { tokenUrl: s.tokenUrl, clientId: s.clientId, @@ -60,24 +59,28 @@ async function main(): Promise { }; }); const getSaAccessToken = () => saTokenProvider.getAccessToken(); - const needsProxyAuth = config.storageServiceUrl.includes("proxy"); - const storageClient = new StorageClient( - config.storageServiceUrl, - needsProxyAuth ? getSaAccessToken : undefined, - ); + const storageClient = new StorageClient({ + resolver: async () => { + const s = await settingsService.getNyxid(); + return { baseUrl: s.chronoStorageUrl, bucket: s.chronoStorageBucket }; + }, + getAccessToken: getSaAccessToken, + }); const skillService = new SkillService({ skillRepo, skillVersionRepo, storageClient, - storageBucket: config.storageBucket, + storageBucketResolver: async () => + (await settingsService.getNyxid()).chronoStorageBucket, }); // Self-gates on disabled/incomplete config — exits cleanly with a - // zero-count result in either case so the cron's exit code stays 0. - const runtime = await platformSettingsService.getGithubMirrorConfig(); + // zero-count result in either case so the manual run's exit code + // stays 0 when the operator just wants to verify the wiring. + const runtime = await settingsService.getMirror(); if (!runtime.enabled) { - logger.warn("Mirror is disabled in platform_settings — reconcile is a no-op. Exiting."); + logger.warn("Mirror is disabled in settings — reconcile is a no-op. Exiting."); return; } if ( @@ -85,7 +88,7 @@ async function main(): Promise { !runtime.owner || !runtime.repo ) { logger.warn( - "Mirror is enabled but credentials/coords are incomplete in platform_settings — reconcile is a no-op. Exiting.", + "Mirror is enabled but credentials/coords are incomplete in settings — reconcile is a no-op. Exiting.", ); return; } @@ -94,7 +97,7 @@ async function main(): Promise { skillRepo, skillService, ornnPublicOrigin: config.ornnPublicOrigin, - platformSettingsService, + settingsService, }); const t0 = Date.now(); diff --git a/ornn-api/src/bootstrap.ts b/ornn-api/src/bootstrap.ts index 71ba227a..2aab3cc8 100644 --- a/ornn-api/src/bootstrap.ts +++ b/ornn-api/src/bootstrap.ts @@ -94,6 +94,10 @@ import { createFormatRoutes } from "./domains/skills/format/routes"; // Domain: GitHub Mirror (public + system skill auto-mirror) import { MirrorService } from "./domains/skills/mirror/mirrorService"; import { createMirrorRoutes } from "./domains/skills/mirror/routes"; +import { + createMirrorScheduler, + type MirrorScheduler, +} from "./domains/skills/mirror/scheduler"; // Domain: Me (caller-scoped endpoints) import { createMeRoutes } from "./domains/me/routes"; @@ -110,6 +114,7 @@ import { createPlatformSettingsRoutes } from "./domains/platform/routes"; import { SettingsRepository } from "./domains/settings/repository"; import { SettingsServiceImpl } from "./domains/settings/service"; import { createSettingsRoutes } from "./domains/settings/routes"; +import { migrateLegacyMirrorIntoSettings } from "./domains/settings/sections/mirror.migration"; import { LlmProvidersRepository } from "./domains/settings/llmProviders/repository"; import { LlmProvidersService } from "./domains/settings/llmProviders/service"; import { createLlmProvidersRoutes } from "./domains/settings/llmProviders/routes"; @@ -212,6 +217,21 @@ export async function bootstrap(config: SkillConfig): Promise { encryptionKey: config.encryptionKey, }); + // One-shot migration: copy any non-default mirror config from the + // legacy `platform_settings:{_id:"ornn"}.githubMirror` field into the + // new per-section `platform_settings:{_id:"mirror"}` doc. Idempotent; + // no-op when the new doc already exists or the legacy field is + // absent. Must run BEFORE the first `settingsService.getMirror()` + // call (none happen during boot, but be defensive). Failure is + // logged + non-fatal — operators can still set mirror config via + // the admin UI after boot. + await migrateLegacyMirrorIntoSettings(db, logger).catch((err) => + logger.error( + { err: err instanceof Error ? err.message : String(err) }, + "legacy mirror migration failed — admin must re-save mirror config", + ), + ); + // ---- SA Token Provider (shared by proxy-authenticated clients) ---- // Credentials live in admin Settings → Integrations → NyxID and are // resolved lazily on every refresh; an empty section throws a clear @@ -626,12 +646,35 @@ export async function bootstrap(config: SkillConfig): Promise { skillRepo, skillService, ornnPublicOrigin: config.ornnPublicOrigin, - platformSettingsService, + settingsService, }); + // In-process mirror reconcile scheduler. Multi-pod-safe (Agenda's + // per-fire row lock on `agendaJobs`); schedule is driven by + // `settings.mirror.reconcileSchedule` and updated dynamically by the + // scheduler's own 1-minute sync tick. Replaces the legacy k8s + // CronJob (#437). Constructed before `createMirrorRoutes` so the + // status endpoint can read scheduled-run history through it (#475). + let mirrorScheduler: MirrorScheduler | null = null; + try { + mirrorScheduler = createMirrorScheduler({ + db, + logger, + mirrorService, + settingsService, + }); + await mirrorScheduler.start(); + } catch (err) { + logger.error( + { err: err instanceof Error ? err.message : String(err) }, + "mirror scheduler failed to start — scheduled reconciles will not run on this pod", + ); + mirrorScheduler = null; + } const mirrorRoutes = createMirrorRoutes({ mirrorService, - platformSettingsService, + settingsService, skillRepo, + mirrorScheduler, }); // Skill routes — sharing is now a direct PUT /permissions write; the @@ -942,6 +985,16 @@ export async function bootstrap(config: SkillConfig): Promise { // ---- Shutdown ---- async function shutdown(): Promise { logger.info("Shutting down ornn-api..."); + // Stop the scheduler first so no new mirror reconciles start while + // we're tearing the Mongo connection down. `stop()` is idempotent + + // already swallows its own errors. + if (mirrorScheduler) { + try { + await mirrorScheduler.stop(); + } catch (err) { + logger.warn({ err }, "Mirror scheduler stop failed — continuing"); + } + } // Drain PostHog buffer before closing Mongo — losing buffered events // is the most common cause of "missing api.error" complaints. try { diff --git a/ornn-api/src/domains/platform/repository.ts b/ornn-api/src/domains/platform/repository.ts index 9e122bdf..3259ad7e 100644 --- a/ornn-api/src/domains/platform/repository.ts +++ b/ornn-api/src/domains/platform/repository.ts @@ -8,7 +8,7 @@ */ import type { Collection, Db, Document } from "mongodb"; -import type { GithubMirrorConfig, LlmProviderConfig, PlatformSettings } from "./types"; +import type { LlmProviderConfig, PlatformSettings } from "./types"; const SETTINGS_ID = "ornn"; @@ -33,18 +33,6 @@ export class PlatformSettingsRepository { if (typeof doc.auditWaiverThreshold === "number") { out.auditWaiverThreshold = doc.auditWaiverThreshold; } - if (doc.githubMirror && typeof doc.githubMirror === "object") { - const m = doc.githubMirror as Partial; - out.githubMirror = { - enabled: typeof m.enabled === "boolean" ? m.enabled : false, - owner: typeof m.owner === "string" ? m.owner : "", - repo: typeof m.repo === "string" ? m.repo : "", - branch: typeof m.branch === "string" ? m.branch : "", - appId: typeof m.appId === "string" ? m.appId : "", - installationId: typeof m.installationId === "string" ? m.installationId : "", - appPrivateKey: typeof m.appPrivateKey === "string" ? m.appPrivateKey : "", - }; - } if (doc.llmProvider && typeof doc.llmProvider === "object") { const p = doc.llmProvider as Partial; out.llmProvider = { @@ -57,26 +45,15 @@ export class PlatformSettingsRepository { /** * Partial upsert. Pass only the fields you want to change; nothing - * else is touched. `githubMirror` and `llmProvider` are written as full - * objects (atomic) — the service layer assembles complete shapes - * (including encrypting any sensitive fields) before calling here. + * else is touched. `llmProvider` is written as a full object (atomic) + * — the service layer assembles complete shapes (including encrypting + * any sensitive fields) before calling here. */ async patch(partial: Partial): Promise> { const set: Record = {}; if (typeof partial.auditWaiverThreshold === "number") { set.auditWaiverThreshold = partial.auditWaiverThreshold; } - if (partial.githubMirror) { - set.githubMirror = { - enabled: !!partial.githubMirror.enabled, - owner: partial.githubMirror.owner ?? "", - repo: partial.githubMirror.repo ?? "", - branch: partial.githubMirror.branch ?? "", - appId: partial.githubMirror.appId ?? "", - installationId: partial.githubMirror.installationId ?? "", - appPrivateKey: partial.githubMirror.appPrivateKey ?? "", - } satisfies GithubMirrorConfig; - } if (partial.llmProvider) { set.llmProvider = { gatewayUrl: partial.llmProvider.gatewayUrl ?? "", diff --git a/ornn-api/src/domains/platform/routes.ts b/ornn-api/src/domains/platform/routes.ts index e7f56408..9e841334 100644 --- a/ornn-api/src/domains/platform/routes.ts +++ b/ornn-api/src/domains/platform/routes.ts @@ -35,10 +35,6 @@ export function maskSensitiveSettings(settings: PlatformSettings): PlatformSetti gatewayUrl: settings.llmProvider.gatewayUrl, apiKey: midMaskSecret(settings.llmProvider.apiKey), }, - githubMirror: { - ...settings.githubMirror, - appPrivateKey: midMaskSecret(settings.githubMirror.appPrivateKey), - }, }; } diff --git a/ornn-api/src/domains/platform/service.ts b/ornn-api/src/domains/platform/service.ts index 6776ef57..0986c8df 100644 --- a/ornn-api/src/domains/platform/service.ts +++ b/ornn-api/src/domains/platform/service.ts @@ -1,14 +1,17 @@ /** * PlatformSettingsService — thin in-memory cache on top of the - * repository so hot code paths (the audit-gated permissions handler, - * MirrorService's per-commit repo-coords lookup) don't hit Mongo on - * every call. + * repository so hot code paths (the audit-gated permissions handler) + * don't hit Mongo on every call. * - * Decrypts at-rest secrets (LLM provider `apiKey`, GitHub App - * `appPrivateKey`) at the service boundary on read; encrypts on write. - * Every downstream consumer sees plaintext. Failures are non-fatal: an - * unreadable secret degrades to "no value set" so the rest of the - * system keeps working. + * Decrypts at-rest secrets (LLM provider `apiKey`) at the service + * boundary on read; encrypts on write. Every downstream consumer sees + * plaintext. Failures are non-fatal: an unreadable secret degrades to + * "no value set" so the rest of the system keeps working. + * + * Mirror config now lives in `SettingsService.getMirror()` — + * `MirrorService` consumes it from there directly (#437). This service + * remains the home for `auditWaiverThreshold` and the legacy + * `llmProvider` override only. * * @module domains/platform/service */ @@ -17,7 +20,6 @@ import { decryptSecret, encryptSecret } from "../../infra/crypto"; import type { PlatformSettingsRepository } from "./repository"; import { DEFAULT_PLATFORM_SETTINGS, - type GithubMirrorConfig, type LlmProviderConfig, type PlatformSettings, } from "./types"; @@ -27,8 +29,8 @@ const logger = pino({ level: "info" }).child({ module: "platformSettingsService" export interface PlatformSettingsDefaults { /** * Master passphrase used to encrypt/decrypt at-rest secrets (LLM - * `apiKey`, GitHub App `appPrivateKey`). Sourced from `ENCRYPTION_KEY` - * env. Required — the service never sees plaintext at the DB layer. + * `apiKey`). Sourced from `ENCRYPTION_KEY` env. Required — the + * service never sees plaintext at the DB layer. */ encryptionKey: string; } @@ -61,34 +63,11 @@ export class PlatformSettingsService { ); } - const mirrorRaw = stored.githubMirror ?? DEFAULT_PLATFORM_SETTINGS.githubMirror; - let appPrivateKey = ""; - try { - appPrivateKey = decryptSecret( - mirrorRaw.appPrivateKey ?? "", - this.defaults.encryptionKey, - ); - } catch (err) { - logger.error( - { err: (err as Error).message }, - "Failed to decrypt GitHub App private key — treating as unset", - ); - } - const settings: PlatformSettings = { auditWaiverThreshold: typeof stored.auditWaiverThreshold === "number" ? stored.auditWaiverThreshold : DEFAULT_PLATFORM_SETTINGS.auditWaiverThreshold, - githubMirror: { - enabled: typeof mirrorRaw.enabled === "boolean" ? mirrorRaw.enabled : false, - owner: mirrorRaw.owner ?? "", - repo: mirrorRaw.repo ?? "", - branch: mirrorRaw.branch ?? "", - appId: mirrorRaw.appId ?? "", - installationId: mirrorRaw.installationId ?? "", - appPrivateKey, - }, llmProvider: { gatewayUrl: llmRaw.gatewayUrl ?? "", apiKey: llmApiKey, @@ -102,11 +81,6 @@ export class PlatformSettingsService { return (await this.get()).auditWaiverThreshold; } - /** Full mirror config — kill switch + repo coords + App credentials. */ - async getGithubMirrorConfig(): Promise { - return (await this.get()).githubMirror; - } - /** Convenience accessor used by `NyxLlmClient` on every LLM call. */ async getLlmProviderConfig(): Promise { return (await this.get()).llmProvider; @@ -127,21 +101,6 @@ export class PlatformSettingsService { apiKey: enc, }; } - if (partial.githubMirror) { - const enc = encryptSecret( - partial.githubMirror.appPrivateKey ?? "", - this.defaults.encryptionKey, - ); - toStore.githubMirror = { - enabled: !!partial.githubMirror.enabled, - owner: partial.githubMirror.owner ?? "", - repo: partial.githubMirror.repo ?? "", - branch: partial.githubMirror.branch ?? "", - appId: partial.githubMirror.appId ?? "", - installationId: partial.githubMirror.installationId ?? "", - appPrivateKey: enc, - }; - } await this.repo.patch(toStore); // Bust the cache so the next read pulls the fresh value through the // merge layer (which re-applies defaults for fields the admin diff --git a/ornn-api/src/domains/platform/types.ts b/ornn-api/src/domains/platform/types.ts index 611f6089..69bc8d0e 100644 --- a/ornn-api/src/domains/platform/types.ts +++ b/ornn-api/src/domains/platform/types.ts @@ -18,17 +18,6 @@ export interface PlatformSettings { * share request flow (owner justification → reviewer decision). */ readonly auditWaiverThreshold: number; - /** - * Full GitHub mirror config — kill switch, repo coords, App credentials. - * The MirrorService reads this on every operation; an admin patch via - * the admin UI takes effect on the next sync without a redeploy. - * - * `appPrivateKey` is encrypted at rest (AES-256-GCM via `infra/crypto`, - * scrypt-derived from `ENCRYPTION_KEY`); the routes layer mid-masks it - * on read so the operator can sanity-check which key is in place - * without exposing the body. - */ - readonly githubMirror: GithubMirrorConfig; /** * LLM provider override. Empty fields fall back to env (the * Chrono LLM gateway via NyxID SA token exchange). When `gatewayUrl` @@ -54,45 +43,18 @@ export interface LlmProviderConfig { readonly apiKey: string; } -/** - * Full GitHub mirror config. Operator-flippable end-to-end via the admin - * UI; no env / configmap fallback — empty fields mean "not configured" - * and the MirrorService no-ops. - */ -export interface GithubMirrorConfig { - /** Master kill switch. When false, every mirror op is a no-op. */ - readonly enabled: boolean; - readonly owner: string; - readonly repo: string; - readonly branch: string; - /** GitHub App numeric id (visible on the App settings page). */ - readonly appId: string; - /** Installation id for `/` (org-wide installation). */ - readonly installationId: string; - /** - * RSA private key in PEM format. Encrypted at rest; the routes layer - * mid-masks it on read. Empty string = "no key set". - */ - readonly appPrivateKey: string; -} - /** * Sentinel default. Returned by `PlatformSettingsService.get()` when the - * DB row is missing fields — empty strings + `enabled: false` so a - * fresh deployment with no admin-set settings has the mirror cleanly - * disabled until an operator flips it on via the UI. + * DB row is missing fields — empty strings so a fresh deployment with + * no admin-set settings boots cleanly. + * + * Mirror config has moved to `SettingsService.getMirror()` + * (`platform_settings:{_id:"mirror"}`); the legacy `githubMirror` field + * on this PlatformSettings doc was dropped in #437 after a one-shot + * boot migration copied any existing values into the new section. */ export const DEFAULT_PLATFORM_SETTINGS: PlatformSettings = { auditWaiverThreshold: 6.0, - githubMirror: { - enabled: false, - owner: "", - repo: "", - branch: "", - appId: "", - installationId: "", - appPrivateKey: "", - }, llmProvider: { gatewayUrl: "", apiKey: "", diff --git a/ornn-api/src/domains/settings/exportImport/importer.test.ts b/ornn-api/src/domains/settings/exportImport/importer.test.ts index 26125c1b..5e98006f 100644 --- a/ornn-api/src/domains/settings/exportImport/importer.test.ts +++ b/ornn-api/src/domains/settings/exportImport/importer.test.ts @@ -74,6 +74,7 @@ describe("SettingsImporter", () => { appId: "1", installationId: "2", appPrivateKey: "real-pem-from-db", + reconcileSchedule: "0 2 * * *", }, }); const importer = new SettingsImporter({ settingsService: svc }); @@ -89,6 +90,7 @@ describe("SettingsImporter", () => { appId: "1", installationId: "2", appPrivateKey: redactSentinel("appPrivateKey"), + reconcileSchedule: "0 2 * * *", }, }, }, diff --git a/ornn-api/src/domains/settings/sections/mirror.migration.test.ts b/ornn-api/src/domains/settings/sections/mirror.migration.test.ts new file mode 100644 index 00000000..af4a884f --- /dev/null +++ b/ornn-api/src/domains/settings/sections/mirror.migration.test.ts @@ -0,0 +1,193 @@ +/** + * Tests for the legacy mirror config -> per-section migration. + * + * Uses `mongodb-memory-server` (already a test dep) so the actual + * Mongo update behaviour is exercised end-to-end rather than relying + * on a hand-rolled in-memory fake. + * + * @module domains/settings/sections/mirror.migration.test + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { MongoMemoryServer } from "mongodb-memory-server"; +import { MongoClient, type Db, type Document } from "mongodb"; +import pino from "pino"; +import { migrateLegacyMirrorIntoSettings } from "./mirror.migration"; + +let mongo: MongoMemoryServer; +let client: MongoClient; +let db: Db; +const logger = pino({ level: "silent" }); + +beforeAll(async () => { + mongo = await MongoMemoryServer.create(); + client = new MongoClient(mongo.getUri()); + await client.connect(); + db = client.db("mirror_migration_test"); +}); + +afterAll(async () => { + await client.close(); + await mongo.stop(); +}); + +beforeEach(async () => { + await db.collection("platform_settings").deleteMany({}); +}); + +describe("migrateLegacyMirrorIntoSettings", () => { + test("copies legacy githubMirror -> settings.mirror when new doc absent", async () => { + await db.collection("platform_settings").insertOne({ + _id: "ornn" as unknown as Document["_id"], + githubMirror: { + enabled: true, + owner: "ChronoAIProject", + repo: "ornn-skills", + branch: "main", + appId: "12345", + installationId: "67890", + appPrivateKey: "ciphertext-abc", + }, + auditWaiverThreshold: 6, + }); + + await migrateLegacyMirrorIntoSettings(db, logger); + + const newDoc = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + expect(newDoc).not.toBeNull(); + expect((newDoc as unknown as { value: Record }).value).toEqual({ + enabled: true, + owner: "ChronoAIProject", + repo: "ornn-skills", + branch: "main", + appId: "12345", + installationId: "67890", + // ciphertext copied byte-for-byte — not re-encrypted + appPrivateKey: "ciphertext-abc", + }); + expect((newDoc as unknown as { updatedBy: string }).updatedBy).toBe( + "system:legacy-mirror-migration", + ); + }); + + test("no-op when new mirror doc already exists (idempotent)", async () => { + // pre-seed both legacy and new docs; the new one is "authoritative" + await db.collection("platform_settings").insertOne({ + _id: "ornn" as unknown as Document["_id"], + githubMirror: { + enabled: true, + owner: "legacy-owner", + repo: "legacy-repo", + branch: "main", + appId: "1", + installationId: "2", + appPrivateKey: "legacy-ct", + }, + }); + await db.collection("platform_settings").insertOne({ + _id: "mirror" as unknown as Document["_id"], + value: { + enabled: false, + owner: "new-owner", + repo: "new-repo", + branch: "develop", + appId: "", + installationId: "", + appPrivateKey: "", + }, + updatedAt: new Date(), + updatedBy: "admin@example", + }); + + await migrateLegacyMirrorIntoSettings(db, logger); + + const newDoc = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + expect((newDoc as unknown as { value: Record }).value.owner).toBe( + "new-owner", + ); + expect((newDoc as unknown as { updatedBy: string }).updatedBy).toBe("admin@example"); + }); + + test("no-op when legacy doc absent entirely", async () => { + await migrateLegacyMirrorIntoSettings(db, logger); + const newDoc = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + expect(newDoc).toBeNull(); + }); + + test("no-op when legacy doc exists but has no githubMirror field", async () => { + await db.collection("platform_settings").insertOne({ + _id: "ornn" as unknown as Document["_id"], + auditWaiverThreshold: 6, + }); + await migrateLegacyMirrorIntoSettings(db, logger); + const newDoc = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + expect(newDoc).toBeNull(); + }); + + test("defaults missing/wrong-typed fields to empty/false", async () => { + await db.collection("platform_settings").insertOne({ + _id: "ornn" as unknown as Document["_id"], + githubMirror: { + // intentionally partial / malformed: enabled wrong type, branch missing + enabled: "yes" as unknown as boolean, + owner: "x", + repo: "y", + }, + }); + await migrateLegacyMirrorIntoSettings(db, logger); + const newDoc = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + const value = (newDoc as unknown as { value: Record }).value; + expect(value).toEqual({ + enabled: false, + owner: "x", + repo: "y", + branch: "", + appId: "", + installationId: "", + appPrivateKey: "", + }); + }); + + test("running twice is a no-op after first success (full idempotency)", async () => { + await db.collection("platform_settings").insertOne({ + _id: "ornn" as unknown as Document["_id"], + githubMirror: { + enabled: true, + owner: "o", + repo: "r", + branch: "main", + appId: "1", + installationId: "2", + appPrivateKey: "ct", + }, + }); + + await migrateLegacyMirrorIntoSettings(db, logger); + const after1 = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + const ts1 = (after1 as unknown as { updatedAt: Date }).updatedAt.getTime(); + + // tiny delay so any new updatedAt would be visibly different + await new Promise((r) => setTimeout(r, 5)); + + await migrateLegacyMirrorIntoSettings(db, logger); + const after2 = await db + .collection("platform_settings") + .findOne({ _id: "mirror" as unknown as Document["_id"] }); + const ts2 = (after2 as unknown as { updatedAt: Date }).updatedAt.getTime(); + + // second run must NOT touch the doc — updatedAt unchanged + expect(ts2).toBe(ts1); + }); +}); diff --git a/ornn-api/src/domains/settings/sections/mirror.migration.ts b/ornn-api/src/domains/settings/sections/mirror.migration.ts new file mode 100644 index 00000000..b2c601c1 --- /dev/null +++ b/ornn-api/src/domains/settings/sections/mirror.migration.ts @@ -0,0 +1,101 @@ +/** + * One-shot boot migration: copy the legacy + * `platform_settings:{ _id: "ornn" }.githubMirror` field into the new + * per-section `platform_settings:{ _id: "mirror" }` doc consumed by + * `SettingsService`. + * + * Idempotent. Re-runs are no-ops in two cases: + * 1. The new `mirror` section doc already exists (admin has saved it, + * or a previous migration run already ran). We do NOT overwrite — + * treating any pre-existing new doc as authoritative protects + * operators who deliberately saved an empty MirrorSection. + * 2. The legacy doc has no `githubMirror` field (e.g. fresh cluster). + * + * Crypto: `appPrivateKey` is stored as AES-256-GCM ciphertext on both + * sides, derived from the same `ENCRYPTION_KEY`. We copy the ciphertext + * byte-for-byte without going through encrypt/decrypt — round-tripping + * would just produce a different IV. Failure to decrypt later (e.g., + * key rotation drift) degrades to "no key set" via the existing + * `SettingsServiceImpl.loadSection` fallback. + * + * @module domains/settings/sections/mirror.migration + */ + +import type { Db, Document } from "mongodb"; +import type pino from "pino"; + +const LEGACY_DOC_ID = "ornn"; +const NEW_DOC_ID = "mirror"; + +interface LegacyMirrorShape { + enabled?: boolean; + owner?: string; + repo?: string; + branch?: string; + appId?: string; + installationId?: string; + /** Already-encrypted ciphertext (AES-256-GCM). */ + appPrivateKey?: string; +} + +export async function migrateLegacyMirrorIntoSettings( + db: Db, + logger: pino.Logger, +): Promise { + const coll = db.collection("platform_settings"); + + const existingNew = await coll.findOne({ + _id: NEW_DOC_ID as unknown as Document["_id"], + }); + if (existingNew) { + logger.debug( + { docId: NEW_DOC_ID }, + "mirror migration: new section doc already present — skipping", + ); + return; + } + + const legacy = (await coll.findOne({ + _id: LEGACY_DOC_ID as unknown as Document["_id"], + })) as (Document & { githubMirror?: LegacyMirrorShape }) | null; + if (!legacy?.githubMirror || typeof legacy.githubMirror !== "object") { + logger.info( + "mirror migration: no legacy githubMirror field — nothing to migrate", + ); + return; + } + + const m = legacy.githubMirror; + const value = { + enabled: typeof m.enabled === "boolean" ? m.enabled : false, + owner: typeof m.owner === "string" ? m.owner : "", + repo: typeof m.repo === "string" ? m.repo : "", + branch: typeof m.branch === "string" ? m.branch : "", + appId: typeof m.appId === "string" ? m.appId : "", + installationId: typeof m.installationId === "string" ? m.installationId : "", + appPrivateKey: typeof m.appPrivateKey === "string" ? m.appPrivateKey : "", + }; + + const now = new Date(); + await coll.updateOne( + { _id: NEW_DOC_ID as unknown as Document["_id"] }, + { + $set: { + value, + updatedAt: now, + updatedBy: "system:legacy-mirror-migration", + }, + $setOnInsert: { _id: NEW_DOC_ID, createdAt: now }, + }, + { upsert: true }, + ); + logger.info( + { + owner: value.owner, + repo: value.repo, + enabled: value.enabled, + hasAppKey: !!value.appPrivateKey, + }, + "mirror migration: copied legacy githubMirror -> settings.mirror", + ); +} diff --git a/ornn-api/src/domains/settings/sections/mirror.ts b/ornn-api/src/domains/settings/sections/mirror.ts index 71605551..8cce1344 100644 --- a/ornn-api/src/domains/settings/sections/mirror.ts +++ b/ornn-api/src/domains/settings/sections/mirror.ts @@ -1,15 +1,43 @@ /** - * GitHub mirror section schema (Story 7.4). + * GitHub mirror section schema. * * `appPrivateKey` is encrypted at rest by the SettingsService before it * lands in Mongo. It is exposed mid-masked on GET and replaced with a * redaction sentinel on settings export. * + * `reconcileSchedule` is a cron expression interpreted in + * `Asia/Singapore` (UTC+8, no DST — see + * `domains/skills/mirror/scheduler.ts`). Empty string disables the + * scheduled reconcile entirely (publish-time webhooks still fire). + * Default `0 2 * * *` = daily at 02:00 SGT. + * * @module domains/settings/sections/mirror */ import { z } from "zod"; +import { CronExpressionParser } from "cron-parser"; import type { SectionMeta } from "./index"; +/** + * Validates that the input is either an empty string (disabled) or a + * cron expression accepted by `cron-parser`. We do NOT require a + * 5-field UNIX cron specifically — `cron-parser` also accepts 6-field + * forms with seconds; either is fine for our purposes. + */ +const cronSchedule = z + .string() + .refine( + (s) => { + if (s.length === 0) return true; + try { + CronExpressionParser.parse(s); + return true; + } catch { + return false; + } + }, + { message: "must be a valid cron expression or empty (disabled)" }, + ); + export const mirrorSchema = z.object({ enabled: z.boolean(), owner: z.string(), @@ -18,6 +46,7 @@ export const mirrorSchema = z.object({ appId: z.string(), installationId: z.string(), appPrivateKey: z.string(), + reconcileSchedule: cronSchedule, }); export type MirrorSection = z.infer; @@ -30,6 +59,10 @@ export const mirrorDefaults: MirrorSection = { appId: "", installationId: "", appPrivateKey: "", + // Daily at 02:00 SGT (UTC+8). Schedule is applied by the in-process + // scheduler with `timezone: "Asia/Singapore"` so this reads literally + // as 2am Singapore time. + reconcileSchedule: "0 2 * * *", }; export const mirrorSection: SectionMeta = { diff --git a/ornn-api/src/domains/settings/sections/sections.test.ts b/ornn-api/src/domains/settings/sections/sections.test.ts index 4ffc16d3..3394de94 100644 --- a/ornn-api/src/domains/settings/sections/sections.test.ts +++ b/ornn-api/src/domains/settings/sections/sections.test.ts @@ -276,6 +276,7 @@ describe("section schemas", () => { appId: "12345", installationId: "67890", appPrivateKey: "-----BEGIN PRIVATE KEY-----\n...\n", + reconcileSchedule: "0 2 * * *", }).success, ).toBe(true); expect( @@ -287,10 +288,53 @@ describe("section schemas", () => { appId: "", installationId: "", appPrivateKey: "", + reconcileSchedule: "", }).success, ).toBe(false); }); + it("UT-SCHEMA-MIRROR-002: reconcileSchedule accepts valid crons + empty string", () => { + const base = { + ...mirrorSection.defaults, + }; + for (const cron of [ + "0 2 * * *", + "0 */6 * * *", + "*/30 * * * *", + "17 * * * *", + "", // disabled + ]) { + const result = mirrorSection.schema.safeParse({ + ...base, + reconcileSchedule: cron, + }); + expect(result.success).toBe(true); + } + }); + + it("UT-SCHEMA-MIRROR-003: reconcileSchedule rejects invalid cron expressions", () => { + const base = { ...mirrorSection.defaults }; + for (const bad of [ + "not-a-cron", + "61 * * * *", // minute out of range + "* * * * * * *", // too many fields + "0 25 * * *", // hour out of range + ]) { + const result = mirrorSection.schema.safeParse({ + ...base, + reconcileSchedule: bad, + }); + expect(result.success).toBe(false); + } + }); + + it("UT-SCHEMA-MIRROR-004: mirror defaults are valid", () => { + expect(mirrorSection.schema.safeParse(mirrorSection.defaults).success).toBe( + true, + ); + expect(mirrorSection.defaults.reconcileSchedule).toBe("0 2 * * *"); + }); + // -------- telemetry -------- it("telemetry schema accepts placeholder defaults", () => { expect( diff --git a/ornn-api/src/domains/settings/service.test.ts b/ornn-api/src/domains/settings/service.test.ts index d383fcac..8512d991 100644 --- a/ornn-api/src/domains/settings/service.test.ts +++ b/ornn-api/src/domains/settings/service.test.ts @@ -95,6 +95,7 @@ describe("SettingsServiceImpl", () => { appId: "12345", installationId: "67890", appPrivateKey: "-----BEGIN PRIVATE KEY-----\nXYZ\n-----END PRIVATE KEY-----", + reconcileSchedule: "0 2 * * *", }, ACTOR, ); @@ -117,6 +118,7 @@ describe("SettingsServiceImpl", () => { appId: "1", installationId: "2", appPrivateKey: "real-secret-pem", + reconcileSchedule: "0 2 * * *", }, ACTOR, ); @@ -131,6 +133,7 @@ describe("SettingsServiceImpl", () => { appId: "1", installationId: "2", appPrivateKey: redactSentinel("appPrivateKey"), + reconcileSchedule: "0 2 * * *", }, ACTOR, ); @@ -151,6 +154,7 @@ describe("SettingsServiceImpl", () => { appId: "1", installationId: "2", appPrivateKey: "another-real-secret-value", + reconcileSchedule: "0 2 * * *", }, ACTOR, ); @@ -165,6 +169,7 @@ describe("SettingsServiceImpl", () => { appId: "1", installationId: "2", appPrivateKey: masked, + reconcileSchedule: "0 2 * * *", }, ACTOR, ); diff --git a/ornn-api/src/domains/skills/mirror/mirrorService.test.ts b/ornn-api/src/domains/skills/mirror/mirrorService.test.ts index 5836e07d..d9e2ddaf 100644 --- a/ornn-api/src/domains/skills/mirror/mirrorService.test.ts +++ b/ornn-api/src/domains/skills/mirror/mirrorService.test.ts @@ -18,23 +18,23 @@ import { describe, expect, it, mock } from "bun:test"; import { createHash } from "node:crypto"; -import { MirrorService } from "./mirrorService"; +import { MirrorService, type MirrorSettingsReader } from "./mirrorService"; import type { GitHubMirrorClient, TreeEntry } from "./githubMirrorClient"; import type { SkillRepository } from "../crud/repository"; import type { SkillService } from "../crud/service"; import type { SkillDocument } from "../../../shared/types/index"; -import type { PlatformSettingsService } from "../../platform/service"; +import type { MirrorSection } from "../../settings/sections/mirror"; -/** Stub for the PlatformSettingsService dep — returns a fixed mirror config. */ -function makeFakePlatformSettings( +/** Stub SettingsService surface used by MirrorService — fixed mirror config. */ +function makeFakeSettings( overrides: { enabled?: boolean; owner?: string; repo?: string; branch?: string; } = {}, -): PlatformSettingsService { - const cfg = { +): MirrorSettingsReader { + const cfg: MirrorSection = { enabled: overrides.enabled ?? true, owner: overrides.owner ?? "ChronoAIProject", repo: overrides.repo ?? "ornn-skills", @@ -42,10 +42,11 @@ function makeFakePlatformSettings( appId: "12345", installationId: "67890", appPrivateKey: "test-key", + reconcileSchedule: "0 2 * * *", }; return { - getGithubMirrorConfig: mock(async () => cfg), - } as unknown as PlatformSettingsService; + getMirror: mock(async () => cfg), + }; } /** @@ -194,7 +195,7 @@ describe("MirrorService disabled", () => { skillRepo: makeFakeRepo([makeSkill()]), skillService: makeFakeSkillService({}), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings({ enabled: false }), + settingsService: makeFakeSettings({ enabled: false }), }); await svc.syncSkill("guid-1"); expect(calls.blobs.length).toBe(0); @@ -209,7 +210,7 @@ describe("MirrorService disabled", () => { skillRepo: makeFakeRepo([makeSkill()]), skillService: makeFakeSkillService({}), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings({ enabled: false }), + settingsService: makeFakeSettings({ enabled: false }), }); const result = await svc.reconcileAll(); expect(result).toEqual({ added: 0, updated: 0, removed: 0, unchanged: 0 }); @@ -233,7 +234,7 @@ describe("MirrorService privacy regression", () => { "g-pub": { "SKILL.md": "# pub" }, }), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings(), + settingsService: makeFakeSettings(), }); await svc.reconcileAll(); // Verify no blob payload contains the private skill's name as a path prefix. @@ -254,7 +255,7 @@ describe("MirrorService privacy regression", () => { skillRepo: makeFakeRepo([skill]), skillService: makeFakeSkillService({}), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings(), + settingsService: makeFakeSettings(), }); await svc.publishSkill("g-priv"); expect(calls.blobs.length).toBe(0); @@ -273,7 +274,7 @@ describe("MirrorService privacy regression", () => { skillRepo: makeFakeRepo([skill]), skillService: makeFakeSkillService({}), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings(), + settingsService: makeFakeSettings(), }); await svc.syncSkill("g-flip"); // Expect one tree create with a sha:null entry for flip/SKILL.md. @@ -320,7 +321,7 @@ describe("MirrorService idempotency", () => { skillRepo: makeFakeRepo([skill]), skillService: makeFakeSkillService({ g1: skillFiles }), ornnPublicOrigin: "https://example", - platformSettingsService: makeFakePlatformSettings(), + settingsService: makeFakeSettings(), }); const result = await svc.reconcileAll(); // SKILL.md should be unchanged. The two READMEs (skill + repo) embed diff --git a/ornn-api/src/domains/skills/mirror/mirrorService.ts b/ornn-api/src/domains/skills/mirror/mirrorService.ts index 01f21803..92154a98 100644 --- a/ornn-api/src/domains/skills/mirror/mirrorService.ts +++ b/ornn-api/src/domains/skills/mirror/mirrorService.ts @@ -29,10 +29,19 @@ import { GitHubMirrorClient, type TreeEntry } from "./githubMirrorClient"; import type { SkillRepository } from "../crud/repository"; import type { SkillService } from "../crud/service"; import type { SkillDocument } from "../../../shared/types/index"; -import type { PlatformSettingsService } from "../../platform/service"; +import type { MirrorSection } from "../../settings/sections/mirror"; const logger = pino({ level: "info" }).child({ module: "mirrorService" }); +/** + * Narrow surface MirrorService needs from SettingsService. Decouples + * the dep from the full SettingsService interface so tests can stub + * just this method. + */ +export interface MirrorSettingsReader { + getMirror(): Promise; +} + export interface MirrorServiceDeps { skillRepo: SkillRepository; skillService: SkillService; @@ -42,10 +51,10 @@ export interface MirrorServiceDeps { * Source of truth for the mirror config — kill switch, repo coords, * App credentials. Read on every sync so an admin patch via the admin * UI takes effect on the next operation without a redeploy. Cached - * for 30s by the service itself, so repeated reads inside one sync - * are cheap. + * for 30s by SettingsService itself, so repeated reads inside one + * sync are cheap. */ - platformSettingsService: PlatformSettingsService; + settingsService: MirrorSettingsReader; /** * Optional override — used by tests to inject a stub * `GitHubMirrorClient` without going through the GitHub App auth @@ -64,8 +73,8 @@ export interface ReconcileResult { /** * Mirror service — runtime-aware. Every public method first asks - * `PlatformSettingsService` for the current mirror config; if disabled - * or missing any of the four App fields, the call no-ops. The active + * `SettingsService` for the current mirror config; if disabled or + * missing any of the four App fields, the call no-ops. The active * `GitHubMirrorClient` is cached by credential fingerprint so admin- * pasted creds take effect on the next call without recreating the * client on every blob/tree request. @@ -89,10 +98,10 @@ export class MirrorService { if (this.deps.githubClientForTest) { // Test seam: still gate on `enabled` so the disabled-short-circuit // assertion can be exercised without rebuilding the auth chain. - const cfg = await this.deps.platformSettingsService.getGithubMirrorConfig(); + const cfg = await this.deps.settingsService.getMirror(); return cfg.enabled ? this.deps.githubClientForTest : null; } - const cfg = await this.deps.platformSettingsService.getGithubMirrorConfig(); + const cfg = await this.deps.settingsService.getMirror(); if (!cfg.enabled) return null; if (!cfg.appId || !cfg.installationId || !cfg.appPrivateKey) return null; if (!cfg.owner || !cfg.repo || !cfg.branch) return null; @@ -126,7 +135,7 @@ export class MirrorService { repo: string; branch: string; }> { - const cfg = await this.deps.platformSettingsService.getGithubMirrorConfig(); + const cfg = await this.deps.settingsService.getMirror(); const configured = !!cfg.appId && !!cfg.installationId && !!cfg.appPrivateKey && !!cfg.owner && !!cfg.repo; return { @@ -362,14 +371,14 @@ export class MirrorService { * call time from the platform-settings cache so an admin re-point * propagates into READMEs on the next sync. */ private async getRepoSlug(): Promise { - const cfg = await this.deps.platformSettingsService.getGithubMirrorConfig(); + const cfg = await this.deps.settingsService.getMirror(); return `${cfg.owner}/${cfg.repo}`; } /** Mirror repo name on its own. Used as the H1 in the top-level * README (`# ornn-skills`). */ private async getRepoName(): Promise { - const cfg = await this.deps.platformSettingsService.getGithubMirrorConfig(); + const cfg = await this.deps.settingsService.getMirror(); return cfg.repo; } diff --git a/ornn-api/src/domains/skills/mirror/routes.ts b/ornn-api/src/domains/skills/mirror/routes.ts index 06c336ff..c3387652 100644 --- a/ornn-api/src/domains/skills/mirror/routes.ts +++ b/ornn-api/src/domains/skills/mirror/routes.ts @@ -39,7 +39,9 @@ import { import { AppError } from "../../../shared/types/index"; import { isMidMaskSentinel, midMaskSecret } from "../../../infra/crypto"; import type { MirrorService, ReconcileResult } from "./mirrorService"; -import type { PlatformSettingsService } from "../../platform/service"; +import type { MirrorScheduler, ScheduledRunStatus } from "./scheduler"; +import type { SettingsService, SettingsActor } from "../../settings/types"; +import type { MirrorSection } from "../../settings/sections/mirror"; import type { SkillRepository } from "../crud/repository"; const logger = pino({ level: "info" }).child({ module: "mirrorRoutes" }); @@ -68,15 +70,24 @@ export interface MirrorRoutesConfig { */ mirrorService: MirrorService; /** - * Platform-settings service for runtime-mutable mirror config. - * Source of truth for enabled + repo coords + App credentials. + * Settings service — single source of truth for mirror config + * (enabled + repo coords + App credentials + reconcile schedule). + * Write path goes through `putSection("mirror", ...)`. */ - platformSettingsService: PlatformSettingsService; + settingsService: SettingsService; /** * Skill repository for the abandon-confirm pre-flight check + the * status endpoint's mirror-counts aggregation. */ skillRepo: SkillRepository; + /** + * In-process scheduler — the status endpoint reads the last scheduled + * fire's outcome from here (persisted in `agendaJobs`, multipod-safe). + * Optional: when the scheduler failed to start at boot, this is + * `null` and the status endpoint reports `never_run` for the + * scheduled-run block. + */ + mirrorScheduler: MirrorScheduler | null; } interface ReconcileRunState { @@ -91,7 +102,7 @@ interface ReconcileRunState { export function createMirrorRoutes( config: MirrorRoutesConfig, ): Hono<{ Variables: AuthVariables }> { - const { mirrorService, platformSettingsService, skillRepo } = config; + const { mirrorService, settingsService, skillRepo, mirrorScheduler } = config; const app = new Hono<{ Variables: AuthVariables }>(); const auth = nyxidAuthMiddleware(); @@ -117,7 +128,7 @@ export function createMirrorRoutes( * surfaced here. */ app.get("/github/repo", async (c) => { - const cfg = await platformSettingsService.getGithubMirrorConfig(); + const cfg = await settingsService.getMirror(); return c.json({ data: { owner: cfg.owner, @@ -149,7 +160,7 @@ export function createMirrorRoutes( requirePermission("ornn:admin:skill"), async (c) => { const body = (await c.req.json().catch(() => ({}))) as Record; - const current = await platformSettingsService.getGithubMirrorConfig(); + const current = await settingsService.getMirror(); const confirmAbandonOldRepo = body.confirmAbandonOldRepo === true; // ---- enabled ---- @@ -251,17 +262,26 @@ export function createMirrorRoutes( } } - const updated = await platformSettingsService.patch({ - githubMirror: { - enabled, - owner, - repo, - branch, - appId, - installationId, - appPrivateKey, - }, - }); + // Preserve any fields the legacy operational endpoint doesn't + // edit (e.g., `reconcileSchedule`) by reading them off `current` + // and re-passing them through. Settings PUT replaces the whole + // section value. + const next: MirrorSection = { + ...current, + enabled, + owner, + repo, + branch, + appId, + installationId, + appPrivateKey, + }; + const actor = mirrorActor(c); + const { value: updated } = await settingsService.putSection( + "mirror", + next, + actor, + ); if (wouldAbandonOldRepo) { // Existing stamps point at commit SHAs in the now-abandoned // repo. Clearing them resets every eligible skill to "Never @@ -274,13 +294,13 @@ export function createMirrorRoutes( } return c.json({ data: { - enabled: updated.githubMirror.enabled, - owner: updated.githubMirror.owner, - repo: updated.githubMirror.repo, - branch: updated.githubMirror.branch, - appId: updated.githubMirror.appId, - installationId: updated.githubMirror.installationId, - appPrivateKey: midMaskSecret(updated.githubMirror.appPrivateKey), + enabled: updated.enabled, + owner: updated.owner, + repo: updated.repo, + branch: updated.branch, + appId: updated.appId, + installationId: updated.installationId, + appPrivateKey: midMaskSecret(updated.appPrivateKey), }, error: null, }); @@ -389,19 +409,30 @@ export function createMirrorRoutes( // ────────────────────────── Admin: GET /admin/mirror/status ────────────────────────── /** - * Snapshot for the admin overview UI. Combines the in-process - * reconcile state, the DB-side mirror counts, and the full mirror - * config (App private key mid-masked) so the page can render the - * settings form pre-populated without a second round-trip. + * Snapshot for the admin overview UI. Combines the persisted + * scheduled-run status (from the in-process scheduler reading + * Agenda's `agendaJobs` doc), the DB-side mirror counts, and the + * full mirror config (App private key mid-masked) so the page can + * render the settings form pre-populated without a second round-trip. + * + * Manual `POST /admin/mirror/reconcile` runs do NOT update + * `scheduledRun` — they're tracked in the in-process `reconcileState` + * which lives on this pod only and feeds the 409 "already running" + * guard. If the dashboard needs to surface manual-run progress, it + * should consult that out of band; `scheduledRun` is the canonical, + * cluster-wide, persisted view of *scheduled* fires. */ app.get( "/admin/mirror/status", auth, requirePermission("ornn:admin:skill"), async (c) => { - const [counts, cfg] = await Promise.all([ + const [counts, cfg, scheduledRun] = await Promise.all([ skillRepo.getMirrorCounts(), - platformSettingsService.getGithubMirrorConfig(), + settingsService.getMirror(), + mirrorScheduler + ? mirrorScheduler.getScheduledRunStatus() + : Promise.resolve(emptyScheduledRun()), ]); return c.json({ data: { @@ -419,14 +450,7 @@ export function createMirrorRoutes( ? counts.oldestUnsyncedAt.toISOString() : null, }, - lastReconcile: { - status: reconcileState.status, - startedAt: reconcileState.startedAt?.toISOString() ?? null, - finishedAt: reconcileState.finishedAt?.toISOString() ?? null, - durationMs: reconcileState.durationMs, - result: reconcileState.result, - error: reconcileState.error, - }, + scheduledRun: serializeScheduledRun(scheduledRun), }, error: null, }); @@ -435,3 +459,49 @@ export function createMirrorRoutes( return app; } + +function emptyScheduledRun(): ScheduledRunStatus { + return { + status: "never_run", + lastRunAt: null, + lastFinishedAt: null, + lastDurationMs: null, + lastError: null, + nextRunAt: null, + }; +} + +function serializeScheduledRun(s: ScheduledRunStatus): { + status: ScheduledRunStatus["status"]; + lastRunAt: string | null; + lastFinishedAt: string | null; + lastDurationMs: number | null; + lastError: string | null; + nextRunAt: string | null; +} { + return { + status: s.status, + lastRunAt: s.lastRunAt?.toISOString() ?? null, + lastFinishedAt: s.lastFinishedAt?.toISOString() ?? null, + lastDurationMs: s.lastDurationMs, + lastError: s.lastError, + nextRunAt: s.nextRunAt?.toISOString() ?? null, + }; +} + +/** + * Build a SettingsActor from the request's auth context. Mirrors the + * helper in `domains/settings/routes.ts` so settings writes from the + * legacy `/github/repo` POST attribute to the same caller shape that + * the new `/admin/settings/mirror` PUT records. + */ +function mirrorActor(c: { get: (k: string) => unknown }): SettingsActor { + const a = c.get("auth") as + | { userId?: string; email?: string; displayName?: string } + | undefined; + return { + userId: a?.userId ?? "unknown", + email: a?.email ?? "unknown@local", + displayName: a?.displayName, + }; +} diff --git a/ornn-api/src/domains/skills/mirror/scheduler.test.ts b/ornn-api/src/domains/skills/mirror/scheduler.test.ts new file mode 100644 index 00000000..72925235 --- /dev/null +++ b/ornn-api/src/domains/skills/mirror/scheduler.test.ts @@ -0,0 +1,427 @@ +/** + * Mirror scheduler unit tests. + * + * These tests focus on our wiring of Agenda — *what we ask Agenda to + * do* on every sync tick — not on Agenda's internal job-running + * machinery, which is Agenda's own test suite's job. We mock the + * Agenda surface so the assertions stay deterministic and fast. + * + * The multipod safety claim (only one pod fires per cron tick) is a + * property of Agenda's per-fire row lock — verified upstream in the + * `agenda` package's tests, not duplicated here. + * + * @module domains/skills/mirror/scheduler.test + */ + +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import pino from "pino"; +import type { MirrorService } from "./mirrorService"; +import type { SettingsService } from "../../settings/types"; +import type { MirrorSection } from "../../settings/sections/mirror"; + +const logger = pino({ level: "silent" }); + +// Module-level mutable refs that the mocked Agenda instance reads. Each +// test resets them in beforeEach. +let agendaCalls: { + define: Array<{ name: string }>; + every: Array<{ interval: string | number; name: string; options?: { timezone?: string } }>; + cancel: Array<{ name?: string }>; + now: string[]; + started: boolean; + stopped: boolean; +}; +const jobHandlers = new Map Promise>(); + +// Mutable canned `queryJobs` result — tests set this per case. +let queryJobsResult: { jobs: Array> } = { jobs: [] }; +let queryJobsThrows: Error | null = null; + +// Mock the `agenda` + `@agendajs/mongo-backend` modules BEFORE the +// scheduler module is imported. We re-import the scheduler in each +// test via dynamic `import()` to ensure it picks up the mocks. +mock.module("agenda", () => ({ + Agenda: class FakeAgenda { + on() {} + define(name: string, fn: () => Promise) { + agendaCalls.define.push({ name }); + jobHandlers.set(name, fn); + } + async every( + interval: string | number, + name: string, + _data: unknown, + options?: { timezone?: string }, + ) { + agendaCalls.every.push({ interval, name, options }); + } + async cancel(opts: { name?: string }) { + agendaCalls.cancel.push(opts); + return 1; + } + async now(name: string) { + agendaCalls.now.push(name); + // Execute the handler synchronously so tests can assert on the + // resulting `every`/`cancel` calls immediately. Mirrors the + // production sequence: enqueued one-shot → handler runs → may + // call `every` to register the recurring job. + const fn = jobHandlers.get(name); + if (fn) await fn(); + } + async start() { + agendaCalls.started = true; + } + async stop() { + agendaCalls.stopped = true; + } + async queryJobs(_opts: { name: string }) { + if (queryJobsThrows) throw queryJobsThrows; + return queryJobsResult; + } + }, +})); +mock.module("@agendajs/mongo-backend", () => ({ + MongoBackend: class FakeBackend { + constructor(_: unknown) {} + }, +})); + +// Lazy-load the scheduler module AFTER mocks are set up. +const { createMirrorScheduler } = await import("./scheduler"); + +function resetAgendaCalls() { + agendaCalls = { + define: [], + every: [], + cancel: [], + now: [], + started: false, + stopped: false, + }; + jobHandlers.clear(); + queryJobsResult = { jobs: [] }; + queryJobsThrows = null; +} + +function makeSettings(initial: string): SettingsService & { + setSchedule(s: string): void; +} { + let cur: MirrorSection = { + enabled: true, + owner: "o", + repo: "r", + branch: "main", + appId: "1", + installationId: "2", + appPrivateKey: "k", + reconcileSchedule: initial, + }; + return { + getMirror: mock(async () => cur), + setSchedule(next: string) { + cur = { ...cur, reconcileSchedule: next }; + }, + } as unknown as SettingsService & { setSchedule(s: string): void }; +} + +function makeMirrorService(): MirrorService { + return { + reconcileAll: mock(async () => ({ added: 0, updated: 0, removed: 0, unchanged: 0 })), + } as unknown as MirrorService; +} + +const FAKE_DB = {} as Parameters[0]["db"]; + +beforeEach(() => { + resetAgendaCalls(); +}); + +afterEach(async () => { + // every test stops the scheduler explicitly, but be defensive. +}); + +describe("createMirrorScheduler", () => { + test("on start, registers both jobs + eager-syncs schedule from settings (SGT)", async () => { + const settings = makeSettings("0 2 * * *"); + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: settings, + }); + await sched.start(); + + // Both jobs defined + expect(agendaCalls.define.map((d) => d.name).sort()).toEqual([ + "mirror-reconcile", + "mirror-sync-schedule", + ]); + + // Eager `agenda.now("mirror-sync-schedule")` ran during start, which + // in turn called `agenda.every("0 2 * * *", "mirror-reconcile", + // ..., { timezone: "Asia/Singapore" })`. + const everyReconcile = agendaCalls.every.find( + (e) => e.name === "mirror-reconcile", + ); + expect(everyReconcile).toBeDefined(); + expect(everyReconcile!.interval).toBe("0 2 * * *"); + expect(everyReconcile!.options?.timezone).toBe("Asia/Singapore"); + + // Recurring sync tick registered. + const everySync = agendaCalls.every.find( + (e) => e.name === "mirror-sync-schedule", + ); + expect(everySync).toBeDefined(); + expect(everySync!.interval).toBe("1 minute"); + + await sched.stop(); + expect(agendaCalls.stopped).toBe(true); + }); + + test("settings change → next sync tick re-registers with new cron, no spurious calls", async () => { + const settings = makeSettings("0 2 * * *"); + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: settings, + }); + await sched.start(); + + const beforeChange = agendaCalls.every.filter( + (e) => e.name === "mirror-reconcile", + ).length; + + // Admin saves new cron + settings.setSchedule("*/30 * * * *"); + await sched.runSyncNow(); + + const afterChange = agendaCalls.every.filter( + (e) => e.name === "mirror-reconcile", + ); + expect(afterChange.length).toBe(beforeChange + 1); + expect(afterChange.at(-1)!.interval).toBe("*/30 * * * *"); + expect(afterChange.at(-1)!.options?.timezone).toBe("Asia/Singapore"); + + await sched.stop(); + }); + + test("unchanged schedule on subsequent sync → no second every() call", async () => { + const settings = makeSettings("0 2 * * *"); + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: settings, + }); + await sched.start(); + const baseline = agendaCalls.every.filter( + (e) => e.name === "mirror-reconcile", + ).length; + + // Two more sync ticks with the SAME schedule — should be no-ops. + await sched.runSyncNow(); + await sched.runSyncNow(); + + const after = agendaCalls.every.filter( + (e) => e.name === "mirror-reconcile", + ).length; + expect(after).toBe(baseline); + + await sched.stop(); + }); + + test("empty schedule → cancels recurring job", async () => { + const settings = makeSettings("0 2 * * *"); + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: settings, + }); + await sched.start(); + + settings.setSchedule(""); + await sched.runSyncNow(); + + const cancels = agendaCalls.cancel.filter( + (c) => c.name === "mirror-reconcile", + ); + expect(cancels.length).toBe(1); + + // Re-enable + settings.setSchedule("0 3 * * *"); + await sched.runSyncNow(); + const everyReconcile = agendaCalls.every.filter( + (e) => e.name === "mirror-reconcile", + ); + expect(everyReconcile.at(-1)!.interval).toBe("0 3 * * *"); + + await sched.stop(); + }); + + test("mirror-reconcile handler delegates to MirrorService.reconcileAll", async () => { + const settings = makeSettings("0 2 * * *"); + const mirror = makeMirrorService(); + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: mirror, + settingsService: settings, + }); + await sched.start(); + + // The fake Agenda recorded the defined handler — call it. + const fn = jobHandlers.get("mirror-reconcile"); + expect(fn).toBeDefined(); + await fn!(); + expect((mirror.reconcileAll as unknown as { mock: { calls: unknown[] } }).mock.calls.length).toBe(1); + + await sched.stop(); + }); + + test("settings read failure on sync tick is swallowed (no crash)", async () => { + const broken = { + getMirror: mock(async () => { + throw new Error("db down"); + }), + } as unknown as SettingsService; + const sched = createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: broken, + }); + // Start must not throw even though the eager initial sync hits the + // broken settings read. + await sched.start(); + // And another sync tick should also be tolerated. + await sched.runSyncNow(); + await sched.stop(); + }); +}); + +describe("MirrorScheduler.getScheduledRunStatus", () => { + function makeScheduler() { + return createMirrorScheduler({ + db: FAKE_DB, + logger, + mirrorService: makeMirrorService(), + settingsService: makeSettings("0 2 * * *"), + }); + } + + test("never_run when no agendaJobs doc exists yet", async () => { + queryJobsResult = { jobs: [] }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("never_run"); + expect(s.lastRunAt).toBeNull(); + expect(s.lastFinishedAt).toBeNull(); + expect(s.lastDurationMs).toBeNull(); + expect(s.lastError).toBeNull(); + expect(s.nextRunAt).toBeNull(); + await sched.stop(); + }); + + test("succeeded when lastFinishedAt set and no recent failure", async () => { + const lastRunAt = new Date("2026-05-13T18:00:00.000Z"); + const lastFinishedAt = new Date("2026-05-13T18:00:04.213Z"); + const nextRunAt = new Date("2026-05-14T18:00:00.000Z"); + queryJobsResult = { + jobs: [{ lastRunAt, lastFinishedAt, nextRunAt }], + }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("succeeded"); + expect(s.lastRunAt?.toISOString()).toBe(lastRunAt.toISOString()); + expect(s.lastFinishedAt?.toISOString()).toBe(lastFinishedAt.toISOString()); + expect(s.lastDurationMs).toBe(4213); + expect(s.lastError).toBeNull(); + expect(s.nextRunAt?.toISOString()).toBe(nextRunAt.toISOString()); + await sched.stop(); + }); + + test("failed when failedAt is the most recent terminal stamp", async () => { + const lastRunAt = new Date("2026-05-13T18:00:00.000Z"); + const lastFinishedAt = new Date("2026-05-13T17:00:01.000Z"); // older + const failedAt = new Date("2026-05-13T18:00:02.000Z"); // newer than lastFinishedAt + queryJobsResult = { + jobs: [ + { + lastRunAt, + lastFinishedAt, + failedAt, + failReason: "github 502: Bad Gateway", + }, + ], + }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("failed"); + expect(s.lastError).toBe("github 502: Bad Gateway"); + await sched.stop(); + }); + + test("failed when failedAt set and lastFinishedAt never set", async () => { + const lastRunAt = new Date("2026-05-13T18:00:00.000Z"); + const failedAt = new Date("2026-05-13T18:00:02.000Z"); + queryJobsResult = { + jobs: [{ lastRunAt, failedAt, failReason: "boom" }], + }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("failed"); + expect(s.lastError).toBe("boom"); + await sched.stop(); + }); + + test("running when lockedAt is set (regardless of other stamps)", async () => { + queryJobsResult = { + jobs: [ + { + lastRunAt: new Date("2026-05-13T18:00:00.000Z"), + lockedAt: new Date("2026-05-14T18:00:00.000Z"), + // Even a stale failedAt doesn't override `running`. + failedAt: new Date("2026-05-13T18:01:00.000Z"), + failReason: "old failure", + }, + ], + }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("running"); + expect(s.lastError).toBeNull(); + await sched.stop(); + }); + + test("lastDurationMs is null when only lastRunAt set (mid-flight, no finish yet)", async () => { + queryJobsResult = { + jobs: [ + { + lastRunAt: new Date("2026-05-13T18:00:00.000Z"), + lockedAt: new Date("2026-05-13T18:00:00.000Z"), + }, + ], + }; + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.lastDurationMs).toBeNull(); + await sched.stop(); + }); + + test("queryJobs throw → returns never_run, doesn't propagate", async () => { + queryJobsThrows = new Error("mongo unreachable"); + const sched = makeScheduler(); + await sched.start(); + const s = await sched.getScheduledRunStatus(); + expect(s.status).toBe("never_run"); + await sched.stop(); + }); +}); diff --git a/ornn-api/src/domains/skills/mirror/scheduler.ts b/ornn-api/src/domains/skills/mirror/scheduler.ts new file mode 100644 index 00000000..67a87af0 --- /dev/null +++ b/ornn-api/src/domains/skills/mirror/scheduler.ts @@ -0,0 +1,304 @@ +/** + * In-process mirror reconcile scheduler. + * + * Replaces the k8s `CronJob` (`deployment/ornn-api/mirror-cronjob.yaml`, + * removed in this PR) with an Agenda-backed scheduler that runs inside + * the long-running `ornn-api` pod. Multi-pod-safe via Agenda's per-fire + * row lock on the `agendaJobs` collection — exactly one pod claims each + * scheduled fire, the rest skip; this is the same per-trigger DB lock + * pattern Quartz/Hangfire use. + * + * Two recurring Agenda jobs are registered: + * + * 1. `mirror-reconcile` — the actual work. Schedule (cron string) is + * driven by `settings.mirror.reconcileSchedule`. Interpreted in + * `Asia/Singapore` (UTC+8, no DST) so admins typing `0 2 * * *` + * get literal 2am Singapore time. Empty schedule = unregistered + * (no scheduled fires; publish-time webhooks still work). + * + * 2. `mirror-sync-schedule` — runs every minute on every pod. Reads + * `settings.mirror.reconcileSchedule` from DB and (re-)registers + * `mirror-reconcile` via `agenda.every(cron, name, ...)`. Because + * `every()` is an upsert on the recurring-job doc keyed by name, + * all pods' Agenda instances converge on the new cadence via the + * shared `agendaJobs` collection — no cross-pod messaging needed, + * max ~65s lag from admin-save to effect. + * + * Crash recovery: `defaultLockLifetime` (10 min) means if a pod dies + * mid-reconcile, the row's lock expires and another pod can re-claim. + * `reconcileAll` is idempotent (diffs against current mirror tree), + * so the worst case of two reconciles racing is a noisy tag-conflict + * log entry — same gap the existing `POST /admin/mirror/reconcile` + * route already accepts. + * + * @module domains/skills/mirror/scheduler + */ + +import { Agenda } from "agenda"; +import { MongoBackend } from "@agendajs/mongo-backend"; +import type { Db } from "mongodb"; +import type pino from "pino"; +import type { MirrorService } from "./mirrorService"; +import type { SettingsService } from "../../settings/types"; + +const JOB_RECONCILE = "mirror-reconcile"; +const JOB_SYNC_SCHEDULE = "mirror-sync-schedule"; + +/** Interpreted timezone for every cron expression we register. */ +const DEFAULT_TIMEZONE = "Asia/Singapore"; + +/** Sync tick cadence — how often each pod re-reads settings. */ +const DEFAULT_SYNC_INTERVAL = "1 minute"; + +/** Per-fire safety lock TTL. */ +const DEFAULT_LOCK_LIFETIME_MS = 10 * 60 * 1000; + +/** Agenda's internal poll cadence for the recurring-job table. */ +const PROCESS_EVERY = "5 seconds"; + +export interface MirrorSchedulerDeps { + /** Shared MongoDB connection — Agenda uses our existing client/pool. */ + db: Db; + logger: pino.Logger; + mirrorService: MirrorService; + settingsService: SettingsService; + /** Override the per-fire lock TTL (mostly for tests). */ + lockLifetimeMs?: number; + /** Override the sync tick cadence (mostly for tests). */ + syncInterval?: string | number; + /** Override the pinned cron timezone (mostly for tests). */ + timezone?: string; + /** Override Agenda's internal poll cadence (mostly for tests). */ + processEvery?: string | number; +} + +/** + * Snapshot of the recurring `mirror-reconcile` job's last execution. + * Derived from the timestamps Agenda stamps on the recurring-job doc; + * survives pod restarts and aggregates correctly across replicas. + */ +export interface ScheduledRunStatus { + /** + * Derived status of the most recent scheduled fire: + * - `succeeded` — last fire finished cleanly. + * - `failed` — last fire threw; `lastError` carries the message. + * - `running` — a fire is currently in flight on some pod. + * - `never_run` — no doc yet (fresh boot / schedule disabled). + */ + status: "succeeded" | "failed" | "running" | "never_run"; + lastRunAt: Date | null; + lastFinishedAt: Date | null; + lastDurationMs: number | null; + /** Failure message from the last fire; null when not in `failed` state. */ + lastError: string | null; + nextRunAt: Date | null; +} + +export interface MirrorScheduler { + /** Spin up Agenda + register both jobs + kick off the first sync. */ + start(): Promise; + /** Stop Agenda's polling loop. Idempotent. */ + stop(): Promise; + /** + * Test hook — force an immediate run of the sync tick instead of + * waiting for the next minute. Returns once the tick completes. + */ + runSyncNow(): Promise; + /** + * Read-only snapshot of the last scheduled fire's outcome, for the + * admin UI. Persisted source (Agenda's `agendaJobs` doc), so it + * survives pod restarts and reflects the cluster-wide latest run + * rather than per-pod in-process state. + */ + getScheduledRunStatus(): Promise; +} + +export function createMirrorScheduler(deps: MirrorSchedulerDeps): MirrorScheduler { + const { db, mirrorService, settingsService, logger } = deps; + const lockLifetime = deps.lockLifetimeMs ?? DEFAULT_LOCK_LIFETIME_MS; + const syncInterval = deps.syncInterval ?? DEFAULT_SYNC_INTERVAL; + const timezone = deps.timezone ?? DEFAULT_TIMEZONE; + + const agenda = new Agenda({ + backend: new MongoBackend({ mongo: db }), + processEvery: deps.processEvery ?? PROCESS_EVERY, + defaultLockLifetime: lockLifetime, + }); + + // Per-process memo of the schedule we last registered with Agenda. + // Lets the sync tick early-return when nothing changed, avoiding an + // `every()` upsert each minute. NOT cross-pod; convergence still + // happens via the shared `agendaJobs` doc. + let currentSchedule: string | null = null; + + agenda.define(JOB_RECONCILE, async () => { + const t0 = Date.now(); + try { + const result = await mirrorService.reconcileAll(); + logger.info( + { ...result, durationMs: Date.now() - t0 }, + "scheduled mirror reconcile completed", + ); + } catch (err) { + logger.error( + { + err: err instanceof Error ? err.message : String(err), + durationMs: Date.now() - t0, + }, + "scheduled mirror reconcile failed", + ); + throw err; + } + }); + + agenda.define(JOB_SYNC_SCHEDULE, async () => { + let mirror; + try { + mirror = await settingsService.getMirror(); + } catch (err) { + logger.error( + { err: err instanceof Error ? err.message : String(err) }, + "mirror-sync-schedule: failed to read settings — skipping tick", + ); + return; + } + const next = mirror.reconcileSchedule; + if (next === currentSchedule) return; + + if (next === "") { + await agenda.cancel({ name: JOB_RECONCILE }); + currentSchedule = ""; + logger.info( + "mirror schedule: cancelled (settings.mirror.reconcileSchedule is empty)", + ); + return; + } + + try { + await agenda.every(next, JOB_RECONCILE, undefined, { timezone }); + currentSchedule = next; + logger.info({ cron: next, timezone }, "mirror schedule: registered"); + } catch (err) { + // cron-parser already validated at settings-write time, so this + // should only fire on an Agenda-internal failure (e.g., Mongo + // unreachable mid-write). Log and try again next tick. + logger.error( + { err: err instanceof Error ? err.message : String(err), cron: next }, + "mirror schedule: failed to register — will retry on next sync tick", + ); + } + }); + + // Surface Agenda's own error events so an internal Mongo hiccup is + // visible in pod logs rather than swallowed. + agenda.on("error", (err: unknown) => { + logger.error( + { err: err instanceof Error ? err.message : String(err) }, + "agenda error event", + ); + }); + + return { + async start() { + await agenda.start(); + // Eager initial sync — don't wait a minute for the schedule to + // come up after a fresh boot. + try { + await agenda.now(JOB_SYNC_SCHEDULE); + } catch (err) { + logger.warn( + { err: err instanceof Error ? err.message : String(err) }, + "mirror scheduler: initial sync enqueue failed — recurring sync will catch up", + ); + } + // Recurring sync tick. `every` upserts by job name, so this is + // safe across multiple pods and across restarts. + await agenda.every(syncInterval, JOB_SYNC_SCHEDULE); + logger.info( + { syncInterval, timezone, lockLifetimeMs: lockLifetime }, + "mirror scheduler started", + ); + }, + async stop() { + try { + // `false` = don't close the Mongo client (we own it). + await agenda.stop(false); + } catch (err) { + logger.warn( + { err: err instanceof Error ? err.message : String(err) }, + "mirror scheduler: agenda.stop failed — continuing", + ); + } + }, + async runSyncNow() { + await agenda.now(JOB_SYNC_SCHEDULE); + }, + async getScheduledRunStatus(): Promise { + let result; + try { + result = await agenda.queryJobs({ name: JOB_RECONCILE }); + } catch (err) { + // Querying agenda is a single Mongo find; failure here means + // the DB is unreachable. Treat as `never_run` rather than + // crashing the admin endpoint — the dashboard's poll will + // recover on the next tick when DB comes back. + logger.warn( + { err: err instanceof Error ? err.message : String(err) }, + "getScheduledRunStatus: queryJobs failed — returning never_run", + ); + return emptyStatus(); + } + const job = result.jobs[0]; + if (!job) return emptyStatus(); + + const lastRunAt = job.lastRunAt ?? null; + const lastFinishedAt = job.lastFinishedAt ?? null; + const failedAt = job.failedAt ?? null; + const lockedAt = job.lockedAt ?? null; + + // Derivation priority: + // 1. Locked → running (a pod is executing it right now). + // 2. failedAt is the most recent terminal stamp → failed. + // 3. lastFinishedAt set → succeeded. + // 4. Nothing set yet → never_run. + let status: ScheduledRunStatus["status"]; + if (lockedAt) { + status = "running"; + } else if ( + failedAt && + (!lastFinishedAt || failedAt.getTime() >= lastFinishedAt.getTime()) + ) { + status = "failed"; + } else if (lastFinishedAt) { + status = "succeeded"; + } else { + status = "never_run"; + } + + const lastDurationMs = + lastFinishedAt && lastRunAt + ? lastFinishedAt.getTime() - lastRunAt.getTime() + : null; + + return { + status, + lastRunAt, + lastFinishedAt, + lastDurationMs, + lastError: status === "failed" ? (job.failReason ?? null) : null, + nextRunAt: job.nextRunAt ?? null, + }; + }, + }; +} + +function emptyStatus(): ScheduledRunStatus { + return { + status: "never_run", + lastRunAt: null, + lastFinishedAt: null, + lastDurationMs: null, + lastError: null, + nextRunAt: null, + }; +} diff --git a/ornn-api/tests/integration/settings_exportImport.test.ts b/ornn-api/tests/integration/settings_exportImport.test.ts index 98ac23ae..9f7ceeb0 100644 --- a/ornn-api/tests/integration/settings_exportImport.test.ts +++ b/ornn-api/tests/integration/settings_exportImport.test.ts @@ -82,6 +82,7 @@ describe("IT-SETTINGS export/import", () => { appId: "12345", installationId: "67890", appPrivateKey: "real-pem-content", + reconcileSchedule: "0 2 * * *", }, ACTOR, ); @@ -114,6 +115,7 @@ describe("IT-SETTINGS export/import", () => { appId: "99999", installationId: "00000", appPrivateKey: redactSentinel("appPrivateKey"), // keep DB + reconcileSchedule: "0 2 * * *", }, ACTOR, ); diff --git a/ornn-web/package.json b/ornn-web/package.json index 2a816518..0fe6a8f8 100644 --- a/ornn-web/package.json +++ b/ornn-web/package.json @@ -14,6 +14,7 @@ "dependencies": { "@hookform/resolvers": "^5.2.2", "@tanstack/react-query": "^5.62.0", + "cron-parser": "^5.5.0", "diff": "^9.0.0", "framer-motion": "^12.38.0", "highlight.js": "^11.10.0", diff --git a/ornn-web/src/components/admin/MirrorSetupHelp.tsx b/ornn-web/src/components/admin/MirrorSetupHelp.tsx index 1708f155..6f1a083c 100644 --- a/ornn-web/src/components/admin/MirrorSetupHelp.tsx +++ b/ornn-web/src/components/admin/MirrorSetupHelp.tsx @@ -186,8 +186,9 @@ export function MirrorSetupHelp({ className = "" }: MirrorSetupHelpProps) { (e.g. ChronoAIProject / ornn-skills / main), flip the Enable toggle, Save. Then hit{" "} Reconcile now — the first run pushes every - public + system skill to the repo. The hourly cron at{" "} - :17 takes over from there. + public + system skill to the repo. The in-process scheduler + (configurable on the settings page; default daily at 02:00 SGT) + takes over from there.

diff --git a/ornn-web/src/hooks/useGithubMirror.ts b/ornn-web/src/hooks/useGithubMirror.ts index 16cbe28d..042524b5 100644 --- a/ornn-web/src/hooks/useGithubMirror.ts +++ b/ornn-web/src/hooks/useGithubMirror.ts @@ -52,10 +52,10 @@ export function useMirrorStatus() { return useQuery({ queryKey: STATUS_KEY, queryFn: fetchMirrorStatus, - // Poll fast while a reconcile is running so the UI shows progress - // landing without a manual refresh; back off when idle. + // Poll fast while a scheduled reconcile is running so the UI shows + // progress landing without a manual refresh; back off when idle. refetchInterval: (q) => - q.state.data?.lastReconcile.status === "running" ? 5_000 : 30_000, + q.state.data?.scheduledRun.status === "running" ? 5_000 : 30_000, staleTime: 0, }); } diff --git a/ornn-web/src/i18n/en.json b/ornn-web/src/i18n/en.json index bd9b995c..52019f7b 100644 --- a/ornn-web/src/i18n/en.json +++ b/ornn-web/src/i18n/en.json @@ -1038,7 +1038,7 @@ "sections": { "mirror": { "title": "GitHub mirror", - "description": "Repo coords + App credentials. Run controls are on the legacy mirror dashboard.", + "description": "Repo coords + App credentials + scheduled reconcile cadence. Run controls are on the legacy mirror dashboard.", "savedToast": "Mirror config saved", "label": { "enabled": "Mirror enabled", @@ -1047,7 +1047,30 @@ "branch": "Branch", "appId": "App ID", "installationId": "Installation ID", - "privateKey": "App private key (PEM)" + "privateKey": "App private key (PEM)", + "schedule": "Reconcile schedule" + }, + "schedule": { + "preset": { + "disabled": "Disabled", + "daily2am": "Daily at 02:00", + "every6h": "Every 6 hours", + "every12h": "Every 12 hours", + "hourly": "Hourly", + "custom": "Custom (cron expression)…" + }, + "tzNote": "Schedules are interpreted in Singapore time (UTC+8, no DST).", + "disabledHint": "Scheduled reconciles are paused. Publish-time webhooks still fire.", + "customHint": "5-field cron expression (minute hour day month weekday).", + "invalidHint": "Invalid cron expression.", + "nextRun": "Next run: {{at}}", + "nextRunUnavailable": "Next run: —", + "lastRun": "Last run: {{at}}", + "lastRunUnavailable": "Last run: —", + "lastRunSucceeded": "✓ Succeeded", + "lastRunFailed": "✗ Failed", + "lastRunRunning": "⟳ Running…", + "lastRunDuration": "{{seconds}}s" }, "openDashboard": "Open mirror dashboard →" }, diff --git a/ornn-web/src/i18n/zh.json b/ornn-web/src/i18n/zh.json index a3ad609b..34cba74d 100644 --- a/ornn-web/src/i18n/zh.json +++ b/ornn-web/src/i18n/zh.json @@ -1038,7 +1038,7 @@ "sections": { "mirror": { "title": "GitHub 镜像", - "description": "仓库坐标 + GitHub App 凭据。运行控制位于旧版镜像看板。", + "description": "仓库坐标 + GitHub App 凭据 + 定时同步节奏。运行控制位于旧版镜像看板。", "savedToast": "镜像配置已保存", "label": { "enabled": "启用镜像", @@ -1047,7 +1047,30 @@ "branch": "分支", "appId": "App ID", "installationId": "Installation ID", - "privateKey": "App 私钥 (PEM)" + "privateKey": "App 私钥 (PEM)", + "schedule": "同步计划" + }, + "schedule": { + "preset": { + "disabled": "禁用", + "daily2am": "每日 02:00", + "every6h": "每 6 小时", + "every12h": "每 12 小时", + "hourly": "每小时", + "custom": "自定义 (cron 表达式)…" + }, + "tzNote": "计划以新加坡时间 (UTC+8，无夏令时) 解析。", + "disabledHint": "定时同步已暂停。发布时的 webhook 仍会触发。", + "customHint": "5 位 cron 表达式 (分 时 日 月 周)。", + "invalidHint": "无效的 cron 表达式。", + "nextRun": "下次运行：{{at}}", + "nextRunUnavailable": "下次运行：—", + "lastRun": "上次运行：{{at}}", + "lastRunUnavailable": "上次运行：—", + "lastRunSucceeded": "✓ 成功", + "lastRunFailed": "✗ 失败", + "lastRunRunning": "⟳ 进行中…", + "lastRunDuration": "{{seconds}} 秒" }, "openDashboard": "打开镜像看板 →" }, diff --git a/ornn-web/src/pages/admin/MirrorPage.tsx b/ornn-web/src/pages/admin/MirrorPage.tsx index b1635e83..a16e1ed2 100644 --- a/ornn-web/src/pages/admin/MirrorPage.tsx +++ b/ornn-web/src/pages/admin/MirrorPage.tsx @@ -233,8 +233,13 @@ export function MirrorPage() { } }; - const lastRun = status?.lastReconcile; - const reconcileRunning = lastRun?.status === "running"; + // Last *scheduled* reconcile — sourced from the persisted `scheduledRun` + // block (Agenda's `agendaJobs` doc), so it survives pod restarts and + // aggregates across replicas. Manual `Reconcile now` clicks are tracked + // server-side via in-process state for the 409 guard; their progress is + // not surfaced in this widget. + const lastRun = status?.scheduledRun; + const scheduledFireRunning = lastRun?.status === "running"; const credsConfigured = !!status && !!status.appId && !!status.installationId && !!status.appPrivateKey; return ( @@ -296,31 +301,25 @@ export function MirrorPage() {
- {t("adminMirror.lastReconcile", "Last reconcile")} + {t("adminMirror.lastReconcile", "Last scheduled reconcile")}
- {reconcileRunning + {scheduledFireRunning ? t("adminMirror.runningSince", "Running since {{when}}", { - when: formatTime(lastRun?.startedAt ?? null), + when: formatTime(lastRun?.lastRunAt ?? null), }) - : lastRun?.finishedAt - ? formatTime(lastRun.finishedAt) + : lastRun?.lastFinishedAt + ? formatTime(lastRun.lastFinishedAt) : t("adminMirror.never", "Never")}
- {!reconcileRunning && lastRun?.durationMs !== null && lastRun?.durationMs !== undefined && ( + {!scheduledFireRunning && lastRun?.lastDurationMs != null && (
- {t("adminMirror.duration", "Duration")} {formatDuration(lastRun.durationMs)} + {t("adminMirror.duration", "Duration")} {formatDuration(lastRun.lastDurationMs)}
)} - {lastRun?.error && ( -
- {t("adminMirror.lastError", "Last error")}: {lastRun.error} -
- )} - {!reconcileRunning && lastRun?.result && ( -
- +{lastRun.result.added} ~{lastRun.result.updated} −{lastRun.result.removed} ={" "} - {lastRun.result.unchanged} + {lastRun?.lastError && ( +
+ {t("adminMirror.lastError", "Last error")}: {lastRun.lastError}
)}
@@ -368,7 +367,7 @@ export function MirrorPage() {

{t( "adminMirror.reconcileSubtitle", - "The hourly cron at :17 runs the same operation. Hit this when you can't wait — fire-and-forget, the page polls until it lands.", + "The in-process scheduler runs the same operation on the cadence set in mirror settings. Hit this when you can't wait — fire-and-forget, the page polls until it lands.", )}

{!credsConfigured && status.enabled && ( @@ -383,12 +382,12 @@ export function MirrorPage() { @@ -473,7 +472,7 @@ export function MirrorPage() { ) : t( "adminMirror.repoFormHint", - "Saving updates the next sync's target. The cron's next run picks up the new coords automatically.", + "Saving updates the next sync's target. The scheduler's next run picks up the new coords automatically.", )}