From f6f432c86e1fe2d8522c424f2efa6687c1745501 Mon Sep 17 00:00:00 2001 From: transitrix Date: Thu, 27 Aug 2026 12:19:18 +0400 Subject: [PATCH] feat(reg-intel): align discovery model and date validation with spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task transitrix-hq#356 — fix reg-intel documentation and implementation to match 14-codex.md §3.7 spec: - Discovery now reads from operations/config/scan-sources.yaml (the centralised watch-list), not by walking codex/ directory. Updated SKILL.md, CLI README, and package.json descriptions to reflect this. - Codex watch-list fixture creates scan-sources.yaml per the spec schema. When a source is static (monitoring_needed: false), its monitor_instead entries are surfaced as separate scan targets. - Added date validation to reject impossible dates (e.g., 2026-02-31) and malformed ISO-8601 dates (e.g., 2026-6-01). Both addFrequency and isDue now validate date format and reject non-dates rather than silently comparing them lexicographically. - Added test Part M to cover date validation edge cases: impossible dates, malformed ISO dates, and non-date strings. - Created GitHub Actions workflow (.github/workflows/reg-intel-skill-test.yml) to run the integrity test on PRs touching reg-intel skill or CLI. Co-Authored-By: Claude Haiku 4.5 Signed-off-by: transitrix --- .github/workflows/reg-intel-skill-test.yml | 40 +++++++++++++++ packages/reg-intel-cli/README.md | 4 +- packages/reg-intel-cli/package.json | 2 +- packages/reg-intel-cli/src/codex.mjs | 43 +++++++++++----- packages/reg-intel-cli/src/schedule.mjs | 25 +++++++++- transitrix/skills/reg-intel/SKILL.md | 4 +- .../tests/test_reg_intel_integrity.py | 49 ++++++++++++++++++- 7 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/reg-intel-skill-test.yml diff --git a/.github/workflows/reg-intel-skill-test.yml b/.github/workflows/reg-intel-skill-test.yml new file mode 100644 index 00000000..ad3fb98c --- /dev/null +++ b/.github/workflows/reg-intel-skill-test.yml @@ -0,0 +1,40 @@ +name: Reg-intel skill test + +# Integrity test for the Transitrix Reg-Intel skill + @transitrix/reg-intel-cli. +# Deterministic, no API key, no network. Runs the fixture end-to-end through all +# CLI commands (list-due, check-signal, fetch-snapshot, segment, classify, validate, +# amendment, update-scan, digest). + +on: + pull_request: + paths: + - 'transitrix/skills/reg-intel/**' + - 'packages/reg-intel-cli/**' + - '.github/workflows/reg-intel-skill-test.yml' + workflow_dispatch: + schedule: + # Mondays 09:45 UTC — weekly safety net. + - cron: '45 9 * * 1' + +jobs: + integrity: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install PyYAML + run: pip install pyyaml + + - name: Run reg-intel integrity test (deterministic) + run: python transitrix/skills/reg-intel/tests/test_reg_intel_integrity.py diff --git a/packages/reg-intel-cli/README.md b/packages/reg-intel-cli/README.md index 2bd9974d..c2967e40 100644 --- a/packages/reg-intel-cli/README.md +++ b/packages/reg-intel-cli/README.md @@ -11,7 +11,7 @@ Deterministic CLI for the [Transitrix Reg-Intel skill](../../transitrix/skills/r | Command | Status | Purpose | |---|---|---| | `--version` / `--help` | ✅ | Version (the skill's Step-0 pre-check) and usage. | -| `list-due [org-root] [--as-of YYYY-MM-DD] [--json]` | ✅ | List codex sources due for a scan: `monitoring_needed: true` artefacts whose `scan.next_scan_due <= as-of` (or never scanned), plus the `monitor_instead[]` counterparts of static sources. One run filters by date — not N schedules. | +| `list-due [org-root] [--as-of YYYY-MM-DD] [--json]` | ✅ | List sources due for a scan from `operations/config/scan-sources.yaml`: reads the watch-list, then for each entry reads the codex artefact's `scan.next_scan_due` to find which are due. Surfaces the `monitor_instead[]` counterparts of static sources. One run filters by date — not N schedules. | | `update-scan [--today YYYY-MM-DD] [--frequency daily\|weekly\|monthly\|quarterly] [--change ""] [--review]` | ✅ | Write the codex `scan` block (`last_scanned_at`, `next_scan_due` from the cadence, `change_detected`, `change_description`, `review_needed`) — operating state only; never touches any other field. | | `check-signal --observed [--method etag\|last_modified\|api_version\|amended_date\|fragment_hash] [--accept-no-signal] [--today YYYY-MM-DD]` | ✅ | The cheap change-signal gate (network-free — caller supplies the observed value). Compares to the last-seen value in `operations/state/reg-intel/signal-cache.json`: `unchanged` → bump scan; `moved` / `no_signal` → proceed. | | `fetch-snapshot --from [--ext ] [--today YYYY-MM-DD]` | ✅ | Snapshot the caller-fetched bytes to `_intake/snapshots/-.`, fingerprint (`source_hash: sha256:…`), and detect `captured` / `changed` / `cosmetic_change` (bytes differ but normative text unchanged — markup/whitespace/tracker churn) / `bytes_identical` (cross-checkout via the committed cache). Network-free — the caller fetches. | @@ -45,7 +45,7 @@ packages/reg-intel-cli/ batch-path.mjs # non-destructive batch-directory naming ``` -The "source registry" is **the codex artefacts the repo already carries** (`14-codex.md` §3.4–3.5), not a separate database — the schedule rides on each codex YAML's `scan` block, so scan history is auditable via git. The change-signal cache is committed operational **state** at `operations/state/reg-intel/signal-cache.json` (`method/06-team-operations.md` §3.3) — outside the model, disposable, survives clone/CI. Zero runtime dependencies, Node ≥ 18. +The "source registry" is the centralised watch-list at **`operations/config/scan-sources.yaml`** (`14-codex.md` §3.7), not a walk of the `codex/` directory. `list-due` reads this file; for each entry, it reads the codex artefact's `scan` block for runtime state (`14-codex.md` §3.4–3.5) — the schedule and history are auditable via git. The change-signal cache is committed operational **state** at `operations/state/reg-intel/signal-cache.json` (`method/06-team-operations.md` §3.3) — outside the model, disposable, survives clone/CI. Zero runtime dependencies, Node ≥ 18. ## Tests diff --git a/packages/reg-intel-cli/package.json b/packages/reg-intel-cli/package.json index 3d5306a1..65ac9559 100644 --- a/packages/reg-intel-cli/package.json +++ b/packages/reg-intel-cli/package.json @@ -1,7 +1,7 @@ { "name": "@transitrix/reg-intel-cli", "version": "0.0.1", - "description": "Deterministic CLI for the Transitrix reg-intel skill — watch codex sources, run the change-signal gate, segment and classify regulatory text, and stage a review digest. Never writes canon. This package is being built incrementally; the scheduler core (list-due, update-scan) ships first.", + "description": "Deterministic CLI for the Transitrix reg-intel skill — read the codex watch-list, run the change-signal gate, segment and classify regulatory text, and stage a review digest. Never writes canon. All nine run-loop commands ship: list-due, check-signal, fetch-snapshot, segment, classify, validate, amendment, update-scan, digest.", "type": "module", "bin": { "transitrix-reg-intel": "./reg-intel.mjs" diff --git a/packages/reg-intel-cli/src/codex.mjs b/packages/reg-intel-cli/src/codex.mjs index 7233843f..381ebb26 100644 --- a/packages/reg-intel-cli/src/codex.mjs +++ b/packages/reg-intel-cli/src/codex.mjs @@ -112,19 +112,38 @@ export async function listDue(orgRoot, asOf) { try { text = await readFile(codexPath, 'utf8'); } catch { continue; } const art = parseCodexArtefact(text, codexPath); - const next = (art.scan && art.scan.next_scan_due) || null; - if (!isDue(next, asOf)) continue; + // A static source (monitoring_needed: false) does not get scanned directly; + // instead, its monitor_instead counterparts are surfaced as separate scan targets. + if (!art.monitoring_needed && art.monitor_instead) { + for (const alt of art.monitor_instead) { + due.push({ + id: alt.id, + type: null, + name: alt.name || alt.id, + source_url: alt.url, + scan_frequency: entry.cadence || null, + next_scan_due: null, + monitoring_needed: true, + reason: 'monitor_instead', + via: entry.id || art.id, + }); + } + } else { + // A live source (monitoring_needed: true) is scanned directly. + const next = (art.scan && art.scan.next_scan_due) || null; + if (!isDue(next, asOf)) continue; - due.push({ - id: entry.id || art.id, - type: art.type, - name: art.name, - source_url: art.source_url, - scan_frequency: entry.cadence || (art.scan && art.scan.scan_frequency) || null, - next_scan_due: next, - monitoring_needed: true, - reason: next ? 'due' : 'never_scanned', - }); + due.push({ + id: entry.id || art.id, + type: art.type, + name: art.name, + source_url: art.source_url, + scan_frequency: entry.cadence || (art.scan && art.scan.scan_frequency) || null, + next_scan_due: next, + monitoring_needed: true, + reason: next ? 'due' : 'never_scanned', + }); + } } return due; } diff --git a/packages/reg-intel-cli/src/schedule.mjs b/packages/reg-intel-cli/src/schedule.mjs index 8b05b6a3..1acb4ad3 100644 --- a/packages/reg-intel-cli/src/schedule.mjs +++ b/packages/reg-intel-cli/src/schedule.mjs @@ -9,11 +9,26 @@ const ADD_MONTHS = { monthly: 1, quarterly: 3 }; const ISO_RE = /^\d{4}-\d{2}-\d{2}$/; +function validateISODate(isoDate) { + if (!ISO_RE.test(isoDate || '')) throw new Error(`expected YYYY-MM-DD, got ${JSON.stringify(isoDate)}`); + const [y, m, d] = isoDate.split('-').map(Number); + if (m < 1 || m > 12) throw new Error(`invalid month in ${JSON.stringify(isoDate)}`); + if (d < 1 || d > 31) throw new Error(`invalid day in ${JSON.stringify(isoDate)}`); + const temp = new Date(Date.UTC(y, m - 1, d)); + if (temp.getUTCFullYear() !== y || temp.getUTCMonth() !== m - 1 || temp.getUTCDate() !== d) { + throw new Error(`impossible date ${JSON.stringify(isoDate)} (e.g. Feb 31)`); + } +} + // next_scan_due = isoDate + scan_frequency. Month additions clamp the day to the // target month's length (e.g. Jan 31 + monthly → Feb 28/29). UTC throughout so the // result is host-timezone independent. export function addFrequency(isoDate, freq) { - if (!ISO_RE.test(isoDate || '')) throw new Error(`addFrequency: expected YYYY-MM-DD, got ${JSON.stringify(isoDate)}`); + try { + validateISODate(isoDate); + } catch (e) { + throw new Error(`addFrequency: ${e.message}`); + } if (!FREQUENCIES.has(freq)) throw new Error(`addFrequency: unknown scan_frequency ${JSON.stringify(freq)} (expected ${[...FREQUENCIES].join('|')})`); const [y, m, d] = isoDate.split('-').map(Number); const base = new Date(Date.UTC(y, m - 1, d)); @@ -30,8 +45,14 @@ export function addFrequency(isoDate, freq) { } // A source is due when it has never been scheduled (no next_scan_due) or its -// next_scan_due is on/before the as-of date. +// next_scan_due is on/before the as-of date. Rejects non-dates (does not silently +// return false via string comparison). export function isDue(nextScanDue, asOf) { if (!nextScanDue) return true; + try { + validateISODate(String(nextScanDue)); + } catch (e) { + throw new Error(`isDue: ${e.message}`); + } return String(nextScanDue) <= String(asOf); } diff --git a/transitrix/skills/reg-intel/SKILL.md b/transitrix/skills/reg-intel/SKILL.md index 839a7cdd..9d159330 100644 --- a/transitrix/skills/reg-intel/SKILL.md +++ b/transitrix/skills/reg-intel/SKILL.md @@ -1,6 +1,6 @@ --- name: Transitrix Reg-Intel -description: "Collect regulatory-source changes in practice — the operational counterpart to the methodology's codex / SEGMENT / AMENDMENT notation work. Watches codex sources flagged `monitoring_needed: true`, runs a cheap change-signal gate before any expensive extraction, then on signal-moved runs the SCAN → SEGMENT → CLASSIFY → OUTPUT pass: fetches and snapshots the source, slices it into obligation-bearing SEGMENT field artefacts, classifies each segment as a CONSTRAINT or REQUIREMENT candidate (positive obligation → REQUIREMENT by default), emits an AMENDMENT field artefact when an existing source has drifted, updates the codex `scan` block, and stages everything in a review digest for human admission. Never writes canon. Never auto-admits. Never silently drops." +description: "Collect regulatory-source changes in practice — the operational counterpart to the methodology's codex / SEGMENT / AMENDMENT notation work. Reads the centralised watch-list at `operations/config/scan-sources.yaml`, runs a cheap change-signal gate before any expensive extraction, then on signal-moved runs the SCAN → SEGMENT → CLASSIFY → OUTPUT pass: fetches and snapshots the source, slices it into obligation-bearing SEGMENT field artefacts, classifies each segment as a CONSTRAINT or REQUIREMENT candidate (positive obligation → REQUIREMENT by default), emits an AMENDMENT field artefact when an existing source has drifted, updates the codex `scan` block, and stages everything in a review digest for human admission. Never writes canon. Never auto-admits. Never silently drops." when_to_use: 'User says "stand up the regulatory watcher", "run the daily reg-intel scan", "scan my codex sources for amendments", "extract obligations from this regulation", "produce the reg-intel review digest", or has a codex repo with `monitoring_needed: true` sources and wants drift detection plus a draft-obligation pipeline routed through a human gate.' min_version: "1.0.0" allowed-tools: Read, Write, Edit, Bash, Glob, Grep, WebFetch @@ -41,7 +41,7 @@ Also confirm you are operating inside a Transitrix adopter repository (a `transi ## Step 1 — Read the source registry -The "source registry" is **the set of codex artefacts the repository already carries**, not a separate database. A codex artefact's `monitoring_needed`, `scan.scan_frequency`, `scan.last_scanned_at`, `scan.next_scan_due`, and (when present) the `monitor_instead[]` URLs are what the scheduler reads — exactly the operating state defined in [`14-codex.md`](https://raw.githubusercontent.com/transitrix/methodology/main/notations/elements/14-codex.md) §3.4–3.5. Keeping the schedule embedded in each codex YAML rather than in a sidecar database is deliberate: it makes the scan history auditable via git and lets each source carry its own cadence. +The "source registry" is the centralised watch-list at **`operations/config/scan-sources.yaml`** (see [`14-codex.md`](https://raw.githubusercontent.com/transitrix/methodology/main/notations/elements/14-codex.md) §3.7), not a walk of the `codex/` directory. `list-due` reads this file to determine which sources to scan; it does not discover sources by walking the disk. For each entry in the watch-list, the scheduler then reads the codex artefact's `scan` block (`scan.last_scanned_at`, `scan.next_scan_due`, `scan_frequency`) to get the runtime state — exactly the operating state defined in §3.4–3.5. Keeping the scan history embedded in each codex YAML makes it auditable via git; the separate watch-list makes discovery explicit and allows future non-codex connector types (APIs, Confluence, etc.) to be added without requiring codex artefacts. ``` npx @transitrix/reg-intel-cli list-due [--as-of YYYY-MM-DD] diff --git a/transitrix/skills/reg-intel/tests/test_reg_intel_integrity.py b/transitrix/skills/reg-intel/tests/test_reg_intel_integrity.py index f861270d..50143239 100644 --- a/transitrix/skills/reg-intel/tests/test_reg_intel_integrity.py +++ b/transitrix/skills/reg-intel/tests/test_reg_intel_integrity.py @@ -133,7 +133,7 @@ def part_a_bundle(): def _codex_org(work): org = os.path.join(work, "org") - for sub in ("codex/external/eu", "codex/external/us", "codex/internal"): + for sub in ("codex/external/eu", "codex/external/us", "codex/internal", "operations/config"): os.makedirs(os.path.join(org, sub), exist_ok=True) with open(os.path.join(org, "transitrix.yaml"), "w", encoding="utf-8") as fh: fh.write('transitrix: 1\nmethodology_version: "0.5.0"\n') @@ -160,6 +160,20 @@ def w(rel, body): w("codex/external/us/REGULATION-D-1.yaml", 'id: "REGULATION-D-1"\ntype: "REGULATION"\nname: "Final Rule D"\nmonitoring_needed: false\n' 'monitor_instead:\n - id: "REGULATION-CFR-1"\n name: "21 CFR Part 809"\n url: "https://ecfr.gov/x"\n') + + # Create the watch-list at operations/config/scan-sources.yaml (the discovery source). + # D is static (monitoring_needed: false) so it won't be returned as a due source; + # its monitor_instead entries will be surfaced as separate scan targets instead. + w("operations/config/scan-sources.yaml", + 'sources:\n' + ' - id: "REGULATION-A-1"\n type: "codex"\n path_or_url: "codex/external/eu/REGULATION-A-1.yaml"\n' + ' cadence: monthly\n domain: "test"\n notes: "Test regulation A"\n' + ' - id: "REGULATION-B-1"\n type: "codex"\n path_or_url: "codex/external/us/REGULATION-B-1.yaml"\n' + ' cadence: quarterly\n domain: "test"\n notes: "Test regulation B"\n' + ' - id: "POLICY-C-1"\n type: "codex"\n path_or_url: "codex/internal/POLICY-C-1.yaml"\n' + ' cadence: weekly\n domain: "test"\n notes: "Test policy C"\n' + ' - id: "REGULATION-D-1"\n type: "codex"\n path_or_url: "codex/external/us/REGULATION-D-1.yaml"\n' + ' cadence: null\n domain: "test"\n notes: "Static source (monitoring_needed: false)"\n') return org @@ -757,6 +771,39 @@ def part_k_templates(): part_k_templates() +# ── Part M — date validation ──────────────────────────────────── + +def part_m_date_validation(): + """addFrequency and isDue reject impossible or malformed dates.""" + if not shutil.which("node"): + print("SKIP Part M: `node` not found.") + return + work = tempfile.mkdtemp(prefix="regintel-dates-") + try: + org = _codex_org(work) + a_file = os.path.join(org, "codex", "external", "eu", "REGULATION-A-1.yaml") + + # addFrequency rejects impossible dates like Feb 31. + r = run_cli("update-scan", a_file, "--today", "2026-02-31", "--frequency", "daily") + check(r.returncode != 0 and ("impossible" in (r.stdout + r.stderr).lower() or "feb" in (r.stdout + r.stderr).lower() or "invalid" in (r.stdout + r.stderr).lower()), + f"M: addFrequency must reject impossible dates like Feb 31; got {r.stdout + r.stderr}") + + # addFrequency rejects malformed dates (e.g., 2026-6-01 instead of 2026-06-01). + r = run_cli("update-scan", a_file, "--today", "2026-6-01", "--frequency", "daily") + check(r.returncode != 0 and "YYYY-MM-DD" in (r.stdout + r.stderr), + f"M: addFrequency must reject malformed dates like 2026-6-01; got {r.stdout + r.stderr}") + + # isDue rejects non-dates (does not silently return false via string compare). + r = run_cli("check-signal", a_file, "--observed", "x", "--today", "not-a-date") + check(r.returncode != 0 and ("YYYY-MM-DD" in (r.stdout + r.stderr) or "date" in (r.stdout + r.stderr).lower()), + f"M: isDue must reject non-dates; got {r.stdout + r.stderr}") + finally: + shutil.rmtree(work, ignore_errors=True) + + +part_m_date_validation() + + # ── Part L — #837 batch naming (non-destructive digest default) ─ def part_l_batch_naming():