Keep uploaded originals beside Knowledge Store source cards

packages/ingest-cli/src/intake-profile.mjs

@@ -71,10 +71,13 @@ export async function checkIntakeProfile(orgRoot, profile) {
   const sourceDir = join(root, 'sources');
   const sourceDirStat = await inspect(sourceDir);
   if (profile === 'knowledge-store' && sourceDirStat && !sourceDirStat.isDirectory()) fail('sources is not a directory');
-  const sourceFiles = sourceDirStat?.isDirectory()
-    ? (await files(sourceDir)).filter(path => path.endsWith('.md') && !['index.md', 'log.md'].includes(basename(path)))
-    : [];
-  for (const path of sourceFiles) {
+  const sourceFiles = sourceDirStat?.isDirectory() ? await files(sourceDir) : [];
+  const sourceMarkdown = sourceFiles.filter(path => path.endsWith('.md') && !['index.md', 'log.md'].includes(basename(path)));
+  const uploadedOriginals = sourceFiles.filter(path => {
+    const name = basename(path);
+    return name.endsWith('.original') || name.includes('.original.');
+  });
+  for (const path of sourceMarkdown) {
     if (await sourceRecordType(path) === 'source-document') {
       evidence['knowledge-store'].push(relative(root, path));
     }
@@ -103,11 +106,12 @@ export async function checkIntakeProfile(orgRoot, profile) {
 
   if (profile === 'knowledge-store') {
     // Git evaluates nested/global ignores and negations, including tracked files.
-    const paths = ['sources/profile-check.md', ...[...sourceFiles, ...processed].map(p => relative(root, p))];
+    const sourcePaths = new Set([...sourceMarkdown, ...uploadedOriginals, ...processed]);
+    const paths = ['sources/profile-check.md', ...Array.from(sourcePaths, path => relative(root, path))];
     const result = spawnSync('git', ['-C', root, 'check-ignore', '--no-index', '--stdin'], {
       input: paths.join('\n') + '\n', encoding: 'utf8',
     });
-    if (result.status === 0) fail(`OKF source records would be ignored: ${result.stdout.trim()}`);
+    if (result.status === 0) fail(`OKF source records or uploaded originals would be ignored: ${result.stdout.trim()}`);
     if (result.status !== 1) fail('cannot verify OKF record visibility; use a Git workspace and retry the check');
   }
   return { profile, declared, evidence: evidence[profile] };

packages/ingest-cli/src/intake-profile.test.mjs

@@ -123,6 +123,15 @@ test('new layout checks source cards without reserving an unused legacy director
   await assert.rejects(checkIntakeProfile(root, 'knowledge-store'), /would be ignored/);
 });
 
+test('Knowledge Store refuses an ignored uploaded original beside its card', async t => {
+  const root = await workspace(t);
+  await put(root, 'sources/interview.md', okf);
+  await put(root, 'sources/interview.original.pdf', 'original bytes');
+  await checkIntakeProfile(root, 'knowledge-store');
+  await put(root, '.gitignore', '*.pdf\n');
+  await assert.rejects(checkIntakeProfile(root, 'knowledge-store'), /uploaded originals would be ignored/);
+});
+
 test('legacy redirects count as Knowledge Store evidence and remain visible', async t => {
   const root = await workspace(t);
   await put(root, 'sources/source.md', okf);

patterns/knowledge-store-templates/okf-source-document.md

@@ -2,8 +2,8 @@
 type: source-document
 title: ""
 description: ""
-source: ""
-source_hash: ""
+source: ""      # Bundle-root uploaded-original path or external URI
+source_hash: "" # SHA-256 of uploaded bytes, or external content when available
 created_at: YYYY-MM-DD
 confidence: observed | inferred | assumed
 tags: []

patterns/knowledge-store.md

@@ -18,7 +18,7 @@ Choose one intake profile per repository/workspace before setup:
 | Profile | Consumer | Source record and original locations |
 |---|---|---|
 | `ingest` | Ordinary ingest and reg-intel, together | Completed raw source files, private/gitignored |
-| `knowledge-store` | OKF curation | Committed OKF `source-document` records in `sources/`. Uploaded originals stay in gitignored `_intake/originals/`. |
+| `knowledge-store` | OKF curation | Committed OKF `source-document` records in `sources/`, with uploaded originals beside their cards. External documents are referenced by URI. |
 
 These profiles are **not supported together in one workspace**. Use separate
 workspaces if both are needed. This is a deployment boundary, not a migration or
@@ -46,7 +46,7 @@ Detection checks the marker, profile-specific directories, the regulatory watch
 list, `sources/` records, and legacy processed content (OKF records or redirects versus raw files).
 Mixed evidence, unknown intake entries, symlinks or unreadable evidence stop setup
 for adopter inspection. The OKF check also uses Git's effective ignore rules,
-including nested/global rules and tracked files, to refuse hidden source cards and redirects.
+including nested/global rules and tracked files, to refuse hidden source cards, uploaded originals, and redirects.
 It requires an existing Git workspace; if absent, request authorised Git setup
 before retrying. New or empty intake directories and shared `inbox/` alone do not
 choose a profile. Explicit selection is required. No tool silently renames paths,
@@ -93,7 +93,7 @@ Project repositories retain field evidence such as meeting notes, interview tran
 
 The `knowledge/` directory holds curated findings and other knowledge objects in OKF Markdown, with source citations and timestamps. Each object captures a distinct finding or concept. A citation can point directly to field evidence or to a `source-document` record that identifies it.
 
-A card in `sources/` contains a summary and a `source:` path or URI. To locate the original, follow that reference. In the single-repo Knowledge Store profile, an uploaded file is retained in `_intake/originals/` on the ingesting workspace. A card may instead cite a document already held at a shared location.
+A card in `sources/` contains a summary and a `source:` path or URI. For an uploaded file, commit its original bytes beside the card. For a web page or a document already held in corporate storage, retain the card and link to the original, using a version-specific URI when available. To locate the original, follow the card's reference. Existing cards may still point to `_intake/originals/` in the workspace that retained the file.
 
 OKF frontmatter per knowledge object (Google OKF v0.1 fields + Transitrix extensions):
 
@@ -258,10 +258,10 @@ What it MAY NOT vary: the admission rules, the risk-tier table, the mandatory-fi
 | `KS-001` | error | — | Frontmatter block missing or not valid YAML. |
 | `KS-002` | error | — | Missing required field `type:`. |
 | `KS-003` | error | 1 | Source card does not carry `type: source-document` (quarantine violation). |
-| `KS-004` | error | 4 | Missing `source:` on an intake record — blocks admission. |
+| `KS-004` | error | 4 | Missing `source:` or incorrect SHA-256 for an uploaded original. |
 | `KS-005` | error | 4 | Missing or invalid `confidence:` on a knowledge object — blocks promotion. |
 | `KS-006` | warning | 4 | Missing `timestamp:` on a knowledge object (recommended). |
-| `KS-007` | error | 6 | Dangling bundle-relative link, unresolved source card citation, or invalid legacy source redirect. |
+| `KS-007` | error | 6 | Dangling bundle-relative link, missing uploaded original, unresolved source card citation, or invalid legacy source redirect. |
 | `KS-008` | warning | 2 | Title similarity suggests a possible duplicate — confirm Confirms/Extends, not silent mint. |
 | `KS-009` | info | 3 | Blast-radius tier (Low / Medium / High) from dependent-object count. |
 | `KS-010` | warning | 4 | `confidence: assumed` with 3+ dependents — requires review note in `_intake/log.md`. |
@@ -287,11 +287,13 @@ For early adoption where standing up a separate knowledge repo adds too much ove
 ```
 _intake/
   inbox/           ← incoming documents (gitignored — temporary staging)
-  originals/       ← source files archived by reference (gitignored)
   drafts/          ← proposer output (Gate 6 — not cited; not promoted until curator approves)
   log.md           ← all events: [route] / [admit] / [ambig] / [conflicts] / [assert]
 
-sources/           ← committed OKF source-document records
+sources/
+  interview.md              ← OKF source card
+  interview.original.pdf    ← uploaded original
+  company-policy.md         ← OKF source card with a corporate storage link
 
 knowledge/         ← OKF knowledge objects (type: insight / finding / concept / ...)
   index.md         ← auto-generated bundle index
@@ -304,7 +306,18 @@ canon/             ← Transitrix primitives (unchanged)
 - **OKF track** — extract knowledge object drafts → verify with linter → curator reviews → promote to `knowledge/` → update `index.md`
 - **Canon track** — extract Transitrix primitives → open PR to `canon/` → merge gate
 
-Record each incoming document as an OKF source card in `sources/` before routing. The card carries `source:` (path or URI) and `source_hash:` (SHA-256) for integrity. Keep the cards in Git.
+### Preserve uploaded originals
+
+Record each incoming document as an OKF source card in `sources/` before routing. For an upload, use the same stem for the card and original. For example, `sources/interview.md` cites `/sources/interview.original.pdf` in `source:` and carries the SHA-256 of the original bytes in `source_hash:`. Keep the original's extension. For an extensionless upload, use `interview.original`. Commit the pair together and verify that the file is intended for the repository's readers.
+
+Preserve uploaded bytes through Git checkout. Add these rules to the repository's `.gitattributes` when setting up uploads:
+
+```gitattributes
+sources/**/*.original -text
+sources/**/*.original.* -text
+```
+
+For a web page or an existing corporate file, write only the card with its URI. Record a hash if the exact content used was available to hash. A stable revision or versioned URI identifies the original when the storage system supplies one.
 
 Migrate to a separate knowledge repo when the single-repo structure becomes congested or when a dedicated curation role is established.
 
@@ -323,7 +336,7 @@ redirect_to: /sources/interview.md
 [Open the source card](/sources/interview.md)
 ```
 
-Knowledge Store resolves an old citation through the redirect. A reader opening the old file in GitHub can follow its link. Existing knowledge records and citations can remain as written. Run the Knowledge Store linter and the intake profile check before committing the move. Keep the redirects for as long as old citations are in use.
+Knowledge Store resolves an old citation through the redirect. A reader opening the old file in GitHub can follow its link. Existing knowledge records and citations can remain as written. Run the Knowledge Store linter and the intake profile check before committing the move. Keep the redirects for as long as old citations are in use. A card's existing `source:` may still refer to a legacy local original. If the team chooses to publish that file, retain the card's address and update its `source:` after verifying that the bytes and hash match.
 
 ## When to use
 

tools/knowledge_store_lint.py

@@ -17,6 +17,7 @@
 satisfies the same rules.
 """
 
+import hashlib
 import os
 import re
 import sys
@@ -124,8 +125,13 @@ def _load_objects(self):
         ):
             if not directory.exists():
                 continue
-            for path in sorted(directory.rglob("*.md")):
-                if path.name in ("index.md", "log.md"):
+            paths = sorted(directory.rglob("*.md"))
+            originals = set()
+            if directory == self.source_dir:
+                # Read each card before its longer-named Markdown original.
+                paths.sort(key=lambda path: len(path.name))
+            for path in paths:
+                if path in originals or path.name in ("index.md", "log.md"):
                     continue
                 rel = str(path.relative_to(self.root)).replace("\\", "/")
                 text = path.read_text(encoding="utf-8")
@@ -144,9 +150,17 @@ def _load_objects(self):
                         message=f"Frontmatter is not valid YAML: {e}",
                     ))
                     continue
-                self.objects.append(KnowledgeObject(
-                    path=path, rel_path=rel, frontmatter=fm, body=m.group(2), zone=zone,
-                ))
+                if not isinstance(fm, dict):
+                    self.findings.append(Finding(
+                        rel, "KS-001", "error", "Frontmatter must be a YAML mapping.",
+                    ))
+                    continue
+                obj = KnowledgeObject(path=path, rel_path=rel, frontmatter=fm, body=m.group(2), zone=zone)
+                self.objects.append(obj)
+                if directory == self.source_dir and fm.get("type") == "source-document":
+                    original = self._uploaded_original(obj)
+                    if original is not None:
+                        originals.add(original)
 
     @staticmethod
     def _is_source_redirect(obj: KnowledgeObject) -> bool:
@@ -179,6 +193,42 @@ def _index_sources(self):
                 continue
             self.source_records[obj.rel_path] = self.source_records[target]
 
+    def _uploaded_original(self, obj: KnowledgeObject) -> Optional[Path]:
+        source = obj.frontmatter.get("source")
+        if not isinstance(source, str) or not obj.rel_path.startswith(SOURCE_PREFIX):
+            return None
+        source = source.strip().lstrip("/")
+        prefix = obj.rel_path[:-len(".md")] + ".original"
+        if re.fullmatch(re.escape(prefix) + r"(?:\.[^./\\]+)*", source):
+            return self.root / source
+        return None
+
+    def _check_uploaded_original(self, obj: KnowledgeObject):
+        original = self._uploaded_original(obj)
+        if original is None:
+            return
+        if original.is_symlink() or not original.is_file():
+            self.findings.append(Finding(
+                obj.rel_path, "KS-007", "error", f"Uploaded original is missing or is not a regular file: `{obj.frontmatter['source']}`.",
+            ))
+            return
+        digest = hashlib.sha256()
+        try:
+            with original.open("rb") as stream:
+                for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+                    digest.update(chunk)
+        except OSError as error:
+            self.findings.append(Finding(
+                obj.rel_path, "KS-007", "error", f"Cannot read uploaded original: {error}.",
+            ))
+            return
+        actual = digest.hexdigest()
+        expected = str(obj.frontmatter.get("source_hash", "")).lower()
+        if expected not in (actual, f"sha256:{actual}"):
+            self.findings.append(Finding(
+                obj.rel_path, "KS-004", "error", "The source_hash must match the SHA-256 of the uploaded original.",
+            ))
+
     # --- structural (Gate 1, Gate 4, item 5) ----------------------------
 
     def _check_structural(self):
@@ -202,6 +252,8 @@ def _check_structural(self):
                         obj.rel_path, "KS-004", "error",
                         "Missing `source:` — required for source records (Gate 4).",
                     ))
+                if fm.get("type") == "source-document":
+                    self._check_uploaded_original(obj)
 
             if obj.zone in ("knowledge", "draft"):
                 confidence = fm.get("confidence")
@@ -246,7 +298,10 @@ def _check_referential_integrity(self):
             source = self._norm_source_path(obj.frontmatter.get("source") or "")
             if obj.zone in ("knowledge", "draft") and source.startswith((SOURCE_PREFIX, LEGACY_SOURCE_PREFIX)):
                 source_links.append(source)
-            for target in sorted({self._norm_source_path(link) for link in source_links}):
+            targets = {self._norm_source_path(link) for link in source_links}
+            if obj.zone == "intake" and self._uploaded_original(obj) is not None:
+                targets.discard(source)  # The upload is verified separately.
+            for target in sorted(targets):
                 if target not in self.source_records:
                     self.findings.append(Finding(
                         obj.rel_path, "KS-007", "error",

transitrix/skills/ingest/README.md

@@ -15,8 +15,8 @@ An authorised contributor's explicit request to add their own notes is sufficien
 ## Intake profile selection
 
 Choose `ingest` for the ordinary ingest/reg-intel workflow (private processed raw
-files), or `knowledge-store` for OKF curation (committed source-document records,
-private originals). They require separate workspaces. Setup and subsequent intake
+files), or `knowledge-store` for OKF curation (committed source cards and
+uploaded originals). They require separate workspaces. Setup and subsequent intake
 runs must pass the shared CLI preflight; incompatible or ambiguous existing
 content is refused without migration. See the [deployment compatibility contract](../../../patterns/knowledge-store.md#deployment-profiles-and-intake-compatibility)
 for selection, installation and retention rules.

transitrix/skills/knowledge-store/README.md

@@ -6,13 +6,12 @@ This directory is the **`knowledge-store` skill** within the `transitrix` plugin
 
 ## Source cards and original files
 
-New cards go in `sources/`. Open the card's `source:` reference to locate the original. Uploaded files in the single-repo profile are kept in the ingesting workspace's `_intake/originals/`. Legacy cards remain readable and can be moved with [a redirect at the old path](../../../patterns/knowledge-store.md#move-existing-source-cards).
+New cards go in `sources/`. An uploaded file sits beside its card and is committed with it. For a web page or corporate-storage file, the card holds a link to the original. Open the card's `source:` reference to locate either kind of source. Legacy cards remain readable and can be moved with [a redirect at the old path](../../../patterns/knowledge-store.md#move-existing-source-cards).
 
 ## Intake profile selection
 
 Choose `ingest` for the ordinary ingest/reg-intel workflow (private processed raw
-files), or `knowledge-store` for OKF curation (committed source-document records,
-private originals). They require separate workspaces. Setup and subsequent intake
+files), or `knowledge-store` for OKF curation (committed source cards and uploaded originals). They require separate workspaces. Setup and subsequent intake
 runs must pass the shared CLI preflight; incompatible or ambiguous existing
 content is refused without migration. See the [deployment compatibility contract](../../../patterns/knowledge-store.md#deployment-profiles-and-intake-compatibility)
 for selection, installation and retention rules.