feat(slack-bot): daily company-knowledge distiller → PR (no PII)

Adds a harness-side pipeline that distills GENERIC company knowledge from the
Slack bot's agent_logs and opens a daily PR to the bot's GAP repo, with a
deterministic PII scrub + dedup so no user-specific/personal data is learned.

- examples/knowledge-distiller.ts: gather agent_logs since watermark → Anthropic
  distill (forced-tool JSON) → PII scrub → dedup vs knowledge_index → open/append
  one PR/day via GitHub Contents API → advance watermark. Scheduler + pure helpers.
- examples/agent-log-store.ts: listSince() + close().
- examples/slack-bot.ts: opt-in scheduler start (LYRA_LEARN=1) in createSlackBotsApp.
- examples/distill-now.ts: --dry-run CLI.
- examples/knowledge-distiller.test.ts: PII-scrub + dedup + repo-parse unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: kapaleshreyas

examples/agent-log-store.ts

@@ -90,8 +90,26 @@ export class AgentLogStore {
     return (await this.coll()).findOne({ _id: id });
   }
 
+  /**
+   * Successful entries for a bot newer than `since`, oldest-first — the input
+   * for the knowledge distiller's daily batch. Only `ok` turns are returned
+   * (errored turns carry no learnable content). `max` caps the batch size.
+   */
+  async listSince(bot: string, since: Date, max = 5000): Promise<AgentLogEntry[]> {
+    return (await this.coll())
+      .find({ bot, ok: true, ts: { $gt: since } })
+      .sort({ ts: 1 })
+      .limit(Math.min(Math.max(max, 1), 20000))
+      .toArray();
+  }
+
   /** Count entries for an agent (used for the agents-list stats). */
   async count(bot: string): Promise<number> {
     return (await this.coll()).countDocuments({ bot });
   }
+
+  /** Close the underlying Mongo connection (for one-shot CLIs). */
+  async close(): Promise<void> {
+    if (this.connected) await this.client.close();
+  }
 }

examples/distill-now.ts

@@ -0,0 +1,67 @@
+/**
+ * distill-now — run the company-knowledge distiller once, on demand.
+ *
+ *   node --experimental-strip-types examples/distill-now.ts [--dry-run]
+ *   bun run examples/distill-now.ts --dry-run
+ *
+ * Reads the same env the Slack bot uses: MONGO_URL, MONGO_DATABASE,
+ * GITAGENT_ANTHROPIC_API_KEY, and (repo/token) LYRA_LEARN_REPO or SLACK_LYRA_SOURCE
+ * + SLACK_LYRA_GIT_TOKEN / GITHUB_TOKEN. --dry-run distills + scrubs + dedups and
+ * prints what WOULD be proposed, without touching git or the watermark.
+ */
+import { KnowledgeDistiller } from "./knowledge-distiller.ts";
+
+const dryRun = process.argv.includes("--dry-run");
+
+const mongoUrl = process.env.MONGO_URL;
+const mongoDb = process.env.MONGO_DATABASE ?? "computeragent";
+const anthropicKey = process.env.GITAGENT_ANTHROPIC_API_KEY;
+const bot = process.env.LYRA_LEARN_BOT ?? "lyra";
+const repo = process.env.LYRA_LEARN_REPO ?? process.env.SLACK_LYRA_SOURCE;
+const gitToken = process.env.SLACK_LYRA_GIT_TOKEN ?? process.env.GITHUB_TOKEN ?? "";
+
+if (!mongoUrl) {
+  console.error("MONGO_URL is required");
+  process.exit(1);
+}
+if (!anthropicKey) {
+  console.error("GITAGENT_ANTHROPIC_API_KEY is required");
+  process.exit(1);
+}
+if (!repo) {
+  console.error("Set LYRA_LEARN_REPO (or SLACK_LYRA_SOURCE) to the target repo");
+  process.exit(1);
+}
+if (!dryRun && !gitToken) {
+  console.error("A git token (SLACK_LYRA_GIT_TOKEN / GITHUB_TOKEN) is required for a real run");
+  process.exit(1);
+}
+
+const distiller = new KnowledgeDistiller({
+  mongoUrl,
+  mongoDb,
+  bot,
+  repo,
+  gitToken,
+  anthropicKey,
+  model: process.env.LYRA_LEARN_MODEL,
+});
+
+try {
+  console.log(`[distill-now] bot=${bot} repo=${repo} dryRun=${dryRun}`);
+  const r = await distiller.runOnce({ dryRun });
+  console.log(
+    `\nthreads=${r.scannedThreads} extracted=${r.extracted} afterScrub=${r.afterScrub} fresh=${r.fresh}`,
+  );
+  if (r.freshStatements.length) {
+    console.log("\nWould propose:");
+    for (const s of r.freshStatements) console.log(`  - ${s}`);
+  }
+  if (r.prUrl) console.log(`\nPR: ${r.prUrl}`);
+  if (r.skippedReason) console.log(`\n(${r.skippedReason})`);
+} catch (e) {
+  console.error("[distill-now] failed:", (e as Error).message);
+  process.exitCode = 1;
+} finally {
+  await distiller.close();
+}

examples/knowledge-distiller.test.ts

@@ -0,0 +1,75 @@
+/**
+ * Unit tests for the knowledge distiller's pure logic: the PII scrub (the hard
+ * requirement — no personal/user-specific data may survive), dedup, hashing, and
+ * repo parsing. These need no Mongo or network.
+ */
+import { describe, expect, it } from "vitest";
+import { dedupe, normalizeHash, parseRepo, piiReject } from "./knowledge-distiller.ts";
+
+describe("piiReject", () => {
+  it("accepts generic company facts", () => {
+    expect(piiReject("The company deploys agents via a bubblewrap sandbox on EC2.")).toBeNull();
+    expect(piiReject("The billing service charges per active sandbox-hour.")).toBeNull();
+    expect(piiReject("Releases are cut from the deploy branch, which rebuilds the ECR images.")).toBeNull();
+  });
+
+  it("drops emails, tokens, and JWTs", () => {
+    expect(piiReject("Contact the owner at jane.doe@acme.com for access.")).not.toBeNull();
+    expect(piiReject("The service key is sk-ant-api03-abcdef0123456789ABCDEF.")).not.toBeNull();
+    expect(piiReject("Deploy token ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.")).not.toBeNull();
+    expect(piiReject("Session eyJhbGciOiJIUzI1NiIsInR5cCI.eyJzdWIiOiIxMjM0NTY.abcdef123456")).not.toBeNull();
+    expect(piiReject("AWS key AKIAIOSFODNN7EXAMPLE was rotated.")).not.toBeNull();
+  });
+
+  it("drops Slack mentions/user ids and phone numbers", () => {
+    expect(piiReject("Ping <@U012AB3CD> when the build is green.")).not.toBeNull();
+    expect(piiReject("The on-call reaches U08ZZ12QKP for escalation.")).not.toBeNull();
+    expect(piiReject("Call the support line at +1 (415) 555-0198 for urgent issues.")).not.toBeNull();
+  });
+
+  it("drops personal attribution ('X said/wants/prefers')", () => {
+    expect(piiReject("Shreyas prefers the reports in a table format.")).not.toBeNull();
+    expect(piiReject("Abhishek asked for the migration to run overnight.")).not.toBeNull();
+    expect(piiReject("Sarah said the onboarding flow is confusing.")).not.toBeNull();
+    expect(piiReject("My email is on file for the newsletter.")).not.toBeNull();
+  });
+
+  it("drops trivially short statements", () => {
+    expect(piiReject("Yes.")).toBe("too-short");
+    expect(piiReject("ok thanks")).toBe("too-short");
+  });
+});
+
+describe("normalizeHash + dedupe", () => {
+  it("hashes semantically-equal statements to the same value", () => {
+    const a = normalizeHash("The company uses MongoDB Atlas.");
+    const b = normalizeHash("the   company uses mongodb atlas!!!");
+    expect(a).toBe(b);
+  });
+
+  it("skips statements already in the seen set and within the batch", () => {
+    const seen = new Set([normalizeHash("The platform runs on EKS.")]);
+    const { fresh, hashes } = dedupe(
+      [
+        "The platform runs on EKS.", // already seen
+        "The API is rate-limited to 4 concurrent runs.", // new
+        "the api is rate limited to 4 concurrent runs", // dup of the above within-batch
+      ],
+      seen,
+    );
+    expect(fresh).toEqual(["The API is rate-limited to 4 concurrent runs."]);
+    expect(hashes).toHaveLength(1);
+  });
+});
+
+describe("parseRepo", () => {
+  it("parses bare, https, and .git forms", () => {
+    expect(parseRepo("github.com/example-org/agent-repo")).toEqual({ owner: "example-org", name: "agent-repo" });
+    expect(parseRepo("https://github.com/example-org/agent-repo.git")).toEqual({ owner: "example-org", name: "agent-repo" });
+    expect(parseRepo("git@github.com:example-org/agent-repo.git")).toEqual({ owner: "example-org", name: "agent-repo" });
+  });
+
+  it("throws on unparseable input", () => {
+    expect(() => parseRepo("not-a-repo")).toThrow();
+  });
+});