Close final v3 relevance leaks in retrieval (#32)

Require missing-tracker evidence to assert the missing tracker state, require stale-work bridge receipts to independently prove open state, and lock the production failures into broad and focused synthesis regressions.

Author: vaibhav4046

lib/server/synthesis.ts

@@ -216,6 +216,14 @@ function relevance(question: string, text: string, provider?: string) {
   // matches; a citation-backed answer cannot be composed from those fragments.
   if (!supportsCommittedGeneralAvailabilityDate(question, text)) return 0;
   if (/\bpromise\b/.test(q) && !/\b(promise|promised|commit|committed)\b/.test(candidate)) return 0;
+  // Missing-tracker questions ask for negative evidence, not merely a nearby
+  // promise or issue. The retained evidence unit must itself assert absence.
+  const missingTrackerQuestion =
+    /\b(?:no|without|missing|lacks?)\b[^?]{0,70}\b(?:issue|ticket|track(?:ed|ing)?)\b/.test(q) ||
+    /\b(?:issue|ticket)\b[^?]{0,50}\b(?:missing|absent|not tracked)\b/.test(q);
+  const missingTrackerAssertion =
+    /\b(?:no (?:linear )?(?:issue|ticket)|not tracked(?: in [a-z0-9_-]+)?|without (?:an? )?(?:issue|ticket)|lacks? (?:an? )?(?:issue|ticket))\b/.test(candidate);
+  if (missingTrackerQuestion && !missingTrackerAssertion) return 0;
   if (/\b(disagree|conflict)\w*\b/.test(q) && anchors.length >= 2 && anchorMatches < 2 && identifierScore === 0) return 0;
   if (/\bopen\b/.test(q) && /\b(resolved|complete|completed|closed)\b/.test(q)) {
     const hasTrackedState = /\b(issue|ticket|tracked|tracking|open)\b/.test(candidate);
@@ -270,6 +278,9 @@ const staleBridgeQuestion = (question: string) =>
   // exact record ID plus two non-generic anchors before another provider wins.
   /\b(?:tracker|tracking|issue|ticket)\b[^?]{0,90}\b(?:still\s+)?(?:show(?:s|ing)?|remain(?:s|ing)?|stay(?:s|ing)?)?[^?]{0,30}\bopen\b/i.test(question);
 
+const TRACKER_OPEN_ASSERTION =
+  /\b(?:still\s+(?:showing\s+as\s+)?open|remains?\s+open|issue\s+is\s+open|ticket\s+still\s+open|status(?:\s+is|:)?\s+open)\b/i;
+
 const bridgeTokens = (value: string) => new Set(
   tokenise(value).filter((token) => token.length >= 4 && !BRIDGE_GENERIC_TOKENS.has(token)),
 );
@@ -300,6 +311,10 @@ function crossSourceBridgeScore(
   const staleState = staleBridgeQuestion(question);
   if (!questionIds.length && !staleState) return 0;
   const candidateCorpus = clean(candidateText);
+  // A stale-state bridge is independent proof only when the second receipt
+  // itself reports the tracker as open. Shared incident/project context is not
+  // enough to manufacture a cross-provider contradiction.
+  if (staleState && !TRACKER_OPEN_ASSERTION.test(candidateCorpus)) return 0;
   const relationRequired = staleState || /\bwho\s+filed\b|\bwhich\s+project\b|\bproject\b[^?]{0,80}\bagainst\b/i.test(question);
   if (relationRequired && !/\b(?:against|filed|issue|record|ticket|track(?:ed|ing)?)\b/i.test(candidateCorpus)) {
     return 0;
@@ -708,9 +723,26 @@ function contradictions(question: string, relevantEvidence: SynthesisEvidence[])
       "tracked issue";
     const sameReceipt = completed.id === open.id;
     const trackedBridgeCandidates = completed.id === open.id && staleBridgeQuestion(question)
-      ? selected.filter((item) => item.provider !== completed.provider &&
-          /\b(?:against|filed|issue|record|ticket|track(?:ed|ing)?)\b/i.test(`${item.title} ${item.excerpt}`) &&
-          crossSourceBridgeScore(question, `${item.title}. ${item.excerpt}`, item.provider, selected) > 0)
+      ? selected.filter((item) => {
+          if (item.provider === completed.provider) return false;
+          const corpus = `${item.title} ${item.excerpt}`;
+          const namesTrackedEntity = trackedEntity === "tracked issue" ||
+            corpus.toUpperCase().includes(trackedEntity.toUpperCase());
+          const independentlyReportsOpen =
+            /\b(?:still\s+(?:showing\s+as\s+)?open|remains?\s+open|issue\s+is\s+open|ticket\s+still\s+open|status(?:\s+is|:)?\s+open)\b/i.test(corpus);
+          return namesTrackedEntity && independentlyReportsOpen &&
+            /\b(?:against|filed|issue|record|ticket|track(?:ed|ing)?)\b/i.test(corpus) &&
+            crossSourceBridgeScore(question, `${item.title}. ${item.excerpt}`, item.provider, selected) > 0;
+        }).sort((left, right) => {
+          const score = (item: SynthesisEvidence) => {
+            const corpus = `${item.title} ${item.excerpt}`;
+            let value = 0;
+            if (trackedEntity !== "tracked issue" && corpus.toUpperCase().includes(trackedEntity.toUpperCase())) value += 8;
+            if (/\b(?:still\s+(?:showing\s+as\s+)?open|remains?\s+open|issue\s+is\s+open|ticket\s+still\s+open|status(?:\s+is|:)?\s+open)\b/i.test(corpus)) value += 6;
+            return value;
+          };
+          return score(right) - score(left);
+        })
       : [];
     // A Thinking follow-up may add another related provider after the Fast
     // baseline has already found the tracker receipt. Prefer the eligible
@@ -722,7 +754,7 @@ function contradictions(question: string, relevantEvidence: SynthesisEvidence[])
     result.push({
       summary: sameReceipt
         ? trackedBridge
-          ? `${completed.provider} reports the code complete while ${trackedEntity} remains open; ${trackedBridge.provider} independently records the linked incident and tracked work.`
+          ? `${completed.provider} reports the code complete while ${trackedBridge.provider} reports ${trackedEntity} remains open.`
           : `${completed.provider} receipt reports the code complete while ${trackedEntity} remains open in its cited tracking state.`
         : `${completed.provider} reports the work complete while ${open.provider} reports ${trackedEntity} remains open.`,
       evidenceIds: [...new Set([completed.id, open.id, ...(trackedBridge ? [trackedBridge.id] : [])])],

tests/final-v3-retrieval-fix.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+import { synthesiseGroundedAnswer } from "../lib/server/synthesis";
+
+describe("final v3 retrieval relevance regressions", () => {
+  it("keeps an unrelated incident out of a missing-tracker promise answer", () => {
+    const result = synthesiseGroundedAnswer(
+      "Which promise to Northwind has no issue tracking it?",
+      [
+        {
+          id: "slack-promise",
+          provider: "slack",
+          title: "Northwind post-mortem promise",
+          excerpt: "I promised Northwind a written incident post-mortem by 10 August. There is no Linear issue tracking that yet, someone please raise one.",
+        },
+        {
+          id: "github-promise",
+          provider: "github",
+          title: "Post-mortem doc for INC-2031 Northwind outage",
+          excerpt: "Vaibhav promised Northwind a written incident post-mortem by 10 August 2026. Tracking it here since there is no Linear issue for it. Owner unassigned.",
+        },
+        {
+          id: "linear-noise",
+          provider: "linear",
+          title: "AuthShield authentication outage for Northwind",
+          excerpt: "INC-2031: enterprise customer Northwind reported an authentication outage on 29 July 2026. Priya Raman filed this against Atlas Launch. Engineering committed to shipping the AuthShield fix before Friday 7 August 2026.",
+        },
+      ],
+    );
+    expect(result.claims.map((claim) => claim.providers[0])).not.toContain("linear");
+    expect(result.claims.map((claim) => claim.evidenceIds[0])).not.toContain("linear-noise");
+    expect(result.answer).toMatch(/post-mortem/i);
+    expect(result.answer).toMatch(/no Linear issue/i);
+  });
+
+  it("uses the tracker receipt, not a related incident, for an open-vs-complete contradiction", () => {
+    const result = synthesiseGroundedAnswer(
+      "Which open issue appears to be already resolved elsewhere?",
+      [
+        {
+          id: "github-complete",
+          provider: "github",
+          title: "AuthShield fix merged in PR-8871 but ENG-456 still open",
+          excerpt: "The AuthShield authentication fix for the Northwind outage (INC-2031) was merged. Linear issue ENG-456 is still showing as open even though the code is shipped. Someone should close it or explain why it is still tracked.",
+        },
+        {
+          id: "linear-open",
+          provider: "linear",
+          title: "ENG-456 AuthShield tracker",
+          excerpt: "Linear issue ENG-456 remains open for the AuthShield work linked to INC-2031 and Northwind.",
+        },
+        {
+          id: "linear-noise",
+          provider: "linear",
+          title: "AuthShield authentication outage for Northwind",
+          excerpt: "INC-2031: enterprise customer Northwind reported an authentication outage. Priya Raman filed this against Atlas Launch. Engineering committed to shipping the AuthShield fix.",
+        },
+      ],
+    );
+    const conflict = result.contradictions.find((item) => item.providers.includes("github") && item.providers.includes("linear"));
+    expect(conflict).toBeTruthy();
+    expect(conflict?.evidenceIds).toContain("linear-open");
+    expect(conflict?.evidenceIds).not.toContain("linear-noise");
+    expect(conflict?.summary).toMatch(/github reports the code complete while linear reports ENG-456 remains open/i);
+  });
+});

tests/synthesis.test.ts

@@ -204,17 +204,17 @@ describe("evidence-constrained synthesis", () => {
     expect(result.answer).not.toContain("external risks");
   });
 
-  it("joins a stale-state receipt to the independently tracked provider context", () => {
+  it("does not treat related tracker-provider context as independent open-state proof", () => {
     const result = synthesiseGroundedAnswer(
       "Which open issue appears to be already resolved elsewhere?",
       [evidence[2], evidence[0]],
     );
 
-    expect(result.validation.providerCoverage).toEqual(expect.arrayContaining(["github", "linear"]));
+    expect(result.validation.providerCoverage).toEqual(["github"]);
     expect(result.contradictions).toEqual([
       expect.objectContaining({
-        providers: expect.arrayContaining(["github", "linear"]),
-        evidenceIds: expect.arrayContaining(["github-1", "linear-1"]),
+        providers: ["github"],
+        evidenceIds: ["github-1"],
       }),
     ]);
   });
@@ -940,9 +940,9 @@ describe("evidence-constrained synthesis", () => {
         {
           id: "linear-incident",
           provider: "linear",
-          title: "Authentication outage for Northwind",
+          title: "ENG-456 AuthShield tracker remains open",
           excerpt:
-            "INC-2031: Northwind reported an authentication outage. Priya Raman filed this against Atlas Launch.",
+            "Linear issue ENG-456 remains open for the AuthShield work linked to INC-2031 and Northwind.",
           timestamp: "2026-08-02T01:03:55Z",
         },
         {