Skip to content

Commit 6e50a59

Browse files
EoghanOConnorEoghanOConnor
authored
Merge pull request #45 from jerry-ng2/fix/batch-duplicates
fix: Deduplicates within the same batch no longer query to DB
2 parents 25d62cf + 8b4db6e commit 6e50a59

4 files changed

Lines changed: 454 additions & 47 deletions

File tree

‎deploy/batch-rca-automation/batch_rca_headless.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,38 @@ JOB_COUNT=$(echo "$JOB_IDS" | wc -l | tr -d ' ')
110110
JOBS_LIST=$(echo "$JOB_IDS" | tr '\n' ' ' | sed 's/ $//')
111111
echo "[INFO] Found $JOB_COUNT job(s) to analyze: $JOBS_LIST"
112112

113+
#############################################
114+
# Step 1a: Intra-batch dedup
115+
#############################################
116+
echo "[STEP 1a] Deduplicating within batch..."
117+
118+
INTRA_BATCH_DUPES="[]"
119+
DUPE_COUNT=0
120+
121+
DEDUP_OUTPUT=$(echo "$JOB_IDS" | python3 "$SCRIPT_DIR/scripts/pre_filter_jobs.py" --dedup-only 2>/dev/null) || DEDUP_OUTPUT=""
122+
123+
if [ -n "$DEDUP_OUTPUT" ]; then
124+
DUPE_COUNT=$(echo "$DEDUP_OUTPUT" | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('dupes', [])))" 2>/dev/null || echo "0")
125+
126+
if [ "$DUPE_COUNT" -gt 0 ] 2>/dev/null; then
127+
INTRA_BATCH_DUPES=$(echo "$DEDUP_OUTPUT" | python3 -c "
128+
import json, sys
129+
print(json.dumps(json.load(sys.stdin).get('dupes', [])))
130+
")
131+
132+
JOB_IDS=$(echo "$DEDUP_OUTPUT" | python3 -c "
133+
import json, sys
134+
for jid in json.load(sys.stdin).get('representatives', []):
135+
print(jid)
136+
")
137+
JOB_COUNT=$(echo "$JOB_IDS" | wc -l | tr -d ' ')
138+
JOBS_LIST=$(echo "$JOB_IDS" | tr '\n' ' ' | sed 's/ $//')
139+
echo "[INFO] Intra-batch dedup: $DUPE_COUNT duplicate(s) removed, $JOB_COUNT representative(s): $JOBS_LIST"
140+
else
141+
echo "[INFO] No intra-batch duplicates found"
142+
fi
143+
fi
144+
113145
#############################################
114146
# Step 1b: Pre-filter against known issues
115147
#############################################
@@ -167,6 +199,12 @@ for jid in json.load(sys.stdin).get('analyze', []):
167199
echo "[INFO] Remaining: $JOB_COUNT job(s) for full RCA: $JOBS_LIST"
168200
else
169201
echo "[INFO] All jobs matched known issues, skipping Claude invocation"
202+
if [ "$DUPE_COUNT" -gt 0 ] 2>/dev/null && [ "$INTRA_BATCH_DUPES" != "[]" ]; then
203+
echo "[STEP 5b] Linking $DUPE_COUNT intra-batch duplicate(s)..."
204+
python3 "$SCRIPT_DIR/scripts/store_report.py" --link-dupes "$INTRA_BATCH_DUPES" || {
205+
echo "[WARN] Failed to link some intra-batch duplicates (non-fatal)"
206+
}
207+
fi
170208
echo "[SUCCESS] Batch RCA completed at $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
171209
exit 0
172210
fi
@@ -305,5 +343,15 @@ python3 "$SCRIPT_DIR/scripts/store_report.py" "$REPORT_FILE" || {
305343
exit 1
306344
}
307345

346+
#############################################
347+
# Step 5b: Link intra-batch duplicates
348+
#############################################
349+
if [ "$DUPE_COUNT" -gt 0 ] 2>/dev/null && [ "$INTRA_BATCH_DUPES" != "[]" ]; then
350+
echo "[STEP 5b] Linking $DUPE_COUNT intra-batch duplicate(s)..."
351+
python3 "$SCRIPT_DIR/scripts/store_report.py" --link-dupes "$INTRA_BATCH_DUPES" || {
352+
echo "[WARN] Failed to link some intra-batch duplicates (non-fatal)"
353+
}
354+
fi
355+
308356
echo "[SUCCESS] Batch RCA completed at $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
309357
echo "[INFO] Report: $REPORT_FILE"

‎deploy/batch-rca-automation/docs/batch-rca-flow.md‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# Batch RCA Analysis Flow
2+
3+
```mermaid
4+
flowchart TD
5+
Start([Cron trigger every 20 min]) --> LoadEnv[Load env vars from .claude/settings.json]
6+
7+
LoadEnv --> Step1
8+
9+
subgraph Step1["Step 1: Query Source DB"]
10+
Q[query_source_db.py<br/>SELECT job_id WHERE ai_processed = FALSE<br/>AND job_finished >= --since]
11+
end
12+
13+
Step1 --> NoJobs{Jobs found?}
14+
NoJobs -- No --> Done([Exit 0])
15+
NoJobs -- Yes --> Step1a
16+
17+
subgraph Step1a["Step 1a: Intra-Batch Dedup"]
18+
direction TB
19+
D1[pre_filter_jobs.py --dedup-only]
20+
D2[Fetch job_name + error_message<br/>from source table]
21+
D3[Group jobs by catalog_item<br/>extracted from job_name]
22+
D4[Cluster within each group by<br/>error_message similarity >= 0.75]
23+
D5[Pick earliest job_id per cluster<br/>as representative]
24+
D6{Duplicates<br/>found?}
25+
D7[Output representatives list<br/>+ dupes list for later linking]
26+
D8[All jobs are representatives]
27+
28+
D1 --> D2 --> D3 --> D4 --> D5 --> D6
29+
D6 -- Yes --> D7
30+
D6 -- No --> D8
31+
end
32+
33+
Step1a --> Step1b
34+
35+
subgraph Step1b["Step 1b: Pre-Filter Against Known Issues"]
36+
direction TB
37+
F0{--no-pre-filter<br/>flag set?}
38+
F1[fetch_known_issues.py<br/>Query recent high-confidence results<br/>from last 4 hours]
39+
F2{Known issues<br/>found?}
40+
F3["pre_filter_jobs.py<br/>(normal mode)"]
41+
F4["Pass 1: Match on catalog_item<br/>+ error_message similarity >= 0.75"]
42+
F5["Pass 2: Cross-catalog match on<br/>error_message similarity >= 0.90<br/>(catches platform-wide failures)"]
43+
F6{Pre-matched<br/>jobs?}
44+
F7[store_report.py --pre-matched<br/>Set FK + ai_processed = TRUE]
45+
F8{Remaining jobs<br/>to analyze?}
46+
F9[Link dupes via<br/>store_report.py --link-dupes]
47+
F10[Skip pre-filter]
48+
49+
F0 -- Yes --> F10
50+
F0 -- No --> F1 --> F2
51+
F2 -- No --> F10
52+
F2 -- Yes --> F3 --> F4 --> F5 --> F6
53+
F6 -- No --> F10
54+
F6 -- Yes --> F7 --> F8
55+
F8 -- No, all matched --> F9 --> Done2([Exit 0])
56+
F8 -- Yes --> F10
57+
end
58+
59+
Step1b --> Step2
60+
61+
subgraph Step2["Step 2: Build Claude Prompt"]
62+
P1[Inject remaining job IDs,<br/>batch_id, and known issues<br/>into orchestration prompt]
63+
end
64+
65+
Step2 --> Step3
66+
67+
subgraph Step3["Step 3: MLflow Setup"]
68+
M1{MLflow tracing<br/>enabled?}
69+
M2[Create venv + install mlflow<br/>if first run]
70+
M3[Skip]
71+
M1 -- Yes --> M2
72+
M1 -- No --> M3
73+
end
74+
75+
Step3 --> Step4
76+
77+
subgraph Step4["Step 4: Claude Headless Execution"]
78+
direction TB
79+
C1["claude -p --model claude-sonnet-4-6<br/>Sends orchestration prompt"]
80+
C1 --> SpawnAgents["Spawn parallel agents<br/>(one per remaining job)"]
81+
SpawnAgents --> A1["Agent 1"] & A2["Agent 2"] & A3["Agent N"]
82+
A1 & A2 & A3 --> AGG
83+
AGG["Aggregate results:<br/>- Tally categories + confidence<br/>- Deduplicate recommendations<br/>- Semantic historical matching<br/>- Cross-job pattern detection"]
84+
AGG --> WriteReport["Write batch report JSON"]
85+
end
86+
87+
subgraph PerAgent["Each Agent: root-cause-analysis skill"]
88+
direction TB
89+
S1["Step 1: Parse job log<br/>(auto-fetch via SSH)"]
90+
S1check{Error matches<br/>known issue?}
91+
S1early["Early exit:<br/>matched_known_issue"]
92+
S2["Step 2: Query Splunk logs"]
93+
S3["Step 3: Correlate AAP + Splunk"]
94+
S4["Step 4: Fetch GitHub configs"]
95+
S5["Step 5: Claude analysis<br/>+ generate summary"]
96+
S1 --> S1check
97+
S1check -- Yes --> S1early
98+
S1check -- No --> S2 --> S3 --> S4 --> S5
99+
end
100+
101+
SpawnAgents -.->|each agent runs| PerAgent
102+
103+
Step4 --> Step4b
104+
105+
subgraph Step4b["Step 4b: Verify Report"]
106+
V1{Report file<br/>exists?}
107+
V1 -- No --> Fail([Exit 1])
108+
end
109+
110+
Step4b --> Step5
111+
112+
subgraph Step5["Step 5: Store Report in DB"]
113+
direction TB
114+
ST1[store_report.py report.json]
115+
ST2["For each job result:"]
116+
ST3{Agent declared<br/>matched_result_id?}
117+
ST4["Validate match:<br/>id + root_cause_category<br/>+ confidence = high"]
118+
ST5{Valid?}
119+
ST6["Use matched FK<br/>Update source: FK + ai_processed"]
120+
ST7["Fallback: find_match()<br/>difflib similarity >= 0.85<br/>same catalog_item + category"]
121+
ST8{Match found?}
122+
ST9["INSERT into results table<br/>Update source: FK + ai_processed"]
123+
124+
ST1 --> ST2 --> ST3
125+
ST3 -- Yes --> ST4 --> ST5
126+
ST5 -- Yes --> ST6
127+
ST5 -- No --> ST7
128+
ST3 -- No --> ST7
129+
ST7 --> ST8
130+
ST8 -- Yes --> ST6
131+
ST8 -- No --> ST9
132+
end
133+
134+
Step5 --> Step5b
135+
136+
subgraph Step5b["Step 5b: Link Intra-Batch Duplicates"]
137+
direction TB
138+
L1{Dupes from<br/>Step 1a?}
139+
L2["store_report.py --link-dupes<br/>Copy representative's FK<br/>to each duplicate job"]
140+
L3[Skip]
141+
L1 -- Yes --> L2
142+
L1 -- No --> L3
143+
end
144+
145+
Step5b --> Success([Batch RCA Complete])
146+
147+
style Start fill:#4a9eff,color:#fff
148+
style Done fill:#2ecc71,color:#fff
149+
style Done2 fill:#2ecc71,color:#fff
150+
style Success fill:#2ecc71,color:#fff
151+
style Fail fill:#e74c3c,color:#fff
152+
style Step1a fill:#fff3cd
153+
style Step1b fill:#fff3cd
154+
style Step4 fill:#d1ecf1
155+
style PerAgent fill:#e2d6f3
156+
style Step5 fill:#d4edda
157+
style Step5b fill:#d4edda
158+
```

0 commit comments

Comments
 (0)