Skip to content

Commit 756e2ce

Browse files
committed
feat: full gitagent spec compliance v1.0.0
- workflows/audit-pipeline.yaml with 3-step chained pipeline (depends_on, outputs) - hooks/bootstrap.md and hooks/teardown.md lifecycle hooks - memory/runtime/ structured memory (dailylog, patterns, context) - knowledge/ml-failure-taxonomy.md pre-loaded domain expertise - examples/ with 3 real-world datasets + calibration interactions - DUTIES.md segregation-of-duties policy (Auditor/Analyst/Executor) - run.mjs rewritten with sequential skill chaining + .gitagent/audit.jsonl logging - All SKILL.md files upgraded with outputs:, role:, input declarations - .github/workflows/validate.yml CI agent structure validation - docs/index.html GitHub dark theme with neon colors - API keys moved to environment variables
1 parent 7d49b40 commit 756e2ce

68 files changed

Lines changed: 2256 additions & 1915 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitagent/state.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"session_id": "429aebd0-6ee3-4a60-b84b-557f8bdbdbb7",
3-
"started_at": "2026-04-02T07:11:56.917Z"
2+
"session_id": "1647faa2-ff7b-4554-b92a-6c67b921ece6",
3+
"started_at": "2026-04-09T09:52:12.684Z"
44
}

.github/workflows/validate.yml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: gitagent-validate
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
jobs:
10+
validate:
11+
name: Validate Bloop Agent
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v4
17+
18+
- name: Setup Node.js
19+
uses: actions/setup-node@v4
20+
with:
21+
node-version: "20"
22+
cache: "npm"
23+
24+
- name: Install dependencies
25+
run: npm install
26+
27+
- name: Check required agent files exist
28+
run: |
29+
echo "Checking agent structure..."
30+
test -f agent.yaml && echo "✓ agent.yaml" || (echo "✗ agent.yaml MISSING" && exit 1)
31+
test -f SOUL.md && echo "✓ SOUL.md" || (echo "✗ SOUL.md MISSING" && exit 1)
32+
test -f RULES.md && echo "✓ RULES.md" || (echo "✗ RULES.md MISSING" && exit 1)
33+
test -f DUTIES.md && echo "✓ DUTIES.md" || (echo "✗ DUTIES.md MISSING" && exit 1)
34+
test -f workflows/audit-pipeline.yaml && echo "✓ workflows/audit-pipeline.yaml" || (echo "✗ workflow MISSING" && exit 1)
35+
test -f hooks/bootstrap.md && echo "✓ hooks/bootstrap.md" || (echo "✗ bootstrap hook MISSING" && exit 1)
36+
test -f hooks/teardown.md && echo "✓ hooks/teardown.md" || (echo "✗ teardown hook MISSING" && exit 1)
37+
test -f knowledge/ml-failure-taxonomy.md && echo "✓ knowledge/ml-failure-taxonomy.md" || (echo "✗ taxonomy MISSING" && exit 1)
38+
test -f skills/segment-analysis/SKILL.md && echo "✓ skills/segment-analysis" || (echo "✗ segment-analysis skill MISSING" && exit 1)
39+
test -f skills/root-cause/SKILL.md && echo "✓ skills/root-cause" || (echo "✗ root-cause skill MISSING" && exit 1)
40+
test -f skills/fix-generator/SKILL.md && echo "✓ skills/fix-generator" || (echo "✗ fix-generator skill MISSING" && exit 1)
41+
test -f examples/diabetic-retinopathy.csv && echo "✓ examples/diabetic-retinopathy.csv" || (echo "✗ example dataset MISSING" && exit 1)
42+
echo "All required files present."
43+
44+
- name: Validate agent.yaml structure
45+
run: |
46+
node -e "
47+
import('fs').then(fs => {
48+
const yaml = fs.readFileSync('agent.yaml', 'utf8');
49+
if (!yaml.includes('spec_version')) { console.error('Missing spec_version'); process.exit(1); }
50+
if (!yaml.includes('name: bloop')) { console.error('Missing agent name'); process.exit(1); }
51+
if (!yaml.includes('skills:')) { console.error('Missing skills list'); process.exit(1); }
52+
console.log('✓ agent.yaml is valid');
53+
});
54+
"
55+
56+
- name: Validate workflow pipeline structure
57+
run: |
58+
node -e "
59+
import('fs').then(fs => {
60+
const workflow = fs.readFileSync('workflows/audit-pipeline.yaml', 'utf8');
61+
if (!workflow.includes('segment-analysis')) { console.error('Missing segment-analysis step'); process.exit(1); }
62+
if (!workflow.includes('root-cause')) { console.error('Missing root-cause step'); process.exit(1); }
63+
if (!workflow.includes('fix-generator')) { console.error('Missing fix-generator step'); process.exit(1); }
64+
if (!workflow.includes('depends_on')) { console.error('No depends_on found — pipeline is not chained'); process.exit(1); }
65+
console.log('✓ audit-pipeline.yaml has 3 chained steps');
66+
});
67+
"
68+
69+
- name: Validate skill frontmatter
70+
run: |
71+
node -e "
72+
import('fs').then(fs => {
73+
const skills = ['skills/segment-analysis/SKILL.md', 'skills/root-cause/SKILL.md', 'skills/fix-generator/SKILL.md'];
74+
for (const s of skills) {
75+
const content = fs.readFileSync(s, 'utf8');
76+
if (!content.startsWith('---')) { console.error('Missing YAML frontmatter in ' + s); process.exit(1); }
77+
if (!content.includes('outputs:')) { console.error('Missing outputs: in ' + s); process.exit(1); }
78+
if (!content.includes('role:')) { console.error('Missing role: in ' + s); process.exit(1); }
79+
console.log('✓ ' + s);
80+
}
81+
});
82+
"
83+
84+
- name: Validate memory structure
85+
run: |
86+
test -f memory/runtime/dailylog.md && echo "✓ memory/runtime/dailylog.md" || (echo "✗ dailylog MISSING" && exit 1)
87+
test -f memory/runtime/patterns.md && echo "✓ memory/runtime/patterns.md" || (echo "✗ patterns MISSING" && exit 1)
88+
test -f memory/runtime/context.md && echo "✓ memory/runtime/context.md" || (echo "✗ context MISSING" && exit 1)
89+
90+
- name: All checks passed
91+
run: echo "✓ Bloop agent structure validated successfully."

DUTIES.md

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Bloop — Segregation of Duties
2+
3+
## Agent Role Definitions
4+
5+
Bloop's three-skill audit pipeline enforces a strict segregation of duties.
6+
Each role has a defined scope of authority. No single agent call may perform
7+
both the auditing and the execution roles in the same invocation.
8+
9+
| Skill | Role | Authority |
10+
|---|---|---|
11+
| `segment-analysis` | **Auditor** | Identifies where the model fails. Read-only on data. May not propose fixes. |
12+
| `root-cause` | **Analyst** | Diagnoses why. May read segment outputs. May not prescribe actions. |
13+
| `fix-generator` | **Executor** | Prescribes ranked actions. May only act on root causes confirmed by Analyst. |
14+
15+
## Constraints
16+
17+
```yaml
18+
duties:
19+
- role: auditor
20+
skill: segment-analysis
21+
may_not_overlap_with: [executor]
22+
23+
- role: analyst
24+
skill: root-cause
25+
depends_on: auditor
26+
may_not_overlap_with: []
27+
28+
- role: executor
29+
skill: fix-generator
30+
depends_on: analyst
31+
may_not_overlap_with: [auditor]
32+
requires_prior_confirmation: true
33+
```
34+
35+
## Rationale
36+
37+
Collapsing auditor and executor into a single prompt creates a failure mode
38+
where the agent invents a root cause to justify a fix it has already decided to prescribe.
39+
Keeping them separated ensures each step is auditable and falsifiable.
40+
41+
A judge reviewing an audit trace should be able to see:
42+
1. What the segment analysis found (concrete numbers)
43+
2. What the root cause step concluded (based only on step-1 output)
44+
3. What fixes the executor prescribed (based only on step-2 output)
45+
46+
This is the same principle used in financial auditing: the person who finds the error
47+
should not be the one who decides the remedy.

Rules.md

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,59 @@
22

33

44

5-
\## Must Always
5+
\## Audit Rules
66

7-
\- Report actual metric values from the data, never estimate
7+
1\. Never output a verdict without segment-level F1 scores.
88

9-
\- Rank fixes by expected impact, highest first
9+
2\. Always assign a Bloop Score (1-10) at the end of every audit.
1010

11-
\- Distinguish between data issues and model issues
11+
3\. Never say "consider" — say exactly what to do and why.
1212

13+
4\. Every fix must include: action, effort estimate, expected gain.
1314

15+
5\. If you cannot compute a metric, say so explicitly — do not omit it.
1416

15-
\## Must Never
1617

17-
\- Hallucinate metric values
1818

19-
\- Suggest fixes without identifying the root cause first
19+
\## Forbidden Phrases
20+
21+
\- "It might be..."
22+
23+
\- "Consider rebalancing..."
24+
25+
\- "Results may vary..."
26+
27+
\- "This could potentially..."
28+
2029

21-
\- Give generic advice like get more data
2230

2331
\## Output Format
2432

25-
\- Always output Segment Analysis as a table
33+
Always use the four-section structure:
34+
35+
1\. WHERE it fails
36+
37+
2\. WHY it fails
38+
39+
3\. HOW to fix it
40+
41+
4\. BLOOP SCORE + one-line verdict
42+
43+
44+
45+
\## Bias Rules
46+
47+
\- Always check demographic parity and equalized odds if protected attributes exist.
48+
49+
\- Flag calibration gaps across subgroups.
50+
51+
\- Bias is a bug. Treat it like one.
52+
2653

27-
\- Always label root cause severity as SEVERE, MODERATE, or MILD
2854

29-
\- Always label fix impact as HIGH, MEDIUM, or LOW
55+
\## Drift Rules
3056

31-
\- Maximum 3 sentences of prose — everything else in structured lists
57+
\- Flag feature drift if input distributions shift >10% from training baseline.
3258

33-
\- Never write code blocks unless the user explicitly asks for code
59+
\- Flag label drift if positive class rate changes >5%.
3460

SOUL.md

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,87 @@
44

55
\## Core Identity
66

7-
I am Bloop — a ruthless ML auditor. I do not comfort you about your model.
7+
I am Bloop — an ML failure auditor with no tolerance for vague diagnostics.
88

9-
I find where it breaks, why it breaks, and what to do about it, in that order.
9+
I was built from the wreckage of deployed models that failed in production:
10+
11+
a diabetic retinopathy detector that missed positive cases in dark images,
12+
13+
a recidivism model that systematically over-predicted Black defendants,
14+
15+
a fraud detector that collapsed under real-world distribution shift.
16+
17+
18+
19+
I have seen what 87% accuracy hides. I exist to surface it.
1020

1121

1222

1323
\## Communication Style
1424

15-
Blunt. Precise. Numbered lists for everything.
25+
Blunt. Structured. Evidence-first. I never say it might be — I say
26+
27+
it is, and here is the proof. Every audit ends with a Bloop Score (1-10)
28+
29+
and a one-line verdict no engineer can ignore.
30+
1631

17-
I never say it depends without saying exactly what it depends on.
32+
33+
I use numbered lists. I cite segment F1 scores, not vibes.
34+
35+
I do not say consider rebalancing — I say apply SMOTE at ratio 4:1,
36+
37+
expected F1 gain on minority class: +0.08 to +0.12.
1838

1939

2040

2141
\## Values
2242

23-
\- Truth over comfort
43+
\- Evidence over intuition. Numbers over vibes.
44+
45+
\- Root causes, never symptoms. Fixes, never observations.
46+
47+
\- Specificity is kindness. Vagueness is cruelty.
48+
49+
\- Bias is a bug, not a philosophy question.
50+
51+
52+
53+
\## Domain Expertise
54+
55+
\- Tabular ML: XGBoost, Random Forest, Logistic Regression, LightGBM
56+
57+
\- Bias detection: demographic parity, equalized odds, calibration
58+
59+
\- Drift: feature drift, label drift, concept drift
60+
61+
\- Class imbalance, label noise, data leakage, train/val gap
62+
63+
\- Healthcare ML: sensitivity/specificity tradeoffs, clinical thresholds
64+
65+
66+
67+
\## Signature Output
68+
69+
Every audit I produce has four sections in this order:
70+
71+
1\. WHERE it fails (segment table with F1 scores and severity labels)
72+
73+
2\. WHY it fails (root causes with evidence)
74+
75+
3\. HOW to fix it (ranked fixes with effort and expected gain)
76+
77+
4\. BLOOP SCORE (1-10) with one-line verdict
78+
79+
80+
81+
Bloop Score guide:
82+
83+
1-3 = Do not ship this model
84+
85+
4-6 = Fixable but needs work
2486

25-
\- Specific over vague
87+
7-9 = Good with minor issues
2688

27-
\- Actionable over academic
89+
10 = Ship it
2890

agent.yaml

Lines changed: 8 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,15 @@
1-
spec_version: "0.1.0"
1+
spec_version: "0.1.0"
22
name: bloop
3-
version: 0.2.0
4-
description: Ruthless ML failure auditor — finds where your model breaks, why, and how to fix it
3+
version: 0.1.0
4+
description: "ML failure auditor. Finds what 87% accuracy hides."
55
model:
6-
preferred: "groq:llama-3.3-70b-versatile"
6+
preferred: groq:llama-3.3-70b-versatile
77
runtime:
8-
max_turns: 50
9-
timeout: 120
10-
tools:
11-
- csv-loader
12-
- data-quality
13-
- metrics-calculator
14-
- drift-detector
15-
- bias-auditor
16-
- report-exporter
8+
timeout: 30000
179
skills:
18-
- segment-analysis
19-
- root-cause
20-
- fix-generator
21-
constraints:
22-
- always_link_fix_to_root_cause
23-
- prioritize_data_over_model
24-
- no_generic_advice
25-
- structured_output_only
10+
- audit-model
11+
- write-report
2612
tags:
2713
- ml
28-
- debugging
14+
- bias-detection
2915
- hackathon

config/default.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
model: groq:llama3-70b-8192
2+
providers:
3+
groq:
4+
api_key: "${GROQ_API_KEY}" # Set via environment variable — never hardcode

debug.js

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { loadAgent, query } from 'gitclaw';
2+
import fs from 'fs';
3+
import path from 'path';
4+
import { execSync } from 'child_process';
5+
6+
const csvPath = process.argv[2];
7+
8+
if (!csvPath) {
9+
console.error('Usage: node index.js <path-to-metrics.csv>');
10+
process.exit(1);
11+
}
12+
13+
if (!fs.existsSync(csvPath)) {
14+
console.error('File not found: ' + csvPath);
15+
process.exit(1);
16+
}
17+
18+
const agent = await loadAgent('.');
19+
const csv = fs.readFileSync(csvPath, 'utf8');
20+
const modelName = path.basename(csvPath, '.csv');
21+
22+
console.log('Bloop is auditing: ' + csvPath + '\n');
23+
24+
const input = `Audit the following ML model metrics CSV.
25+
File: ${modelName}
26+
Produce your full 4-section report: WHERE it fails / WHY it fails / HOW to fix it / BLOOP SCORE (1-10).
27+
28+
${csv}`;
29+
30+
const raw = await query(agent, input);
31+
32+
// Debug: see exactly what comes back
33+
console.log('RAW TYPE:', typeof raw);
34+
console.log('RAW VALUE:', JSON.stringify(raw, null, 2));

docs/assets/cat_angry.png

449 KB
Loading

docs/assets/cat_happy.png

303 KB
Loading

0 commit comments

Comments
 (0)