Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ vouch evidence import junit .vouch/artifacts/pytest.xml
vouch gate
```

JUnit covers required-test obligations only. Behavior, security, runtime, and
rollback obligations need their own evidence.
JUnit covers required-test obligations only. `security_check` artifacts can use
SARIF 2.1.0 when scanner rules or results reference exact obligation IDs.
Behavior, runtime, and rollback obligations need their own evidence.

## Install

Expand Down
2 changes: 1 addition & 1 deletion ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Implemented today:
- Manifest creation from changed files and owned paths.
- Artifact attachment with obligation inference.
- JUnit test-map adapter for raw pytest-style JUnit evidence.
- SARIF 2.1.0 import for `security_check` evidence with exact obligation-ID mapping.
- Machine-readable gate result artifact output for status checks.
- Release policy files loaded from `.vouch/policy/release-policy.json`.
- Policy simulation command with structured policy input/output.
Expand Down Expand Up @@ -200,7 +201,6 @@ Planned work:
- Typed API/signature obligation suggestions.
- Coverage report import.
- Static analysis import.
- SARIF import.
- Secret scanning import.
- Logging and PII scanner import.
- Migration and external-effect detectors.
Expand Down
5 changes: 5 additions & 0 deletions docs/COMPILER.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,11 @@ vouch --repo DIR gate
JUnit covers `required_test` obligations only. Missing behavior, security,
runtime, or rollback evidence can still block.

`security_check` artifacts can also be SARIF 2.1.0 logs. Vouch treats SARIF as
scanner evidence only when rules or result properties reference exact compiled
obligation IDs. High or critical mapped SARIF results become blocking findings;
unmapped scanner output is not treated as contract evidence.

## What Is Proven

The repo-local benchmark is VouchBench:
Expand Down
8 changes: 8 additions & 0 deletions docs/site/compiler.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,3 +110,11 @@ The current decisions are:
- `auto_merge`

`gate` exits non-zero only when the final decision is `block`.

## Evidence Model

Manifest-backed `security_check` artifacts can be generic exact-ID JSON or
SARIF 2.1.0 scanner logs. SARIF rules and result properties must reference exact
compiled obligation IDs. High or critical mapped SARIF results enter the normal
blocking finding and policy path; unmapped scanner output does not satisfy a
contract obligation.
5 changes: 3 additions & 2 deletions docs/site/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ vouch evidence import junit .vouch/artifacts/pytest.xml
vouch gate
```

JUnit covers required-test obligations only. Behavior, security, runtime, and
rollback obligations need their own evidence.
JUnit covers required-test obligations only. `security_check` artifacts can use
SARIF 2.1.0 when scanner rules or results reference exact obligation IDs.
Behavior, runtime, and rollback obligations need their own evidence.

## Status

Expand Down
5 changes: 2 additions & 3 deletions internal/vouch/evidence.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,10 +162,9 @@ func artifactCoverageByKind(artifacts []ArtifactResult) map[EvidenceKind]map[str

func importVerifierFindings(evidence *Evidence) {
for _, result := range evidence.ArtifactResults {
if result.VerifierOutput == nil {
continue
if result.VerifierOutput != nil {
evidence.VerifierOutputs = append(evidence.VerifierOutputs, *result.VerifierOutput)
}
evidence.VerifierOutputs = append(evidence.VerifierOutputs, *result.VerifierOutput)
evidence.Findings = append(evidence.Findings, result.VerifierFindings...)
}
}
Expand Down
7 changes: 7 additions & 0 deletions internal/vouch/evidence_artifacts.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,13 @@ func LinkEvidenceArtifacts(repo string, manifest Manifest, artifacts []EvidenceA
result.addIssue("junit_import", issue)
}
}
} else if artifact.Kind == EvidenceSecurityCheck && len(data) > 0 && sarifLooksLike(data) {
covered, findings, issues := importSARIFEvidence(data, artifact.Obligations, index)
result.CoveredObligations = covered
result.VerifierFindings = findings
for _, issue := range issues {
result.addIssue("sarif_import", issue)
}
} else if len(data) > 0 {
covered, issues := importGenericEvidence(data, artifact.Obligations)
result.CoveredObligations = covered
Expand Down
118 changes: 118 additions & 0 deletions internal/vouch/evidence_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1026,6 +1026,86 @@ func TestGenericArtifactRequiresExactObligationTokens(t *testing.T) {
}
}

func TestSARIFSecurityEvidenceCoversReferencedObligation(t *testing.T) {
repo, manifestPath, ids := writeFullyCoveredUIScenario(t, nil)
securityID := ids[ObligationSecurity]
manifest := mustLoadManifest(t, manifestPath)
setArtifactPath(t, &manifest, "security", "artifacts/security.sarif")
writeJSON(t, manifestPath, manifest)
writeSARIFArtifact(t, repo, "artifacts/security.sarif", sarifSecurityLog(securityID, nil))

evidence, err := CollectEvidence(repo, manifestPath)
if err != nil {
t.Fatal(err)
}
if evidence.Decision != "auto_merge" {
t.Fatalf("expected SARIF security evidence to pass, got %s: findings=%#v invalid=%#v", evidence.Decision, evidence.Findings, evidence.InvalidEvidence)
}
if !artifactCovered(evidence, "security", securityID) {
t.Fatalf("expected SARIF artifact to cover security obligation: %#v", evidence.ArtifactResults)
}
if hasInvalidEvidence(evidence, "security", "sarif_import") {
t.Fatalf("expected SARIF artifact to import cleanly: %#v", evidence.InvalidEvidence)
}
}

func TestSARIFHighSecurityFindingBlocks(t *testing.T) {
repo, manifestPath, ids := writeFullyCoveredUIScenario(t, nil)
securityID := ids[ObligationSecurity]
manifest := mustLoadManifest(t, manifestPath)
setArtifactPath(t, &manifest, "security", "artifacts/security.sarif")
writeJSON(t, manifestPath, manifest)
writeSARIFArtifact(t, repo, "artifacts/security.sarif", sarifSecurityLog(
"rules.no-hardcoded-secret",
map[string]any{
"severity": "warning",
"tags": []string{securityID},
"security-severity": "8.2",
},
sarifFindingResult("rules.no-hardcoded-secret", "warning", "hardcoded secret in changed file"),
))

evidence, err := CollectEvidence(repo, manifestPath)
if err != nil {
t.Fatal(err)
}
if evidence.Decision != "block" {
t.Fatalf("expected high SARIF finding to block, got %s", evidence.Decision)
}
if artifactCovered(evidence, "security", securityID) {
t.Fatalf("expected blocked SARIF obligation to remain uncovered: %#v", evidence.ArtifactResults)
}
if !hasFinding(evidence, "sarif", "semgrep reported high-severity finding rules.no-hardcoded-secret") {
t.Fatalf("expected imported SARIF finding: %#v", evidence.Findings)
}
if !contains(evidence.PolicyResult.RulesFired, "block_verifier_findings") {
t.Fatalf("expected policy to block on SARIF finding: %#v", evidence.PolicyResult)
}
}

func TestSARIFRequiresExactObligationIDs(t *testing.T) {
data, err := json.Marshal(sarifSecurityLog("rules.near-match", map[string]any{
"tags": []string{"obligation.one_extra"},
}))
if err != nil {
t.Fatal(err)
}
covered, findings, issues := importSARIFEvidence(data, []string{"obligation.one"}, ObligationIndex{
ByID: map[string]Obligation{
"obligation.one": {ID: "obligation.one"},
},
})
if len(covered) != 0 {
t.Fatalf("expected near-match SARIF reference not to cover, got %#v", covered)
}
if len(findings) != 0 {
t.Fatalf("expected no finding for unmapped SARIF result, got %#v", findings)
}
if !containsSubstring(issues, "SARIF does not reference obligation obligation.one") {
t.Fatalf("expected exact-ID issue, got %#v", issues)
}
}

func TestJUnitErrorsAndSkipsInvalidateArtifact(t *testing.T) {
covered, failed, issues := importJUnitEvidence([]byte(`<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="suite" tests="3" failures="0" errors="1" skipped="1">
Expand Down Expand Up @@ -2291,6 +2371,44 @@ func writeVerifierOutputArtifact(t *testing.T, repo string, relPath string, outp
writeArtifact(t, repo, relPath, string(append(data, '\n')))
}

func writeSARIFArtifact(t *testing.T, repo string, relPath string, log map[string]any) {
t.Helper()
data, err := json.MarshalIndent(log, "", " ")
if err != nil {
t.Fatal(err)
}
writeArtifact(t, repo, relPath, string(append(data, '\n')))
}

func sarifSecurityLog(ruleID string, properties map[string]any, results ...map[string]any) map[string]any {
rule := map[string]any{"id": ruleID}
if len(properties) > 0 {
rule["properties"] = properties
}
return map[string]any{
"version": sarifVersion,
"runs": []any{map[string]any{
"tool": map[string]any{
"driver": map[string]any{
"name": "semgrep",
"rules": []any{rule},
},
},
"results": results,
}},
}
}

func sarifFindingResult(ruleID string, level string, message string) map[string]any {
return map[string]any{
"ruleId": ruleID,
"level": level,
"message": map[string]any{
"text": message,
},
}
}

func writeEvidenceBundle(t *testing.T, repo string, relPath string, manifest Manifest, artifact EvidenceArtifact, mutate func(EvidenceArtifact, *EvidenceBundle)) {
t.Helper()
artifactData := mustReadFile(t, filepath.Join(repo, artifact.Path))
Expand Down
3 changes: 3 additions & 0 deletions internal/vouch/onboarding.go
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,9 @@ func importArtifactCoverage(data []byte, kind EvidenceKind, candidateObligations
covered, _, issues := importJUnitEvidence(data, candidateObligations)
return covered, issues
}
if kind == EvidenceSecurityCheck && sarifLooksLike(data) {
return importSARIFReferences(data, candidateObligations)
}
return importGenericEvidence(data, candidateObligations)
}

Expand Down
34 changes: 34 additions & 0 deletions internal/vouch/onboarding_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,40 @@ func TestAttachArtifactInfersCoveredObligations(t *testing.T) {
}
}

func TestAttachArtifactInfersSARIFSecurityObligations(t *testing.T) {
repo := initializedRepo(t)
spec := createSampleContract(t, repo, RiskMedium)
_, err := CreateManifest(repo, ManifestCreateOptions{
TaskID: "agent-1",
Summary: "change app service",
Agent: "codex",
RunID: "run-1",
ChangedFiles: []string{"src/app/service.py"},
Out: ".vouch/manifests/agent-1.json",
})
if err != nil {
t.Fatal(err)
}
securityID := obligationID(t, spec, ObligationSecurity, "project paths stay inside repo")
writeSARIFArtifact(t, repo, ".vouch/artifacts/security.sarif", sarifSecurityLog(securityID, nil))

_, artifact, err := AttachArtifact(repo, AttachArtifactOptions{
ManifestPath: ".vouch/manifests/agent-1.json",
ID: "security",
Kind: EvidenceSecurityCheck,
Path: ".vouch/artifacts/security.sarif",
Command: "semgrep scan --sarif",
ExitCode: 0,
Out: ".vouch/manifests/agent-1.with-security.json",
})
if err != nil {
t.Fatal(err)
}
if !contains(artifact.Obligations, securityID) {
t.Fatalf("expected inferred SARIF obligation %s, got %#v", securityID, artifact.Obligations)
}
}

func TestAttachArtifactRejectsNonZeroExitAndPathEscape(t *testing.T) {
repo := initializedRepo(t)
createSampleContract(t, repo, RiskMedium)
Expand Down
Loading
Loading