From 7546ed62c8d6d240b42cd5124b9fe18eae94802e Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Tue, 3 Dec 2024 20:40:10 +0600 Subject: [PATCH] refactor(test): run Trivy once Signed-off-by: Nikita Pivkin --- Makefile | 2 +- integration/check_examples_test.go | 157 ++++++++++++++++++----------- 2 files changed, 100 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 2f050ec1..1353b715 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ test: .PHONY: integration-test test-integration: - go test -v -timeout 15m -tags=integration ./integration/... + go test -v -timeout 5m -tags=integration ./integration/... .PHONY: rego rego: fmt-rego test-rego diff --git a/integration/check_examples_test.go b/integration/check_examples_test.go index fdc33489..f92141a2 100644 --- a/integration/check_examples_test.go +++ b/integration/check_examples_test.go @@ -3,11 +3,12 @@ package integration import ( - "fmt" "io/fs" "os" "os/exec" "path/filepath" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -22,6 +23,49 @@ import ( func TestValidateCheckExamples(t *testing.T) { cacheDir := setupCache(t) + targetDir := setupTarget(t) + outputFile := filepath.Join(t.TempDir(), "report.json") + + args := []string{ + "conf", + "--skip-check-update", + "--quiet", + "--format", "json", + "--output", outputFile, + "--cache-dir", cacheDir, + targetDir, + } + runTrivy(t, args) + + report := readTrivyReport(t, outputFile) + + verifyExamples(t, report, targetDir) +} + +func setupCache(t *testing.T) string { + t.Helper() + + cmd := exec.Command("make", "create-bundle") + cmd.Dir = ".." + require.NoError(t, cmd.Run()) + defer os.Remove("../bundle.tar.gz") + + cacheDir := t.TempDir() + + policyDir := filepath.Join(cacheDir, "policy", "content") + require.NoError(t, os.MkdirAll(policyDir, os.ModePerm)) + + cmd = exec.Command("tar", "-zxf", "bundle.tar.gz", "-C", policyDir) + cmd.Dir = ".." + require.NoError(t, cmd.Run()) + + return cacheDir +} + +func setupTarget(t *testing.T) string { + t.Helper() + + targetDir := t.TempDir() // TODO(nikpivkin): load examples from fs rego.LoadAndRegister() @@ -32,85 +76,82 @@ func TestValidateCheckExamples(t *testing.T) { continue } - t.Run(r.AVDID, func(t *testing.T) { - examples, path, err := examples.GetCheckExamples(r.Rule) - require.NoError(t, err) + examples, path, err := examples.GetCheckExamples(r.Rule) + require.NoError(t, err) - if path == "" { - return - } + if path == "" { + continue + } - for provider, providerExamples := range examples { - validateExamples(t, providerExamples.Bad.ToStrings(), provider, cacheDir, r.AVDID, true) - validateExamples(t, providerExamples.Good.ToStrings(), provider, cacheDir, r.AVDID, false) - } - }) + for provider, providerExamples := range examples { + writeExamples(t, providerExamples.Bad.ToStrings(), provider, targetDir, r.AVDID, "bad") + writeExamples(t, providerExamples.Good.ToStrings(), provider, targetDir, r.AVDID, "good") + } } + + return targetDir } -func validateExamples(t *testing.T, examples []string, provider, cacheDir, avdID string, expected bool) { +func writeExamples(t *testing.T, examples []string, provider, cacheDir string, id string, typ string) { for i, example := range examples { - fileName := fmt.Sprintf("test-%d%s", i, extensionByProvider(provider)) - t.Run(fileName, func(t *testing.T) { - targetFile := filepath.Join(t.TempDir(), fileName) - - require.NoError(t, os.WriteFile(targetFile, []byte(example), fs.ModePerm)) - - outputFile := filepath.Join(t.TempDir(), "report.json") - - args := []string{ - "conf", - "--skip-check-update", - "--quiet", - "--format", "json", - "--output", outputFile, - "--cache-dir", cacheDir, - targetFile, - } - runTrivy(t, args) - - report := readTrivyReport(t, outputFile) - - assert.Equal(t, expected, reportContainsMisconfig(report, fileName, avdID)) - }) + name := "test" + extensionByProvider(provider) + file := filepath.Join(cacheDir, id, provider, typ, strconv.Itoa(i), name) + require.NoError(t, os.MkdirAll(filepath.Dir(file), fs.ModePerm)) + require.NoError(t, os.WriteFile(file, []byte(example), fs.ModePerm)) } } -func setupCache(t *testing.T) string { - t.Helper() +func verifyExamples(t *testing.T, report types.Report, targetDir string) { + got := getFailureIDs(report) - cmd := exec.Command("make", "create-bundle") - cmd.Dir = ".." + err := filepath.Walk(targetDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } - require.NoError(t, cmd.Run()) - defer os.Remove("bundle.tar.gz") + relPath, err := filepath.Rel(targetDir, path) + require.NoError(t, err) - cacheDir := t.TempDir() + parts := strings.Split(relPath, string(os.PathSeparator)) + require.Len(t, parts, 5) - policyDir := filepath.Join(cacheDir, "policy", "content") - require.NoError(t, os.MkdirAll(policyDir, os.ModePerm)) + id, _, exampleType := parts[0], parts[1], parts[2] - cmd = exec.Command("tar", "-zxf", "bundle.tar.gz", "-C", policyDir) - cmd.Dir = ".." - require.NoError(t, cmd.Run()) + shouldBePresent := exampleType == "bad" - return cacheDir + t.Run(relPath, func(t *testing.T) { + if shouldBePresent { + ids, exists := got[relPath] + assert.True(t, exists) + assert.Contains(t, ids, id) + } else { + ids, exists := got[relPath] + if exists { + assert.NotContains(t, ids, id) + } + } + }) + return nil + }) + + require.NoError(t, err) } -func reportContainsMisconfig(report types.Report, path string, id string) bool { - for _, res := range report.Results { - if res.Target != path { - continue - } +func getFailureIDs(report types.Report) map[string][]string { + ids := make(map[string][]string) - for _, misconf := range res.Misconfigurations { - if misconf.AVDID == id && misconf.Status == types.MisconfStatusFailure { - return true + for _, result := range report.Results { + for _, misconf := range result.Misconfigurations { + if misconf.Status == types.MisconfStatusFailure { + ids[result.Target] = append(ids[result.Target], misconf.AVDID) } } } - return false + return ids } func extensionByProvider(provider string) string {