add command line flag for system test result dumps

Also fix case of failing tests without validation in the case that the
dump fails.

Author: efd6

cmd/testrunner.go

@@ -421,6 +421,8 @@ func getTestRunnerSystemCommand() *cobra.Command {
 	cmd.Flags().Bool(cobraext.TearDownFlagName, false, cobraext.TearDownFlagDescription)
 	cmd.Flags().Bool(cobraext.NoProvisionFlagName, false, cobraext.NoProvisionFlagDescription)
 
+	cmd.Flags().String(cobraext.TestDumpPrefixFlagName, "", cobraext.TestDumpPrefixFlagDescription)
+
 	cmd.MarkFlagsMutuallyExclusive(cobraext.SetupFlagName, cobraext.TearDownFlagName, cobraext.NoProvisionFlagName)
 	cmd.MarkFlagsRequiredTogether(cobraext.ConfigFileFlagName, cobraext.SetupFlagName)
 
@@ -476,6 +478,11 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.TestCoverageFormatFlagName)
 	}
 
+	dumpPrefix, err := cmd.Flags().GetString(cobraext.TestDumpPrefixFlagName)
+	if err != nil {
+		return cobraext.FlagParsingError(err, cobraext.TestDumpPrefixFlagName)
+	}
+
 	if !slices.Contains(testrunner.CoverageFormatsList(), testCoverageFormat) {
 		return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName)
 	}
@@ -585,6 +592,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		WithCoverage:       testCoverage,
 		CoverageType:       testCoverageFormat,
 		RepositoryRoot:     repositoryRoot,
+		DumpPrefix:         dumpPrefix,
 	})
 
 	logger.Debugf("Running suite...")

internal/cobraext/flags.go

@@ -209,6 +209,9 @@ const (
 	TestCoverageFormatFlagName        = "coverage-format"
 	TestCoverageFormatFlagDescription = "set format for coverage reports: %s"
 
+	TestDumpPrefixFlagName        = "dump"
+	TestDumpPrefixFlagDescription = "prefix for system test dump file"
+
 	VariantFlagName        = "variant"
 	VariantFlagDescription = "service variant"
 

internal/testrunner/runners/system/runner.go

@@ -40,6 +40,7 @@ type runner struct {
 	generateTestResult bool
 	withCoverage       bool
 	coverageType       string
+	dumpPrefix         string
 
 	configFilePath string
 	runSetup       bool
@@ -78,6 +79,7 @@ type SystemTestRunnerOptions struct {
 	DeferCleanup       time.Duration
 	WithCoverage       bool
 	CoverageType       string
+	DumpPrefix         string
 }
 
 func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
@@ -100,6 +102,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
 		withCoverage:       options.WithCoverage,
 		coverageType:       options.CoverageType,
 		repositoryRoot:     options.RepositoryRoot,
+		dumpPrefix:         options.DumpPrefix,
 	}
 
 	r.resourcesManager = resources.NewManager()
@@ -265,6 +268,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
 					GlobalTestConfig:   r.globalTestConfig,
 					WithCoverage:       r.withCoverage,
 					CoverageType:       r.coverageType,
+					DumpPrefix:         r.dumpPrefix,
 				})
 				if err != nil {
 					return nil, fmt.Errorf(

internal/testrunner/runners/system/tester.go

@@ -221,6 +221,7 @@ type tester struct {
 	dataStreamManifest *packages.DataStreamManifest
 	withCoverage       bool
 	coverageType       string
+	dumpPrefix         string
 
 	serviceStateFilePath string
 
@@ -253,6 +254,7 @@ type SystemTesterOptions struct {
 	GlobalTestConfig testrunner.GlobalRunnerTestConfig
 	WithCoverage     bool
 	CoverageType     string
+	DumpPrefix       string
 
 	RunSetup     bool
 	RunTearDown  bool
@@ -705,7 +707,6 @@ func (r *tester) runTestPerVariant(ctx context.Context, stackConfig stack.Config
 	logger.Debugf("Using config: %q", testConfig.Name())
 
 	partial, err := r.runTest(ctx, testConfig, stackConfig, svcInfo)
-
 	tdErr := r.tearDownTest(ctx)
 	if err != nil {
 		return partial, err
@@ -1817,32 +1818,26 @@ func (r *tester) runTest(ctx context.Context, config *testConfig, stackConfig st
 		return results, nil
 	}
 
-	if dump, ok := os.LookupEnv(dumpScenarioDocsEnv); ok && dump != "" {
-		err := dumpScenarioDocs(scenario.docs)
+	var dumpPath string
+	switch {
+	case r.dumpPrefix != "":
+		dumpPath = fmt.Sprintf("%s-%s.json", r.dumpPrefix, time.Now().Format("20060102150405"))
+	case os.Getenv(dumpScenarioDocsEnv) != "":
+		dumpPath = filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", time.Now().Format("20060102150405")))
+	}
+	var dumpErr error
+	if dumpPath != "" {
+		err := dumpScenarioDocs(scenario.docs, dumpPath)
 		if err != nil {
-			return nil, fmt.Errorf("failed to dump scenario docs: %w", err)
+			dumpErr = fmt.Errorf("failed to dump scenario docs: %w", err)
 		}
 	}
 
-	return r.validateTestScenario(ctx, result, scenario, config)
-}
-
-func (r *tester) isTestUsingOTELCollectorInput(policyTemplateInput string) bool {
-	// Just supported for input packages currently
-	if r.pkgManifest.Type != "input" {
-		return false
-	}
-
-	if policyTemplateInput != otelCollectorInputName {
-		return false
-	}
-
-	return true
+	results, err := r.validateTestScenario(ctx, result, scenario, config)
+	return results, errors.Join(err, dumpErr)
 }
 
-func dumpScenarioDocs(docs any) error {
-	timestamp := time.Now().Format("20060102150405")
-	path := filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", timestamp))
+func dumpScenarioDocs(docs any, path string) error {
 	f, err := os.Create(path)
 	if err != nil {
 		return fmt.Errorf("failed to create dump file: %w", err)
@@ -1860,6 +1855,19 @@ func dumpScenarioDocs(docs any) error {
 	return nil
 }
 
+func (r *tester) isTestUsingOTELCollectorInput(policyTemplateInput string) bool {
+	// Just supported for input packages currently
+	if r.pkgManifest.Type != "input" {
+		return false
+	}
+
+	if policyTemplateInput != otelCollectorInputName {
+		return false
+	}
+
+	return true
+}
+
 func (r *tester) checkEnrolledAgents(ctx context.Context, agentInfo agentdeployer.AgentInfo, svcInfo servicedeployer.ServiceInfo) (*kibana.Agent, error) {
 	var agents []kibana.Agent