validator_test.go

// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package validator

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"testing"

	cp "github.com/otiai10/copy"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/elastic/package-spec/v3/code/go/internal/linkedfiles"
	"github.com/elastic/package-spec/v3/code/go/internal/validator/common"
	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
)

func TestValidateFile(t *testing.T) {
	// Workaround for error messages that contain OS-dependant base paths.
	osTestBasePath := filepath.Join("..", "..", "..", "..", "test", "packages") + string(filepath.Separator)

	tests := map[string]struct {
		invalidPkgFilePath  string
		expectedErrContains []string
	}{
		"good":                               {},
		"good_v2":                            {},
		"good_v3":                            {},
		"good_input":                         {},
		"good_input_otel":                    {},
		"good_content":                       {},
		"good_lookup_index":                  {},
		"good_alert_rule_templates":          {},
		"deploy_custom_agent":                {},
		"deploy_custom_agent_multi_services": {},
		"deploy_docker":                      {},
		"deploy_terraform":                   {},
		"missing_data_stream":                {},
		"icons_dark_mode":                    {},
		"ignored_malformed":                  {},
		"custom_ilm_policy":                  {},
		"profiling_symbolizer":               {},
		"logs_synthetic_mode":                {},
		"kibana_configuration_links":         {},
		"with_links":                         {},
		"bad_duration_vars": {
			"manifest.yml",
			[]string{
				`field vars.1: Must not be present`,
				`field vars.2: Must not be present`,
			},
		},
		"bad_additional_content": {
			"bad-bad",
			[]string{
				"directory name inside package bad_additional_content contains -: bad-bad",
			},
		},
		"bad_deploy_variants": {
			"_dev/deploy/variants.yml",
			[]string{
				"field (root): default is required",
				"field variants: Invalid type. Expected: object, given: array",
			},
		},
		"missing_pipeline_dashes": {
			"data_stream/foo/elasticsearch/ingest_pipeline/default.yml",
			[]string{
				"document dashes are required (start the document with '---')",
			},
		},
		"missing_image_files": {
			"manifest.yml",
			[]string{
				"field screenshots.0.src: relative path is invalid, target doesn't exist or it exceeds the file size limit",
				"field icons.0.src: relative path is invalid, target doesn't exist or it exceeds the file size limit",
			},
		},
		"integration_benchmarks": {},
		"input_template":         {},
		"input_groups":           {},
		"input_groups_bad_data_stream": {
			"manifest.yml",
			[]string{
				"field policy_templates.2.data_streams.1: data stream doesn't exist",
			},
		},
		"bad_github_owner": {
			"manifest.yml",
			[]string{
				"field owner.github: Does not match pattern '^(([a-zA-Z0-9-_]+)|([a-zA-Z0-9-_]+\\/[a-zA-Z0-9-_]+))$'",
			},
		},
		"bad_owner_type": {
			"manifest.yml",
			[]string{
				`field owner.type: owner.type must be one of the following: "elastic", "partner", "community"`,
			},
		},
		"bad_owner_type_missing": {
			"manifest.yml",
			[]string{
				`field owner: type is required`,
			},
		},
		"missing_version": {
			"manifest.yml",
			[]string{
				"package version undefined in the package manifest file",
			},
		},
		"deploy_custom_agent_invalid_property": {
			"_dev/deploy/agent/custom-agent.yml",
			[]string{
				"field services.docker-custom-agent: Must not be present",
			},
		},
		"invalid_field_for_version": {
			"manifest.yml",
			[]string{
				"field (root): Additional property license is not allowed",
			},
		},
		"bad_release_tag": {
			"manifest.yml",
			[]string{
				"field (root): Additional property release is not allowed",
			},
		},
		"bad_custom_ilm_policy": {
			"data_stream/test/manifest.yml",
			[]string{
				fmt.Sprintf("field ilm_policy: ILM policy \"logs-bad_custom_ilm_policy.test-notexists\" not found in package, expected definition in \"%sbad_custom_ilm_policy/data_stream/test/elasticsearch/ilm/notexists.json\"", osTestBasePath),
			},
		},
		"bad_select": {
			"data_stream/foo_stream/manifest.yml",
			[]string{
				"field streams.0.vars.1: options is required",
				"field streams.0.vars.3: Must not be present",
			},
		},
		"bad_skip_ignored_fields": {
			"data_stream/foo/_dev/test/system/test-default-config.yml",
			[]string{
				"field skip_ignored_fields: Invalid type. Expected: array, given: boolean",
			},
		},
		"bad_profiling_symbolizer": {
			"data_stream/example/manifest.yml",
			[]string{
				"profiling data type cannot be used in GA packages",
			},
		},
		"bad_secret_vars": {
			"manifest.yml",
			[]string{
				"field vars.0: Additional property secret is not allowed",
			},
		},
		"bad_secret_vars_v3": {
			"manifest.yml",
			[]string{
				"field vars.0: variable identified as possible secret, secret parameter required to be set to true or false",
				"field vars.1: variable identified as possible secret, secret parameter required to be set to true or false",
			},
		},
		"bad_lifecycle": {
			"data_stream/test/lifecycle.yml",
			[]string{
				"field (root): Additional property max_age is not allowed",
			},
		},
		"bad_saved_object_tags": {
			"kibana/tags.yml",
			[]string{
				`field 0.asset_types.11: 0.asset_types.11 must be one of the following: "dashboard", "visualization", "search", "map", "lens", "index_pattern", "security_rule", "csp_rule_template", "alerting_rule_template", "slo_template", "ml_module", "osquery_pack_asset", "osquery_saved_query"`,
				`field 0.asset_types.12: 0.asset_types.12 must be one of the following: "dashboard", "visualization", "search", "map", "lens", "index_pattern", "security_rule", "csp_rule_template", "alerting_rule_template", "slo_template", "ml_module", "osquery_pack_asset", "osquery_saved_query"`,
				`field 1.asset_ids.1: Invalid type. Expected: string, given: integer`,
				`field 2: text is required`,
				`field 3: asset_types is required`,
			},
		},
		"bad_dotted_fields": {
			"manifest.yml",
			[]string{
				"field conditions: Additional property elastic.subscription is not allowed",
				"field conditions: Additional property kibana.version is not allowed",
			},
		},
		"bad_dangling_object_ids": {
			"kibana/dashboard/bad_dangling_object_ids-82273ffe-6acc-4f2f-bbee-c1004abba63d.json",
			[]string{
				`dangling reference found: bad_dangling_object_ids-8287a5d5-1576-4f3a-83c4-444e9058439c (search) (SVR00003)`,
			},
		},
		"kibana_legacy_visualizations": {
			"kibana/dashboard/kibana_legacy_visualizations-c36e9b90-596c-11ee-adef-4fe896364076.json",
			[]string{
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"TSVB time series\" (timeseries, TSVB)",
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"TSVB gauge\" (gauge, TSVB)",
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"Aggs-based table\" (table, Aggs-based)",
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"Aggs-based tag cloud\" (tagcloud, Aggs-based)",
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"\" (heatmap, Aggs-based)",
				"\"Dashboard with mixed by-value visualizations\" contains legacy visualization: \"Timelion time series\" (timelion, Timelion)",
			},
		},
		"bad_deployment_mode": {
			"manifest.yml",
			[]string{
				`field policy_templates.0.deployment_modes: Additional property default is not allowed`,
				`field policy_templates.0.inputs.0.vars.0.hide_in_deployment_modes.0: policy_templates.0.inputs.0.vars.0.hide_in_deployment_modes.0 must be one of the following: "agentless"`,
			},
		},
		"bad_deployment_mode_without_identities": {
			"manifest.yml",
			[]string{
				`field policy_templates.0.deployment_modes.agentless: organization is required`,
				`field policy_templates.0.deployment_modes.agentless: division is required`,
				`field policy_templates.0.deployment_modes.agentless: team is required`,
			},
		},
		"bad_deployment_mode_resources": {
			"manifest.yml",
			[]string{
				`field policy_templates.0.deployment_modes.agentless.resources.requests: Additional property disk is not allowed`,
			},
		},
		"bad_input_dataset_vars": {
			"_dev/test/policy/test-vars.yml",
			[]string{
				`field vars.data_stream.dataset: Does not match pattern '^[a-zA-Z0-9]+[a-zA-Z0-9\._]*$'`,
			},
		},
		"bad_integration_dataset_vars": {
			"data_stream/datasets/_dev/test/system/test-vars-config.yml",
			[]string{
				`field vars.data_stream.dataset: Does not match pattern '^[a-zA-Z0-9]+[a-zA-Z0-9\._]*$'`,
				`field data_stream.vars.data_stream.dataset: Does not match pattern '^[a-zA-Z0-9]+[a-zA-Z0-9\._]*$'`,
			},
		},
		"bad_missing_capability_security_rules": {
			"manifest.yml",
			[]string{
				"found security rule assets in package but security capability is missing",
			},
		},
		"bad_policy_template_behavior": {
			"manifest.yml",
			[]string{
				"field policy_templates_behavior: policy_templates_behavior must be one of the following: \"all\"",
			},
		},
		"bad_configuration_links": {
			"manifest.yml",
			[]string{
				"field policy_templates.0.configuration_links: Array must have at least 1 items",
				"field policy_templates.1.configuration_links.0: url is required",
				"field policy_templates.1.configuration_links.1.url: Does not match pattern '^(http(s)?://|kbn:/)'",
				"field policy_templates.1.configuration_links.2.url: Does not match pattern '^(http(s)?://|kbn:/)'",
			},
		},
		"bad_required_vars": {
			"manifest.yml",
			[]string{
				`field policy_templates.0.inputs.0.required_vars.password.1: name is required`,
				`required var "api_key" in optional group is defined as always required`,
				`required var "password" in optional group is not defined`,
			},
		},
		"bad_required_vars_data_streams": {
			"data_stream/test/manifest.yml",
			[]string{
				`field streams.0.required_vars.empty_name.0: name is required`,
				`required var "api_key" in optional group is defined as always required`,
				`required var "password" in optional group is not defined`,
			},
		},
		"bad_input_deployment_modes": {
			"manifest.yml",
			[]string{
				`field policy_templates.0.inputs.0.deployment_modes.0: policy_templates.0.inputs.0.deployment_modes.0 must be one of the following: "default", "agentless"`,
				`field policy_templates.0.inputs.1.deployment_modes: Array must have at least 1 items`,
				`field policy_templates.0.inputs.2.deployment_modes: array items[0,1] must be unique`,
				`input "test/metrics" in policy template "test" specifies unsupported deployment mode "invalid_mode"`,
				`input "test/system" in policy template "test" specifies unsupported deployment mode "agentless"`,
				`policy template "unsupported_modes" enables deployment mode "default" but no input supports this mode`,
			},
		},
		"bad_discovery_fields": {
			"manifest.yml",
			[]string{
				"field discovery.fields.0.name: Invalid type. Expected: string, given: integer",
				"field discovery.fields.1: name is required",
				"field discovery.fields.2: name is required",
				"field discovery.fields.2: Additional property value is not allowed",
				"field discovery.datasets.0.name: Invalid type. Expected: string, given: integer",
				"field discovery.datasets.1: Additional property foo is not allowed",
			},
		},
		"bad_input_otel_old_version": {
			"manifest.yml",
			[]string{
				"field policy_templates.0.input: Must not be present",
			},
		},
		"bad_input_template_path": {
			"manifest.yml",
			[]string{
				"field policy_templates.0: template_path is required",
				"policy template \"sql_query\" references template_path \"\": template_path is required for input type packages",
			},
		},
		"bad_agent_version_v3": {
			"manifest.yml",
			[]string{
				"field conditions.agent: version is required",
			},
		},
		"bad_integration_stream_template_path": {
			"manifest.yml",
			[]string{
				"policy template \"sample\" references input template_path: error validating input from streams \"logfile\": template file not found",
			},
		},
		"bad_integration_stream_template_path_default": {
			"manifest.yml",
			[]string{
				"policy template \"sample\" references input template_path: error validating input from streams \"logfile\": template file not found",
			},
		},
		"bad_integration_input_template_path": {
			"manifest.yml",
			[]string{
				"policy template \"sample\" references input template_path: error validating input \"logfile\": template file not found",
			},
		},
		"bad_esql_view_content": {
			"elasticsearch/esql_view/view.yml",
			[]string{"field query: Invalid type. Expected: string, given: null"},
		},
		"bad_esql_view_integration": {
			"elasticsearch/esql_view/view.yml",
			[]string{"field query: Invalid type. Expected: string, given: null"},
		},
		"bad_content_duplicate_tags": {
			"kibana/tags.yml",
			[]string{"duplicate tag name 'Tag One' found (SVR00007)"},
		},
		"bad_kibana_tag_duplicate": {
			"kibana/tag/bad_tag-security-solution-default.json",
			[]string{"tag name 'Security Solution' is already defined in tags.yml (SVR00007)"},
		},
		"deprecated_integration":              {},
		"deprecated_input":                    {},
		"deprecated_content":                  {},
		"deprecated_integration_policy_input": {},
		"deprecated_integration_policy":       {},
		"deprecated_integration_stream_var":   {},
		"bad_deprecation_description": {
			"manifest.yml",
			[]string{"field deprecated.description: Invalid type. Expected: string, given: null"},
		},
		"bad_deprecation_since": {
			"manifest.yml",
			[]string{"field deprecated: since is required"},
		},
	}

	for pkgName, test := range tests {
		t.Run(pkgName, func(t *testing.T) {
			pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)
			errPrefix := fmt.Sprintf("file \"%s/%s\" is invalid: ", pkgRootPath, test.invalidPkgFilePath)

			errs := ValidateFromPath(pkgRootPath)
			if verrs, ok := errs.(specerrors.ValidationErrors); ok {
				filterConfig, err := specerrors.LoadConfigFilter(os.DirFS(pkgRootPath))
				if !errors.Is(err, os.ErrNotExist) {
					filter := specerrors.NewFilter(filterConfig)
					result, err := filter.Run(verrs)
					require.NoError(t, err)
					assert.Empty(t, result.UnusedProcessors, "There are unused exclusion checks in the validation.yml file")
					errs = result.Processed
				}
			}

			if test.expectedErrContains == nil {
				require.NoError(t, errs)
			} else {
				require.Error(t, errs)
				vErrs, ok := errs.(specerrors.ValidationErrors)
				if ok {
					assert.Len(t, errs, len(test.expectedErrContains))
					var errMessages []string
					for _, vErr := range vErrs {
						errMessages = append(errMessages, vErr.Error())
					}

					for _, expectedErrMessage := range test.expectedErrContains {
						expectedErr := errPrefix + expectedErrMessage
						require.Contains(t, errMessages, expectedErr)
					}
					return
				}
				require.Equal(t, errs.Error(), test.expectedErrContains[0])
			}
		})
	}
}

func TestValidateItemNotAllowed(t *testing.T) {
	tests := map[string]map[string][]string{
		"wrong_kibana_filename": {
			"kibana/dashboard": []string{
				"b7e55b73-97cc-44fd-8555-d01b7e13e70d.json",
				"bad-ecs.json",
				"bad-foobar-ecs.json",
				"bad-Foobaz-ECS.json",
			},
			"kibana/map": []string{
				"06149856-cbc1-4988-a93a-815915c4408e.json",
				"another-package-map.json",
			},
			"kibana/search": []string{
				"691240b5-7ec9-4fd7-8750-4ef97944f960.json",
				"another-package-search.json",
			},
			"kibana/visualization": []string{
				"defa1bcc-1ab6-4069-adec-8c997b069a5e.json",
				"another-package-visualization.json",
			},
		},
		"bad_nested_knowledge_base": {
			"docs/knowledge_base": []string{
				"nested_dir",
				"file.txt",
			},
		},
		"bad_alert_rule_templates": {
			"kibana": []string{
				"alerting_rule_template",
			},
		},
	}

	for pkgName, invalidItemsPerFolder := range tests {
		t.Run(pkgName, func(t *testing.T) {
			requireErrorMessage(t, pkgName, invalidItemsPerFolder, "item [%s] is not allowed in folder [%s/%s]")
		})
	}
}

func TestValidateItemNotExpected(t *testing.T) {
	tests := map[string]map[string][]string{
		"docs_extra_files": {
			"docs": []string{
				".missing",
			},
		},
	}

	for pkgName, invalidItemsPerFolder := range tests {
		t.Run(pkgName, func(t *testing.T) {
			requireErrorMessage(t, pkgName, invalidItemsPerFolder, "item [%s] is not allowed in folder [%s/%s]")
		})
	}
}

func TestValidateBadKibanaIDs(t *testing.T) {
	tests := map[string]map[string][]string{
		"bad_kibana_ids": {
			"kibana/dashboard": []string{
				"bad_kibana_ids-bar-baz.json",
			},
			"kibana/security_rule": []string{
				"bad_kibana_ids-bar-baz.json",
			},
		},
	}

	for pkgName, invalidItemsPerFolder := range tests {
		t.Run(pkgName, func(t *testing.T) {
			pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)

			errs := ValidateFromPath(pkgRootPath)
			require.Error(t, errs)
			vErrs, ok := errs.(specerrors.ValidationErrors)
			require.True(t, ok)

			var errMessages []string
			for _, vErr := range vErrs {
				errMessages = append(errMessages, vErr.Error())
			}

			var c int
			for itemFolder, invalidItems := range invalidItemsPerFolder {
				for _, invalidItem := range invalidItems {
					objectFilePath := path.Join(pkgRootPath, itemFolder, invalidItem)
					expected := fmt.Sprintf("kibana object file [%s] defines non-matching ID", objectFilePath)

					var found bool
					for _, e := range errMessages {
						if strings.HasPrefix(e, expected) {
							found = true
						}
					}
					require.True(t, found, "Missing item: "+expected)
					c++
				}
			}
			require.Equal(t, c, len(errMessages))
		})
	}
}

func TestValidateBadRuleIDs(t *testing.T) {
	tests := map[string]string{
		"bad_rule_ids": "kibana object ID [saved_object_id] should start with rule ID [rule_id]",
	}

	for pkgName, expectedError := range tests {
		t.Run(pkgName, func(t *testing.T) {
			errs := ValidateFromPath(filepath.Join("..", "..", "..", "..", "test", "packages", pkgName))
			require.Error(t, errs)
			vErrs, ok := errs.(specerrors.ValidationErrors)
			require.True(t, ok)

			var errMessages []string
			for _, vErr := range vErrs {
				errMessages = append(errMessages, vErr.Error())
			}
			require.Contains(t, errMessages, expectedError)
		})
	}
}

func TestValidateMissingRequiredFields(t *testing.T) {
	tests := map[string][]string{
		"good":       {},
		"good_v2":    {},
		"good_input": {},
		"missing_required_fields": {
			`expected type "constant_keyword" for required field "data_stream.dataset", found "keyword" in "../../../../test/packages/missing_required_fields/data_stream/foo/fields/base-fields.yml"`,
			`expected field "data_stream.type" with type "constant_keyword" not found in datastream "foo"`,
			// `expected field "data_stream.namespace" with type "constant_keyword" not found in transform "good_example_abc_1"`,
			// `expected type "date" for required field "@timestamp", found "long" in "../../../../test/packages/missing_required_fields/elasticsearch/transform/good_example_abc_1/fields/base-fields.yml"`,
		},
		"missing_required_fields_input": {
			`expected type "constant_keyword" for required field "data_stream.dataset", found "keyword" in "../../../../test/packages/missing_required_fields_input/fields/base-fields.yml"`,
			`expected field "data_stream.type" with type "constant_keyword" not found`,
		},
	}

	for pkgName, expectedErrors := range tests {
		t.Run(pkgName, func(t *testing.T) {
			pkgRootPath := path.Join("..", "..", "..", "..", "test", "packages", pkgName)
			err := ValidateFromPath(pkgRootPath)
			if len(expectedErrors) == 0 {
				assert.NoError(t, err)
				return
			}
			assert.Error(t, err)

			errs, ok := err.(specerrors.ValidationErrors)
			require.True(t, ok)
			assert.Len(t, errs, len(expectedErrors))

			for _, expectedError := range expectedErrors {
				found := false
				for _, foundError := range errs {
					if foundError.Error() == expectedError {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("expected error: %q (%v)", expectedError, errs)
				}
			}
		})
	}
}

func TestValidateVersionIntegrity(t *testing.T) {
	tests := map[string]string{
		"inconsistent_version": "current manifest version doesn't have changelog entry",
		"same_version_twice":   "versions in changelog must be unique, found at least two same versions (0.0.2)",
	}

	for pkgName, expectedErrorMessage := range tests {
		t.Run(pkgName, func(t *testing.T) {
			errs := ValidateFromPath(filepath.Join("..", "..", "..", "..", "test", "packages", pkgName))
			require.Error(t, errs)
			vErrs, ok := errs.(specerrors.ValidationErrors)
			require.True(t, ok)

			var errMessages []string
			for _, vErr := range vErrs {
				errMessages = append(errMessages, vErr.Error())
			}
			require.Contains(t, errMessages, expectedErrorMessage)
		})
	}
}

func TestValidateDuplicatedFields(t *testing.T) {
	tests := map[string][]string{
		"good":       {},
		"good_v2":    {},
		"good_input": {},
		"bad_duplicated_fields": {
			"field \"event.dataset\" is defined multiple times for data stream \"wrong\", found in: ../../../../test/packages/bad_duplicated_fields/data_stream/wrong/fields/base-fields.yml, ../../../../test/packages/bad_duplicated_fields/data_stream/wrong/fields/ecs.yml",
			"field \"field1\" is defined multiple times for transform \"good_example_abc_1\", found in: ../../../../test/packages/bad_duplicated_fields/elasticsearch/transform/good_example_abc_1/fields/fields.yml, ../../../../test/packages/bad_duplicated_fields/elasticsearch/transform/good_example_abc_1/fields/more-fields.yml",
		},
		"bad_duplicated_fields_input": {
			"field \"event.dataset\" is defined multiple times, found in: ../../../../test/packages/bad_duplicated_fields_input/fields/base-fields.yml, ../../../../test/packages/bad_duplicated_fields_input/fields/ecs.yml",
		},
	}

	for pkgName, expectedErrorMessages := range tests {
		t.Run(pkgName, func(t *testing.T) {
			errs := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName))
			if len(expectedErrorMessages) == 0 {
				assert.NoError(t, errs)
				return
			}
			require.Error(t, errs)
			vErrs, ok := errs.(specerrors.ValidationErrors)
			require.True(t, ok)

			assert.Len(t, vErrs, len(expectedErrorMessages))

			for _, expectedError := range expectedErrorMessages {
				found := false
				for _, foundError := range vErrs {
					if foundError.Error() == expectedError {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("expected error: %q (%v)", expectedError, errs)
				}
			}
		})
	}
}

func TestValidateMinimumKibanaVersions(t *testing.T) {
	tests := map[string][]string{
		"good":       {},
		"good_input": {},
		"good_v2":    {},
		"custom_logs": {
			"conditions.kibana.version must be ^8.8.0 or greater for non experimental input packages (version > 1.0.0)",
		},
		"httpjson_input": {
			"conditions.kibana.version must be ^8.8.0 or greater for non experimental input packages (version > 1.0.0)",
		},
		"sql_input": {
			"conditions.kibana.version must be ^8.8.0 or greater for non experimental input packages (version > 1.0.0)",
		},
		"bad_runtime_kibana_version": {
			"conditions.kibana.version must be ^8.10.0 or greater to include runtime fields",
		},
	}

	for pkgName, expectedErrorMessages := range tests {
		t.Run(pkgName, func(t *testing.T) {
			err := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName))
			if len(expectedErrorMessages) == 0 {
				assert.NoError(t, err)
				return
			}
			assert.Error(t, err)

			errs, ok := err.(specerrors.ValidationErrors)
			require.True(t, ok)
			assert.Len(t, errs, len(expectedErrorMessages))

			for _, expectedError := range expectedErrorMessages {
				found := false
				for _, foundError := range errs {
					if foundError.Error() == expectedError {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("expected error: %q, found: %q", expectedError, errs)
				}
			}
		})
	}
}

func TestValidateWarnings(t *testing.T) {
	tests := map[string][]string{
		"good":    {},
		"good_v2": {},
		"visualizations_by_reference": {
			"references found in dashboard kibana/dashboard/visualizations_by_reference-82273ffe-6acc-4f2f-bbee-c1004abba63d.json: visualizations_by_reference-5e1a01ff-6f9a-41c1-b7ad-326472db42b6 (visualization), visualizations_by_reference-8287a5d5-1576-4f3a-83c4-444e9058439b (lens) (SVR00004)",
		},
		"bad_saved_object_tags_kibana_version": {
			"conditions.kibana.version must be ^8.10.0 or greater to include saved object tags file: kibana/tags.yml (SVR00005)",
		},
		"bad_readme_structure": {
			"missing required section 'Overview' in file 'README_part1.md'\nmissing required section 'How do I deploy this integration?' in file 'README_part2.md'",
		},
		"good_readme_structure": {},
	}
	if err := common.EnableWarningsAsErrors(); err != nil {
		require.NoError(t, err)
	}
	defer common.DisableWarningsAsErrors()

	for pkgName, expectedWarnContains := range tests {
		t.Run(pkgName, func(t *testing.T) {
			pkgRootPath := path.Join("..", "..", "..", "..", "test", "packages", pkgName)
			errs := ValidateFromPath(pkgRootPath)
			if len(expectedWarnContains) == 0 {
				require.NoError(t, errs)
			} else {
				require.Error(t, errs)
				vErrs, ok := errs.(specerrors.ValidationErrors)
				if ok {
					require.Len(t, errs, len(expectedWarnContains))
					var warnMessages []string
					for _, vErr := range vErrs {
						warnMessages = append(warnMessages, vErr.Error())
					}

					for _, expectedWarnMessage := range expectedWarnContains {
						require.Contains(t, warnMessages, expectedWarnMessage)
					}
					return
				}
				require.Equal(t, errs.Error(), expectedWarnContains[0])
			}
		})
	}
}

func TestValidateExternalFieldsWithoutDevFolder(t *testing.T) {
	pkgName := "bad_external_without_dev_build"
	tests := []struct {
		title               string
		invalidPkgFilePath  string
		buildFileContents   string
		expectedErrContains []string
	}{
		{
			"valid definition",
			"data_stream/foo/fields/ecs.yml",
			"dependencies:\n  ecs:\n    reference: git@v8.6.0\n",
			[]string{},
		},
		{
			"build file not exist",
			"data_stream/foo/fields/ecs.yml",
			"",
			[]string{
				"field container.id with external key defined (\"ecs\") but no _dev/build/build.yml found",
			},
		},
		{
			"ecs not defined",
			"data_stream/foo/fields/ecs.yml",
			"dependencies: {}\n",
			[]string{
				"field container.id with external key defined (\"ecs\") but no definition found for it (_dev/build/build.yml)",
			},
		},
	}

	pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)

	for _, test := range tests {
		t.Run(test.title, func(t *testing.T) {
			tempDir := t.TempDir()

			devFolderPath := filepath.Join(tempDir, "_dev")
			buildFolderPath := filepath.Join(devFolderPath, "build")
			buildFilePath := filepath.Join(buildFolderPath, "build.yml")

			errPrefix := fmt.Sprintf("file \"%s/%s\" is invalid: ", tempDir, test.invalidPkgFilePath)

			err := cp.Copy(pkgRootPath, tempDir)
			require.NoError(t, err)

			err = os.RemoveAll(devFolderPath)
			require.NoError(t, err)

			if test.buildFileContents != "" {
				err := os.MkdirAll(buildFolderPath, 0755)
				require.NoError(t, err)

				err = os.WriteFile(buildFilePath, []byte(test.buildFileContents), 0644)
				require.NoError(t, err)
			}

			errs := ValidateFromPath(tempDir)
			if len(test.expectedErrContains) == 0 {
				require.NoError(t, errs)
			} else {
				require.Error(t, errs)
				vErrs, ok := errs.(specerrors.ValidationErrors)
				if ok {
					require.Len(t, errs, len(test.expectedErrContains))
					var errMessages []string
					for _, vErr := range vErrs {
						errMessages = append(errMessages, vErr.Error())
					}

					for _, expectedErrMessage := range test.expectedErrContains {
						expectedErr := errPrefix + expectedErrMessage
						require.Contains(t, errMessages, expectedErr)
					}
					return
				}
				require.Equal(t, errs.Error(), test.expectedErrContains[0])
			}
		})
	}
}

func TestValidateRoutingRules(t *testing.T) {
	tests := map[string][]string{
		"good":    {},
		"good_v2": {},
		"bad_routing_rules": {
			`routing rules defined in data stream "rules" but dataset field is missing: dataset field is required in manifest for data stream "rules"`,
		},
		"bad_routing_rules_wrong_spec": {
			`item [routing_rules.yml] is not allowed in folder [../../../../test/packages/bad_routing_rules_wrong_spec/data_stream/rules]`,
		},
		"bad_routing_rules_missing_if": {
			`file "../../../../test/packages/bad_routing_rules_missing_if/data_stream/rules/routing_rules.yml" is invalid: field 0.rules.0: if is required`,
		},
		"bad_routing_rules_missing_target_dataset": {
			`file "../../../../test/packages/bad_routing_rules_missing_target_dataset/data_stream/rules/routing_rules.yml" is invalid: field 0.rules.0: target_dataset is required`,
		},
	}

	for pkgName, expectedErrorMessages := range tests {
		t.Run(pkgName, func(t *testing.T) {
			err := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName))
			if len(expectedErrorMessages) == 0 {
				assert.NoError(t, err)
				return
			}
			assert.Error(t, err)

			errs, ok := err.(specerrors.ValidationErrors)
			require.True(t, ok)
			assert.Len(t, errs, len(expectedErrorMessages))

			for _, foundError := range errs {
				require.Contains(t, expectedErrorMessages, foundError.Error())
			}
		})
	}
}

func TestValidateIngestPipelines(t *testing.T) {
	tests := map[string]map[string][]string{
		"good_v3":                         {},
		"skip_pipeline_rename_validation": {},
		"bad_ingest_pipeline": {
			"test": []string{
				"field processors.1: Additional property reroute is not allowed",
				"field processors.2.foreach.processor: Additional property paint is not allowed",
			},
			"bad_rename_message": []string{
				"field processors.1.rename: rename \"message\" to \"event.original\" processor requires if: 'ctx.event?.original == null' (JSE00001)",
				"field processors.0: rename \"message\" to \"event.original\" processor requires remove \"message\" processor (JSE00001)",
			},
			"bad_rename_message_2": []string{
				"field processors.2.remove.field: rename \"message\" to \"event.original\" processor requires remove \"message\" processor (JSE00001)",
				"field processors.2.remove.if: rename \"message\" to \"event.original\" processor requires remove \"message\" processor with if: 'ctx.event?.original != null' (JSE00001)",
			},
		},
		"bad_pipeline_tags": {
			"example": []string{
				"set processor at line 4 missing required tag (SVR00006)",
				"set processor at line 15 has duplicate tag value: \"set_sample_field\"",
			},
		},
	}

	for pkgName, pipelines := range tests {
		t.Run(pkgName, func(t *testing.T) {
			pkgRootPath := path.Join("..", "..", "..", "..", "test", "packages", pkgName)
			var allErrorMessages []string
			for pipeline, expectedErrorMessages := range pipelines {
				pipelineFilePath := path.Join(pkgRootPath, "data_stream", pipeline, "elasticsearch", "ingest_pipeline", "default.yml")
				errPrefix := fmt.Sprintf("file \"%s\" is invalid: ", pipelineFilePath)
				for _, errMsg := range expectedErrorMessages {
					allErrorMessages = append(allErrorMessages, errPrefix+errMsg)
				}
			}

			errs := ValidateFromPath(pkgRootPath)

			if verrs, ok := errs.(specerrors.ValidationErrors); ok {
				filterConfig, err := specerrors.LoadConfigFilter(os.DirFS(pkgRootPath))
				if !errors.Is(err, os.ErrNotExist) {
					filter := specerrors.NewFilter(filterConfig)
					result, err := filter.Run(verrs)
					require.NoError(t, err)
					errs = result.Processed
				}
			}

			if len(allErrorMessages) == 0 {
				require.NoError(t, errs)
			} else {
				require.Error(t, errs)
				vErrs, ok := errs.(specerrors.ValidationErrors)
				if ok {
					var errMessages []string
					for _, vErr := range vErrs {
						errMessages = append(errMessages, vErr.Error())
					}
					assert.Len(t, errs, len(allErrorMessages))
					for _, expectedErrMessage := range allErrorMessages {
						require.Contains(t, errMessages, expectedErrMessage)
					}
				}
			}
		})
	}
}

func TestValidateForbiddenDataStreamName(t *testing.T) {
	testPackagesPath := path.Join("..", "..", "..", "..", "test", "packages")
	pkgPath := path.Join(testPackagesPath, "bad_data_stream_name")
	err := ValidateFromPath(pkgPath)
	require.Error(t, err)

	expectedErrorMessages := []string{
		fmt.Sprintf(`item [integration] is not allowed in folder [%s]`, path.Join(pkgPath, "data_stream")),
	}
	errs, ok := err.(specerrors.ValidationErrors)
	require.True(t, ok)
	assert.Len(t, errs, len(expectedErrorMessages))

	for _, foundError := range errs {
		assert.Contains(t, expectedErrorMessages, foundError.Error())
	}
}

func TestLinksAreBlocked(t *testing.T) {
	err := ValidateFromFS("test-package", newMockFS().WithLink())
	errs, ok := err.(specerrors.ValidationErrors)
	require.True(t, ok)
	for _, err := range errs {
		// we want at least one error indicating links are blocked
		if errors.Is(err, linkedfiles.ErrUnsupportedLinkFile) {
			return
		}
	}
	t.Error("links should not be allowed in package")
}

func TestValidateHandlebarsFiles(t *testing.T) {
	tests := map[string]string{
		"bad_input_hbs":              "invalid handlebars template: error validating agent/input/input.yml.hbs: Parse error on line 10:\nExpecting OpenEndBlock, got: 'EOF'",
		"bad_integration_hbs":        "invalid handlebars template: error validating data_stream/foo/agent/stream/filestream.yml.hbs: Parse error on line 43:\nExpecting OpenEndBlock, got: 'EOF'",
		"bad_integration_hbs_linked": "invalid handlebars template: error validating ../bad_integration_hbs/data_stream/foo/agent/stream/filestream.yml.hbs: Parse error on line 43:\nExpecting OpenEndBlock, got: 'EOF'",
	}

	for pkgName, expectedErrorMessage := range tests {
		t.Run(pkgName, func(t *testing.T) {
			errs := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName))
			require.Error(t, errs)
			vErrs, ok := errs.(specerrors.ValidationErrors)
			require.True(t, ok)

			var errMessages []string
			for _, vErr := range vErrs {
				errMessages = append(errMessages, vErr.Error())
			}
			require.Contains(t, errMessages, expectedErrorMessage)
		})
	}
}

func requireErrorMessage(t *testing.T, pkgName string, invalidItemsPerFolder map[string][]string, expectedErrorMessage string) {
	pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)

	errs := ValidateFromPath(pkgRootPath)
	require.Error(t, errs)
	vErrs, ok := errs.(specerrors.ValidationErrors)
	require.True(t, ok)

	var errMessages []string
	for _, vErr := range vErrs {
		errMessages = append(errMessages, vErr.Error())
	}

	var c int
	for itemFolder, invalidItems := range invalidItemsPerFolder {
		for _, invalidItem := range invalidItems {
			c++
			expected := fmt.Sprintf(expectedErrorMessage, invalidItem, pkgRootPath, itemFolder)
			require.Contains(t, errMessages, expected)
		}
	}
	require.Len(t, errs, c)
}