Support deprecated packages or individual features

code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go

@@ -0,0 +1,76 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"io/fs"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// ValidateIntegrationInputsDeprecation checks that if all inputs in an integration package are deprecated,
+// then the integration package itself must be marked as deprecated.
+func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationErrors {
+
+	type manifest struct {
+		Type       string `yaml:"type,omitempty"`
+		Deprecated *struct {
+			Since       string `yaml:"since,omitempty"`
+			Description string `yaml:"description,omitempty"`
+		} `yaml:"deprecated,omitempty"`
+		PolicyTemplates []struct {
+			Inputs []struct {
+				Deprecated *struct {
+					Since       string `yaml:"since,omitempty"`
+					Description string `yaml:"description,omitempty"`
+				} `yaml:"deprecated,omitempty"`
+			} `yaml:"inputs,omitempty"`
+		} `yaml:"policy_templates,omitempty"`
+	}
+
+	manifestPath := "manifest.yml"
+	data, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
+	}
+
+	var m manifest
+	err = yaml.Unmarshal(data, &m)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
+	}
+	// skip if not an integration package
+	if m.Type != packageTypeIntegration {
+		return nil
+	}
+
+	// if package is deprecated, skip checks
+	if m.Deprecated != nil {
+		return nil
+	}
+
+	total := 0
+	deprecated := 0
+	for _, pt := range m.PolicyTemplates {
+		for _, input := range pt.Inputs {
+			total++
+			if input.Deprecated != nil {
+				deprecated++
+			}
+		}
+	}
+	if deprecated == total && total > 0 {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: all inputs are deprecated but the integration package is not marked as deprecated", fsys.Path(manifestPath)),
+		}
+	}
+
+	return nil
+}

code/go/internal/validator/semantic/validate_integration_inputs_deprecated_test.go

@@ -0,0 +1,137 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+func TestValidateIntegrationInputsDeprecation(t *testing.T) {
+
+	t.Run("integration_deprecated_all_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+deprecated:
+  since: '1.0.0'
+  description: 'This integration is deprecated.'
+policy_templates:
+  - inputs:
+    - type: udp
+      deprecated:
+        since: '0.5.0'
+        description: 'This input is deprecated.'
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+
+	})
+
+	t.Run("integration_deprecated_none_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+deprecated:
+  since: '1.0.0'
+  description: 'This integration is deprecated.'
+policy_templates:
+  - inputs:
+    - type: udp
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("integration_deprecated_partial_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+deprecated:
+  since: '1.0.0'
+  description: 'This integration is deprecated.'
+policy_templates:
+  - inputs:
+    - type: udp
+      deprecated:
+        since: '0.5.0'
+        description: 'This input is deprecated.'
+    - type: tcp
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("integration_not_deprecated_all_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - inputs:
+    - type: udp
+      deprecated:
+        since: '0.5.0'
+        description: 'This input is deprecated.'
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		require.Len(t, errs, 1)
+		assert.ErrorContains(t, errs[0], "all inputs are deprecated but the integration package is not marked as deprecated")
+
+	})
+
+	t.Run("integration_not_deprecated_none_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - inputs:
+    - type: udp
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("integration_not_deprecated_partial_inputs_deprecated", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - inputs:
+    - type: udp
+      deprecated:
+        since: '0.5.0'
+        description: 'This input is deprecated.'
+    - type: tcp
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+}

code/go/pkg/validator/validator_test.go

@@ -358,6 +358,20 @@ func TestValidateFile(t *testing.T) {
 			"kibana/tag/bad_tag-security-solution-default.json",
 			[]string{"tag name 'Security Solution' is already defined in tags.yml (SVR00007)"},
 		},
+		"deprecated_integration":              {},
+		"deprecated_input":                    {},
+		"deprecated_content":                  {},
+		"deprecated_integration_policy_input": {},
+		"deprecated_integration_policy":       {},
+		"deprecated_integration_stream_var":   {},
+		"bad_deprecation_description": {
+			"manifest.yml",
+			[]string{"field deprecated.description: Invalid type. Expected: string, given: null"},
+		},
+		"bad_deprecation_since": {
+			"manifest.yml",
+			[]string{"field deprecated: since is required"},
+		},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -21,6 +21,9 @@
     - description: Add support for Kibana `slo_template` assets
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1037
+    - description: Add support for deprecating packages or individual features (datastreams, inputs, policies).
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1053
 - version: 3.5.6-next
   changes:
     - description: Add validation for Kibana tag duplicates.

spec/content/manifest.spec.yml

@@ -82,6 +82,8 @@ spec:
       $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
     owner:
       $ref: "../integration/manifest.spec.yml#/definitions/owner"
+    deprecated:
+      $ref: "../integration/manifest.spec.yml#/definitions/deprecated"
   required:
   - format_version
   - name
@@ -90,3 +92,10 @@ spec:
   - version
   - type
   - owner
+
+# JSON patches for newer versions should be placed on top
+versions:
+  - before: 3.6.0
+    patch:
+      - op: remove # remove deprecated field for package
+        path: "/properties/deprecated"

spec/input/manifest.spec.yml

@@ -112,6 +112,8 @@ spec:
       properties:
         index_template:
           $ref: "../integration/data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template"
+    deprecated:
+      $ref: "../integration/manifest.spec.yml#/definitions/deprecated"
   required:
   - format_version
   - name
@@ -128,6 +130,8 @@ versions:
     patch:
       - op: remove
         path: "/required/7" # removes requirement for conditions
+      - op: remove
+        path: "/properties/deprecated" # removes deprecated field for package
   # Reserve otelcol input name before 3.5.0.
   - before: 3.5.0
     patch:

spec/integration/changelog.spec.yml

@@ -30,6 +30,7 @@ spec:
               - "breaking-change"
               - "bugfix"
               - "enhancement"
+              - "deprecation"
             link:
               description: Link to issue or PR describing change in detail.
               type: string

spec/integration/data_stream/manifest.spec.yml

@@ -164,6 +164,8 @@ spec:
           default:
             description: Default value(s) for variable
             $ref: "#/definitions/input_variable_value"
+          deprecated:
+            $ref: "../../integration/manifest.spec.yml#/definitions/deprecated"
         allOf:
          - if:
              properties:
@@ -649,6 +651,10 @@ spec:
   - title
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 3.6.0
+    patch:
+      - op: remove # remove deprecated field for vars
+        path: /definitions/vars/items/properties/deprecated
   - before: 3.5.0
     patch:
       # Require >=3.5.0 to use the duration variable type.

spec/integration/manifest.spec.yml

@@ -495,6 +495,20 @@ spec:
       required:
         - github
         - type
+    deprecated:
+      description: Information on deprecation of a package or an individual feature.
+      type: object
+      additionalProperties: false
+      properties:
+        since:
+          description: Version since when is deprecated.
+          $ref: "#/definitions/version"
+        description:
+          description: Reason of deprecation.
+          type: string
+      required:
+        - since
+        - description
   properties:
     format_version:
       description: The version of the package specification format used by this package.
@@ -634,6 +648,8 @@ spec:
                   $ref: "./data_stream/manifest.spec.yml#/definitions/required_vars"
                 vars:
                   $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+                deprecated:
+                  $ref: "#/definitions/deprecated"
               required:
                 - type
                 - title
@@ -646,6 +662,8 @@ spec:
             $ref: "#/definitions/screenshots"
           vars:
             $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+          deprecated:
+            $ref: "#/definitions/deprecated"
         required:
           - name
           - title
@@ -676,6 +694,8 @@ spec:
               type: array
               items:
                 type: string
+    deprecated:
+      $ref: "#/definitions/deprecated"
   required:
     - format_version
     - name
@@ -715,6 +735,14 @@ versions:
         path: "/definitions/conditions/required/0" # removes requirement for agent
       - op: remove
         path: "/definitions/conditions/properties/agent/required/0" # removes requirement for version
+      - op: remove # removes package deprecation info
+        path: "/properties/deprecated"
+      - op: remove # removes policy template input deprecation info
+        path: "/properties/policy_templates/items/properties/inputs/items/properties/deprecated"
+      - op: remove # removes policy template deprecation info
+        path: "/properties/policy_templates/items/properties/deprecated"
+      - op: remove # remove deprecated defintion
+        path: "/definitions/deprecated"
   - before: 3.3.2
     patch:
       - op: remove