[Security Rules][Backport] Add execution permission for the security_detection_engine package script (8.19)

.buildkite/hooks/pre-command

@@ -11,6 +11,8 @@ git config --global core.pager 'cat'
 export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
+export ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
+export STACK_LOGSDB_ENABLED=${STACK_LOGSDB_ENABLED:-"false"}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 
@@ -29,15 +31,16 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
-PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
-
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
 
-EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 
-# variables required for terraform
+export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
+
+# -------------
+# variables required by packages using Terraform as a service deployer
+# https://github.com/elastic/elastic-package/blob/f8f2f15a04bcc25eca00887fb147bd7f8a0f32b3/internal/servicedeployer/_static/terraform_deployer.yml#L8
+
 export ENVIRONMENT="ci"
 export REPO="${REPO_NAME}"
 
@@ -68,6 +71,15 @@ export BUILD_ID="${BUILDKITE_BUILD_NUMBER}"
 CREATED_DATE=$(date +%s%3N)
 export CREATED_DATE
 
+# -------------
+
+if [[ "${ELASTIC_PACKAGE_CUSTOMIZE_SERVICE_TEST_RUN_ID:-"false"}" == "true" ]]; then
+    # Required to customize the RunID value mainly for those packages creating resources in cloud providers
+    # via the terraform service deployer.
+    # Get the latest 4 digits of the BUILDKITE_STEP_ID
+    export ELASTIC_PACKAGE_PREFIX_SERVICE_TEST_RUN_ID="${BUILDKITE_STEP_ID: -4}"
+fi
+
 if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     # first upload pipeline does not have the environment variables defined in the YAML
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
@@ -78,7 +90,7 @@ if [[ ( "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$
     # This step MUST be the first one and not run in parallel with any other step to ensure
     # that there is just one value for this variable
     if is_pr ; then
-        git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+        git fetch -v origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
         commit_main=$(git rev-parse --verify FETCH_HEAD)
         buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
     fi
@@ -100,62 +112,21 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
     if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
-        GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
-        export GITHUB_TOKEN
     fi
 
     if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
-        ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_SECRET_KEY
-        ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
-        export ELASTIC_PACKAGE_AWS_ACCESS_KEY
-
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
-        # Environment variables required by the service deployer
-        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
-        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
     fi
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
-        # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
-        # add the AWS credentials as in the integrations pipeline.
-
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
 
-        EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
-        export EC_API_KEY_SECRET
-        EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
-        export EC_HOST_SECRET
         EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
         export EC_REGION_SECRET
     fi
 fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-    GITHUB_USERNAME="elastic-vault-github-plugin-prod"
-    GITHUB_EMAIL="[email protected]"
-    GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
-    export GITHUB_TOKEN GITHUB_EMAIL GITHUB_USERNAME
-  fi
-fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" || "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "report-failed-tests" ]]; then
-    export GITHUB_TOKEN="${VAULT_GITHUB_TOKEN}"
-  fi
-fi
-

.buildkite/hooks/pre-exit

@@ -4,49 +4,52 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
-    # FIXME: update condition depending on the pipeline steps triggered
-    if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
+should_run_stack_down() {
+    # as first check, ensure that the elastic-package binary exists
+    if [ ! -f "${ELASTIC_PACKAGE_BIN}" ]; then
+        return 1
+    fi
 
-        # Ensure that kind cluster is deleted
-        delete_kind_cluster
+    if is_serverless; then
+        return 0
+    fi
 
-        # Ensure elastic stack is stopped
-        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
-            echo "--- Take down the Elastic stack"
-            ${ELASTIC_PACKAGE_BIN} stack down -v
-        fi
+    if is_stack_created; then
+        return 0
     fi
+
+    return 1
+}
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
+  cd "${WORKSPACE}"
+  git config remote.origin.url "https://github.com/elastic/integrations.git"
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
+exit_code=0
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack|integrations-serverless)$ ]]; then
+    # it should match "^test-integration-" steps created in the integrations and integrations-test-stack pipelines (e.g. test-integration-apache or test-integration-aws)
+    # as well as the step ID "test-integrations-serverless-project" from the "integrations-serverless" pipeline
+    if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
 
         # Ensure that kind cluster is deleted
         delete_kind_cluster
 
         # Ensure elastic stack is stopped
-        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
+        if should_run_stack_down; then
             echo "--- Take down the Elastic stack"
-            EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v
+            if ! ${ELASTIC_PACKAGE_BIN} stack down -v ; then
+                exit_code=1
+            fi
         fi
+
+        echo "+++ :bookmark: Documentation to access logs"
+        inline_link "https://docs.elastic.dev/ingest-dev-docs/elastic-packages/ecosystem-ci-pipelines#private-logs"
     fi
 fi
 
+echo "--- Cleaning up"
 unset_secrets
 cleanup
 
-google_cloud_logout_active_account
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-  cd "${WORKSPACE}"
-  git config remote.origin.url "https://github.com/elastic/integrations.git"
-fi
+exit "${exit_code}"

.buildkite/pipeline.backport.yml

@@ -4,6 +4,8 @@ name: "integrations-backport"
 
 env:
   YQ_VERSION: 'v4.35.2'
+  # Agent images used in pipeline steps
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
 
 steps:
 
@@ -29,10 +31,14 @@ steps:
       key: "BASE_COMMIT"
       required: true
       default: ""
-    - text: "Enter package name"
+    - text: "Enter package name (as defined in manifest.yml)"
       key: "PACKAGE_NAME"
       required: true
       default: ""
+    - text: "Enter name of the folder for the package (in most cases coincides with PACKAGE_NAME)"
+      key: "PACKAGE_FOLDER_NAME"
+      required: true
+      default: ""
     - text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)"
       key: "PACKAGE_VERSION"
       required: true
@@ -49,6 +55,13 @@ steps:
   - label: "Creating the backport branch"
     key: "create-backport-branch"
     command: ".buildkite/scripts/backport_branch.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+    env:
+      GITHUB_EMAIL: "[email protected]"
+      GITHUB_USERNAME: "elastic-vault-github-plugin-prod"
+    plugins:
+      - elastic/vault-github-token#v0.1.0:
     depends_on:
       - step: "input-variables"
         allow_failure: false

.buildkite/pipeline.schedule-daily.yml

@@ -21,47 +21,84 @@ steps:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 7.17.28
+        STACK_VERSION: 7.17.29
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_7_BRANCH') == "true"
 
   - label: "Check integrations local stacks - Stack Version v8.19"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.19.0-SNAPSHOT
+        STACK_VERSION: 8.19.6-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_8_BRANCH') == "true"
 
   - label: "Check integrations local stacks - Stack Version v8.19 - LogsDB"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.19.0-SNAPSHOT
+        STACK_VERSION: 8.19.6-SNAPSHOT
         STACK_LOGSDB_ENABLED: "true"
         PUBLISH_COVERAGE_REPORTS: "false"
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_8_BRANCH') == "true"
 
-  - label: "Check integrations local stacks - Stack Version v9.1"
+  - label: "Check integrations local stacks and basic subscription and LogsDB"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 9.1.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_SUBSCRIPTION: "basic"
+        STACK_LOGSDB_ENABLED: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true"
+
+  - label: "Check integrations local stacks and basic subscription"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_SUBSCRIPTION: "basic"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true"
+
+  - label: "Check integrations local stacks - Stack Version v9.3"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 9.3.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_9_BRANCH') == "true"
 
   - label: "Check integrations in serverless - project: Observability"
     key: "trigger-integrations-serverless-obs"
@@ -72,6 +109,8 @@ steps:
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_SERVERLESS') == "true"
 
   - label: "Check integrations in serverless - project: Security"
     key: "trigger-integrations-serverless-security"
@@ -82,10 +121,14 @@ steps:
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('TEST_PACKAGES_SERVERLESS') == "true"
 
   - label: ":package: Publish missing packages"
     key: "trigger-integrations-publish"
     trigger: "integrations-publish"
     depends_on:
       - step: "check"
         allow_failure: false
+    if: |
+      build.env('REPUBLISH_PACKAGES') == "true"

.buildkite/pipeline.schedule-weekly.yml

@@ -21,20 +21,20 @@ steps:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.19.0-SNAPSHOT
+        STACK_VERSION: 8.19.6-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
         ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.1"
+  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.3"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 9.1.0-SNAPSHOT
+        STACK_VERSION: 9.3.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
         ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
     depends_on: