Merge branch 'master' into powerumc/odbc-source

# Conflicts:
#	scripts/integration/Dockerfile

Author: powerumc

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,6 +19,8 @@ This should help the reviewers give feedback faster and with higher quality. -->
 - [ ] No
 
 ## Does this PR include user facing changes?
+<!-- If this PR alters Vector behavior in any way, for example, it adds a new config field or changes internal metrics it is considered a user facing change.
+Changes to CI, website, playground and similar are generally not considered user facing -->
 
 - [ ] Yes. Please add a changelog fragment based on our [guidelines](https://github.com/vectordotdev/vector/blob/master/changelog.d/README.md).
 - [ ] No. A maintainer will apply the `no-changelog` label to this PR.

.github/actions/setup/action.yml

@@ -90,11 +90,19 @@ runs:
                    "${{ inputs.wasm-pack }}"; do
           if [[ "$val" == "true" ]]; then
             rust="true"
+            break
           fi
         done
 
         echo "NEEDS_RUST=$rust" >> "$GITHUB_ENV"
 
+        if [[ "$rust" == "true" ]]; then
+          echo "RUST_BACKTRACE=full" >> "$GITHUB_ENV"
+          # Set nice CI colors
+          echo "CARGO_TERM_COLOR=always" >> "$GITHUB_ENV"
+        fi
+
+
     - name: Enable rust matcher
       if: ${{ env.NEEDS_RUST == 'true' }}
       run: echo "::add-matcher::.github/matchers/rust.json"

.github/actions/spelling/allow.txt

@@ -554,3 +554,4 @@ zst
 zstandard
 ZTE
 Zync
+sighup

.github/workflows/ci-integration-review.yml

@@ -34,6 +34,7 @@ on:
 
 permissions:
   statuses: write
+  packages: write
 
 env:
   AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }}
@@ -80,8 +81,17 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           status: pending
 
-  integration-tests:
+  build-test-runner:
     needs: prep-pr
+    uses: ./.github/workflows/build-test-runner.yml
+    with:
+      commit_sha: ${{ github.event.review.commit_id }}
+      checkout_ref: ${{ github.event.review.commit_id }}
+
+  integration-tests:
+    needs:
+      - prep-pr
+      - build-test-runner
     runs-on: ubuntu-24.04
     timeout-minutes: 90
     strategy:
@@ -99,9 +109,13 @@ jobs:
           submodules: "recursive"
           ref: ${{ github.event.review.commit_id }}
 
-      - run: bash scripts/environment/prepare.sh --modules=datadog-ci
+      - name: Pull test runner image
+        uses: ./.github/actions/pull-test-runner
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ github.event.review.commit_id }}
 
-      - run: docker image prune -af ; docker container prune -f
+      - run: bash scripts/environment/prepare.sh --modules=datadog-ci
 
       - name: Integration Tests - ${{ matrix.service }}
         if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all')
@@ -114,7 +128,9 @@ jobs:
           command: bash scripts/run-integration-test.sh int ${{ matrix.service }}
 
   e2e-tests:
-    needs: prep-pr
+    needs:
+      - prep-pr
+      - build-test-runner
     runs-on: ubuntu-24.04-8core
     timeout-minutes: 30
     steps:
@@ -123,9 +139,14 @@ jobs:
           submodules: "recursive"
           ref: ${{ github.event.review.commit_id }}
 
+      - name: Pull test runner image
+        uses: ./.github/actions/pull-test-runner
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ github.event.review.commit_id }}
+
       - run: bash scripts/environment/prepare.sh --modules=datadog-ci
 
-      - run: docker image prune -af ; docker container prune -f
       - name: e2e-datadog-logs
         if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-logs')
           || startsWith(github.event.review.body, '/ci-run-e2e-all')

.github/workflows/ci-review-trigger.yml

@@ -38,9 +38,6 @@ env:
   VERBOSE: true
   CI: true
   PROFILE: debug
-  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
-  # can be removed when we switch back to the upstream openssl-sys crate
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
 
 # The below concurrency group settings would let us cancel in progress runs that were triggered with the
 # same comment on a given PR, which could save on time consuming runs.

.github/workflows/cleanup-ghcr-images.yml

@@ -0,0 +1,38 @@
+# Cleanup GHCR Images
+#
+# This workflow cleans up old images from GitHub Container Registry
+# to prevent unlimited storage growth. It runs weekly and removes:
+# 1. Untagged images from all packages (intermediate build artifacts)
+# 2. Old test-runner versions (keeps 50 most recent, deletes up to 50 oldest)
+
+name: Cleanup Untagged GHCR Images
+
+on:
+  schedule:
+    # Run weekly on Sundays at 2 AM UTC
+    - cron: '0 2 * * 0'
+  workflow_dispatch:
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Delete untagged vector images
+        uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0
+        with:
+          package-name: vector
+          package-type: container
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: true
+        continue-on-error: true
+
+      - name: Delete old vector-test-runner images
+        uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0
+        with:
+          package-name: test-runner
+          package-type: container
+          min-versions-to-keep: 5
+          num-old-versions-to-delete: 50

.github/workflows/environment.yml

@@ -3,12 +3,18 @@ name: Environment Suite
 on:
   workflow_call:
   workflow_dispatch:
+    inputs:
+      push:
+        description: "Push image when manually triggered"
+        type: boolean
+        default: false
   schedule:
     - cron: '0 6 * * 1'  # Every Monday at 06:00 UTC
 
 env:
   VERBOSE: true
   CI: true
+  SHOULD_PUBLISH: ${{ (github.event_name != 'workflow_dispatch' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.push == 'true') }}
 
 permissions:
   statuses: write
@@ -43,7 +49,7 @@ jobs:
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
       - name: Login to DockerHub
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
-        if: github.ref == 'refs/heads/master'
+        if: env.SHOULD_PUBLISH == 'true'
         with:
           username: ${{ secrets.CI_DOCKER_USERNAME }}
           password: ${{ secrets.CI_DOCKER_PASSWORD }}
@@ -60,12 +66,37 @@ jobs:
             org.opencontainers.image.source=https://github.com/vectordotdev/vector/tree/master/scripts/environment/Dockerfile
             org.opencontainers.image.title=Vector development environment
             org.opencontainers.image.url=https://github.com/vectordotdev/vector
-      - name: Build and push
+
+      - name: Free disk space
+        shell: bash
+        run: sudo -E bash scripts/ci-free-disk-space.sh
+
+      - name: Build image (no push)
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: .
+          file: ./scripts/environment/Dockerfile
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Test built image
+        run: |
+          IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
+          echo "Running build test inside container: ${IMAGE_TAG}"
+          docker run --rm \
+            -v "${GITHUB_WORKSPACE}:/src" \
+            -w /src \
+            "${IMAGE_TAG}" \
+            cargo run --bin vector -- --version
+
+      - name: Push image
+        if: env.SHOULD_PUBLISH == 'true'
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
         with:
           context: .
           file: ./scripts/environment/Dockerfile
-          push: ${{ github.ref == 'refs/heads/master' }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

.github/workflows/integration.yml

@@ -17,6 +17,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.head_sha }}
   cancel-in-progress: true
 
+permissions:
+  packages: write
+  contents: read
+
 env:
   CONTAINER_TOOL: "docker"
   DD_ENV: "ci"
@@ -29,9 +33,6 @@ env:
   VERBOSE: true
   CI: true
   PROFILE: debug
-  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
-  # can be removed when we switch back to the upstream openssl-sys crate
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
 
 jobs:
   changes:
@@ -43,10 +44,26 @@ jobs:
       e2e_tests: true
     secrets: inherit
 
+  build-test-runner:
+    needs: changes
+    if: |
+      ${{
+        github.event_name == 'workflow_dispatch' ||
+        (github.event_name == 'merge_group' &&
+         (needs.changes.outputs.dependencies == 'true' ||
+          needs.changes.outputs.integration-yml == 'true' ||
+          needs.changes.outputs.int-tests-any == 'true' ||
+          needs.changes.outputs.e2e-tests-any == 'true'))
+      }}
+    uses: ./.github/workflows/build-test-runner.yml
+    with:
+      commit_sha: ${{ github.sha }}
+
   integration-tests:
     runs-on: ubuntu-24.04-8core
     needs:
       - changes
+      - build-test-runner
 
     if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
     strategy:
@@ -72,6 +89,12 @@ jobs:
         with:
           name: int_tests_changes
 
+      - name: Pull test runner image
+        uses: ./.github/actions/pull-test-runner
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ github.sha }}
+
       - name: Run Integration Tests for ${{ matrix.service }}
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
         with:
@@ -109,6 +132,7 @@ jobs:
     runs-on: ubuntu-24.04-8core
     needs:
       - changes
+      - build-test-runner
     if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
     strategy:
       matrix:
@@ -126,6 +150,12 @@ jobs:
         with:
           name: e2e_tests_changes
 
+      - name: Pull test runner image
+        uses: ./.github/actions/pull-test-runner
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ github.sha }}
+
       - name: Run E2E Tests for ${{ matrix.service }}
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
         with:
@@ -163,6 +193,7 @@ jobs:
     runs-on: ubuntu-24.04
     if: always()
     needs:
+      - build-test-runner
       - integration-tests
       - e2e-tests
     steps:

.github/workflows/master_merge_queue.yml

@@ -36,9 +36,6 @@ env:
   VERBOSE: true
   CI: true
   PROFILE: debug
-  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
-  # can be removed when we switch back to the upstream openssl-sys crate
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
 
 jobs:
   # This is the entry job which is required for all the actual tests in this workflow.

.github/workflows/msrv.yml

@@ -13,9 +13,6 @@ env:
   RUST_BACKTRACE: full
   CI: true
   PROFILE: debug
-  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
-  # can be removed when we switch back to the upstream openssl-sys crate
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
 
 jobs:
   check-msrv: