Merge branch 'main' into fl/primitive-delta-support

Author: fionaliao

.github/workflows/changelog-check.yml

@@ -15,8 +15,9 @@ jobs:
         id: check-label
         run: |
           LABELS=$(gh pr view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
-          
-          if echo "$LABELS" | grep -qE "(changelog-not-needed|dependency-update|vendored-mimir-prometheus-update|helm-weekly-release)"; then
+
+          # "grep -w" to check for the whole words
+          if echo "$LABELS" | grep -w -qE "(changelog-not-needed|dependency-update|vendored-mimir-prometheus-update|helm-weekly-release|backport)"; then
             echo "skip=true" >> $GITHUB_OUTPUT
             echo "PR has a label that skips changelog check, skipping changelog check"
           else
@@ -30,10 +31,23 @@ jobs:
         if: steps.check-label.outputs.skip == 'false'
         run: |
           PR_NUMBER=${{ github.event.pull_request.number }}
+          MAIN_CHANGELOG_FOUND=false
+          HELM_CHANGELOG_FOUND=false
+          
           if grep -q "#${PR_NUMBER}" CHANGELOG.md; then
             echo "✅ PR #${PR_NUMBER} is referenced in CHANGELOG.md"
+            MAIN_CHANGELOG_FOUND=true
+          fi
+          
+          if grep -q "#${PR_NUMBER}" operations/helm/charts/mimir-distributed/CHANGELOG.md; then
+            echo "✅ PR #${PR_NUMBER} is referenced in helm CHANGELOG.md"
+            HELM_CHANGELOG_FOUND=true
+          fi
+          
+          if [ "$MAIN_CHANGELOG_FOUND" = true ] || [ "$HELM_CHANGELOG_FOUND" = true ]; then
+            echo "✅ PR #${PR_NUMBER} is referenced in at least one changelog"
           else
-            echo "❌ PR #${PR_NUMBER} is not referenced in CHANGELOG.md"
-            echo "Please add an entry for this PR in CHANGELOG.md or add one of these labels if this change doesn't require a changelog entry: 'changelog-not-needed', 'dependency-update', 'vendored-mimir-prometheus-update', 'helm-weekly-release'"
+            echo "❌ PR #${PR_NUMBER} is not referenced in CHANGELOG.md or operations/helm/charts/mimir-distributed/CHANGELOG.md"
+            echo "Please add an entry for this PR in the appropriate changelog or add one of these labels if this change doesn't require a changelog entry: 'changelog-not-needed', 'dependency-update', 'vendored-mimir-prometheus-update', 'helm-weekly-release'"
             exit 1
-          fi
+          fi

.github/workflows/flaky-tests.yml

@@ -0,0 +1,59 @@
+name: Flaky Tests Detection
+
+on:
+  schedule:
+    - cron: "0 2 * * 1" # Run every Monday at 2 AM
+  workflow_dispatch:
+    inputs:
+      skip-posting-issues:
+        description: 'Skip creating/updating GitHub issues (preview mode)'
+        required: false
+        default: 'true'
+        type: boolean
+
+permissions:
+  contents: read
+  issues: write
+  id-token: write
+
+jobs:
+  analyze-flaky-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '0' # we need all commits so that the git history check returns valid commits
+
+      - name: Retrieve Secrets from Vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # get-vault-secrets-v1.2.0
+        with:
+          repo_secrets: |
+            LOKI_URL=flaky-tests-bot:loki-url
+            LOKI_USERNAME=flaky-tests-bot:loki-username
+            LOKI_PASSWORD=flaky-tests-bot:loki-password
+            APP_ID=mimir-github-bot:app_id
+            PRIVATE_KEY=mimir-github-bot:private_key
+
+      - name: Generate GitHub App Token
+        id: github-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ env.APP_ID }}
+          private-key: ${{ env.PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Run Flaky Tests Analysis
+        uses: dimitarvdimitrov/shared-workflows/actions/go-flaky-tests@0e7ecbf2ae0a5b76e24e31beed04d63e610e6b1d # dimitar/analyze-test-failures/exclude-tests
+        with:
+          loki-url: ${{ env.LOKI_URL }}
+          loki-username: ${{ env.LOKI_USERNAME }}
+          loki-password: ${{ env.LOKI_PASSWORD }}
+          repository: ${{ github.repository }}
+          time-range: "7d"
+          top-k: "3"
+          skip-posting-issues: ${{ github.event_name == 'workflow_dispatch' && inputs.skip-posting-issues || 'false' }}
+          ignored-tests: "TestOurUpstreamTestCasesAreInSyncWithUpstream" # This is supposed to block upstream mimir-prometheus updates, so it's also expected to fail.
+        env:
+          GITHUB_TOKEN: ${{ steps.github-token.outputs.token }}

.github/workflows/push-mimir-build-image.yml

@@ -39,7 +39,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
       - name: Login to Docker Hub
         uses: grafana/shared-workflows/actions/dockerhub-login@13fb504e3bfe323c1188bf244970d94b2d336e86 # v1.0.1
@@ -109,6 +109,26 @@ jobs:
             echo "isDifferent=true" >> $GITHUB_OUTPUT
           fi
 
+      # This job uses "Mimir bot" instead of "github-actions bot" (secrets.GITHUB_TOKEN)
+      # because any events triggered by the latter don't spawn GitHub actions.
+      # Refer to https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
+      - name: Retrieve GitHub App Credentials from Vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760 # v1.1.0
+        with:
+          repo_secrets: |
+            APP_ID=mimir-github-bot:app_id
+            PRIVATE_KEY=mimir-github-bot:private_key
+
+      - name: Generate GitHub App Token
+        id: app-token
+        uses: actions/create-github-app-token@v1
+        with:
+          # Variables generated by the previous step get-secrets
+          app-id: ${{ env.APP_ID }}
+          private-key: ${{ env.PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
       - name: Add commit to PR in order to update Build Image version
         if: steps.compare_tag.outputs.isDifferent == 'true'
         run: |
@@ -122,12 +142,13 @@ jobs:
             sed -i "s/$MAIN_TAG/${{ steps.compute_hash.outputs.tag }}/g" Makefile
             git config --global user.email "${GITHUB_LOGIN}@users.noreply.github.com"
             git config --global user.name "${GITHUB_LOGIN}"
+            git config --global url.https://x-access-token:${GITHUB_TOKEN}@github.com/.insteadOf https://github.com/
             git add Makefile
             git commit -m "Update build image version to ${{ steps.compute_hash.outputs.tag }}"
             git push origin HEAD
           fi
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
           GITHUB_LOGIN: ${{ github.event.pull_request.user.login }}
           TAG: ${{ steps.compute_hash.outputs.tag }}
           MAIN_TAG: ${{ steps.prepare.outputs.main_image_tag }}

.github/workflows/scripts/run-unit-tests-group.sh

@@ -47,4 +47,4 @@ echo "$GROUP_TESTS"
 echo ""
 
 # shellcheck disable=SC2086 # we *want* word splitting of GROUP_TESTS.
-exec go test -tags=netgo,stringlabels -timeout 30m -race -count 1 ${GROUP_TESTS}
+exec go test -tags=netgo,stringlabels -timeout 30m -race ${GROUP_TESTS}

.github/workflows/test-build-deploy.yml

@@ -205,7 +205,7 @@ jobs:
         uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
       - name: Symlink Expected Path to Workspace
         run: |
           mkdir -p /go/src/github.com/grafana/mimir

.github/workflows/trigger-backend-enterprise-update.yml

@@ -0,0 +1,39 @@
+name: Update vendored versions in grafana/backend-enterprise weekly branches
+
+on:
+  push:
+    branches:
+      - 'r[0-9]*'
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  trigger:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Retrieve GitHub App Credentials from Vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760 # v1.1.0
+        with:
+          repo_secrets: |
+            APP_ID=mimir-github-bot:app_id
+            PRIVATE_KEY=mimir-github-bot:private_key
+
+      - name: Generate GitHub App Token
+        id: app-token
+        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0
+        with:
+          # Variables generated by the previous step get-secrets
+          app-id: ${{ env.APP_ID }}
+          private-key: ${{ env.PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Send repository_dispatch to grafana/backend-enterprise
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+          repository: grafana/backend-enterprise
+          event-type: mimir-branch-push
+          client-payload: '{"branch_name": "${{ github.ref_name }}"}'

.github/workflows/update-vendored-mimir-prometheus.yml

@@ -14,6 +14,9 @@ on:
         default: 'main'
         type: string
 
+  repository_dispatch:
+    types: [ mimir-prometheus-branch-push ]
+
 permissions:
   contents: write
   pull-requests: write
@@ -23,10 +26,44 @@ jobs:
   update-mimir-prometheus:
     runs-on: ubuntu-latest
     steps:
+      - name: Determine branch names (repository_dispatch)
+        id: branch-names-dispatch
+        if: github.event_name == 'repository_dispatch'
+        env:
+          MIMIR_PROMETHEUS_BRANCH: ${{ github.event.client_payload.branch_name }}
+        run: |
+          # For repository_dispatch, use the env-injected mimir_prometheus_branch.
+          # The main use case is for weekly branches, so we just use the same branch name here too.
+          echo "Triggered by repository_dispatch"
+          echo "Using mimir-prometheus branch from payload: $MIMIR_PROMETHEUS_BRANCH"
+          
+          echo "mimir_branch=$MIMIR_PROMETHEUS_BRANCH" >> $GITHUB_OUTPUT
+          echo "mimir_prometheus_branch=$MIMIR_PROMETHEUS_BRANCH" >> $GITHUB_OUTPUT
+
+      - name: Determine branch names (workflow_dispatch)
+        id: branch-names-manual
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          MIMIR_BRANCH: ${{ inputs.mimir_branch }}
+          MIMIR_PROMETHEUS_BRANCH: ${{ inputs.mimir_prometheus_branch }}
+        run: |
+          # For workflow_dispatch, use the env-injected branch names.
+          echo "Triggered by workflow_dispatch"
+          echo "Using input values - mimir branch: $MIMIR_BRANCH, mimir-prometheus branch: $MIMIR_PROMETHEUS_BRANCH"
+
+          echo "mimir_branch=$MIMIR_BRANCH" >> $GITHUB_OUTPUT
+          echo "mimir_prometheus_branch=$MIMIR_PROMETHEUS_BRANCH" >> $GITHUB_OUTPUT
+
+      - name: Determine branch names (combined)
+        id: branch-names
+        run: |
+          echo "mimir_branch=${{ steps.branch-names-dispatch.outputs.mimir_branch || steps.branch-names-manual.outputs.mimir_branch }}" >> $GITHUB_OUTPUT
+          echo "mimir_prometheus_branch=${{ steps.branch-names-dispatch.outputs.mimir_prometheus_branch || steps.branch-names-manual.outputs.mimir_prometheus_branch }}" >> $GITHUB_OUTPUT
+
       - name: Check out repository
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.mimir_branch }}
+          ref: ${{ steps.branch-names.outputs.mimir_branch }}
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
           set-safe-directory: '*'
@@ -60,7 +97,7 @@ jobs:
       - name: Get mimir-prometheus commit info
         id: get-commit-info
         env:
-          MIMIR_PROMETHEUS_BRANCH: ${{ inputs.mimir_prometheus_branch }}
+          MIMIR_PROMETHEUS_BRANCH: ${{ steps.branch-names.outputs.mimir_prometheus_branch }}
         run: |
           # Fetch the latest commit hash from the specified branch
           NEW_HASH=$(git ls-remote https://github.com/grafana/mimir-prometheus.git "${MIMIR_PROMETHEUS_BRANCH}" | cut -f1)
@@ -106,23 +143,43 @@ jobs:
           
           echo "Dependencies updated successfully"
 
+      - name: Regenerate OTLP files
+        run: make generate-otlp
+
       - name: Create Pull Request
+        id: create_pull_request
         uses: peter-evans/create-pull-request@4e1beaa7521e8b457b572c090b25bd3db56bf1c5 # v5.0.3
         with:
           token: ${{ steps.app-token.outputs.token }}
-          title: \[${{ inputs.mimir_branch }}\] Update mimir-prometheus to ${{ steps.get-commit-info.outputs.short_hash }}
+          title: '[${{ steps.branch-names.outputs.mimir_branch }}] Update mimir-prometheus to ${{ steps.get-commit-info.outputs.short_hash }}'
           body: |
             ## Update mimir-prometheus dependency
 
             *This PR was automatically created by the [update-vendored-mimir-prometheus.yml](https://github.com/grafana/mimir/blob/main/.github/workflows/update-vendored-mimir-prometheus.yml) workflow.*
 
             ### Details:
-            - **Source branch/ref**: ${{ format('[`{0}`](https://github.com/grafana/mimir-prometheus/tree/{0})', inputs.mimir_prometheus_branch) }}
+            - **Source branch/ref**: ${{ format('[`{0}`](https://github.com/grafana/mimir-prometheus/tree/{0})', steps.branch-names.outputs.mimir_prometheus_branch) }}
             - **Previous commit**: ${{ format('[`{0}`](https://github.com/grafana/mimir-prometheus/commit/{0})', steps.get-commit-info.outputs.old_hash) }}
             - **New commit**: ${{ format('[`{0}`](https://github.com/grafana/mimir-prometheus/commit/{0})', steps.get-commit-info.outputs.new_hash) }}
             - **Changes**: ${{ format('[`{0}...{1}`](https://github.com/grafana/mimir-prometheus/compare/{0}...{1})', steps.get-commit-info.outputs.old_hash, steps.get-commit-info.outputs.new_hash) }}
           commit-message: Update mimir-prometheus to `${{ steps.get-commit-info.outputs.short_hash }}`
-          branch: bot/${{ inputs.mimir_branch }}/update-mimir-prometheus-${{ steps.get-commit-info.outputs.short_hash }}-${{ steps.get-commit-info.outputs.timestamp }}
-          base: ${{ inputs.mimir_branch }}
+          branch: bot/${{ steps.branch-names.outputs.mimir_branch }}/update-mimir-prometheus-${{ steps.get-commit-info.outputs.short_hash }}-${{ steps.get-commit-info.outputs.timestamp }}
+          base: ${{ steps.branch-names.outputs.mimir_branch }}
           labels: vendored-mimir-prometheus-update
           delete-branch: true
+
+      - name: Close Old Vendor PRs
+        run: |
+          prs=$(gh pr list --state open --json number --search 'label:vendored-mimir-prometheus-update' --base $BASE_BRANCH --jq '.[].number')
+          echo "The following Mimir automatic vendor PRs are open: $prs"
+          for pr in $prs; do
+            if [ $pr == ${PULL_REQUEST_NUMBER} ]; then
+              continue
+            fi
+            gh pr comment $pr -b "This pull request has been automatically closed because it has been superseded by #${PULL_REQUEST_NUMBER}."
+            gh pr close $pr --delete-branch
+          done
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+          BASE_BRANCH: ${{ steps.branch-names.outputs.mimir_branch }}
+          PULL_REQUEST_NUMBER: ${{ steps.create_pull_request.outputs.pull-request-number }}