ci.yml

name: CI

on:
  push:
    branches:
      - main
      - stable/*
      - release-*
      - trying
      - staging
  pull_request: { }
  workflow_dispatch: { }
  workflow_call: { }

defaults:
  run:
    # use bash shell by default to ensure pipefail behavior is the default
    # see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
    shell: bash

jobs:
  integration-tests:
    name: Integration tests
    runs-on: [ self-hosted, linux, "16" ]
    timeout-minutes: 45
    env:
      TC_CLOUD_LOGS_VERBOSE: true
      TC_CLOUD_TOKEN: ${{ secrets.TC_CLOUD_TOKEN }}
      TC_CLOUD_CONCURRENCY: 4
      ZEEBE_TEST_DOCKER_IMAGE: localhost:5000/camunda/zeebe:current-test
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          maven-cache: 'true'
      - uses: ./.github/actions/build-zeebe
        id: build-zeebe
        with:
          maven-extra-args: -T1C
      - uses: ./.github/actions/build-docker
        with:
          repository: localhost:5000/camunda/zeebe
          version: current-test
          push: true
          distball: ${{ steps.build-zeebe.outputs.distball }}
      - name: Prepare Testcontainers Cloud agent
        if: env.TC_CLOUD_TOKEN != ''
        run: |
          curl -L -o agent https://app.testcontainers.cloud/download/testcontainers-cloud-agent_linux_x86-64
          chmod +x agent
          ./agent --private-registry-url=http://localhost:5000 '--private-registry-allowed-image-name-globs=*,*/*' > .testcontainers-agent.log 2>&1 &
          ./agent wait
      - name: Create build output log file
        run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
      - name: Maven Test Build
        run: >
          mvn -B -T2 --no-snapshot-updates
          -D forkCount=5
          -D maven.javadoc.skip=true
          -D skipUTs -D skipChecks
          -D failsafe.rerunFailingTestsCount=3 -D flaky.test.reportDir=failsafe-reports
          -P parallel-tests,extract-flaky-tests
          -pl '!qa/update-tests'
          verify
          | tee "${BUILD_OUTPUT_FILE_PATH}"
      - name: Duplicate Test Check
        uses: ./.github/actions/check-duplicate-tests
        with:
          buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: Integration Tests
  qa-update-tests:
    name: QA Update tests
    runs-on: [ self-hosted, linux, "16" ]
    timeout-minutes: 45
    env:
      TC_CLOUD_LOGS_VERBOSE: true
      TC_CLOUD_TOKEN: ${{ secrets.TC_CLOUD_TOKEN }}
      TC_CLOUD_CONCURRENCY: 4
      ZEEBE_TEST_DOCKER_IMAGE: localhost:5000/camunda/zeebe:current-test
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          maven-cache: 'true'
      - uses: ./.github/actions/build-zeebe
        id: build-zeebe
        with:
          maven-extra-args: -T1C
      - uses: ./.github/actions/build-docker
        with:
          repository: localhost:5000/camunda/zeebe
          version: current-test
          push: true
          distball: ${{ steps.build-zeebe.outputs.distball }}
      - name: Prepare Testcontainers Cloud agent
        if: env.TC_CLOUD_TOKEN != ''
        run: |
          curl -L -o agent https://app.testcontainers.cloud/download/testcontainers-cloud-agent_linux_x86-64
          chmod +x agent
          ./agent --private-registry-url=http://localhost:5000 '--private-registry-allowed-image-name-globs=*,*/*' > .testcontainers-agent.log 2>&1 &
          ./agent wait
      - name: Create build output log file
        run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
      - name: Maven Test Build
        run: >
          mvn -B -T2 --no-snapshot-updates
          -D maven.javadoc.skip=true
          -D skipUTs -D skipChecks
          -D failsafe.rerunFailingTestsCount=3 -D flaky.test.reportDir=failsafe-reports
          -P parallel-tests,extract-flaky-tests
          -pl qa/update-tests
          verify
          | tee "${BUILD_OUTPUT_FILE_PATH}"
      - name: Duplicate Test Check
        uses: ./.github/actions/check-duplicate-tests
        with:
          buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: QA Update Tests
  unit-tests:
    name: Unit tests
    runs-on: [ self-hosted, linux, "16" ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          maven-cache: 'true'
      - uses: ./.github/actions/build-zeebe
        with:
          go: false
          maven-extra-args: -T1C
      - name: Create build output log file
        run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
      - name: Maven Test Build
        # we use the verify goal here as flaky test extraction is bound to the post-integration-test
        # phase of Maven https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html#default-lifecycle
        run: >
          mvn -T2 -B --no-snapshot-updates
          -D skipITs -D skipChecks -D surefire.rerunFailingTestsCount=3
          -D junitThreadCount=16
          -P skip-random-tests,parallel-tests,extract-flaky-tests
          verify
          | tee "${BUILD_OUTPUT_FILE_PATH}"
      - name: Normalize artifact name
        run: echo "ARTIFACT_NAME=$(echo ${{ matrix.project }} | sed 's/\//-/g')" >> $GITHUB_ENV
      - name: Duplicate Test Check
        uses: ./.github/actions/check-duplicate-tests
        with:
          buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: "unit tests"
  smoke-tests:
    name: Smoke tests on ${{ matrix.os }}
    timeout-minutes: 20
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ macos-latest, windows-latest, ubuntu-latest ]
    env:
      JAVA_TOOL_OPTIONS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64M
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          # setting up maven often times out on macOS
          maven: ${{ matrix.os != 'macos-latest' }}
      - uses: ./.github/actions/build-zeebe
        with:
          go: false
          maven-extra-args: -am -pl qa/integration-tests -T1C
      - name: Run smoke test
        run: >
          mvn -B --no-snapshot-updates
          -DskipUTs -DskipChecks -Dsurefire.rerunFailingTestsCount=3
          -pl qa/integration-tests
          -P smoke-test,extract-flaky-tests
          verify
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: Smoke Tests ${{ matrix.os }}
  property-tests:
    name: Property Tests
    runs-on: [ self-hosted, linux, "16" ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          maven-cache: 'true'
      - uses: ./.github/actions/build-zeebe
        with:
          go: false
          maven-extra-args: -T1C
      - name: Create build output log file
        run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
      - name: Maven Test Build
        run: >
          mvn -T1C -B --no-snapshot-updates
          -P parallel-tests,include-random-tests
          -D junitThreadCount=16
          -D skipChecks
          test
          | tee "${BUILD_OUTPUT_FILE_PATH}"
      - name: Duplicate Test Check
        uses: ./.github/actions/check-duplicate-tests
        with:
          buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: Property Tests
  go-client:
    name: Go client tests
    runs-on: "n1-standard-8-netssd-preempt"
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
      - uses: ./.github/actions/build-zeebe
        id: build-zeebe
      # Once we're on Go 1.18, use the official gorelease to do this
      - name: Check backwards compatibility
        working-directory: clients/go/
        run: |
          go install github.com/smola/gocompat/...@v0.3.0
          PREFIX=github.com/camunda/zeebe/clients/go/v8
          EXCLUDE=""
          for file in {internal,cmd/zbctl/internal}/*; do
            EXCLUDE="$EXCLUDE --exclude-package $PREFIX/$file"
          done
          gocompat compare --go1compat ${EXCLUDE} ./...
      - uses: ./.github/actions/build-docker
        id: build-docker
        with:
          repository: camunda/zeebe
          version: current-test
          distball: ${{ steps.build-zeebe.outputs.distball }}
      - name: Run Go tests
        working-directory: clients/go
        run: go test -mod=vendor -v ./...
  java-client:
    name: Java client tests
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v3
      # First package the complete application
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          maven-cache: 'true'
          maven-cache-key-modifier: java-client
      - uses: ./.github/actions/build-zeebe
        with:
          go: false
          maven-extra-args: -am -pl clients/java -T1C
      # This is a workaround for java 8, which does not support the --add-exports options
      - run: rm .mvn/jvm.config
      # Then run client tests with JDK 8
      - uses: actions/setup-java@v3.5.1
        with:
          java-version: '8'
          distribution: 'temurin'
          cache: maven
      - name: Create build output log file
        run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
      - name: Maven Test Build
        run: >
          mvn -B --no-snapshot-updates
          -P disableCheckstyle,extract-flaky-tests
          -D skipChecks -D skipITs
          -D surefire.rerunFailingTestsCount=3
          -pl clients/java
          verify
          | tee "${BUILD_OUTPUT_FILE_PATH}"
      - name: Duplicate Test Check
        uses: ./.github/actions/check-duplicate-tests
        with:
          buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
      - name: Upload test artifacts
        uses: ./.github/actions/collect-test-artifacts
        if: always()
        with:
          name: Java 8 Client
  codeql:
    name: CodeQL
    runs-on: [ self-hosted, linux, "16" ]
    permissions:
      security-events: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          maven-cache: 'true'
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: java
          queries: +security-and-quality
      - uses: ./.github/actions/build-zeebe
        with:
          maven-extra-args: -T1C
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2
        with:
          upload: False
          output: sarif-results
      - name: Remove results for generated code
        uses: advanced-security/filter-sarif@main
        with:
          patterns: |
            +**/*.java
            -**/generated-sources/**/*.java
            -**/generated-test-sources/**/*.java
          input: sarif-results/java.sarif
          output: sarif-results/java.sarif
      - name: Upload CodeQL Results
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: sarif-results/java.sarif
  go-lint:
    name: Go linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          java: false
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v3
        with:
          version: v1.32
          working-directory: clients/go
  java-checks:
    name: Java checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-zeebe
        with:
          go: false
          maven-cache: 'true'
          maven-cache-key-modifier: java-checks
      - run: mvn -T1C -B -D skipTests -P !autoFormat,checkFormat,spotbugs verify
  docker-checks:
    name: Docker checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: hadolint/hadolint-action@v2.1.0
        with:
          config: ./.hadolint.yaml
          dockerfile: ./Dockerfile
          format: sarif
          output-file: ./hadolint.sarif
          no-color: true
      - name: Upload Hadolint Results
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: ./hadolint.sarif
      - uses: ./.github/actions/setup-zeebe
      - uses: ./.github/actions/build-zeebe
        id: build-zeebe
      - uses: ./.github/actions/build-docker
        id: build-docker
        with:
          # give it a fake name to ensure we never try pushing it
          repository: localhost:5000/camunda/zeebe
          distball: ${{ steps.build-zeebe.outputs.distball }}
      - name: Verify Docker image
        env:
          DATE: ${{ steps.build-docker.outputs.date }}
          REVISION: ${{ github.sha }}
          VERSION: ${{ steps.build-docker.outputs.version }}
        run: ${PWD}/docker/test/verify.sh '${{ steps.build-docker.outputs.image }}'
  test-summary:
    # Used by bors to check all tests, including the unit test matrix.
    # New test jobs must be added to the `needs` lists!
    # This name is hard-referenced from bors.toml; remember to update that if this name changes
    name: Test summary
    runs-on: ubuntu-latest
    needs:
      - integration-tests
      - qa-update-tests
      - unit-tests
      - smoke-tests
      - property-tests
      - go-client
      - java-client
      - codeql
      - java-checks
      - go-lint
      - docker-checks
    steps:
      - run: exit 0
  event_file:
    # We need to upload the event file as an artifact in order to support
    # publishing the results of forked repositories
    # https://github.com/EnricoMi/publish-unit-test-result-action#support-fork-repositories-and-dependabot-branches
    name: "Event File"
    runs-on: ubuntu-latest
    needs:
      - integration-tests
      - qa-update-tests
      - unit-tests
      - smoke-tests
      - property-tests
      - go-client
      - java-client
    if: always()
    steps:
      - name: Upload
        uses: actions/upload-artifact@v3
        with:
          name: Event File
          path: ${{ github.event_path }}
          retention-days: 1
  deploy-snapshots:
    name: Deploy snapshot artifacts
    needs: [ test-summary ]
    runs-on: ubuntu-latest
    if: github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
    concurrency:
      group: deploy-maven-snapshot
      cancel-in-progress: false
    steps:
      - uses: actions/checkout@v3
      - name: Import Secrets
        id: secrets
        uses: hashicorp/vault-action@v2.4.2
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/zeebe/ci/zeebe ARTIFACTS_USR;
            secret/data/products/zeebe/ci/zeebe ARTIFACTS_PSW;
      - uses: actions/setup-java@v3.5.1
        with:
          distribution: 'temurin'
          java-version: '17'
          server-id: camunda-nexus
          server-username: MAVEN_USERNAME
          server-password: MAVEN_PASSWORD
      # compile and generate-sources to ensure that the Javadoc can be properly generated; compile is
      # necessary when using annotation preprocessors for code generation, as otherwise the symbols are
      # not resolve-able by the Javadoc generator
      - run: mvn -B -D skipTests -D skipChecks compile generate-sources source:jar javadoc:jar deploy
        env:
          MAVEN_USERNAME: ${{ steps.secrets.outputs.ARTIFACTS_USR }}
          MAVEN_PASSWORD: ${{ steps.secrets.outputs.ARTIFACTS_PSW }}
  deploy-docker-snapshot:
    name: Deploy snapshot Docker image
    needs: [ test-summary ]
    runs-on: ubuntu-latest
    if: github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
    concurrency:
      group: deploy-docker-snapshot
      cancel-in-progress: false
    steps:
      - uses: actions/checkout@v3
      - name: Import Secrets
        id: secrets
        uses: hashicorp/vault-action@v2.4.2
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/zeebe/ci/zeebe REGISTRY_HUB_DOCKER_COM_USR;
            secret/data/products/zeebe/ci/zeebe REGISTRY_HUB_DOCKER_COM_PSW;
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ steps.secrets.outputs.REGISTRY_HUB_DOCKER_COM_USR }}
          password: ${{ steps.secrets.outputs.REGISTRY_HUB_DOCKER_COM_PSW }}
      - uses: ./.github/actions/setup-zeebe
      - uses: ./.github/actions/build-zeebe
        id: build-zeebe
      - uses: ./.github/actions/build-docker
        id: build-docker
        with:
          repository: camunda/zeebe
          version: SNAPSHOT
          push: true
          distball: ${{ steps.build-zeebe.outputs.distball }}
  notify-if-failed:
    name: Send slack notification on build failure
    runs-on: ubuntu-latest
    needs: [ test-summary, deploy-snapshots, deploy-docker-snapshot ]
    if: failure() && github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
    steps:
      - id: slack-notify
        name: Send slack notification
        uses: slackapi/slack-github-action@v1.22.0
        with:
          # For posting a rich message using Block Kit
          payload: |
            {
              "text": ":alarm: Build on `main` failed! :alarm:\n${{ github.event.head_commit.url }}",
             	"blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": ":alarm: Build on `main` failed! :alarm:"
                  }
                },
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "Please check the related commit: ${{ github.event.head_commit.url }}\n \\cc @zeebe-medic"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
  auto-merge:
    # This workflow will auto merge a PR authored by dependabot[bot]. It runs only on open PRs ready for
    # review.
    #
    # It will merge the PR only if: it is authored by dependabot[bot], is a patch semantic update, and
    # all CI checks are successful (ignoring the soon-to-be-removed Jenkins check).
    #
    # The workflow is divided into multiple sequential jobs to allow giving only minimal permissions to
    # the GitHub token passed around.
    #
    # Once we're using the merge queue feature, I think we can simplify this workflow a lot by relying
    # on dependabot merging PRs via its commands, as it will always wait for checks to be green before
    # merging.
    name: Auto-merge dependabot PRs
    runs-on: ubuntu-latest
    needs: [ test-summary ]
    if: github.repository == 'camunda/zeebe' && github.actor == 'dependabot[bot]'
    permissions:
      checks: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v3
      - id: metadata
        name: Fetch dependency metadata
        uses: dependabot/fetch-metadata@v1.3.4
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"
      - id: approve-and-merge
        name: Approve and merge PR
        if: steps.metadata.outputs.update-type == 'version-update:semver-patch'
        run: gh pr review ${{ github.event.pull_request.number }} --approve -b "bors merge"
        env:
          GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}"