ci(publish): pass GitHub token to registry artifact generation (#84415) #92
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Connectors | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - "airbyte-integrations/connectors/**/metadata.yaml" | |
| workflow_call: | |
| inputs: | |
| connectors: | |
| description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres" | |
| default: "--name=source-pokeapi" | |
| type: string | |
| with-semver-suffix: | |
| description: | | |
| Semver suffix to apply ephemerally (not committed back to the repo): | |
| - default: Resolves to 'none' on push to master, 'preview' on manual trigger | |
| - none: Use exact version from metadata.yaml (for production releases) | |
| - preview: Append '-preview.{sha}' suffix (for development builds) | |
| - rc: Append '-rc1' suffix (for release candidates) | |
| default: default | |
| type: string | |
| publish-java-tars: | |
| description: "Whether to publish Java connector tar files." | |
| required: false | |
| default: false | |
| type: boolean | |
| gitref: | |
| description: "Git ref (branch or SHA) to build connectors from. Used by pre-release workflow to build from PR branches." | |
| required: false | |
| type: string | |
| dry-run: | |
| description: "Dry run mode: run all logic but skip actual uploads (Docker, GCS, registry). Useful for testing workflow changes." | |
| required: false | |
| default: false | |
| type: boolean | |
| pr: | |
| description: "Pull request number that triggered this workflow (used for Slack notifications and run URL linking)." | |
| required: false | |
| default: "" | |
| type: string | |
| outputs: | |
| docker-image-tag: | |
| description: "Docker image tag used when publishing. For single-connector callers only; multi-connector callers should not rely on this output." | |
| value: ${{ jobs.publish_connectors.outputs.docker-image-tag }} | |
| workflow_dispatch: | |
| inputs: | |
| connectors: | |
| description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres" | |
| default: "--name=source-pokeapi" | |
| type: string | |
| with-semver-suffix: | |
| description: | | |
| Semver suffix to apply ephemerally (not committed back to the repo): | |
| - default: Resolves to 'none' on push to master, 'preview' on manual trigger | |
| - none: Use exact version from metadata.yaml (for production releases) | |
| - preview: Append '-preview.{sha}' suffix (for development builds) | |
| - rc: Append '-rc1' suffix (for release candidates) | |
| default: default | |
| type: choice | |
| options: | |
| - default | |
| - none | |
| - preview | |
| - rc | |
| publish-java-tars: | |
| description: "Whether to publish Java connector tar files." | |
| required: false | |
| default: false | |
| type: boolean | |
| dry-run: | |
| description: "Dry run mode: run all logic but skip actual uploads (Docker, GCS, registry). Useful for testing workflow changes." | |
| required: false | |
| default: false | |
| type: boolean | |
| registry-refresh-only: | |
| description: "Registry Refresh Only. Generate and publish registry metadata artifacts. Skips connector image/package publishing." | |
| required: false | |
| default: false | |
| type: boolean | |
| env: | |
| RELEASE_IMMEDIATELY_PHRASE: Release immediately (bypasses automatic progressive rollout) | |
| jobs: | |
| publish_options: | |
| name: Resolve options for connector publishing | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Airbyte | |
| # v4 | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ inputs.gitref || '' }} | |
| fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed. | |
| submodules: true # Required for the enterprise repo since it uses a submodule that needs to exist for this workflow to run successfully. | |
| - name: List connectors to publish [manual] | |
| id: list-connectors-manual | |
| if: github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| # When invoked manually, we run on the connectors specified in the input. | |
| run: echo connectors-to-publish=$(./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors }}) | tee -a $GITHUB_OUTPUT | |
| - name: List connectors to publish [On merge to master] | |
| id: list-connectors-master | |
| if: github.event_name == 'push' | |
| shell: bash | |
| # When merging to master, we run on connectors that have changed since the previous commit. | |
| run: | | |
| connectors_json="$(./poe-tasks/get-modified-connectors.sh --prev-commit --require-version-bump --json)" | |
| echo "connectors-to-publish=${connectors_json}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Count connectors to publish | |
| id: count-connectors | |
| shell: bash | |
| env: | |
| CONNECTORS_JSON: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }} | |
| run: | | |
| set -euo pipefail | |
| echo "connector-count=$(jq '[.connector[] | select(. != "")] | length' <<< "$CONNECTORS_JSON")" | tee -a "$GITHUB_OUTPUT" | |
| - name: Resolve semver suffix | |
| id: resolve-semver-suffix | |
| shell: bash | |
| run: | | |
| # Determine the semver suffix mode. | |
| # Priority: with-semver-suffix input > defaults | |
| # Note: On push events, inputs.* is undefined (empty string), so we treat empty as 'default' | |
| SEMVER_SUFFIX="${{ inputs.with-semver-suffix || 'default' }}" | |
| # Resolve 'default' based on trigger type | |
| if [[ "$SEMVER_SUFFIX" == "default" ]]; then | |
| if [[ "${{ github.event_name }}" == "push" ]]; then | |
| # On merge to master, use exact version (no suffix) | |
| SEMVER_SUFFIX="none" | |
| elif [[ "${{ inputs.registry-refresh-only || 'false' }}" == "true" ]]; then | |
| # Registry refresh-only publishes use the exact version already declared in metadata.yaml. | |
| SEMVER_SUFFIX="none" | |
| else | |
| # On manual trigger, use preview suffix | |
| SEMVER_SUFFIX="preview" | |
| fi | |
| fi | |
| echo "with-semver-suffix=$SEMVER_SUFFIX" | tee -a $GITHUB_OUTPUT | |
| - name: Validate registry-refresh-only options | |
| if: > | |
| inputs.registry-refresh-only == true | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]; then | |
| echo "registry-refresh-only publishes are only supported from workflow_dispatch." | |
| exit 1 | |
| fi | |
| CONNECTOR_COUNT=$(echo '${{ steps.list-connectors-manual.outputs.connectors-to-publish }}' | jq '.connector | length') | |
| if [[ "$CONNECTOR_COUNT" -eq 0 ]]; then | |
| echo "registry-refresh-only publishes require at least one connector." | |
| exit 1 | |
| fi | |
| if [[ "$CONNECTOR_COUNT" -gt 1 ]]; then | |
| echo "registry-refresh-only publishes currently support exactly one connector." | |
| exit 1 | |
| fi | |
| if [[ "${{ steps.resolve-semver-suffix.outputs.with-semver-suffix }}" != "none" ]]; then | |
| echo "registry-refresh-only publishes must use with-semver-suffix=none or default." | |
| exit 1 | |
| fi | |
| - name: Resolve publish-java-tars | |
| id: resolve-publish-java-tars | |
| shell: bash | |
| run: | | |
| # If workflow is triggered on merge to master, we can assume | |
| # the workflow is running in the open-source repo and we always want to publish. | |
| if [[ "${{ inputs.publish-java-tars }}" == "" ]]; then | |
| echo "publish-java-tars=true" | tee -a $GITHUB_OUTPUT | |
| # if workflow is triggered manually, any other way, use the input publish-java-tars. | |
| else | |
| echo "publish-java-tars=${{ inputs.publish-java-tars }}" | tee -a $GITHUB_OUTPUT | |
| fi | |
| outputs: | |
| # Exactly one of the manual/master steps will run, so just OR them together. | |
| connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }} | |
| with-semver-suffix: ${{ steps.resolve-semver-suffix.outputs.with-semver-suffix }} | |
| # publishing java tars is not optional if triggered by push to master (in the non-enterprise repo). | |
| publish-java-tars: ${{ steps.resolve-publish-java-tars.outputs.publish-java-tars }} | |
| # dry-run mode skips all uploads but runs all logic | |
| dry-run: ${{ inputs.dry-run || 'false' }} | |
| # registry-refresh-only skips connector image/package publishing but still publishes registry artifacts | |
| registry-refresh-only: ${{ inputs.registry-refresh-only || 'false' }} | |
| # registry-refresh-only is validated to exactly one connector, so this is safe to pass to scoped registry compile. | |
| registry-refresh-connector-name: ${{ fromJson(steps.list-connectors-manual.outputs.connectors-to-publish || '{"connector":[""]}').connector[0] }} | |
| connector-count: ${{ steps.count-connectors.outputs.connector-count }} | |
| publish_connectors: | |
| name: Publish connectors | |
| needs: [publish_options] | |
| if: needs.publish_options.outputs.connector-count != '0' | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: ${{ fromJson(needs.publish_options.outputs.connectors-to-publish) }} | |
| max-parallel: 5 | |
| # Allow all jobs to run, even if one fails | |
| fail-fast: false | |
| outputs: | |
| docker-image-tag: ${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }} | |
| steps: | |
| - name: Authenticate as GitHub App | |
| uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 | |
| id: get-app-token | |
| with: | |
| owner: "airbytehq" | |
| # Use dynamic repo name since this workflow is also called from airbyte-enterprise | |
| repositories: ${{ github.event.repository.name }} | |
| app-id: ${{ secrets.OCTAVIA_PUBLISH_BOT_APP_ID }} | |
| private-key: ${{ secrets.OCTAVIA_PUBLISH_BOT_PRIVATE_KEY }} | |
| - name: Checkout Airbyte | |
| # v4 | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ inputs.gitref || '' }} | |
| fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed. | |
| submodules: true # Required for the enterprise repo since it uses a submodule that needs to exist for this workflow to run successfully. | |
| - name: Create docker buildx builder | |
| id: create-buildx-builder | |
| shell: bash | |
| run: docker buildx create --use --driver=docker-container --name builder --platform linux/amd64,linux/arm64 | |
| - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 | |
| with: | |
| distribution: zulu | |
| java-version: 21 | |
| cache: gradle | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| - name: Set up Python | |
| # v5 | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 | |
| with: | |
| python-version: "3.11" | |
| check-latest: true | |
| update-environment: true | |
| - name: Install and configure Poetry | |
| # v1 | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a | |
| with: | |
| # There are a few uses of `poetry run --directory` in various scripts. | |
| # If we upgrade to 2.x, those need to be switched to `--project`. | |
| version: 1.8.5 | |
| - name: Install the latest version of uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| # We pass a token with dedicated GH rate limit | |
| github-token: ${{ steps.get-app-token.outputs.token }} | |
| - name: Install Poe and Ops CLI | |
| run: | | |
| # Install Poe so we can run the connector tasks: | |
| uv tool install poethepoet | |
| # Install the ops CLI for version resolution: | |
| uv tool install airbyte-internal-ops | |
| - name: Get connector metadata | |
| id: connector-metadata | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| set -euo pipefail | |
| echo "connector-language=$(poe -qq get-language)" | tee -a $GITHUB_OUTPUT | |
| echo "connector-version=$(poe -qq get-version)" | tee -a $GITHUB_OUTPUT | |
| - name: Resolve ephemeral version tag (preview) | |
| id: resolve-ephemeral-version | |
| if: needs.publish_options.outputs.with-semver-suffix == 'preview' | |
| run: | | |
| RESOLVED_TAG=$( | |
| airbyte-ops local connector get-version \ | |
| --name "${{ matrix.connector }}" \ | |
| --repo-path "$GITHUB_WORKSPACE" \ | |
| --next --prerelease | |
| ) | |
| echo "connector-version-tag=${RESOLVED_TAG}" | tee -a $GITHUB_OUTPUT | |
| - name: Resolve ephemeral version tag (rc) | |
| id: resolve-ephemeral-version-rc | |
| if: needs.publish_options.outputs.with-semver-suffix == 'rc' | |
| run: | | |
| RESOLVED_TAG=$( | |
| airbyte-ops local connector get-version \ | |
| --name "${{ matrix.connector }}" \ | |
| --repo-path "$GITHUB_WORKSPACE" \ | |
| --next --bump-type rc | |
| ) | |
| echo "connector-version-tag=${RESOLVED_TAG}" | tee -a $GITHUB_OUTPUT | |
| - name: Resolve connector version tag | |
| id: resolve-version | |
| run: | | |
| echo "connector-version-tag=${{ | |
| steps.resolve-ephemeral-version.outputs.connector-version-tag | |
| || steps.resolve-ephemeral-version-rc.outputs.connector-version-tag | |
| || steps.connector-metadata.outputs.connector-version | |
| }}" | tee -a $GITHUB_OUTPUT | |
| - name: Resolve docker image tag | |
| id: resolve-docker-image-tag | |
| env: | |
| # Disable Segment.io Telemetry in PyAirbyte | |
| DO_NOT_TRACK: "1" | |
| run: | | |
| set -euo pipefail | |
| CONNECTOR_VERSION="${{ steps.connector-metadata.outputs.connector-version }}" | |
| SEMVER_SUFFIX="${{ needs.publish_options.outputs.with-semver-suffix }}" | |
| # Generate docker image tag based on semver suffix mode | |
| case "$SEMVER_SUFFIX" in | |
| preview) | |
| DOCKER_TAG=$(airbyte-ops registry connector-version next \ | |
| --name "${{ matrix.connector }}" \ | |
| --sha "$(git rev-parse HEAD)" \ | |
| --base-version "${CONNECTOR_VERSION}") | |
| echo "docker-image-tag=${DOCKER_TAG}" | tee -a $GITHUB_OUTPUT | |
| echo "release-type-flag=--pre-release" | tee -a $GITHUB_OUTPUT | |
| ;; | |
| rc) | |
| echo "docker-image-tag=${CONNECTOR_VERSION}-rc1" | tee -a $GITHUB_OUTPUT | |
| echo "release-type-flag=--main-release" | tee -a $GITHUB_OUTPUT | |
| ;; | |
| none|*) | |
| echo "docker-image-tag=${CONNECTOR_VERSION}" | tee -a $GITHUB_OUTPUT | |
| echo "release-type-flag=--main-release" | tee -a $GITHUB_OUTPUT | |
| ;; | |
| esac | |
| # --- Release block check --- | |
| # If a block-release.yaml marker exists for this connector, skip all | |
| # build and publish steps with a WARNING. Other connectors in the | |
| # same matrix run are unaffected. | |
| # See: https://github.com/airbytehq/airbyte-ops-mcp/issues/686 | |
| - name: Check for release block | |
| id: check-release-block | |
| if: needs.publish_options.outputs.with-semver-suffix == 'none' | |
| shell: bash | |
| run: | | |
| BLOCK_FILE="airbyte-integrations/connectors/${{ matrix.connector }}/block-release.yaml" | |
| if [[ -f "$BLOCK_FILE" ]]; then | |
| REASON=$(cat "$BLOCK_FILE" | head -1 | sed 's/^reason: *//' | tr -d '"') | |
| echo "::warning::⚠️ Release BLOCKED for ${{ matrix.connector }}: ${REASON}" | |
| echo "::warning::Skipping all publish steps. Remove block-release.yaml to unblock." | |
| echo "blocked=true" >> $GITHUB_OUTPUT | |
| cat "$BLOCK_FILE" | |
| else | |
| echo "blocked=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Publish to Python Registry | |
| id: publish-python-registry | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language == 'python' && | |
| needs.publish_options.outputs.dry-run != 'true' && | |
| needs.publish_options.outputs.with-semver-suffix == 'none' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| shell: bash | |
| run: | | |
| ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} --with-semver-suffix ${{ needs.publish_options.outputs.with-semver-suffix }} | |
| env: | |
| PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }} | |
| CONNECTOR_VERSION_TAG: ${{ steps.resolve-version.outputs.connector-version-tag }} | |
| - name: Build and publish JVM connectors images | |
| id: build-and-publish-JVM-connectors-images | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language == 'java' && | |
| needs.publish_options.outputs.dry-run != 'true' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| shell: bash | |
| env: | |
| CONNECTOR_VERSION_TAG: ${{ steps.resolve-version.outputs.connector-version-tag }} | |
| run: | | |
| ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --name ${{ matrix.connector }} --with-semver-suffix ${{ needs.publish_options.outputs.with-semver-suffix }} --publish | |
| - name: "[DRY-RUN] Build JVM connectors images (no publish)" | |
| id: build-JVM-connectors-images-dry-run | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language == 'java' && | |
| needs.publish_options.outputs.dry-run == 'true' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| shell: bash | |
| env: | |
| CONNECTOR_VERSION_TAG: ${{ steps.resolve-version.outputs.connector-version-tag }} | |
| run: | | |
| echo "DRY-RUN: Building JVM connector without publishing..." | |
| ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --name ${{ matrix.connector }} --with-semver-suffix ${{ needs.publish_options.outputs.with-semver-suffix }} | |
| - name: Publish JVM connectors tar file | |
| id: publish-JVM-connectors-tar-file | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language == 'java' && | |
| needs.publish_options.outputs.publish-java-tars == 'true' && | |
| needs.publish_options.outputs.dry-run != 'true' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| shell: bash | |
| run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --with-semver-suffix ${{ needs.publish_options.outputs.with-semver-suffix }} | |
| env: | |
| GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }} | |
| CONNECTOR_VERSION_TAG: ${{ steps.resolve-version.outputs.connector-version-tag }} | |
| # we allow it to fail because we are testing this step. We should remove this once we are sure it works. | |
| continue-on-error: true | |
| - name: "[DRY-RUN] Skip Publish JVM connectors tar file" | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language == 'java' && | |
| needs.publish_options.outputs.publish-java-tars == 'true' && | |
| needs.publish_options.outputs.dry-run == 'true' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| run: | | |
| echo "DRY-RUN: Skipping JVM connector tar file upload for ${{ matrix.connector }}" | |
| - name: Build and publish Python and Manifest-Only connectors images | |
| id: build-and-publish-python-manifest-only-connectors-images | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| steps.connector-metadata.outputs.connector-language != 'java' && | |
| needs.publish_options.outputs.registry-refresh-only != 'true' | |
| uses: ./.github/actions/connector-image-build-push | |
| with: | |
| connector-name: ${{ matrix.connector }} | |
| tag-override: ${{ steps.resolve-version.outputs.connector-version-tag }} | |
| with-semver-suffix: ${{ needs.publish_options.outputs.with-semver-suffix }} | |
| dry-run: ${{ needs.publish_options.outputs.dry-run }} | |
| docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| # --- Registry artifact generation and publishing --- | |
| - name: Pull existing connector image for registry refresh | |
| if: > | |
| needs.publish_options.outputs.registry-refresh-only == 'true' | |
| shell: bash | |
| run: | | |
| DOCKER_REPOSITORY=$(yq -r '.data.dockerRepository' "airbyte-integrations/connectors/${{ matrix.connector }}/metadata.yaml") | |
| DOCKER_IMAGE="${DOCKER_REPOSITORY}:${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" | |
| docker pull "${DOCKER_IMAGE}" | |
| - name: Enable progressive rollout for RC builds | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| needs.publish_options.outputs.with-semver-suffix == 'rc' | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| echo "Enabling progressive rollout for RC build..." | |
| echo "=== enableProgressiveRollout BEFORE update ===" | |
| grep -n "enableProgressiveRollout" metadata.yaml || echo "(not found)" | |
| # Ephemerally enable progressive rollout in metadata.yaml (not committed back) | |
| yq -i '.data.releases.rolloutConfiguration.enableProgressiveRollout = true' metadata.yaml | |
| echo "=== enableProgressiveRollout AFTER update ===" | |
| grep -n "enableProgressiveRollout" metadata.yaml | |
| echo "Progressive rollout enabled in metadata.yaml" | |
| # --- Ephemeral version bump for pre-release builds --- | |
| # For preview builds, the metadata.yaml still contains the base version (e.g. 3.2.3) | |
| # but the actual published tag is the preview version (e.g. 3.2.3-preview.820ce48). | |
| # The artifact generator reads dockerImageTag from metadata.yaml, so we need to | |
| # bump it in-place before generating artifacts. This is ephemeral — not committed. | |
| # See: https://github.com/airbytehq/airbyte-ops-mcp/issues/604 | |
| - name: Ephemeral version bump for pre-release | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| needs.publish_options.outputs.with-semver-suffix == 'preview' | |
| run: | | |
| echo "Bumping metadata.yaml version to ${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" | |
| airbyte-ops local connector bump-version \ | |
| --name "${{ matrix.connector }}" \ | |
| --repo-path "$GITHUB_WORKSPACE" \ | |
| --new-version "${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" \ | |
| --no-changelog \ | |
| --progressive-rollout-enabled=false | |
| - name: Resolve release-immediately checkbox | |
| id: release-immediately | |
| if: > | |
| github.event_name == 'push' && | |
| needs.publish_options.outputs.with-semver-suffix == 'none' && | |
| steps.check-release-block.outputs.blocked != 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.get-app-token.outputs.token }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| body_file="${RUNNER_TEMP}/merged-pr-body.md" | |
| gh api "repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" --jq '.[0].body // ""' > "$body_file" || { | |
| echo "::warning::Could not read the merged pull request body; using the safe default of progressive rollout enabled." | |
| echo "checked=false" | tee -a "$GITHUB_OUTPUT" | |
| exit 0 | |
| } | |
| checked='false' | |
| tr -d '*_' < "$body_file" | | |
| grep -Ei "^[[:space:]>]*-[[:space:]]*\\[[xX]\\]" | | |
| grep -Fqi -- "$RELEASE_IMMEDIATELY_PHRASE" && checked='true' | |
| echo "checked=$checked" | tee -a "$GITHUB_OUTPUT" | |
| - name: Resolve progressive rollout setting | |
| id: progressive-rollout | |
| if: steps.release-immediately.outputs.checked == 'true' | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| set -euo pipefail | |
| enabled="$(yq -r '.data.releases.rolloutConfiguration.enableProgressiveRollout // false' metadata.yaml)" | |
| echo "enabled=$enabled" | tee -a "$GITHUB_OUTPUT" | |
| - name: Disable progressive rollout for immediate release | |
| if: > | |
| steps.release-immediately.outputs.checked == 'true' && | |
| steps.progressive-rollout.outputs.enabled == 'true' | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| env: | |
| CONNECTOR: ${{ matrix.connector }} | |
| run: | | |
| set -euo pipefail | |
| yq -i '.data.releases.rolloutConfiguration.enableProgressiveRollout = false' metadata.yaml | |
| echo "::notice::Immediate release requested for ${CONNECTOR}; progressive rollout disabled for this publish." | |
| - name: Keep progressive rollout metadata unchanged | |
| if: > | |
| steps.release-immediately.outputs.checked == 'true' && | |
| steps.progressive-rollout.outputs.enabled != 'true' | |
| env: | |
| CONNECTOR: ${{ matrix.connector }} | |
| run: echo "::notice::Immediate release requested for ${CONNECTOR}, but progressive rollout is not enabled; metadata.yaml remains unchanged." | |
| # --- Registry artifact generation and publishing --- | |
| - name: Generate Registry Artifacts | |
| id: generate-registry-artifacts | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' | |
| shell: bash | |
| env: | |
| GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ inputs.pr }} | |
| run: | | |
| echo "Generating registry artifacts for ${{ matrix.connector }}" | |
| DOCKER_REPOSITORY=$(yq -r '.data.dockerRepository' "airbyte-integrations/connectors/${{ matrix.connector }}/metadata.yaml") | |
| DOCKER_IMAGE="${DOCKER_REPOSITORY}:${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" | |
| PR_NUMBER_ARGS=() | |
| PR_NUMBER="${PR_NUMBER//[[:space:]]/}" | |
| if [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then | |
| PR_NUMBER_ARGS+=(--pr-number "$PR_NUMBER") | |
| elif [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "0" ]]; then | |
| echo "::warning::Ignoring non-numeric PR number '$PR_NUMBER' for release attribution." | |
| fi | |
| airbyte-ops \ | |
| registry connector-version artifacts generate \ | |
| --metadata-file "airbyte-integrations/connectors/${{ matrix.connector }}/metadata.yaml" \ | |
| --docker-image "${DOCKER_IMAGE}" \ | |
| --output-dir ./registry-artifacts/${{ matrix.connector }} \ | |
| --with-validate \ | |
| --with-sbom \ | |
| --with-dependency-dump \ | |
| "${PR_NUMBER_ARGS[@]}" | |
| - name: Upload Registry CI Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: registry-artifacts-${{ matrix.connector }}-${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }} | |
| path: ./registry-artifacts/${{ matrix.connector }}/ | |
| retention-days: 7 | |
| - name: Publish Registry Artifacts | |
| id: publish-registry-artifacts | |
| if: > | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| needs.publish_options.outputs.dry-run != 'true' && | |
| steps.generate-registry-artifacts.outcome == 'success' | |
| shell: bash | |
| env: | |
| GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| REGISTRY_STORE: "coral:prod" | |
| run: | | |
| echo "Publishing registry artifacts for ${{ matrix.connector }}" | |
| airbyte-ops \ | |
| registry connector-version artifacts publish \ | |
| --name "${{ matrix.connector }}" \ | |
| --version "${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" \ | |
| --artifacts-dir ./registry-artifacts/${{ matrix.connector }} \ | |
| --store "${REGISTRY_STORE}" \ | |
| --with-validate | |
| - name: Build success notification context | |
| id: success-context | |
| if: > | |
| success() && | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| needs.publish_options.outputs.dry-run != 'true' | |
| run: | | |
| CONNECTOR="${{ matrix.connector }}" | |
| DOCKER_TAG="${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}" | |
| SEMVER_SUFFIX="${{ needs.publish_options.outputs.with-semver-suffix }}" | |
| PR_NUMBER="${{ inputs.pr }}" | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "0" ]]; then | |
| RUN_URL="${RUN_URL}?pr=${PR_NUMBER}" | |
| fi | |
| # Determine release type label and emoji | |
| if [[ "$SEMVER_SUFFIX" == "none" || -z "$SEMVER_SUFFIX" ]]; then | |
| RELEASE_TYPE="GA" | |
| EMOJI="🟢" | |
| else | |
| RELEASE_TYPE="Pre-release" | |
| EMOJI="🔵" | |
| fi | |
| if [[ "${{ needs.publish_options.outputs.registry-refresh-only }}" == "true" ]]; then | |
| RELEASE_TYPE="Registry refresh-only artifact" | |
| EMOJI="🟡" | |
| fi | |
| # Build URLs for hyperlinks | |
| DOCKERHUB_URL="https://hub.docker.com/r/airbyte/${CONNECTOR}/tags?name=${DOCKER_TAG}" | |
| CLOUD_REGISTRY_URL="https://connectors.airbyte.com/files/metadata/airbyte/${CONNECTOR}/${DOCKER_TAG}/cloud.json" | |
| OSS_REGISTRY_URL="https://connectors.airbyte.com/files/metadata/airbyte/${CONNECTOR}/${DOCKER_TAG}/oss.json" | |
| # Build PR text | |
| PR_TEXT="" | |
| if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "0" ]]; then | |
| PR_TEXT="\nPR: <${{ github.server_url }}/${{ github.repository }}/pull/${PR_NUMBER}|#${PR_NUMBER}>" | |
| fi | |
| echo "release-type=$RELEASE_TYPE" >> $GITHUB_OUTPUT | |
| echo "emoji=$EMOJI" >> $GITHUB_OUTPUT | |
| echo "run-url=$RUN_URL" >> $GITHUB_OUTPUT | |
| echo "dockerhub-url=$DOCKERHUB_URL" >> $GITHUB_OUTPUT | |
| echo "cloud-registry-url=$CLOUD_REGISTRY_URL" >> $GITHUB_OUTPUT | |
| echo "oss-registry-url=$OSS_REGISTRY_URL" >> $GITHUB_OUTPUT | |
| echo "pr-text=$PR_TEXT" >> $GITHUB_OUTPUT | |
| - name: Notify Slack on connector publish success | |
| if: > | |
| success() && | |
| steps.check-release-block.outputs.blocked != 'true' && | |
| needs.publish_options.outputs.dry-run != 'true' && | |
| steps.success-context.outcome == 'success' | |
| uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "channel": "#connector-publish-updates", | |
| "username": "Connectors CI/CD Bot", | |
| "text": "${{ steps.success-context.outputs.emoji }} *${{ steps.success-context.outputs.release-type }} Publish SUCCESS:*\nConnector: <${{ steps.success-context.outputs.dockerhub-url }}|airbyte/${{ matrix.connector }}:${{ steps.resolve-docker-image-tag.outputs.docker-image-tag }}>\nReference: <${{ steps.success-context.outputs.cloud-registry-url }}|cloud.json> · <${{ steps.success-context.outputs.oss-registry-url }}|oss.json> · <${{ steps.success-context.outputs.run-url }}|CI Logs>${{ steps.success-context.outputs.pr-text }}" | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} | |
| generate_connector_registry: | |
| name: Generate connector registry | |
| needs: [publish_options, publish_connectors] | |
| if: needs.publish_options.outputs.dry-run != 'true' | |
| uses: ./.github/workflows/generate-connector-registries.yml | |
| with: | |
| pr: ${{ inputs.pr }} | |
| force: ${{ needs.publish_options.outputs.registry-refresh-only == 'true' }} | |
| connector-name: ${{ needs.publish_options.outputs.registry-refresh-only == 'true' && needs.publish_options.outputs.registry-refresh-connector-name || '' }} | |
| secrets: inherit | |
| notify-failure-slack-channel: | |
| name: "Notify Slack Channel on Publish Failures" | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - publish_connectors | |
| - publish_options | |
| if: ${{ always() && needs.publish_connectors.result == 'failure' }} | |
| steps: | |
| - name: Authenticate as GitHub App | |
| uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 | |
| id: get-app-token | |
| with: | |
| owner: "airbytehq" | |
| # Use dynamic repo name since this workflow is also called from airbyte-enterprise | |
| repositories: ${{ github.event.repository.name }} | |
| app-id: ${{ secrets.OCTAVIA_PUBLISH_BOT_APP_ID }} | |
| private-key: ${{ secrets.OCTAVIA_PUBLISH_BOT_PRIVATE_KEY }} | |
| - name: Checkout Airbyte | |
| # v4 | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ inputs.gitref || '' }} | |
| submodules: true # Required for the enterprise repo since it uses a submodule that needs to exist for this workflow to run successfully. | |
| - name: Match GitHub User to Slack User | |
| id: match-github-to-slack-user | |
| continue-on-error: true | |
| uses: ./.github/actions/match-github-to-slack-user | |
| env: | |
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | |
| GITHUB_API_TOKEN: ${{ steps.get-app-token.outputs.token }} | |
| - name: Build notification context | |
| id: context | |
| run: | | |
| PR_NUMBER="${{ inputs.pr }}" | |
| SEMVER_SUFFIX="${{ needs.publish_options.outputs.with-semver-suffix }}" | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Append ?pr=N for "Back to pull request" link in GitHub Actions UI | |
| if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "0" ]]; then | |
| RUN_URL="${RUN_URL}?pr=${PR_NUMBER}" | |
| fi | |
| # Determine release type label | |
| if [[ "$SEMVER_SUFFIX" == "none" || -z "$SEMVER_SUFFIX" ]]; then | |
| RELEASE_TYPE="GA" | |
| else | |
| RELEASE_TYPE="Pre-release (${SEMVER_SUFFIX})" | |
| fi | |
| # Extract connector names from the matrix JSON | |
| CONNECTORS_JSON='${{ needs.publish_options.outputs.connectors-to-publish }}' | |
| CONNECTOR_NAMES=$(echo "$CONNECTORS_JSON" | jq -r '.connector | join(", ")') | |
| if [[ -n "$CONNECTOR_NAMES" ]]; then | |
| CONNECTOR_TEXT="\nConnector(s): \`${CONNECTOR_NAMES}\`" | |
| else | |
| CONNECTOR_TEXT="" | |
| fi | |
| # Build trigger description | |
| if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "0" ]]; then | |
| TRIGGER_TEXT="triggered from PR <${{ github.server_url }}/${{ github.repository }}/pull/${PR_NUMBER}|#${PR_NUMBER}>" | |
| elif [[ "${{ github.ref }}" == "refs/heads/master" ]]; then | |
| TRIGGER_TEXT="merged to master" | |
| elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| TRIGGER_TEXT="triggered via workflow_dispatch" | |
| elif [[ "${{ github.event_name }}" == "workflow_call" ]]; then | |
| TRIGGER_TEXT="triggered via workflow_call" | |
| else | |
| TRIGGER_TEXT="triggered via ${{ github.event_name }}" | |
| fi | |
| echo "run-url=$RUN_URL" >> $GITHUB_OUTPUT | |
| echo "release-type=$RELEASE_TYPE" >> $GITHUB_OUTPUT | |
| echo "trigger-text=$TRIGGER_TEXT" >> $GITHUB_OUTPUT | |
| echo "connector-text=$CONNECTOR_TEXT" >> $GITHUB_OUTPUT | |
| - name: Send publish failures to connector-publish-failures channel | |
| id: slack | |
| uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 | |
| with: | |
| # This data can be any valid JSON from a previous step in the GitHub Action | |
| payload: | | |
| { | |
| "channel": "#connector-publish-failures", | |
| "username": "Connectors CI/CD Bot", | |
| "text": "🚨 ${{ steps.context.outputs.release-type }} publish workflow failed:${{ steps.context.outputs.connector-text }}\n <${{ steps.context.outputs.run-url }}|View workflow run>\n ${{ steps.context.outputs.trigger-text }} by ${{ github.actor }}${{ steps.match-github-to-slack-user.outputs.slack_user_ids && format(' (<@{0}>)', steps.match-github-to-slack-user.outputs.slack_user_ids) || '' }}." | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} |