diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index bbadd3a5fa..67a9e3c8a1 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -26,11 +26,48 @@ global_job_config: - checkout - sem-version java 8 - sem-version python 3.9 + - . vault-setup - . cache-maven restore - pip install tox==3.28.0 - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-") - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -) + # For PR Builds using Packaging + - pip install confluent-release-tools + - if [ $BRANCH_TAG == "master" ]; then export BUILD_KEY=$(pinto get-master-version); else export BUILD_KEY=$BRANCH_TAG; fi + - export LATEST_PACKAGING_BUILD_NUMBER=$(aws s3 ls s3://jenkins-confluent-packages/$BRANCH_TAG/ --no-paginate --recursive | grep "$BRANCH_TAG/[0-9]" | sort | tail -n 1 | awk '{print $4}' | awk -F + / '{print $2}') + - export CONFLUENT_VERSION=$(pinto get-version --build $BUILD_KEY --key confluent.version) + - export DEFAULT_OS_TYPE="ubi" + - export URL_CONFLUENT_VERSION=$(echo $CONFLUENT_VERSION | awk -F . '{print $1"."$2}') + - export PACKAGES_URL="https://s3-us-west-2.amazonaws.com/jenkins-confluent-packages/$BRANCH_TAG/$BUILD_NUMBER/PACKAGE_TYPE/$URL_CONFLUENT_VERSION" + - export PACKAGING_BUILD_NUMBER=$LATEST_PACKAGING_BUILD_NUMBER + - >- + if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then + export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}') + elif [[ ! $IS_RELEASE ]]; then + export PLATFORM_LABEL="~SNAPSHOT" + else + export PLATFORM_LABEL="" + fi + - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION -DALLOW_UNSIGNED=$ALLOW_UNSIGNED" + - >- + if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then + if [[ $IS_BETA ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-beta-maven/$BRANCH_TAG/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_PREVIEW ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-alpha-maven/$BRANCH_NAME/$PACKAGING_BUILD_NUMBER/maven" + elif [[ $IS_RC ]]; then + export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven/v$BRANCH_NAME/maven" + if [[ $PACKAGES_MAVEN_URL ]]; then + export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL + fi + fi + # Overwrite maven global configuration + . vault-sem-get-secret maven-settings-cp-dockerfile + else + echo "This job is not a isBetaJob, isPreviewJob, isHotfixJob, or isRcJob (What we know how to handle) - and we don't know how to handle it" + fi - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY @@ -44,8 +81,7 @@ blocks: - name: Build, Test, & Scan AMD dependencies: [] run: - # don't run the tests on non-functional changes... - when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})" + when: "pull_request =~ '.*'" task: jobs: - name: Build, Test, & Scan ubi8 @@ -54,8 +90,11 @@ blocks: - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH - ci-tools ci-update-version + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS - . cache-maven store - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; @@ -69,74 +108,10 @@ blocks: - . publish-test-results - artifact push workflow target/test-results - artifact push workflow target --destination target-AMD - - name: Deploy AMD confluentinc/cp-base-new - dependencies: ["Build, Test, & Scan AMD"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - jobs: - - name: Deploy AMD confluentinc/cp-base-new ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy AMD confluentinc/cp-base-lite - dependencies: ["Build, Test, & Scan AMD"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - jobs: - - name: Deploy AMD confluentinc/cp-base-lite ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy AMD confluentinc/cp-jmxterm - dependencies: ["Build, Test, & Scan AMD"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - jobs: - - name: Deploy AMD confluentinc/cp-jmxterm ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - name: Build & Test ARM dependencies: [] run: - # don't run the tests on non-functional changes... - when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})" + when: "pull_request =~ '.*'" task: agent: machine: @@ -147,9 +122,12 @@ blocks: - export OS_TAG="-ubi8" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 + $PACKAGING_BUILD_ARGS - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: @@ -158,107 +136,6 @@ blocks: - . publish-test-results - artifact push workflow target/test-results - artifact push workflow target --destination target-ARM - - name: Deploy ARM confluentinc/cp-base-new - dependencies: ["Build & Test ARM"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - agent: - machine: - type: s1-prod-ubuntu20-04-arm64-1 - jobs: - - name: Deploy ARM confluentinc/cp-base-new ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy ARM confluentinc/cp-base-lite - dependencies: ["Build & Test ARM"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - agent: - machine: - type: s1-prod-ubuntu20-04-arm64-1 - jobs: - - name: Deploy ARM confluentinc/cp-base-lite ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Deploy ARM confluentinc/cp-jmxterm - dependencies: ["Build & Test ARM"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - agent: - machine: - type: s1-prod-ubuntu20-04-arm64-1 - jobs: - - name: Deploy ARM confluentinc/cp-jmxterm ubi8 - commands: - - export OS_TAG="-ubi8" - - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm - - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH - - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH - - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - - docker pull $DEV_IMAGE_FULL - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG - - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG - - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - - name: Create Manifest and Maven Deploy - dependencies: ["Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", - "Deploy ARM confluentinc/cp-jmxterm"] - run: - when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'" - task: - jobs: - - name: Create Manifest and Maven Deploy - commands: - - export DOCKER_PROD_IMAGE_NAME=$DOCKER_PROD_REGISTRY${DOCKER_REPOS// / $DOCKER_PROD_REGISTRY} - - ci-tools ci-update-version - - ci-tools ci-push-tag - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/ - -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests -Ddocker.skip-build=true -Ddocker.skip-test=true - # Create manifest - - >- - for image in $DOCKER_PROD_IMAGE_NAME; - do - export OS_TAG="-ubi8" - export GIT_TAG=$GIT_COMMIT$OS_TAG - docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH - docker manifest push $image:$GIT_TAG - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG - docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH - docker manifest push $image:$BRANCH_BUILD_TAG - export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG - docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH - docker manifest push $image:$LATEST_MANIFEST_TAG - done after_pipeline: task: agent: