confluentinc · jkao97 · Apr 5, 2024 · Apr 5, 2024
@@ -26,11 +26,48 @@ global_job_config:
       - checkout
       - sem-version java 8
       - sem-version python 3.9
+      - . vault-setup
       - . cache-maven restore
       - pip install tox==3.28.0
       - export GIT_COMMIT=$(git rev-parse --verify HEAD --short)
       - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-")
       - export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -)
+      # For PR Builds using Packaging
+      - pip install confluent-release-tools
+      - if [ $BRANCH_TAG == "master" ]; then export BUILD_KEY=$(pinto get-master-version); else export BUILD_KEY=$BRANCH_TAG; fi
+      - export LATEST_PACKAGING_BUILD_NUMBER=$(aws s3 ls s3://jenkins-confluent-packages/$BRANCH_TAG/ --no-paginate --recursive | grep "$BRANCH_TAG/[0-9]" | sort | tail -n 1 | awk '{print $4}' | awk -F
+        / '{print $2}')
+      - export CONFLUENT_VERSION=$(pinto get-version --build $BUILD_KEY --key confluent.version)
+      - export DEFAULT_OS_TYPE="ubi"
+      - export URL_CONFLUENT_VERSION=$(echo $CONFLUENT_VERSION | awk -F . '{print $1"."$2}')
+      - export PACKAGES_URL="https://s3-us-west-2.amazonaws.com/jenkins-confluent-packages/$BRANCH_TAG/$BUILD_NUMBER/PACKAGE_TYPE/$URL_CONFLUENT_VERSION"
+      - export PACKAGING_BUILD_NUMBER=$LATEST_PACKAGING_BUILD_NUMBER
+      - >-
+        if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then
+            export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}')
+        elif [[ ! $IS_RELEASE ]]; then
+            export PLATFORM_LABEL="~SNAPSHOT"
+        else
+            export PLATFORM_LABEL=""
+        fi
+      - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION  -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION  -DALLOW_UNSIGNED=$ALLOW_UNSIGNED"
+      - >-
+        if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then
+          if [[ $IS_BETA ]]; then
+              export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-beta-maven/$BRANCH_TAG/$PACKAGING_BUILD_NUMBER/maven"
+          elif [[ $IS_PREVIEW ]]; then
+              export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-alpha-maven/$BRANCH_NAME/$PACKAGING_BUILD_NUMBER/maven"
+          elif [[ $IS_RC ]]; then
+              export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven/v$BRANCH_NAME/maven"
+              if [[ $PACKAGES_MAVEN_URL ]]; then
+                  export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL
+              fi
+          fi
+          # Overwrite maven global configuration
+          . vault-sem-get-secret maven-settings-cp-dockerfile
+        else
+          echo "This job is not a isBetaJob, isPreviewJob, isHotfixJob, or isRcJob (What we know how to handle) - and we don't know how to handle it"
+        fi
       - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/"
       - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/"
       - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY
@@ -44,8 +81,7 @@ blocks:
   - name: Build, Test, & Scan AMD
     dependencies: []
     run:
-      # don't run the tests on non-functional changes...
-      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})"
+      when: "pull_request =~ '.*'"
     task:
       jobs:
         - name: Build, Test, & Scan ubi8
@@ -54,8 +90,11 @@ blocks:
             - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
             - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH
             - ci-tools ci-update-version
+            - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
+            - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
+              $PACKAGING_BUILD_ARGS
             - . cache-maven store
             - >-
               for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
@@ -69,74 +108,10 @@ blocks:
             - . publish-test-results
             - artifact push workflow target/test-results
             - artifact push workflow target --destination target-AMD
-  - name: Deploy AMD confluentinc/cp-base-new
-    dependencies: ["Build, Test, & Scan AMD"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      jobs:
-        - name: Deploy AMD confluentinc/cp-base-new ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-  - name: Deploy AMD confluentinc/cp-base-lite
-    dependencies: ["Build, Test, & Scan AMD"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      jobs:
-        - name: Deploy AMD confluentinc/cp-base-lite ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-  - name: Deploy AMD confluentinc/cp-jmxterm
-    dependencies: ["Build, Test, & Scan AMD"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      jobs:
-        - name: Deploy AMD confluentinc/cp-jmxterm ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$AMD_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$AMD_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
   - name: Build & Test ARM
     dependencies: []
     run:
-      # don't run the tests on non-functional changes...
-      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})"
+      when: "pull_request =~ '.*'"
     task:
       agent:
         machine:
@@ -147,9 +122,12 @@ blocks:
             - export OS_TAG="-ubi8"
             - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
             - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH
+            - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
+            - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
             - ci-tools ci-update-version
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
+              $PACKAGING_BUILD_ARGS
             - . cache-maven store
             - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
@@ -158,107 +136,6 @@ blocks:
             - . publish-test-results
             - artifact push workflow target/test-results
             - artifact push workflow target --destination target-ARM
-  - name: Deploy ARM confluentinc/cp-base-new
-    dependencies: ["Build & Test ARM"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      agent:
-        machine:
-          type: s1-prod-ubuntu20-04-arm64-1
-      jobs:
-        - name: Deploy ARM confluentinc/cp-base-new ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-new
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-  - name: Deploy ARM confluentinc/cp-base-lite
-    dependencies: ["Build & Test ARM"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      agent:
-        machine:
-          type: s1-prod-ubuntu20-04-arm64-1
-      jobs:
-        - name: Deploy ARM confluentinc/cp-base-lite ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-lite
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-lite:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-  - name: Deploy ARM confluentinc/cp-jmxterm
-    dependencies: ["Build & Test ARM"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      agent:
-        machine:
-          type: s1-prod-ubuntu20-04-arm64-1
-      jobs:
-        - name: Deploy ARM confluentinc/cp-jmxterm ubi8
-          commands:
-            - export OS_TAG="-ubi8"
-            - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-jmxterm
-            - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$ARM_ARCH
-            - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$ARM_ARCH
-            - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-jmxterm:$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH
-            - docker pull $DEV_IMAGE_FULL
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
-            - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
-            - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
-            - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-            - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
-  - name: Create Manifest and Maven Deploy
-    dependencies: ["Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite",
-      "Deploy ARM confluentinc/cp-jmxterm"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.[0-9]+'"
-    task:
-      jobs:
-        - name: Create Manifest and Maven Deploy
-          commands:
-            - export DOCKER_PROD_IMAGE_NAME=$DOCKER_PROD_REGISTRY${DOCKER_REPOS// / $DOCKER_PROD_REGISTRY}
-            - ci-tools ci-update-version
-            - ci-tools ci-push-tag
-            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
-              -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests -Ddocker.skip-build=true -Ddocker.skip-test=true
-            # Create manifest
-            - >-
-              for image in $DOCKER_PROD_IMAGE_NAME;
-              do
-                export OS_TAG="-ubi8"
-                export GIT_TAG=$GIT_COMMIT$OS_TAG
-                docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH
-                docker manifest push $image:$GIT_TAG
-                export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG
-                docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH
-                docker manifest push $image:$BRANCH_BUILD_TAG
-                export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG
-                docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH
-                docker manifest push $image:$LATEST_MANIFEST_TAG
-              done
 after_pipeline:
   task:
     agent: