From 7dde26b58756da2f203ecab181233ddb6bdbab1f Mon Sep 17 00:00:00 2001 From: KrishVora01 Date: Thu, 6 Jun 2024 17:42:21 +0530 Subject: [PATCH 01/47] testing out rhel 9 fips enabled --- .semaphore/semaphore.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 755d825835..fb3056da2d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -86,6 +86,9 @@ blocks: run: when: "pull_request =~ '.*'" task: + agent: + machine: + type: s1-prod-rhel-9-amd64-1 jobs: - name: Build, Test, & Scan ubi8 commands: @@ -118,7 +121,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu20-04-arm64-1 + type: s1-prod-rhel-9-arm64-1 jobs: - name: Build & Test ubi8 commands: From 32153c7b27ff4caab3da046eb2beb014d982ceb4 Mon Sep 17 00:00:00 2001 From: KrishVora01 Date: Tue, 11 Jun 2024 09:51:54 +0530 Subject: [PATCH 02/47] modifying sem-version java" --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fb3056da2d..9d173dc991 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -24,7 +24,7 @@ global_job_config: prologue: commands: - checkout - - sem-version java 8 + - sem-version java 8 || sem-version java 1.8 - sem-version python 3.9 - . vault-setup - . cache-maven restore From 45ed3e25d7ec52ffa993aff8fef342a83e6f3c98 Mon Sep 17 00:00:00 2001 From: Corey Christous Date: Wed, 12 Jun 2024 09:42:54 -0400 Subject: [PATCH 03/47] Move pip install of confluent-release-tools up --- .semaphore/semaphore.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9d173dc991..69143bba7c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -26,6 +26,7 @@ global_job_config: - checkout - sem-version java 8 || sem-version java 1.8 - sem-version python 3.9 + - pip install confluent-release-tools - . vault-setup - . cache-maven restore - pip install tox==3.28.0 From 823039b6922833e6343975cc457a431b93ff9e46 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 04:49:51 -0700 Subject: [PATCH 04/47] Update semaphore.yml --- .semaphore/semaphore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 69143bba7c..ed2d5d86dd 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,6 +99,7 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - sudo chmod 666 /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -132,6 +133,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + - sudo chmod 666 /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 151e8e7eac57e004f5288c790e6994821ec6e3c4 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 05:06:06 -0700 Subject: [PATCH 05/47] Update semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ed2d5d86dd..b7b43a0887 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,7 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - sudo chmod 666 /var/run/docker.sock + - sudo chmod g+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -133,7 +133,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - sudo chmod 666 /var/run/docker.sock + - sudo chmod g+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From e22567c285de2121f6c4bb097e135c91f9c8e0cb Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 05:16:16 -0700 Subject: [PATCH 06/47] Update semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index b7b43a0887..16229c1291 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,7 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - sudo chmod g+rw /var/run/docker.sock + - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -133,7 +133,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - sudo chmod g+rw /var/run/docker.sock + - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From a1f4894b6c473666eec053a5526e5e72a5b6c930 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 05:31:04 -0700 Subject: [PATCH 07/47] Update semaphore.yml --- .semaphore/semaphore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 16229c1291..ea00e6a896 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,6 +99,8 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 @@ -133,6 +135,8 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 From 403d5c1027b4753998db405681dce2f946675216 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 06:04:48 -0700 Subject: [PATCH 08/47] Update semaphore.yml --- .semaphore/semaphore.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ea00e6a896..69143bba7c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,9 +99,6 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - sudo chown $(whoami) /var/run/docker.sock - - sudo chown -R $(whoami) /var/run/docker - - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -135,9 +132,6 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - sudo chown $(whoami) /var/run/docker.sock - - sudo chown -R $(whoami) /var/run/docker - - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 5363b6f26d686c7a3ff25b82957c22581f05bbd4 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:32:42 -0700 Subject: [PATCH 09/47] Update semaphore.yml --- .semaphore/semaphore.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 69143bba7c..20e3a7d0b0 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,6 +99,10 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock + - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -132,6 +136,10 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock + - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 6550a0533c83d4d6fb971de25f25a369c2577a5a Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:48:15 -0700 Subject: [PATCH 10/47] Update semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 20e3a7d0b0..5bbbd6cb54 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -102,7 +102,7 @@ blocks: - sudo chown $(whoami) /var/run/docker.sock - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - sudo rpm -ivh https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -139,7 +139,7 @@ blocks: - sudo chown $(whoami) /var/run/docker.sock - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - sudo rpm -ivh https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 592076421563ecdeac5070c6a3b9564f4eb043f3 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:57:51 -0700 Subject: [PATCH 11/47] Update semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5bbbd6cb54..20e3a7d0b0 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -102,7 +102,7 @@ blocks: - sudo chown $(whoami) /var/run/docker.sock - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -139,7 +139,7 @@ blocks: - sudo chown $(whoami) /var/run/docker.sock - sudo chown -R $(whoami) /var/run/docker - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 0f2e09cca9c52084132271da65b3ecf1f3f3e320 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:19:15 -0700 Subject: [PATCH 12/47] Update semaphore.yml --- .semaphore/semaphore.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 20e3a7d0b0..80c2c051e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,10 +99,7 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - sudo chown $(whoami) /var/run/docker.sock - - sudo chown -R $(whoami) /var/run/docker - - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - echo "Running provisioning script as $(whoami)." - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -136,10 +133,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - sudo chown $(whoami) /var/run/docker.sock - - sudo chown -R $(whoami) /var/run/docker - - sudo chmod g+rw,u+rw /var/run/docker.sock - - sudo rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm + - echo "Running provisioning script as $(whoami)." - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From e97436feee0e824cf0c404ee4e66dd54dbd29962 Mon Sep 17 00:00:00 2001 From: supark-cflt <164888392+supark-cflt@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:27:21 -0700 Subject: [PATCH 13/47] Update semaphore.yml --- .semaphore/semaphore.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 80c2c051e5..69143bba7c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,6 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - echo "Running provisioning script as $(whoami)." - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -133,7 +132,6 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - echo "Running provisioning script as $(whoami)." - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 67f366a146bd8b2b0414194d2362b552fa27b059 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 12:34:26 +0530 Subject: [PATCH 14/47] skipping maven tests when building the image --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 69143bba7c..cd6f35e18a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,7 @@ blocks: - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store @@ -132,7 +132,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store From 2bddc17207a8c5e4d1def325543ec167aa3313d0 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 14:12:17 +0530 Subject: [PATCH 15/47] chown docker related things --- .semaphore/semaphore.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index cd6f35e18a..84472d2872 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -96,9 +96,12 @@ blocks: - export OS_TAG="-ubi8" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH - - ci-tools ci-update-version +# - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS @@ -131,7 +134,10 @@ blocks: - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - ci-tools ci-update-version + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock +# - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS From 40c3fd303a77847b80bf058160d63d91d1f2209f Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 18:25:59 +0530 Subject: [PATCH 16/47] trying installing jdk --- base/Dockerfile.ubi8 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index e946cfbd2e..69910ef1b9 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,9 +76,9 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" +RUN microdnf --nodocs install java-17-openjdk + RUN microdnf --nodocs install yum \ - && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ @@ -98,7 +98,7 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ + "java-17-openjdk" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ @@ -109,6 +109,10 @@ RUN microdnf --nodocs install yum \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs +# && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ +# && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ +# "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ + # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left From 0aaea5e5330919d4eef3c7e248f99741271e2e04 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 18:33:34 +0530 Subject: [PATCH 17/47] cleaning up dockerfile --- base/Dockerfile.ubi8 | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 69910ef1b9..ef2b513c1f 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,8 +76,6 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" -RUN microdnf --nodocs install java-17-openjdk - RUN microdnf --nodocs install yum \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ From ba29cc439892e911a66113c9d288709e58b747d0 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 18:50:56 +0530 Subject: [PATCH 18/47] installing eclipse temurin jdk --- base/Dockerfile.ubi8 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index ef2b513c1f..28fe5463f4 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,6 +76,15 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" +RUN bash -c 'cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +baseurl=https://packages.adoptium.net/artifactory/rpm/${DISTRIBUTION_NAME:-$(. /etc/os-release; echo $ID)}/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF' + RUN microdnf --nodocs install yum \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ @@ -96,7 +105,7 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - "java-17-openjdk" \ + "temurin-17-jdk" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ From 1dc5859ac4ee26c848d69342be5c590cd76d44e6 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Mon, 17 Jun 2024 19:03:31 +0530 Subject: [PATCH 19/47] using temurin jdk --- base/Dockerfile.ubi8 | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 28fe5463f4..b10087d341 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,14 +76,13 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" -RUN bash -c 'cat < /etc/yum.repos.d/adoptium.repo -[Adoptium] -name=Adoptium -baseurl=https://packages.adoptium.net/artifactory/rpm/${DISTRIBUTION_NAME:-$(. /etc/os-release; echo $ID)}/\$releasever/\$basearch -enabled=1 -gpgcheck=1 -gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public -EOF' +RUN printf "[Adoptium] \n\ +name=Adoptium \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo RUN microdnf --nodocs install yum \ && yum --nodocs update -y \ From 605372863b62a071c915334bb46bc460ca9a2107 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 20 Jun 2024 13:40:49 +0530 Subject: [PATCH 20/47] directly installing zulu jdk --- base/Dockerfile.ubi8 | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index b10087d341..0a2ed837f3 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,15 +76,9 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" -RUN printf "[Adoptium] \n\ -name=Adoptium \n\ -baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ -enabled=1 \n\ -gpgcheck=1 \n\ -gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ -" > /etc/yum.repos.d/adoptium.repo - RUN microdnf --nodocs install yum \ + && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ @@ -104,7 +98,6 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - "temurin-17-jdk" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ @@ -115,15 +108,11 @@ RUN microdnf --nodocs install yum \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs -# && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ -# && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ -# "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ - # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left # hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a -# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons # such as identified bugs in ZuluJDK's software. ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" @@ -138,4 +127,4 @@ RUN mkdir /licenses COPY license.txt /licenses USER appuser -WORKDIR /home/appuser +WORKDIR /home/appuser \ No newline at end of file From 01c92620bac559f14fd6e6a541e663daa1a7b378 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 26 Jun 2024 11:36:38 +0530 Subject: [PATCH 21/47] using community issue solution --- base/Dockerfile.ubi8 | 4 +++- pom.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 0a2ed837f3..6b8b444684 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -75,10 +75,11 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" +#&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ + && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ @@ -98,6 +99,7 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ + "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ diff --git a/pom.xml b/pom.xml index caa8bf1bc8..c28a12bd81 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ utility-belt docker-utils base - jmxterm + From 1c0ff4e7a5de32b5f4b2cba46a2288295c98bf51 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 26 Jun 2024 12:05:46 +0530 Subject: [PATCH 22/47] adding nogpgcheck flag --- base/Dockerfile.ubi8 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 6b8b444684..09ae8bb333 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -99,7 +99,6 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ @@ -110,6 +109,9 @@ RUN microdnf --nodocs install yum \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs +RUN yum --nodocs install -y --nogpgcheck --setopt=install_weak_deps=False \ + "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" + # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left From b5c552ed6d8bbfdb4033f40d1b7691a3c2fcca5e Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Tue, 2 Jul 2024 15:51:27 +0530 Subject: [PATCH 23/47] installing rpm directly for azul jdk --- base/Dockerfile.ubi8 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 09ae8bb333..2c0a703512 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -76,10 +76,10 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" #&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ +# && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ @@ -99,6 +99,8 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ + "https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm" \ + "https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ From 3726855fc4eed0187aa3935f00950ba22fde4497 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Tue, 2 Jul 2024 17:20:17 +0530 Subject: [PATCH 24/47] installing from gs --- base/Dockerfile.ubi8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 2c0a703512..ebb05f777f 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -99,8 +99,8 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - "https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm" \ - "https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm" \ + https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ + https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ From c017aca548b94d19e92a84de91fa486c0371caca Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Tue, 2 Jul 2024 17:20:25 +0530 Subject: [PATCH 25/47] installing jdk on top --- base/Dockerfile.ubi8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index ebb05f777f..7488d6dfeb 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -80,6 +80,8 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluent RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ + && yum --nodocs -y https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ @@ -99,8 +101,6 @@ RUN microdnf --nodocs install yum \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ - https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ From 0c7275232b22dc3fca853c104325355e6692f743 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Tue, 2 Jul 2024 17:30:32 +0530 Subject: [PATCH 26/47] fixing yum install command --- base/Dockerfile.ubi8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 7488d6dfeb..70d6599211 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -81,7 +81,7 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluent RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - && yum --nodocs -y https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ From 4e0c502419dd2008a61c64a14d7c6fcf8f8ccd5d Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 00:40:44 +0530 Subject: [PATCH 27/47] removing unwanted zulu jdk installation --- base/Dockerfile.ubi8 | 3 --- 1 file changed, 3 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 70d6599211..1d9daac2eb 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -111,9 +111,6 @@ RUN microdnf --nodocs install yum \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs -RUN yum --nodocs install -y --nogpgcheck --setopt=install_weak_deps=False \ - "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" - # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left From d9cbcedeed33a96ddca3a1dd69e93b4e9361ccff Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 01:05:11 +0530 Subject: [PATCH 28/47] updating version of krb5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c28a12bd81..afa88bc2c3 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ 3.9.19-1.module+el8.10.0+21815+bb024982 1.30-9.el8 3.3.15-14.el8 - 1.18.2-27.el8_10 + 1.18.2-28.el8_10 20180629-11.el8 3.20-6.el8 5.2.4-4.el8_6 From 64b45cf428e24ef7bdbfa189053c539b9260f1be Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 01:06:50 +0530 Subject: [PATCH 29/47] using URL from yumdownloader --- base/Dockerfile.ubi8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 1d9daac2eb..24e278bf0d 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -80,8 +80,8 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluent RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ && yum --nodocs update -y \ && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ From e00987880c73ce12371a5efd87ee4a0b474c77ce Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 01:16:53 +0530 Subject: [PATCH 30/47] separating out docker command for easy debugging --- base/Dockerfile.ubi8 | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 24e278bf0d..a123a7a784 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -80,10 +80,12 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluent RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ + && yum --nodocs update -y + +RUN yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ - && yum --nodocs update -y \ - && yum --nodocs install -y --setopt=install_weak_deps=False \ + +RUN yum --nodocs install -y --setopt=install_weak_deps=False \ git \ "openssl${OPENSSL_VERSION}" \ "wget${WGET_VERSION}" \ @@ -118,7 +120,8 @@ RUN microdnf --nodocs install yum \ # security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons # such as identified bugs in ZuluJDK's software. ARG SKIP_SECURITY_UPDATE_CHECK="false" -RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" +#RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ From ff51115996eeee6a17c51241c27ed57dffb31df9 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 01:35:10 +0530 Subject: [PATCH 31/47] fixing installation of jdk --- base/Dockerfile.ubi8 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index a123a7a784..87c9557d17 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -82,9 +82,6 @@ RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ && yum --nodocs update -y -RUN yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ - && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ - RUN yum --nodocs install -y --setopt=install_weak_deps=False \ git \ "openssl${OPENSSL_VERSION}" \ @@ -113,6 +110,9 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs +RUN yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm + # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left From 82c2bd8ae529dec4f7d78bfabbdeed1529a17e24 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 01:47:02 +0530 Subject: [PATCH 32/47] using azul cdn URL --- base/Dockerfile.ubi8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 87c9557d17..09db332d65 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -110,8 +110,8 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs -RUN yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm \ - && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}.$(uname -m).rpm +RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which From f18943b8361b85df5a303e030d4d9caf09584820 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 12:10:29 +0530 Subject: [PATCH 33/47] removing cve-scan --- .semaphore/semaphore.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 84472d2872..5770f8a68a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -106,11 +106,11 @@ blocks: -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - - >- - for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; - do - cve-scan $dev_image - done +# - >- +# for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; +# do +# cve-scan $dev_image +# done - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: From 9df70993d4c6a2917a7284ac707080247d4d62a6 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 13:40:25 +0530 Subject: [PATCH 34/47] build jmxterm also --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index afa88bc2c3..a132c41af7 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ utility-belt docker-utils base - + jmxterm From 45d178eab3e3cda2c195247b2ea89863c7ed2da8 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 16:10:03 +0530 Subject: [PATCH 35/47] manually pushing docker image for debugging docker push issues --- .semaphore/semaphore.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5770f8a68a..59465ecf28 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -142,7 +142,9 @@ blocks: -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" + - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH +# - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: From f57bedf2d34a9f454eb13c18f661d364f9b7916b Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 16:24:29 +0530 Subject: [PATCH 36/47] putting debug docker logs command in amd blocks --- .semaphore/semaphore.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 59465ecf28..199ff3faf3 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -111,7 +111,9 @@ blocks: # do # cve-scan $dev_image # done - - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" + - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH +# - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: @@ -142,9 +144,7 @@ blocks: -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" - - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -# - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: From b83cdbc970276e30987cd9e5f6f3818352c4852a Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 3 Jul 2024 16:35:53 +0530 Subject: [PATCH 37/47] using semaphore oidc role to push images --- .semaphore/semaphore.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 199ff3faf3..146d440402 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -112,6 +112,7 @@ blocks: # cve-scan $dev_image # done - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" + - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH # - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: From efc6c65c2b82221480e11ebcc45b546733979791 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Fri, 5 Jul 2024 12:44:48 +0530 Subject: [PATCH 38/47] adding separate azul and temurin dockerfiles for development --- base/Dockerfile.ubi8.azul | 136 ++++++++++++++++++++++++++++++++++ base/Dockerfile.ubi8.temurin | 137 +++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 base/Dockerfile.ubi8.azul create mode 100644 base/Dockerfile.ubi8.temurin diff --git a/base/Dockerfile.ubi8.azul b/base/Dockerfile.ubi8.azul new file mode 100644 index 0000000000..09db332d65 --- /dev/null +++ b/base/Dockerfile.ubi8.azul @@ -0,0 +1,136 @@ +# +# Copyright 2017 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBI_MINIMAL_VERSION="latest" +FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION} + +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for Confluent's Docker images." +LABEL description="Common base image for Confluent's Docker images." +LABEL io.confluent.docker=true + +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +# Set the classpath for JARs required by `cub` +ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' + +# These ARGs are left blank indicating to the Dnf package manager to install the latest package +# version that happens to be availible at this time. For reproducible builds, versions should be specified +# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used +# where these arguments are set in base/pom.xml under the elements based on the commit you're +# building from. + +# Redhat Package Versions +ARG OPENSSL_VERSION="" +ARG WGET_VERSION="" +ARG NETCAT_VERSION="" +ARG PYTHON39_VERSION="" +ARG TAR_VERSION="" +ARG PROCPS_VERSION="" +ARG KRB5_WORKSTATION_VERSION="" +ARG IPUTILS_VERSION="" +ARG HOSTNAME_VERSION="" +ARG XZ_LIBS_VERSION="" +ARG GLIBC_VERSION="" +ARG CURL_VERSION="" + +# Zulu OpenJDK version +ARG ZULU_OPENJDK_VERSION="" + +# Python Module Versions +ARG PYTHON_PIP_VERSION="" +ARG PYTHON_SETUPTOOLS_VERSION="" + +# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" + +# This can be overriden for an offline/air-gapped builds +ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" +#&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ +# && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ + +RUN microdnf --nodocs install yum \ + && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ + && yum --nodocs update -y + +RUN yum --nodocs install -y --setopt=install_weak_deps=False \ + git \ + "openssl${OPENSSL_VERSION}" \ + "wget${WGET_VERSION}" \ + "nmap-ncat${NETCAT_VERSION}" \ + "python39${PYTHON39_VERSION}" \ + "python39-pip${PYTHON_PIP_VERSION}" \ + "tar${TAR_VERSION}" \ + "procps-ng${PROCPS_VERSION}" \ + "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ + "iputils${IPUTILS_VERSION}" \ + "hostname${HOSTNAME_VERSION}" \ + "xz-libs${XZ_LIBS_VERSION}" \ + "glibc${GLIBC_VERSION}" \ + "glibc-common${GLIBC_VERSION}" \ + "glibc-minimal-langpack${GLIBC_VERSION}" \ + "curl${CURL_VERSION}" \ + "libcurl${CURL_VERSION}" \ + && alternatives --set python /usr/bin/python3 \ + && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && yum remove -y git \ + && yum clean all \ + && rm -rf /tmp/* \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs + +RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm + +# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. +# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which +# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left +# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# such as identified bugs in ZuluJDK's software. +ARG SKIP_SECURITY_UPDATE_CHECK="false" +#RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new + +RUN mkdir /licenses +COPY license.txt /licenses + +USER appuser +WORKDIR /home/appuser \ No newline at end of file diff --git a/base/Dockerfile.ubi8.temurin b/base/Dockerfile.ubi8.temurin new file mode 100644 index 0000000000..91b238d4ab --- /dev/null +++ b/base/Dockerfile.ubi8.temurin @@ -0,0 +1,137 @@ +# +# Copyright 2017 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBI_MINIMAL_VERSION="latest" +FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION} + +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for Confluent's Docker images." +LABEL description="Common base image for Confluent's Docker images." +LABEL io.confluent.docker=true + +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +# Set the classpath for JARs required by `cub` +ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' + +# These ARGs are left blank indicating to the Dnf package manager to install the latest package +# version that happens to be availible at this time. For reproducible builds, versions should be specified +# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used +# where these arguments are set in base/pom.xml under the elements based on the commit you're +# building from. + +# Redhat Package Versions +ARG OPENSSL_VERSION="" +ARG WGET_VERSION="" +ARG NETCAT_VERSION="" +ARG PYTHON39_VERSION="" +ARG TAR_VERSION="" +ARG PROCPS_VERSION="" +ARG KRB5_WORKSTATION_VERSION="" +ARG IPUTILS_VERSION="" +ARG HOSTNAME_VERSION="" +ARG XZ_LIBS_VERSION="" +ARG GLIBC_VERSION="" +ARG CURL_VERSION="" + +# Zulu OpenJDK version +ARG ZULU_OPENJDK_VERSION="" + +# Python Module Versions +ARG PYTHON_PIP_VERSION="" +ARG PYTHON_SETUPTOOLS_VERSION="" + +# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" + +# This can be overriden for an offline/air-gapped builds +ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" + +RUN printf "[Adoptium] \n\ +name=Adoptium \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN microdnf --nodocs install yum \ + && yum --nodocs update -y \ + && yum --nodocs install -y --setopt=install_weak_deps=False \ + git \ + "openssl${OPENSSL_VERSION}" \ + "wget${WGET_VERSION}" \ + "nmap-ncat${NETCAT_VERSION}" \ + "python39${PYTHON39_VERSION}" \ + "python39-pip${PYTHON_PIP_VERSION}" \ + "tar${TAR_VERSION}" \ + "procps-ng${PROCPS_VERSION}" \ + "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ + "iputils${IPUTILS_VERSION}" \ + "hostname${HOSTNAME_VERSION}" \ + "xz-libs${XZ_LIBS_VERSION}" \ + "glibc${GLIBC_VERSION}" \ + "glibc-common${GLIBC_VERSION}" \ + "glibc-minimal-langpack${GLIBC_VERSION}" \ + "curl${CURL_VERSION}" \ + "libcurl${CURL_VERSION}" \ + "temurin-11-jdk" \ + && alternatives --set python /usr/bin/python3 \ + && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && yum remove -y git \ + && yum clean all \ + && rm -rf /tmp/* \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs + +# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. +# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which +# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left +# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# such as identified bugs in ZuluJDK's software. +ARG SKIP_SECURITY_UPDATE_CHECK="false" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new + +RUN mkdir /licenses +COPY license.txt /licenses + +USER appuser +WORKDIR /home/appuser \ No newline at end of file From b04565fd2f348456dd9fc938d889264b5f9f0da0 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Fri, 5 Jul 2024 12:45:41 +0530 Subject: [PATCH 39/47] disabling docker push as it is not supported right now --- .semaphore/semaphore.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 146d440402..1dbbb16de9 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -111,9 +111,9 @@ blocks: # do # cve-scan $dev_image # done - - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" - - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc - - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH +# - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" +# - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc +# - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH # - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -144,8 +144,8 @@ blocks: - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - - . cache-maven store - - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done +# - . cache-maven store +# - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: From 08af03b5478d828e68e617e0b9488867d96c6017 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Fri, 5 Jul 2024 12:49:16 +0530 Subject: [PATCH 40/47] using temurin dockerfile to build the image --- base/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/base/pom.xml b/base/pom.xml index 2465215e3c..da8a993734 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -37,6 +37,7 @@ false false true + Dockerfile.${docker.os_type}.temurin From f6a563d05fe75e4f5ef0d0a253db7aeb8d581f47 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Fri, 5 Jul 2024 13:41:17 +0530 Subject: [PATCH 41/47] reverting dockerfile.ubi8 to original --- base/Dockerfile.ubi8 | 15 +++++---------- base/Dockerfile.ubi8.azul | 2 +- base/pom.xml | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index 09db332d65..fd4bfc5e5c 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -75,14 +75,12 @@ ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" -#&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ -# && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ RUN microdnf --nodocs install yum \ && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ - && yum --nodocs update -y - -RUN yum --nodocs install -y --setopt=install_weak_deps=False \ + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ + && yum --nodocs update -y \ + && yum --nodocs install -y --setopt=install_weak_deps=False \ git \ "openssl${OPENSSL_VERSION}" \ "wget${WGET_VERSION}" \ @@ -100,6 +98,7 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ "glibc-minimal-langpack${GLIBC_VERSION}" \ "curl${CURL_VERSION}" \ "libcurl${CURL_VERSION}" \ + "zulu11-ca-jdk-headless${ZULU_OPENJDK_VERSION}" "zulu11-ca-jre-headless${ZULU_OPENJDK_VERSION}" \ && alternatives --set python /usr/bin/python3 \ && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ @@ -110,9 +109,6 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs -RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm - # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left @@ -120,8 +116,7 @@ RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11. # security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons # such as identified bugs in ZuluJDK's software. ARG SKIP_SECURITY_UPDATE_CHECK="false" -#RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" -RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" +RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ diff --git a/base/Dockerfile.ubi8.azul b/base/Dockerfile.ubi8.azul index 09db332d65..d016167f0e 100644 --- a/base/Dockerfile.ubi8.azul +++ b/base/Dockerfile.ubi8.azul @@ -111,7 +111,7 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ && chown appuser:appuser -R /etc/confluent/ /usr/logs RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm + && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which diff --git a/base/pom.xml b/base/pom.xml index da8a993734..67f5570cbd 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -37,7 +37,7 @@ false false true - Dockerfile.${docker.os_type}.temurin + Dockerfile.${docker.os_type}.azul From a5b3463c9bc2e5591a8fcfd4d50d847394f3ac17 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Fri, 5 Jul 2024 14:11:56 +0530 Subject: [PATCH 42/47] installing zulu jdk using yum URLs directly --- base/Dockerfile.ubi8.azul | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8.azul b/base/Dockerfile.ubi8.azul index d016167f0e..e766a31f27 100644 --- a/base/Dockerfile.ubi8.azul +++ b/base/Dockerfile.ubi8.azul @@ -110,8 +110,12 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs -RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ - && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm +#RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ +# && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm + +RUN echo "Installing zulu jdk rpms directly from URL" \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless-11.0.23-1.aarch64.rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless-11.0.23-1.aarch64.rpm # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which From 058b6c092fa547fdb6b0d6b6ac6cfae5b61abf20 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 10 Jul 2024 10:42:15 +0530 Subject: [PATCH 43/47] enabling docker image push for amd64 images --- .semaphore/semaphore.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 1dbbb16de9..93ff14795a 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -111,10 +111,10 @@ blocks: # do # cve-scan $dev_image # done -# - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" -# - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc -# - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -# - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" + - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc + - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH + - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: From 98154df42fb722c5d31a0bcec55b25f34a0f1c7d Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 10 Jul 2024 11:49:14 +0530 Subject: [PATCH 44/47] fixing arch value in installing jdk --- base/Dockerfile.ubi8.azul | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile.ubi8.azul b/base/Dockerfile.ubi8.azul index e766a31f27..b4cbfd97ff 100644 --- a/base/Dockerfile.ubi8.azul +++ b/base/Dockerfile.ubi8.azul @@ -114,8 +114,8 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False \ # && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm RUN echo "Installing zulu jdk rpms directly from URL" \ - && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless-11.0.23-1.aarch64.rpm \ - && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless-11.0.23-1.aarch64.rpm + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless-11.0.23-1.$(uname -m).rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless-11.0.23-1.$(uname -m).rpm # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which From 38e9d88af9804e74681cceedcebb164da42d9a8d Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 10 Jul 2024 16:52:54 +0530 Subject: [PATCH 45/47] using temurin jdk for cp-base-new --- base/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pom.xml b/base/pom.xml index 67f5570cbd..da8a993734 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -37,7 +37,7 @@ false false true - Dockerfile.${docker.os_type}.azul + Dockerfile.${docker.os_type}.temurin From 7d00f990c91856ecc80b56b8701b42ca8ddcad91 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 10 Jul 2024 17:25:30 +0530 Subject: [PATCH 46/47] removing assume iam-role command --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 93ff14795a..58e67b0425 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -112,7 +112,7 @@ blocks: # cve-scan $dev_image # done - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" - - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc +# - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: From a0307a2d7580a02efc849136465d3dbd8f45dc64 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Wed, 10 Jul 2024 17:53:58 +0530 Subject: [PATCH 47/47] enabling deploy for arm64 images also --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 58e67b0425..94a01a8a52 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -145,7 +145,7 @@ blocks: -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS # - . cache-maven store -# - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: commands: