CPBR-1524: Testing out docker image build on FIPS + RHEL semaphore agent

.semaphore/semaphore.yml

@@ -24,8 +24,9 @@ global_job_config:
   prologue:
     commands:
       - checkout
-      - sem-version java 8
+      - sem-version java 8 || sem-version java 1.8 
       - sem-version python 3.9
+      - pip install confluent-release-tools
       - . vault-setup
       - . cache-maven restore
       - pip install tox==3.28.0
@@ -86,24 +87,33 @@ blocks:
     run:
       when: "pull_request =~ '.*'"
     task:
+      agent:
+        machine:
+          type: s1-prod-rhel-9-amd64-1
       jobs:
         - name: Build, Test, & Scan ubi8
           commands:
             - export OS_TAG="-ubi8"
             - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
             - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH
-            - ci-tools ci-update-version
+#            - ci-tools ci-update-version
             - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
-            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
+            - sudo chown $(whoami) /var/run/docker.sock
+            - sudo chown -R $(whoami) /var/run/docker
+            - sudo chmod g+rw,u+rw /var/run/docker.sock
+            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
               $PACKAGING_BUILD_ARGS
             - . cache-maven store
-            - >-
-              for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
-              do
-                cve-scan $dev_image
-              done
+#            - >-
+#              for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
+#              do
+#                cve-scan $dev_image
+#              done
+            - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH"
+#            - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc
+            - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH
             - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
         always:
@@ -118,7 +128,7 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-ubuntu20-04-arm64-1
+          type: s1-prod-rhel-9-arm64-1
       jobs:
         - name: Build & Test ubi8
           commands:
@@ -127,11 +137,14 @@ blocks:
             - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH
             - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"
-            - ci-tools ci-update-version
-            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
+            - sudo chown $(whoami) /var/run/docker.sock
+            - sudo chown -R $(whoami) /var/run/docker
+            - sudo chmod g+rw,u+rw /var/run/docker.sock
+#            - ci-tools ci-update-version
+            - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
               $PACKAGING_BUILD_ARGS
-            - . cache-maven store
+#            - . cache-maven store
             - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
         always:

base/Dockerfile.ubi8

@@ -113,7 +113,7 @@ RUN microdnf --nodocs install yum \
 # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which
 # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left
 # hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a
-# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons 
+# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons
 # such as identified bugs in ZuluJDK's software.
 ARG SKIP_SECURITY_UPDATE_CHECK="false"
 RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}"
@@ -128,4 +128,4 @@ RUN mkdir /licenses
 COPY license.txt /licenses
 
 USER appuser
-WORKDIR /home/appuser
+WORKDIR /home/appuser

base/Dockerfile.ubi8.azul

@@ -0,0 +1,140 @@
+#
+# Copyright 2017 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG UBI_MINIMAL_VERSION="latest"
+FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION}
+
+ARG PROJECT_VERSION
+ARG ARTIFACT_ID
+
+# Remember where we came from
+LABEL io.confluent.docker.git.repo="confluentinc/common-docker"
+
+ARG GIT_COMMIT
+LABEL io.confluent.docker.git.id=$GIT_COMMIT
+
+ARG BUILD_NUMBER=-1
+LABEL io.confluent.docker.build.number=$BUILD_NUMBER
+
+LABEL maintainer="[email protected]"
+LABEL vendor="Confluent"
+LABEL version=$GIT_COMMIT
+LABEL release=$PROJECT_VERSION
+LABEL name=$ARTIFACT_ID
+LABEL summary="Common base image for Confluent's Docker images."
+LABEL description="Common base image for Confluent's Docker images."
+LABEL io.confluent.docker=true
+
+# This affects how strings in Java class files are interpreted.  We want UTF-8 and this is the only locale in the
+# base image that supports it
+ENV LANG="C.UTF-8"
+
+# Set the classpath for JARs required by `cub`
+ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"'
+
+# These ARGs are left blank indicating to the Dnf package manager to install the latest package
+# version that happens to be availible at this time. For reproducible builds, versions should be specified
+# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used
+# where these arguments are set in base/pom.xml under the <buildArgs> elements based on the commit you're
+# building from.
+
+# Redhat Package Versions
+ARG OPENSSL_VERSION=""
+ARG WGET_VERSION=""
+ARG NETCAT_VERSION=""
+ARG PYTHON39_VERSION=""
+ARG TAR_VERSION=""
+ARG PROCPS_VERSION=""
+ARG KRB5_WORKSTATION_VERSION=""
+ARG IPUTILS_VERSION=""
+ARG HOSTNAME_VERSION=""
+ARG XZ_LIBS_VERSION=""
+ARG GLIBC_VERSION=""
+ARG CURL_VERSION=""
+
+# Zulu OpenJDK version
+ARG ZULU_OPENJDK_VERSION=""
+
+# Python Module Versions
+ARG PYTHON_PIP_VERSION=""
+ARG PYTHON_SETUPTOOLS_VERSION=""
+
+# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install)
+ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master"
+
+# This can be overriden for an offline/air-gapped builds
+ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}"
+#&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \
+# && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \
+
+RUN microdnf --nodocs install yum \
+    && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \
+    && yum --nodocs update -y
+
+RUN yum --nodocs install -y --setopt=install_weak_deps=False \
+        git \
+        "openssl${OPENSSL_VERSION}" \
+        "wget${WGET_VERSION}" \
+        "nmap-ncat${NETCAT_VERSION}" \
+        "python39${PYTHON39_VERSION}" \
+        "python39-pip${PYTHON_PIP_VERSION}" \
+        "tar${TAR_VERSION}" \
+        "procps-ng${PROCPS_VERSION}" \
+        "krb5-workstation${KRB5_WORKSTATION_VERSION}" \
+        "iputils${IPUTILS_VERSION}" \
+        "hostname${HOSTNAME_VERSION}" \
+        "xz-libs${XZ_LIBS_VERSION}" \
+        "glibc${GLIBC_VERSION}" \
+        "glibc-common${GLIBC_VERSION}" \
+        "glibc-minimal-langpack${GLIBC_VERSION}" \
+        "curl${CURL_VERSION}" \
+        "libcurl${CURL_VERSION}" \
+    && alternatives --set python /usr/bin/python3 \
+    && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \
+    && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \
+    && yum remove -y git \
+    && yum clean all \
+    && rm -rf /tmp/* \
+    && mkdir -p /etc/confluent/docker /usr/logs \
+    && useradd --no-log-init --create-home --shell /bin/bash appuser \
+    && chown appuser:appuser -R /etc/confluent/ /usr/logs
+
+#RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \
+#    && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm
+
+RUN echo "Installing zulu jdk rpms directly from URL" \
+    && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless-11.0.23-1.$(uname -m).rpm \
+    && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless-11.0.23-1.$(uname -m).rpm
+
+# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed.
+# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which
+# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left
+# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a
+# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons
+# such as identified bugs in ZuluJDK's software.
+ARG SKIP_SECURITY_UPDATE_CHECK="false"
+#RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}"
+RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}"
+
+COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/
+COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
+
+COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker
+COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new
+
+RUN mkdir /licenses
+COPY license.txt /licenses
+
+USER appuser
+WORKDIR /home/appuser

base/Dockerfile.ubi8.temurin

@@ -0,0 +1,137 @@
+#
+# Copyright 2017 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG UBI_MINIMAL_VERSION="latest"
+FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION}
+
+ARG PROJECT_VERSION
+ARG ARTIFACT_ID
+
+# Remember where we came from
+LABEL io.confluent.docker.git.repo="confluentinc/common-docker"
+
+ARG GIT_COMMIT
+LABEL io.confluent.docker.git.id=$GIT_COMMIT
+
+ARG BUILD_NUMBER=-1
+LABEL io.confluent.docker.build.number=$BUILD_NUMBER
+
+LABEL maintainer="[email protected]"
+LABEL vendor="Confluent"
+LABEL version=$GIT_COMMIT
+LABEL release=$PROJECT_VERSION
+LABEL name=$ARTIFACT_ID
+LABEL summary="Common base image for Confluent's Docker images."
+LABEL description="Common base image for Confluent's Docker images."
+LABEL io.confluent.docker=true
+
+# This affects how strings in Java class files are interpreted.  We want UTF-8 and this is the only locale in the
+# base image that supports it
+ENV LANG="C.UTF-8"
+
+# Set the classpath for JARs required by `cub`
+ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"'
+
+# These ARGs are left blank indicating to the Dnf package manager to install the latest package
+# version that happens to be availible at this time. For reproducible builds, versions should be specified
+# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used
+# where these arguments are set in base/pom.xml under the <buildArgs> elements based on the commit you're
+# building from.
+
+# Redhat Package Versions
+ARG OPENSSL_VERSION=""
+ARG WGET_VERSION=""
+ARG NETCAT_VERSION=""
+ARG PYTHON39_VERSION=""
+ARG TAR_VERSION=""
+ARG PROCPS_VERSION=""
+ARG KRB5_WORKSTATION_VERSION=""
+ARG IPUTILS_VERSION=""
+ARG HOSTNAME_VERSION=""
+ARG XZ_LIBS_VERSION=""
+ARG GLIBC_VERSION=""
+ARG CURL_VERSION=""
+
+# Zulu OpenJDK version
+ARG ZULU_OPENJDK_VERSION=""
+
+# Python Module Versions
+ARG PYTHON_PIP_VERSION=""
+ARG PYTHON_SETUPTOOLS_VERSION=""
+
+# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install)
+ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master"
+
+# This can be overriden for an offline/air-gapped builds
+ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}"
+
+RUN printf "[Adoptium] \n\
+name=Adoptium \n\
+baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\
+enabled=1 \n\
+gpgcheck=1 \n\
+gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\
+" > /etc/yum.repos.d/adoptium.repo
+
+RUN microdnf --nodocs install yum \
+    && yum --nodocs update -y \
+    && yum --nodocs install -y --setopt=install_weak_deps=False \
+        git \
+        "openssl${OPENSSL_VERSION}" \
+        "wget${WGET_VERSION}" \
+        "nmap-ncat${NETCAT_VERSION}" \
+        "python39${PYTHON39_VERSION}" \
+        "python39-pip${PYTHON_PIP_VERSION}" \
+        "tar${TAR_VERSION}" \
+        "procps-ng${PROCPS_VERSION}" \
+        "krb5-workstation${KRB5_WORKSTATION_VERSION}" \
+        "iputils${IPUTILS_VERSION}" \
+        "hostname${HOSTNAME_VERSION}" \
+        "xz-libs${XZ_LIBS_VERSION}" \
+        "glibc${GLIBC_VERSION}" \
+        "glibc-common${GLIBC_VERSION}" \
+        "glibc-minimal-langpack${GLIBC_VERSION}" \
+        "curl${CURL_VERSION}" \
+        "libcurl${CURL_VERSION}" \
+        "temurin-11-jdk" \
+    && alternatives --set python /usr/bin/python3 \
+    && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \
+    && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \
+    && yum remove -y git \
+    && yum clean all \
+    && rm -rf /tmp/* \
+    && mkdir -p /etc/confluent/docker /usr/logs \
+    && useradd --no-log-init --create-home --shell /bin/bash appuser \
+    && chown appuser:appuser -R /etc/confluent/ /usr/logs
+
+# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed.
+# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which
+# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left
+# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a
+# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons
+# such as identified bugs in ZuluJDK's software.
+ARG SKIP_SECURITY_UPDATE_CHECK="false"
+RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}"
+
+COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/
+COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
+
+COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker
+COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new
+
+RUN mkdir /licenses
+COPY license.txt /licenses
+
+USER appuser
+WORKDIR /home/appuser

base/pom.xml

@@ -37,6 +37,7 @@
         <docker.skip-build>false</docker.skip-build>
         <docker.skip-test>false</docker.skip-test>
         <docker.pull-image>true</docker.pull-image>
+        <docker.file>Dockerfile.${docker.os_type}.temurin</docker.file>
     </properties>
 
     <dependencies>

pom.xml

@@ -40,7 +40,7 @@
         <ubi.python39.version>3.9.19-1.module+el8.10.0+21815+bb024982</ubi.python39.version>
         <ubi.tar.version>1.30-9.el8</ubi.tar.version>
         <ubi.procps.version>3.3.15-14.el8</ubi.procps.version>
-        <ubi.krb5.workstation.version>1.18.2-27.el8_10</ubi.krb5.workstation.version>
+        <ubi.krb5.workstation.version>1.18.2-28.el8_10</ubi.krb5.workstation.version>
         <ubi.iputils.version>20180629-11.el8</ubi.iputils.version>
         <ubi.hostname.version>3.20-6.el8</ubi.hostname.version>
         <ubi.xzlibs.version>5.2.4-4.el8_6</ubi.xzlibs.version>