diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 755d825835..94a01a8a52 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -24,8 +24,9 @@ global_job_config: prologue: commands: - checkout - - sem-version java 8 + - sem-version java 8 || sem-version java 1.8 - sem-version python 3.9 + - pip install confluent-release-tools - . vault-setup - . cache-maven restore - pip install tox==3.28.0 @@ -86,24 +87,33 @@ blocks: run: when: "pull_request =~ '.*'" task: + agent: + machine: + type: s1-prod-rhel-9-amd64-1 jobs: - name: Build, Test, & Scan ubi8 commands: - export OS_TAG="-ubi8" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH - - ci-tools ci-update-version +# - ci-tools ci-update-version - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - - >- - for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; - do - cve-scan $dev_image - done +# - >- +# for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; +# do +# cve-scan $dev_image +# done + - echo "Pushing docker image 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH" +# - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc + - docker push 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cp-base-new:$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -118,7 +128,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu20-04-arm64-1 + type: s1-prod-rhel-9-arm64-1 jobs: - name: Build & Test ubi8 commands: @@ -127,11 +137,14 @@ blocks: - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - ci-tools ci-update-version - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock +# - ci-tools ci-update-version + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DskipTests clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - - . cache-maven store +# - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: diff --git a/base/Dockerfile.ubi8 b/base/Dockerfile.ubi8 index e946cfbd2e..fd4bfc5e5c 100644 --- a/base/Dockerfile.ubi8 +++ b/base/Dockerfile.ubi8 @@ -113,7 +113,7 @@ RUN microdnf --nodocs install yum \ # The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which # is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left # hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a -# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons # such as identified bugs in ZuluJDK's software. ARG SKIP_SECURITY_UPDATE_CHECK="false" RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" @@ -128,4 +128,4 @@ RUN mkdir /licenses COPY license.txt /licenses USER appuser -WORKDIR /home/appuser +WORKDIR /home/appuser \ No newline at end of file diff --git a/base/Dockerfile.ubi8.azul b/base/Dockerfile.ubi8.azul new file mode 100644 index 0000000000..b4cbfd97ff --- /dev/null +++ b/base/Dockerfile.ubi8.azul @@ -0,0 +1,140 @@ +# +# Copyright 2017 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBI_MINIMAL_VERSION="latest" +FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION} + +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for Confluent's Docker images." +LABEL description="Common base image for Confluent's Docker images." +LABEL io.confluent.docker=true + +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +# Set the classpath for JARs required by `cub` +ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' + +# These ARGs are left blank indicating to the Dnf package manager to install the latest package +# version that happens to be availible at this time. For reproducible builds, versions should be specified +# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used +# where these arguments are set in base/pom.xml under the elements based on the commit you're +# building from. + +# Redhat Package Versions +ARG OPENSSL_VERSION="" +ARG WGET_VERSION="" +ARG NETCAT_VERSION="" +ARG PYTHON39_VERSION="" +ARG TAR_VERSION="" +ARG PROCPS_VERSION="" +ARG KRB5_WORKSTATION_VERSION="" +ARG IPUTILS_VERSION="" +ARG HOSTNAME_VERSION="" +ARG XZ_LIBS_VERSION="" +ARG GLIBC_VERSION="" +ARG CURL_VERSION="" + +# Zulu OpenJDK version +ARG ZULU_OPENJDK_VERSION="" + +# Python Module Versions +ARG PYTHON_PIP_VERSION="" +ARG PYTHON_SETUPTOOLS_VERSION="" + +# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" + +# This can be overriden for an offline/air-gapped builds +ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" +#&& yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ +# && rpm -ivh --nodigest --nofiledigest https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm \ + +RUN microdnf --nodocs install yum \ + && rpm --import https://www.azul.com/files/0xB1998361219BD9C9.txt \ + && yum --nodocs update -y + +RUN yum --nodocs install -y --setopt=install_weak_deps=False \ + git \ + "openssl${OPENSSL_VERSION}" \ + "wget${WGET_VERSION}" \ + "nmap-ncat${NETCAT_VERSION}" \ + "python39${PYTHON39_VERSION}" \ + "python39-pip${PYTHON_PIP_VERSION}" \ + "tar${TAR_VERSION}" \ + "procps-ng${PROCPS_VERSION}" \ + "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ + "iputils${IPUTILS_VERSION}" \ + "hostname${HOSTNAME_VERSION}" \ + "xz-libs${XZ_LIBS_VERSION}" \ + "glibc${GLIBC_VERSION}" \ + "glibc-common${GLIBC_VERSION}" \ + "glibc-minimal-langpack${GLIBC_VERSION}" \ + "curl${CURL_VERSION}" \ + "libcurl${CURL_VERSION}" \ + && alternatives --set python /usr/bin/python3 \ + && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && yum remove -y git \ + && yum clean all \ + && rm -rf /tmp/* \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs + +#RUN yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jre11.0.23-linux.$(uname -m).rpm \ +# && yum --nodocs -y install https://cdn.azul.com/zulu/bin/zulu11.72.19-ca-jdk11.0.23-linux.$(uname -m).rpm + +RUN echo "Installing zulu jdk rpms directly from URL" \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jre-headless-11.0.23-1.$(uname -m).rpm \ + && yum --nodocs -y install https://repos.azul.com/zulu/rpm/RPMS/zulu11-ca-jdk-headless-11.0.23-1.$(uname -m).rpm + +# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. +# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which +# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left +# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# such as identified bugs in ZuluJDK's software. +ARG SKIP_SECURITY_UPDATE_CHECK="false" +#RUN yum --disablerepo="zulu-openjdk" check-update || "${SKIP_SECURITY_UPDATE_CHECK}" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new + +RUN mkdir /licenses +COPY license.txt /licenses + +USER appuser +WORKDIR /home/appuser \ No newline at end of file diff --git a/base/Dockerfile.ubi8.temurin b/base/Dockerfile.ubi8.temurin new file mode 100644 index 0000000000..91b238d4ab --- /dev/null +++ b/base/Dockerfile.ubi8.temurin @@ -0,0 +1,137 @@ +# +# Copyright 2017 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBI_MINIMAL_VERSION="latest" +FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION} + +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for Confluent's Docker images." +LABEL description="Common base image for Confluent's Docker images." +LABEL io.confluent.docker=true + +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +# Set the classpath for JARs required by `cub` +ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' + +# These ARGs are left blank indicating to the Dnf package manager to install the latest package +# version that happens to be availible at this time. For reproducible builds, versions should be specified +# as '-1.2.3-4.el8' on the command line. Or more preferibly the 'dockerfile-maven-plugin' is used +# where these arguments are set in base/pom.xml under the elements based on the commit you're +# building from. + +# Redhat Package Versions +ARG OPENSSL_VERSION="" +ARG WGET_VERSION="" +ARG NETCAT_VERSION="" +ARG PYTHON39_VERSION="" +ARG TAR_VERSION="" +ARG PROCPS_VERSION="" +ARG KRB5_WORKSTATION_VERSION="" +ARG IPUTILS_VERSION="" +ARG HOSTNAME_VERSION="" +ARG XZ_LIBS_VERSION="" +ARG GLIBC_VERSION="" +ARG CURL_VERSION="" + +# Zulu OpenJDK version +ARG ZULU_OPENJDK_VERSION="" + +# Python Module Versions +ARG PYTHON_PIP_VERSION="" +ARG PYTHON_SETUPTOOLS_VERSION="" + +# Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="master" + +# This can be overriden for an offline/air-gapped builds +ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" + +RUN printf "[Adoptium] \n\ +name=Adoptium \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN microdnf --nodocs install yum \ + && yum --nodocs update -y \ + && yum --nodocs install -y --setopt=install_weak_deps=False \ + git \ + "openssl${OPENSSL_VERSION}" \ + "wget${WGET_VERSION}" \ + "nmap-ncat${NETCAT_VERSION}" \ + "python39${PYTHON39_VERSION}" \ + "python39-pip${PYTHON_PIP_VERSION}" \ + "tar${TAR_VERSION}" \ + "procps-ng${PROCPS_VERSION}" \ + "krb5-workstation${KRB5_WORKSTATION_VERSION}" \ + "iputils${IPUTILS_VERSION}" \ + "hostname${HOSTNAME_VERSION}" \ + "xz-libs${XZ_LIBS_VERSION}" \ + "glibc${GLIBC_VERSION}" \ + "glibc-common${GLIBC_VERSION}" \ + "glibc-minimal-langpack${GLIBC_VERSION}" \ + "curl${CURL_VERSION}" \ + "libcurl${CURL_VERSION}" \ + "temurin-11-jdk" \ + && alternatives --set python /usr/bin/python3 \ + && python3 -m pip install --upgrade "setuptools${PYTHON_SETUPTOOLS_VERSION}" \ + && python3 -m pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ + && yum remove -y git \ + && yum clean all \ + && rm -rf /tmp/* \ + && mkdir -p /etc/confluent/docker /usr/logs \ + && useradd --no-log-init --create-home --shell /bin/bash appuser \ + && chown appuser:appuser -R /etc/confluent/ /usr/logs + +# This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. +# The ARG SKIP_SECURITY_UPDATE_CHECK is an "escape" hatch if you want to by-pass this check and build the container anyways, which +# is not advisable in terms of security posture. If set to false (which triggers a shell exit(1) if the check fails from the left +# hand of ||) this check will fail. If true (which triggers a right-hand || shell exit(0)), then this check will pass even if a +# security update is availible. We skip checks from ZuluJDK repos because Confluent pins those upstream versions for various reasons +# such as identified bugs in ZuluJDK's software. +ARG SKIP_SECURITY_UPDATE_CHECK="false" +RUN yum check-update || "${SKIP_SECURITY_UPDATE_CHECK}" + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-new /etc/cp-base-new + +RUN mkdir /licenses +COPY license.txt /licenses + +USER appuser +WORKDIR /home/appuser \ No newline at end of file diff --git a/base/pom.xml b/base/pom.xml index 2465215e3c..da8a993734 100644 --- a/base/pom.xml +++ b/base/pom.xml @@ -37,6 +37,7 @@ false false true + Dockerfile.${docker.os_type}.temurin diff --git a/pom.xml b/pom.xml index caa8bf1bc8..a132c41af7 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ 3.9.19-1.module+el8.10.0+21815+bb024982 1.30-9.el8 3.3.15-14.el8 - 1.18.2-27.el8_10 + 1.18.2-28.el8_10 20180629-11.el8 3.20-6.el8 5.2.4-4.el8_6