Make docker images fully reproducible [DI-741]

.github/scripts/verify-layer-reproducibility.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail ${RUNNER_DEBUG:+-x}
+
+# Verifies Docker image build reproducibility by building twice and comparing layer digests.
+# Exits 0 if all layers are identical across both builds, 1 if any differ.
+#
+# Usage:
+#   verify-layer-reproducibility.sh -f <dockerfile> [-- <extra buildx build args...>]
+#
+# Example:
+#   verify-layer-reproducibility.sh -f hazelcast-oss/Dockerfile -- hazelcast-oss/
+
+RANDOM_SUFFIX="$(head -c 8 /dev/urandom | od -An -tx1 | tr -d ' \n')"
+readonly RANDOM_SUFFIX
+readonly TAG_A="repro-check-a-${RANDOM_SUFFIX}"
+readonly TAG_B="repro-check-b-${RANDOM_SUFFIX}"
+
+cleanup() {
+    docker rmi "${TAG_A}" "${TAG_B}" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+dockerfile=""
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -f) dockerfile="$2"; shift 2 ;;
+        --) shift; break ;;
+        *) echo "Unknown option: $1" >&2; exit 1 ;;
+    esac
+done
+
+if [[ -z "${dockerfile}" ]]; then
+    echo "Error: -f <dockerfile> is required" >&2
+    exit 1
+fi
+
+extra_args=("$@")
+
+build_image() {
+    local tag="$1"
+    echo "==> Building image '${tag}'..."
+    docker buildx build \
+        --no-cache \
+        --load \
+        -f "${dockerfile}" \
+        -t "${tag}" \
+        "${extra_args[@]}"
+}
+
+get_layers() {
+    docker inspect --format '{{json .RootFS.Layers}}' "$1"
+}
+
+build_image "${TAG_A}"
+echo ""
+build_image "${TAG_B}"
+echo ""
+
+layers_a=$(get_layers "${TAG_A}")
+layers_b=$(get_layers "${TAG_B}")
+
+count_a=$(echo "${layers_a}" | jq 'length')
+count_b=$(echo "${layers_b}" | jq 'length')
+
+echo "=== Layer Reproducibility Report ==="
+echo ""
+
+if [[ "${count_a}" -ne "${count_b}" ]]; then
+    echo "FAIL: Layer count mismatch (${count_a} vs ${count_b})"
+    exit 1
+fi
+
+has_diff=false
+for i in $(seq 0 $((count_a - 1))); do
+    digest_a=$(echo "${layers_a}" | jq -r ".[$i]")
+    digest_b=$(echo "${layers_b}" | jq -r ".[$i]")
+    if [[ "${digest_a}" == "${digest_b}" ]]; then
+        echo "  Layer $((i + 1))/${count_a}: MATCH"
+    else
+        echo "  Layer $((i + 1))/${count_a}: DIFFER"
+        echo "    A: ${digest_a}"
+        echo "    B: ${digest_b}"
+        has_diff=true
+    fi
+done
+
+echo ""
+if [[ "${has_diff}" == "true" ]]; then
+    echo "RESULT: FAIL - some layers differ between builds"
+    exit 1
+fi
+
+echo "RESULT: PASS - all ${count_a} layers are identical"

.github/workflows/build-pr.yml

@@ -26,6 +26,12 @@ jobs:
       ENVIRONMENT: test
     secrets: inherit
 
+  verify-reproducibility:
+    name: Verify layer reproducibility
+    uses: ./.github/workflows/verify-layer-reproducibility.yml
+    with:
+      SOURCE_REF: ${{ github.ref }}
+
   test-push-nlc:
     name: Test pushing image
     uses: ./.github/workflows/ee-nlc-tag-push.yml

.github/workflows/verify-layer-reproducibility.yml

@@ -0,0 +1,46 @@
+name: Verify layer reproducibility
+
+on:
+  workflow_call:
+    inputs:
+      SOURCE_REF:
+        description: 'The hazelcast-docker branch to verify'
+        required: true
+        type: string
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - context: hazelcast-oss
+            dist-file: hazelcast-distribution.zip
+          - context: hazelcast-enterprise
+            dist-file: hazelcast-enterprise-distribution.zip
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.SOURCE_REF }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create test distribution
+        run: |
+          dist_dir=$(mktemp -d)
+          mkdir -p "$dist_dir/hazelcast-0.0.0/bin" "$dist_dir/hazelcast-0.0.0/lib"
+          printf '#!/bin/bash\n' > "$dist_dir/hazelcast-0.0.0/bin/hz"
+          printf '#!/bin/bash\n' > "$dist_dir/hazelcast-0.0.0/bin/hz-healthcheck"
+          chmod +x "$dist_dir/hazelcast-0.0.0/bin/"*
+          touch "$dist_dir/hazelcast-0.0.0/lib/placeholder"
+          (cd "$dist_dir" && zip -qr "$GITHUB_WORKSPACE/${{ matrix.context }}/${{ matrix.dist-file }}" hazelcast-0.0.0/)
+          rm -rf "$dist_dir"
+
+      - name: Verify layer reproducibility
+        run: |
+          .github/scripts/verify-layer-reproducibility.sh \
+            -f ${{ matrix.context }}/Dockerfile \
+            -- ${{ matrix.context }}/

hazelcast-enterprise/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.7
+
 # Used for image metadata only
 # Describes the version of the Dockerfile, *not* the version of the bundled Hazelcast binary as this is/can be controlled externally
 # Dockerfile needs some concept of versioning so that the release pipeline can tag/archive with an appropriate label
@@ -8,40 +10,47 @@ ARG HZ_HOME="/opt/hazelcast"
 ARG USER_NAME="hazelcast"
 ARG USER_GROUP="hazelcast"
 ARG JDK_VERSION="21"
+ARG SOURCE_DATE_EPOCH=1688223600
+ARG BUILD_ROOT=/build_root
 
 FROM alpine:3 AS get-distribution
 
 ARG HZ_HOME
+ARG SOURCE_DATE_EPOCH
 ARG HAZELCAST_ZIP_FILE_NAME="hazelcast-enterprise-distribution.zip"
+ARG BUILD_ROOT
 
 # Expects distribution ZIP to be on the local filesystem within the build context
 COPY ${HAZELCAST_ZIP_FILE_NAME} /tmp/
 
-RUN mkdir -p ${HZ_HOME} \
+RUN mkdir -p ${BUILD_ROOT}${HZ_HOME} \
     && apk add --no-cache unzip \
     && unzip -qq /tmp/${HAZELCAST_ZIP_FILE_NAME} -d /tmp/hz \
     # Distribution ZIP structure is a single folder (e.g. "hazelcast-5.4.0-slim") containing the content
     # Move all the content up a level so that the path contains no version and is constant
-    && mv /tmp/hz/*/* ${HZ_HOME}/ \
+    && mv /tmp/hz/*/* ${BUILD_ROOT}${HZ_HOME}/ \
     && apk del unzip \
     && rm -rf /tmp/* \
     && echo "Setting Pardot ID to 'docker'" \
-    && echo 'hazelcastDownloadId=docker' > "${HZ_HOME}/lib/hazelcast-download.properties" \
-    && echo "Granting read permission to ${HZ_HOME}" \
-    && chmod -R +r ${HZ_HOME} \
+    && echo 'hazelcastDownloadId=docker' > "${BUILD_ROOT}${HZ_HOME}/lib/hazelcast-download.properties" \
+    && echo "Granting read permission to ${BUILD_ROOT}${HZ_HOME}" \
+    && chmod -R +r ${BUILD_ROOT}${HZ_HOME} \
     && echo "Grant execute permission to scripts in order to address the issue of permissions not being accurately propagated on Windows OS" \
-    && chmod +x ${HZ_HOME}/bin/*
+    && chmod +x ${BUILD_ROOT}${HZ_HOME}/bin/*
 
-COPY log4j2.properties log4j2-json.properties jmx_agent_config.yaml ${HZ_HOME}/config/
+COPY log4j2.properties log4j2-json.properties jmx_agent_config.yaml ${BUILD_ROOT}${HZ_HOME}/config/
 
+# Normalize timestamps for reproducible layer
+RUN find ${BUILD_ROOT} -exec touch -h -d "@$SOURCE_DATE_EPOCH" {} +
 
-FROM redhat/ubi9-minimal:9.7
+FROM eclipse-temurin:${JDK_VERSION}-jre-ubi9-minimal
 
-ARG HZ_VERSION
 ARG HZ_HOME
+ARG HZ_VERSION
 ARG USER_NAME
 ARG USER_GROUP
-ARG JDK_VERSION
+ARG SOURCE_DATE_EPOCH
+ARG BUILD_ROOT
 
 # Runtime variables
 ENV HZ_HOME="${HZ_HOME}" \
@@ -53,7 +62,8 @@ ENV HZ_HOME="${HZ_HOME}" \
     JAVA_OPTS="" \
     HAZELCAST_CONFIG=config/hazelcast-docker.xml \
     LANG=C.UTF-8 \
-    PATH=${HZ_HOME}/bin:$PATH
+    PATH=${HZ_HOME}/bin:$PATH \
+    SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 
 LABEL name="Hazelcast Enterprise" \
       maintainer="info@hazelcast.com" \
@@ -67,27 +77,29 @@ LABEL name="Hazelcast Enterprise" \
 EXPOSE 5701
 
 COPY licenses /licenses
-COPY --from=get-distribution ${HZ_HOME} ${HZ_HOME}/
 
-# Install
-RUN echo "Upgrading packages" \
+COPY --link --from=get-distribution ${BUILD_ROOT}/ /
+
+RUN touch /tmp/.timestamp-marker && sleep 1 \
+    && echo "Adding non-root user" \
+    && groupadd --system ${USER_GROUP} \
+    && useradd --no-log-init --system --gid ${USER_GROUP} --create-home ${USER_NAME} \
+    && chage --lastday 0 ${USER_NAME} \
+    && echo "Upgrading packages" \
     && microdnf -y update --nodocs \
-    && echo "Installing new packages" \
-    && microdnf -y --nodocs --disablerepo=* --enablerepo=ubi-9-appstream-rpms --enablerepo=ubi-9-baseos-rpms \
-        --disableplugin=subscription-manager install \
-        java-${JDK_VERSION}-openjdk-headless \
-        shadow-utils \
-        tar \
-        tzdata-java \
-        util-linux \
     && echo "Removing unnecessary packages and redundant files/folders" \
-    && microdnf -y clean all
+    && microdnf -y clean all \
+    && truncate --size 0 /var/cache/ldconfig/aux-cache \
+    && truncate --size 0 /var/lib/dnf/history.sqlite* \
+    && echo "Fixing non-deterministic files for reproducibility" \
+    && find / -newer /tmp/.timestamp-marker -xdev \
+            -exec touch -h -d "@$SOURCE_DATE_EPOCH" {} + 2>/dev/null \
+    && rm -f /tmp/.timestamp-marker \
+    && touch -d "@${SOURCE_DATE_EPOCH}" /tmp
+
 
 WORKDIR ${HZ_HOME}
 
-RUN echo "Adding non-root user" \
-    && groupadd --system ${USER_GROUP} \
-    && useradd --no-log-init --system --gid ${USER_GROUP} --create-home ${USER_NAME}
 USER ${USER_NAME}
 
 HEALTHCHECK CMD ["hz-healthcheck"]

hazelcast-oss/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.7
+
 # Used for image metadata only
 # Describes the version of the Dockerfile, *not* the version of the bundled Hazelcast binary as this is/can be controlled externally
 # Dockerfile needs some concept of versioning so that the release pipeline can tag/archive with an appropriate label
@@ -8,39 +10,47 @@ ARG HZ_HOME="/opt/hazelcast"
 ARG USER_NAME="hazelcast"
 ARG USER_GROUP="hazelcast"
 ARG JDK_VERSION="21"
+ARG SOURCE_DATE_EPOCH=1688223600
+ARG BUILD_ROOT=/build_root
 
 FROM alpine:3 AS get-distribution
 
 ARG HZ_HOME
+ARG SOURCE_DATE_EPOCH
 ARG HAZELCAST_ZIP_FILE_NAME="hazelcast-distribution.zip"
+ARG BUILD_ROOT
 
 # Expects distribution ZIP to be on the local filesystem within the build context
 COPY ${HAZELCAST_ZIP_FILE_NAME} /tmp/
 
-RUN mkdir -p ${HZ_HOME} \
+RUN mkdir -p ${BUILD_ROOT}${HZ_HOME} \
     && apk add --no-cache unzip \
     && unzip -qq /tmp/${HAZELCAST_ZIP_FILE_NAME} -d /tmp/hz \
     # Distribution ZIP structure is a single folder (e.g. "hazelcast-5.4.0-slim") containing the content
     # Move all the content up a level so that the path contains no version and is constant
-    && mv /tmp/hz/*/* ${HZ_HOME}/ \
+    && mv /tmp/hz/*/* ${BUILD_ROOT}${HZ_HOME}/ \
     && apk del unzip \
     && rm -rf /tmp/* \
     && echo "Setting Pardot ID to 'docker'" \
-    && echo 'hazelcastDownloadId=docker' > "${HZ_HOME}/lib/hazelcast-download.properties" \
-    && echo "Granting read permission to ${HZ_HOME}" \
-    && chmod -R +r ${HZ_HOME} \
+    && echo 'hazelcastDownloadId=docker' > "${BUILD_ROOT}${HZ_HOME}/lib/hazelcast-download.properties" \
+    && echo "Granting read permission to ${BUILD_ROOT}${HZ_HOME}" \
+    && chmod -R +r ${BUILD_ROOT}${HZ_HOME} \
     && echo "Grant execute permission to scripts in order to address the issue of permissions not being accurately propagated on Windows OS" \
-    && chmod +x ${HZ_HOME}/bin/*
+    && chmod +x ${BUILD_ROOT}${HZ_HOME}/bin/*
 
-COPY log4j2.properties log4j2-json.properties jmx_agent_config.yaml ${HZ_HOME}/config/
+COPY log4j2.properties log4j2-json.properties jmx_agent_config.yaml ${BUILD_ROOT}${HZ_HOME}/config/
 
+# Normalize timestamps for reproducible layer
+RUN find ${BUILD_ROOT} -exec touch -h -d "@$SOURCE_DATE_EPOCH" {} +
 
 FROM alpine:3
 
+ARG JDK_VERSION
 ARG HZ_HOME
 ARG USER_NAME
 ARG USER_GROUP
-ARG JDK_VERSION
+ARG SOURCE_DATE_EPOCH
+ARG BUILD_ROOT
 
 # Runtime variables
 ENV HZ_HOME="${HZ_HOME}" \
@@ -52,29 +62,42 @@ ENV HZ_HOME="${HZ_HOME}" \
     JAVA_OPTS="" \
     HAZELCAST_CONFIG=config/hazelcast-docker.xml \
     LANG=C.UTF-8 \
-    PATH=${HZ_HOME}/bin:$PATH
+    PATH=${HZ_HOME}/bin:$PATH \
+    SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
 
 # Expose port
 EXPOSE 5701
 
-COPY --from=get-distribution ${HZ_HOME} ${HZ_HOME}/
+COPY --link --from=get-distribution ${BUILD_ROOT}/ /
 
-# Install
-RUN echo "Upgrading packages" \
-    && apk upgrade --no-cache \
+RUN touch /tmp/.timestamp-marker && sleep 1 \
+    && echo "Adding non-root user" \
+    && addgroup -S ${USER_GROUP} \
+    && adduser -S ${USER_NAME} -G ${USER_GROUP} \
     && echo "Installing new packages" \
-    && apk add --no-cache \
+    && apk add --no-cache --no-logfile \
+        openjdk${JDK_VERSION}-jre-headless \
         bash \
         curl \
-        openjdk${JDK_VERSION}-jre-headless \
-    && echo "Removing unnecessary packages and redundant files/folders" \
-    && rm -rf /var/cache/apk/* 
+    && echo "Regenerating Java cacerts with SOURCE_DATE_EPOCH for reproducibility" \
+    && trust extract --overwrite --format=java-cacerts --filter=ca-anchors \
+        --purpose server-auth /etc/ssl/certs/java/cacerts \
+    && echo "Fixing non-deterministic files for reproducibility" \
+    && find / -newer /tmp/.timestamp-marker -xdev \
+            -exec touch -h -d "@$SOURCE_DATE_EPOCH" {} + 2>/dev/null; \
+    rm -f /tmp/.timestamp-marker; touch -d "@$SOURCE_DATE_EPOCH" /tmp
+
+# The explicit touch of / /etc /tmp catches parent directories that overlay2 copies up with stale timestamps during
+# the upgrade - find -newer misses them because their mtime predates the marker.
+RUN touch /tmp/.timestamp-marker && sleep 1 \
+    && echo "Upgrading packages" \
+    && apk upgrade --no-cache --no-logfile \
+    && find / -newer /tmp/.timestamp-marker -xdev \
+            -exec touch -h -d "@$SOURCE_DATE_EPOCH" {} + 2>/dev/null; \
+    rm -f /tmp/.timestamp-marker; touch -h -d "@$SOURCE_DATE_EPOCH" / /etc /tmp 2>/dev/null
 
 WORKDIR ${HZ_HOME}
 
-RUN echo "Adding non-root user" \
-    && addgroup -S ${USER_GROUP} \
-    && adduser -S ${USER_NAME} -G ${USER_GROUP}
 USER ${USER_NAME}
 
 HEALTHCHECK CMD ["hz-healthcheck"]