Skip to content

Commit afa0600

Browse files
nickboldtnickboldt
authored
chore(dockerfile): backport fixes for python, node headers, cachi2 (#2745)
* chore(dockerfile): backport fixes for cachi2 to 1.6 branch - python, playwright, node headers, etc (RHIDP-6561, RHIDP-6868, RHIDP-6755, RHIDP-6756) Signed-off-by: Nick Boldt <[email protected]> * don't include ./e2e-tests/package.json in downstream dockerfile Signed-off-by: Nick Boldt <[email protected]> --------- Signed-off-by: Nick Boldt <[email protected]>
1 parent 9288af9 commit afa0600

13 files changed

+149
-165
lines changed

‎.nvm/releases/README.adoc

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
To determine the specific version of node headers to include in this folder, pull the latest base image used in the Dockerfiles, then check the nodejs version that is installed.
2+
3+
```
4+
podman run -it --rm --entrypoint /bin/bash --user root registry.access.redhat.com/ubi9/nodejs-22 -c "node --version"
5+
6+
v22.13.1
7+
```
8+
9+
Or, go to https://catalog.redhat.com/software/containers/ubi9/nodejs-22/66431d1785c5c3a31edd24f1?container-tabs=packages and search for `nodejs` to get the version.
10+
11+
As of 2025/04/10, this version is `v22.13.1`
12+
13+
To download headers:
14+
15+
```
16+
NODE_HEADERS_VERSION=$(podman run -it --rm --entrypoint /bin/bash --user root registry.access.redhat.com/ubi9/nodejs-22 -c "node --version")
17+
cd .nvm/releases/
18+
curl -sSLO https://nodejs.org/dist/v${NODE_HEADERS_VERSION}/node-v${NODE_HEADERS_VERSION}-headers.tar.gz
19+
git add node-v${NODE_HEADERS_VERSION}-headers.tar.gz
20+
```
21+
22+
Then commit the new file to the appropriate branches of this repo.

‎.nvm/releases/node-v22.13.1-headers.tar.gz

8.39 MB
Binary file not shown.

‎.rhdh/docker/Dockerfile

Lines changed: 52 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,13 @@
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
17-
#
18-
# To transform into Brew-friendly Dockerfile:
19-
# 1. comment out lines with EXTERNAL_SOURCE_NESTED=. and CONTAINER_SOURCE=/opt/app-root/src
20-
# 2. uncomment lines with EXTERNAL_SOURCE_NESTED and CONTAINER_SOURCE pointing at $REMOTE_SOURCES and $REMOTE_SOURCES_DIR instead (Brew defines these paths)
21-
# 3. uncomment lines with RUN source .../cachito.env
22-
# 4. add Brew metadata
2317

2418
# Stage 1 - Build nodejs skeleton
2519
# https://registry.access.redhat.com/ubi9/nodejs-22
2620
FROM registry.access.redhat.com/ubi9/nodejs-22:9.5-1744136606 AS skeleton
2721
# hadolint ignore=DL3002
2822
USER 0
2923

30-
# debug
31-
# RUN cat /cachi2/cachi2.env
32-
3324
# Install isolated-vm dependencies
3425
# hadolint ignore=DL3041
3526
RUN dnf install -q -y --allowerasing --nobest \
@@ -56,11 +47,10 @@ RUN dnf install -q -y --allowerasing --nobest \
5647
python3-subscription-manager-rhsm python3-systemd python3-urllib3 readline redhat-release rootfiles rpm rpm-build-libs rpm-libs rpm-plugin-selinux rpm-sign-libs rsync scl-utils \
5748
sed selinux-policy selinux-policy-targeted setup shadow-utils skopeo slirp4netns sqlite-libs subscription-manager subscription-manager-rhsm-certificates systemd systemd-libs \
5849
systemd-pam systemd-rpm-macros tar tcl tpm2-tss tzdata unzip usermode util-linux util-linux-core vim-filesystem vim-minimal virt-what which xz xz-libs yajl yum zlib zlib-devel && \
59-
50+
6051
# '(micro)dnf update -y' not allowed in Konflux+Cachi2: instead use renovate or https://github.com/konflux-ci/rpm-lockfile-prototype to update the rpms.lock.yaml file
6152
dnf clean all
6253

63-
# sources
6454
ENV EXTERNAL_SOURCE_NESTED=.
6555
ENV CONTAINER_SOURCE=/opt/app-root/src
6656

@@ -85,7 +75,6 @@ COPY $EXTERNAL_SOURCE_NESTED/plugins/dynamic-plugins-info-backend/package.json .
8575
COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json
8676
COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json
8777
COPY $EXTERNAL_SOURCE_NESTED/package.json ./package.json
88-
COPY $EXTERNAL_SOURCE_NESTED/e2e-tests/package.json ./e2e-tests/package.json
8978
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/_utils/package.json ./dynamic-plugins/_utils/package.json
9079
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/wrappers/roadiehq-scaffolder-backend-module-utils-dynamic/package.json ./dynamic-plugins/wrappers/roadiehq-scaffolder-backend-module-utils-dynamic/package.json
9180
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/wrappers/roadiehq-scaffolder-backend-module-http-request-dynamic/package.json ./dynamic-plugins/wrappers/roadiehq-scaffolder-backend-module-http-request-dynamic/package.json
@@ -168,53 +157,18 @@ COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/wrappers/backstage-community-plugin
168157
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/wrappers/backstage-community-plugin-3scale-backend-dynamic/package.json ./dynamic-plugins/wrappers/backstage-community-plugin-3scale-backend-dynamic/package.json
169158
# END COPY package.json files
170159

171-
# Downstream only - debugging
172-
# RUN ls -la /cachi2/output/deps/yarn/*
173-
174-
# Downstream only - enable offline node-gyp install from local headers - see artifacts.lock.yaml for tar.gz
175-
ENV NODE_HEADERS_VERSION="22.13.1"
176-
RUN mkdir -p ~/.cache/node-gyp/${NODE_HEADERS_VERSION}; \
177-
tar -xf /cachi2/output/deps/generic/node-v${NODE_HEADERS_VERSION}-headers.tar.gz --directory ~/.cache/node-gyp/${NODE_HEADERS_VERSION}/ --strip-components 1; \
178-
echo "11" > ~/.cache/node-gyp/${NODE_HEADERS_VERSION}/installVersion
179-
180-
# Downstream only - enable offline playwright - see artifacts.lock.yaml for zip
181-
ENV CHROMIUM_VERSION="1155" \
182-
HEADLESS_VERSION="1155" \
183-
FFMPEG_VERSION="1011" \
184-
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \
185-
PLAYWRIGHT_BROWSERS_PATH=/cachi2/output/deps/generic/pw-browsers \
186-
CHROMIUM_DIR=$PLAYWRIGHT_BROWSERS_PATH/chromium-${CHROMIUM_VERSION} \
187-
HEADLESS_DIR=$PLAYWRIGHT_BROWSERS_PATH/chromium_headless_shell-${HEADLESS_VERSION} \
188-
FFMPEG_DIR=$PLAYWRIGHT_BROWSERS_PATH/ffmpeg-${FFMPEG_VERSION}
189-
190-
# # python3 -m playwright install chromium firefox --dry-run
191-
# browser: chromium version 133.0.6943.16
192-
# Install location: /opt/app-root/src/.cache/ms-playwright/chromium-1155
193-
# Download url: https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1155/chromium-linux.zip
194-
# browser: ffmpeg
195-
# Install location: /opt/app-root/src/.cache/ms-playwright/ffmpeg-1011
196-
# Download url: https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
197-
# browser: chromium-headless-shell version 133.0.6943.16
198-
# Install location: /opt/app-root/src/.cache/ms-playwright/chromium_headless_shell-1155
199-
# Download url: https://cdn.playwright.dev/dbazure/download/playwright/builds/chromium/1155/chromium-headless-shell-linux.zip
200-
# browser: ffmpeg
201-
# Install location: /opt/app-root/src/.cache/ms-playwright/ffmpeg-1011
202-
# Download url: https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip
203-
204-
RUN dnf install -q -y alsa-lib at-spi2-atk at-spi2-core atk cairo cups-libs libdrm libX11 libX11-xcb libXcomposite libXdamage libXext libXfixes libxkbcommon libXrandr mesa-libgbm nspr nss pango; \
205-
export PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH/; \
206-
ls -la /cachi2/output/deps/generic/; \
207-
mkdir -p $CHROMIUM_DIR; unzip -q /cachi2/output/deps/generic/chromium-linux.zip -d $CHROMIUM_DIR/; touch $CHROMIUM_DIR/DEPENDENCIES_VALIDATED $CHROMIUM_DIR/INSTALLATION_COMPLETE; \
208-
mkdir -p $HEADLESS_DIR; unzip -q /cachi2/output/deps/generic/chromium-headless-shell-linux.zip -d $HEADLESS_DIR/; touch $HEADLESS_DIR/DEPENDENCIES_VALIDATED $HEADLESS_DIR/INSTALLATION_COMPLETE; \
209-
mkdir -p $FFMPEG_DIR; unzip -q /cachi2/output/deps/generic/ffmpeg-linux.zip -d $FFMPEG_DIR/; touch $FFMPEG_DIR/DEPENDENCIES_VALIDATED $FFMPEG_DIR/INSTALLATION_COMPLETE; \
210-
# echo "Chromium: "; ls -1 $CHROMIUM_DIR; \
211-
# echo "Chromium Headless Shell:"; ls -1 $HEADLESS_DIR; \
212-
# echo "ffmpeg:"; ls -1 $FFMPEG_DIR; \
213-
pip3 install playwright==1.49.1 && playwright --version
214-
215-
# debugging
216-
# Redirect console output and errors to a log file to make this log shorter
217-
# RUN exec 3>&1 4>&2 1>> /tmp/build.log.txt 2>> /tmp/build.log.txt
160+
# unpack headers from tarball (includes openssl headers not present in /usr/include/node) - see RHIDP-6755 for why we need this upstream
161+
COPY $EXTERNAL_SOURCE_NESTED/.nvm/ .
162+
RUN NODE_HEADERS_VERSION=$(node --version); echo "=== Install node headers $NODE_HEADERS_VERSION from tar.gz ==="; \
163+
if [[ ! -f releases/node-${NODE_HEADERS_VERSION}-headers.tar.gz ]]; then \
164+
echo "[ERROR] Base image includes nodejs $NODE_HEADERS_VERSION but could not find releases/node-${NODE_HEADERS_VERSION}-headers.tar.gz to install!"; \
165+
echo "[ERROR] To fix, upload the node-${NODE_HEADERS_VERSION}-headers.tar.gz file into https://github.com/redhat-developer/rhdh/tree/main/.nvm/releases (or related branch)!"; \
166+
exit 1; \
167+
fi; \
168+
mkdir -p ~/.cache/node-gyp/${NODE_HEADERS_VERSION/v/}; \
169+
tar -xf releases/node-${NODE_HEADERS_VERSION}-headers.tar.gz --directory ~/.cache/node-gyp/${NODE_HEADERS_VERSION/v/}/ --strip-components 1; \
170+
echo "11" > ~/.cache/node-gyp/${NODE_HEADERS_VERSION/v/}/installVersion; \
171+
rm -fr releases/node-${NODE_HEADERS_VERSION}-headers.tar.gz
218172

219173
# Increate timeout for yarn install
220174
RUN "$YARN" config set httpTimeout 600000
@@ -224,8 +178,13 @@ RUN "$YARN" config set httpTimeout 600000
224178
# export YARN_ENABLE_IMMUTABLE_CACHE=false
225179
# export YARN_ENABLE_MIRROR=true
226180
# export YARN_GLOBAL_FOLDER=/cachi2/output/deps/yarn
227-
RUN echo "=== YARN INSTALL ==="; "$YARN" install --immutable | tee /tmp/yarn.install.log.txt 2>&1; \
228-
for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then echo; echo $d; echo "======"; cat ${d}/build.log; fi; done
181+
# RUN cat /cachi2/cachi2.env; echo; ls -la /cachi2/output/deps/yarn/*
182+
183+
RUN echo "=== YARN INSTALL ==="; FAILED=0; "$YARN" install --immutable || true; \
184+
for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then \
185+
(( FAILED = FAILED + 1 )); echo; echo $d; echo "======"; cat ${d}/build.log; \
186+
fi; done; \
187+
if [[ $FAILED -gt 0 ]]; then exit $FAILED; fi
229188

230189
# Stage 3 - Build packages
231190
FROM deps AS build
@@ -246,26 +205,19 @@ RUN \
246205
-e "s/(\"Last Commit\": \"(.+)\")/\1, \"Build Time\": \"$now\"/" && \
247206
cat packages/app/src/build-metadata.json; echo
248207

249-
RUN echo "=== YARN BUILD ==="; "$YARN" build --filter=backend \
250-
>/tmp/yarn.build.log.txt 2>&1 \
251-
|| { for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then echo; echo $d; echo "======"; cat ${d}/build.log; fi; done }
208+
RUN echo "=== YARN BUILD ==="; "$YARN" build --filter=backend
252209

253-
# >/tmp/yarn.export.dynamic.log.txt 2>&1
254-
RUN echo "=== EXPORT DYNAMIC PLUGINS (with --no-install) ==="; "$YARN" export-dynamic --filter=./dynamic-plugins/wrappers/*; \
255-
for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then echo; echo $d; echo "======"; cat ${d}/build.log; fi; done; \
256-
for d in $(find . -name yarn-install.log); do echo; echo "===== EXPORT DYNAMIC PLUGINS: $d =====>"; cat ${d}; echo "<===== EXPORT DYNAMIC PLUGINS: $d ====="; done;
210+
RUN echo "=== EXPORT DYNAMIC PLUGINS (with --no-install) ==="; FAILED=0; "$YARN" export-dynamic --filter=./dynamic-plugins/wrappers/* || true; \
211+
for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then \
212+
(( FAILED = FAILED + 1 )); echo; echo $d; echo "======"; cat ${d}/build.log; \
213+
fi; done; \
214+
if [[ $FAILED -gt 0 ]]; then exit $FAILED; fi
257215

258-
# Stan says we can omit this step - https://redhat-internal.slack.com/archives/C08A1KD7TNY/p1741805639019149?thread_ts=1741720088.366109&cid=C08A1KD7TNY
259-
# RUN echo "=== DIST-DYNAMIC YARN INSTALLS ==="; for f in $(find ./dynamic-plugins/wrappers/ -maxdepth 2 -type d -name dist-dynamic); do g=${f#*/wrappers/}; echo " > $g"; g=${g//\//_}; \
260-
# pushd $f >/dev/null; "$YARN" install --immutable >"/tmp/yarn.install.${g}.log.txt" 2>&1; popd >/dev/null; done
261-
262-
# >/tmp/yarn.copy.dynamic.log.txt 2>&1
263216
RUN echo "=== YARN COPY DYNAMIC PLUGINS in $(pwd) ==="; "$YARN" copy-dynamic-plugins dist
264-
# RUN ls -la dynamic-plugins/*
217+
265218
# Downstream only - clean up dynamic plugins sources:
266219
# Only keep the dist sub-folder in the dynamic-plugins folder
267220
RUN echo "=== DELETE DYNAMIC PLUGINS/* (except dist/) from $(pwd) ==="; find dynamic-plugins -maxdepth 1 -mindepth 1 -type d -not -name dist -exec rm -Rf {} \;
268-
# RUN ls -la dynamic-plugins/dist/*
269221

270222
# Stage 4 - Build the actual backend image and install production dependencies
271223
FROM skeleton AS cleanup
@@ -278,21 +230,31 @@ COPY --from=build $CONTAINER_SOURCE/yarn.lock \
278230
./
279231
ENV TARBALL_PATH=.
280232

281-
# Increate timeout for yarn install
282-
RUN "$YARN" config set httpTimeout 600000
283-
284233
RUN tar xzf "$TARBALL_PATH"/skeleton.tar.gz; tar xzf "$TARBALL_PATH"/bundle.tar.gz; \
285234
rm -f "$TARBALL_PATH"/skeleton.tar.gz "$TARBALL_PATH"/bundle.tar.gz
286235

287236
# Copy app-config files needed in runtime
288237
COPY $EXTERNAL_SOURCE_NESTED/app-config*.yaml ./
289238
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins.default.yaml ./
290239

291-
# >/tmp/yarn.workspaces.focus.log.txt 2>&1
240+
# Hackaround: create library symlinks to build: isolated-vm node-gyp better-sqlite cpu-features
241+
# TODO is this still needed? maintaining it over time will be a chore if the .so files change
242+
RUN echo "=== lib64 symlinks ==="; \
243+
for l in libbrotlidec.so.1 libbrotlienc.so.1 libbrotlicommon.so.1 libc.so.6 ld-linux-x86-64.so.2 libcrypto.so.3 libk5crypto.so.3 libz.so.1 libssl.so.3; do \
244+
ln -s /usr/lib64/$l /usr/lib64/${l%.*} 2>/dev/null || true; \
245+
ls -l /usr/lib64/${l%.*}; \
246+
done
247+
248+
# Increate timeout for yarn install
249+
RUN "$YARN" config set httpTimeout 600000
250+
292251
# Install production dependencies
293252
# hadolint ignore=DL3059
294-
RUN "$YARN" workspaces focus --all --production \
295-
|| { for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then echo; echo $d; echo "======"; cat ${d}/build.log; fi; done }
253+
RUN echo "=== YARN WORKSPACES FOCUS ==="; FAILED=0; "$YARN" workspaces focus --all --production || true; \
254+
for d in /tmp/xfs-*; do if [[ -f ${d}/build.log ]]; then \
255+
(( FAILED = FAILED + 1 )); echo; echo $d; echo "======"; cat ${d}/build.log; \
256+
fi; done; \
257+
if [[ $FAILED -gt 0 ]]; then exit $FAILED; fi
296258

297259
RUN \
298260
# delete all the nested .npmrc files and set default values
@@ -316,30 +278,30 @@ done; if [[ $hadFail -gt 0 ]]; then exit $hadFail; fi
316278
FROM registry.access.redhat.com/ubi9/nodejs-22-minimal:9.5-1742929466 AS runner
317279
USER 0
318280

319-
# sources
320281
ENV EXTERNAL_SOURCE_NESTED=.
321-
322282
ENV CONTAINER_SOURCE=/opt/app-root/src
323283
WORKDIR $CONTAINER_SOURCE/
324284

325-
# Downstream only - install techdocs dependencies using cachito sources
326-
# Using cachi2 env vars:
285+
# cachi2 env vars:
327286
# export PIP_FIND_LINKS=/cachi2/output/deps/pip
328287
# export PIP_NO_INDEX=true
288+
289+
# Install techdocs dependencies using requirements files
329290
# hadolint ignore=DL3013,DL3041,SC2086
330-
COPY "$EXTERNAL_SOURCE_NESTED"/docker/ ./docker
291+
COPY "$EXTERNAL_SOURCE_NESTED"/python/ ./python
331292
RUN microdnf install -y python3.11 python3.11-pip python3.11-devel make cmake cpp gcc gcc-c++ skopeo 1>/dev/null 2>&1; \
332293
alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
333294
alternatives --install /usr/bin/pip pip /usr/bin/pip3.11 1 && \
334295
# fix ownership for pip install folder
335296
mkdir -p /opt/app-root/src/.cache/pip && chown -R root:root /opt/app-root && \
336297
# ls -ld /opt/ /opt/app-root /opt/app-root/src/ /opt/app-root/src/.cache /opt/app-root/src/.cache/pip || true; \
337-
pushd "$EXTERNAL_SOURCE_NESTED"/docker/ >/dev/null && \
298+
pushd "$EXTERNAL_SOURCE_NESTED"/python/ >/dev/null && \
338299
set -e; \
339300
python3.11 -V; pip3.11 -V; \
340301
pip install --no-cache-dir --upgrade pip setuptools pyyaml; \
341302
pip install --no-cache-dir -r requirements.txt -r requirements-build.txt; mkdocs --version; \
342303
popd >/dev/null; \
304+
rm -fr python/; \
343305
microdnf clean all
344306

345307
# Upstream only - copy from cleanup stage
@@ -363,12 +325,12 @@ RUN chmod -R a+r ./dynamic-plugins/ ./install-dynamic-plugins.py; \
363325
# the container.
364326
RUN ln -s /opt/app-root/src/packages/app/dist/index.html /opt/app-root/src/packages/app/dist/index.html.tmpl
365327

366-
# Downstream only - fix for https://issues.redhat.com/browse/RHIDP-728 + some cleanup of unneeded files
367-
RUN mkdir /opt/app-root/src/.npm; chown -R 1001:1001 /opt/app-root/src/.npm; rm -fr docker/
328+
# Fix for https://issues.redhat.com/browse/RHIDP-728
329+
RUN mkdir -p /opt/app-root/src/.npm; chown -R 1001:1001 /opt/app-root/src/.npm
368330

369331
# The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
370332
# The upstream backstage image does not account for this and it causes the container to fail at runtime.
371-
# suppress warnings about dereferencing symlinks
333+
# suppress warnings about dereferencing symlinks
372334
RUN fix-permissions ./ 2>&1 | grep -v "chgrp: cannot dereference" || true
373335

374336
# Switch to nodejs user

0 commit comments

Comments
 (0)