RHOAIENG-21668: chore(gha): implement the rhel building on pull_request_trigger workflow

Author: jiridanek

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -15,6 +15,11 @@ name: Build & Publish Notebook Servers (TEMPLATE)
         required: true
         description: "top workflow's `github`"
         type: string
+      subscription:
+        required: false
+        default: false
+        description: "add RHEL subscription from github secret"
+        type: boolean
 
 jobs:
   build:
@@ -49,9 +54,38 @@ jobs:
           echo "CACHE=${CACHE,,}" >>${GITHUB_ENV}
 
       - uses: actions/checkout@v4
+        if: ${{ fromJson(inputs.github).event_name != 'pull_request_target' }}
+      # we need to checkout the pr branch, not pr target (the default for pull_request_target)
+      # user access check is done in calling workflow
+      - uses: actions/checkout@v4
+        if: ${{ fromJson(inputs.github).event_name == 'pull_request_target' }}
+        with:
+          ref: "refs/pull/${{ fromJson(inputs.github).event.number }}/merge"
 
       - run: mkdir -p $TMPDIR
 
+      # do this early because it's fast and why not
+      - name: Unlock encrypted secrets with git-crypt
+        if: ${{ inputs.subscription }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install git-crypt
+          echo "${GIT_CRYPT_KEY}" | base64 --decode > ./git-crypt-key
+          git-crypt unlock ./git-crypt-key
+          rm ./git-crypt-key
+        env:
+          GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }}
+
+      - name: Add subscriptions from GitHub secret
+        if: ${{ inputs.subscription }}
+        run: |
+          sudo mkdir -p /etc/pki/
+          sudo cp -R ${PWD}/ci/secrets/pki/* /etc/pki/
+          printf "${PWD}/ci/secrets/pki/consumer:/etc/pki/consumer\n${PWD}/ci/secrets/pki/entitlement:/etc/pki/entitlement" | sudo tee /usr/share/containers/mounts.conf
+
+          mkdir -p $HOME/.config/containers/
+          sudo cp ${PWD}/ci/secrets/pull-secret.txt $HOME/.config/containers/auth.json
+
       # for bin/buildinputs in scripts/sandbox.py
       - uses: actions/setup-go@v5
         with:
@@ -257,7 +291,8 @@ jobs:
       - name: "pull_request: make ${{ inputs.target }}"
         run: |
           make ${{ inputs.target }}
-        if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
+        if: "${{ fromJson(inputs.github).event_name == 'pull_request' ||
+          fromJson(inputs.github).event_name == 'pull_request_target' }}"
         env:
           IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"

.github/workflows/build-notebooks-pr-rhel.yaml

@@ -2,7 +2,6 @@
 "name": "Build Notebooks (pr, RHEL images)"
 "on":
   "pull_request_target":
-    "types": ["opened", "synchronize", "reopened", "edited"]
 
 # BEWARE: This GitHub Actions workflow runs on pull_request_target, meaning it has access to our secrets
 # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-secrets
@@ -13,7 +12,7 @@ permissions:
   packages: read
 
 env:
-  #language=json
+  # language=json
   contributors: |
     ["atheo89", "andyatmiami", "caponetto", "daniellutz", "dibryant", "harshad16", "jesuino", "jiridanek", "jstourac", "paulovmr"]
 
@@ -25,31 +24,31 @@ jobs:
       matrix: ${{ steps.gen.outputs.matrix }}
       has_jobs: ${{ steps.gen.outputs.has_jobs }}
     steps:
-      - name: Check permissions (this must be done FIRST, for security, before we checkout)
+
+      - name: Check permissions and deny untrusted users (this must be done FIRST, for security, before we checkout)
         if: ${{ !contains(fromJSON(env.contributors), github.actor) }}
         run: |
           echo "GitHub user ${{ github.actor }} is not a registered project contributor, not allowed to run actions on RHEL!"
           exit 1
 
+      # Here we are checking out the pull request, so that we can build from the new code
+      # We can do this because we already checked that the submitting user is a contributor
       - uses: actions/checkout@v4
+        if: ${{ github.event_name == 'pull_request_target' }}
         with:
           ref: "refs/pull/${{ github.event.number }}/merge"
-
-      - name: Unlock encrypted secrets with git-crypt
-        run: |
-          echo "${GIT_CRYPT_KEY}" | base64 --decode > ./git-crypt-key
-          git-crypt unlock ./git-crypt-key
-          rm ./git-crypt-key
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name != 'pull_request_target' }}
 
       - name: Determine targets to build based on changed files
+        if: ${{ github.event_name == 'pull_request_target' }}
         run: |
           set -x
           git fetch --no-tags origin 'pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}'
           git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
           python3 ci/cached-builds/gen_gha_matrix_jobs.py \
             --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
-            --to-ref '${{ github.event.pull_request.head.ref }}'
-            --only-rhel
+            --to-ref '${{ github.event.pull_request.head.ref }}' \
         id: gen
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -65,5 +64,5 @@ jobs:
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"
-      subscription: true
+      subscription: "${{ matrix.subscription }}"
     secrets: inherit

.github/workflows/build-notebooks-pr.yaml

@@ -33,7 +33,8 @@ jobs:
           git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
           python3 ci/cached-builds/gen_gha_matrix_jobs.py \
             --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
-            --to-ref '${{ github.event.pull_request.head.ref }}'
+            --to-ref '${{ github.event.pull_request.head.ref }}' \
+            --leave-out-rhel '${{ github.event_name == 'pull_request' }}'
         id: gen
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -49,4 +50,5 @@ jobs:
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"
+      subscription: "${{ matrix.subscription }}"
     secrets: inherit

.github/workflows/build-notebooks-push.yaml

@@ -39,4 +39,5 @@ jobs:
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"
+      subscription: "${{ matrix.subscription }}"
     secrets: inherit

base/rhel9-python-3.11/Dockerfile

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import argparse
 import json
 import logging
@@ -6,7 +8,6 @@
 import os
 import pathlib
 import re
-import string
 import sys
 import unittest
 
@@ -21,7 +22,7 @@
 project_dir = pathlib.Path(__file__).parent.parent.parent.absolute()
 
 
-def parse_makefile(target: str, makefile_dir: str) -> str:
+def parse_makefile(target: str, makefile_dir: pathlib.Path | str) -> str:
     # Check if the operating system is macOS
     if platform.system() == 'Darwin':
         make_command = 'gmake'
@@ -30,7 +31,9 @@ def parse_makefile(target: str, makefile_dir: str) -> str:
 
     try:
         # Run the make (or gmake) command and capture the output
-        result = subprocess.run([make_command, '-nps', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, cwd=makefile_dir)
+        result = subprocess.run([make_command, '-nps', target],
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+                                check=True, cwd=makefile_dir)
     except subprocess.CalledProcessError as e:
         # Handle errors if the make command fails
         print(f'{make_command} failed with return code: {e.returncode}:\n{e.stderr}', file=sys.stderr)
@@ -43,7 +46,7 @@ def parse_makefile(target: str, makefile_dir: str) -> str:
     return result.stdout
 
 
-def extract_image_targets(makefile_dir: str = os.getcwd()) -> list[str]:
+def extract_image_targets(makefile_dir: pathlib.Path | str = os.getcwd()) -> list[str]:
     makefile_all_target = 'all-images'
 
     output = parse_makefile(target=makefile_all_target, makefile_dir=makefile_dir)
@@ -68,19 +71,30 @@ def main() -> None:
                            help="Git ref of the base branch (to determine changed files)")
     argparser.add_argument("--to-ref", type=str, required=False,
                            help="Git ref of the PR branch (to determine changed files)")
+    argparser.add_argument("--leave-out-rhel", type=bool, required=False, default=False, action=argparse.BooleanOptionalAction,
+                           help="Does not output rhel-based images even when they have changed files")
     args = argparser.parse_args()
 
-
     targets = extract_image_targets()
 
     if args.from_ref:
         logging.info(f"Skipping targets not modified in the PR")
         changed_files = gha_pr_changed_files.list_changed_files(args.from_ref, args.to_ref)
         targets = gha_pr_changed_files.filter_out_unchanged(targets, changed_files)
 
+    if args.leave_out_rhel:
+        targets = [target for target in targets if "rhel" not in target]
+
+    # https://stackoverflow.com/questions/66025220/paired-values-in-github-actions-matrix
     output = [
-        f"matrix={json.dumps({"target": targets}, separators=(',', ':'))}",
-        f"has_jobs={json.dumps(len(targets) > 0, separators=(',', ':'))}"
+        "matrix=" + json.dumps({
+            "include": [
+                {"target": target, "subscription": "rhel" in target} for target in targets
+            ],
+        }, separators=(',', ':')),
+        "has_jobs=" + json.dumps(
+            len(targets) > 0, separators=(',', ':')
+        ),
     ]
 
     print("targets", targets)

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -48,11 +48,15 @@ LABEL name="odh-notebook-rstudio-server-rhel9-python-3.11" \
 
 USER 0
 
+# Check if we are already entitled
+RUN subscription-manager status || touch /var/tmp/.subscription-initially-not-present
+
 # uncomment the below line if you fall on this error: subscription-manager is disabled when running inside a container. Please refer to your host system for subscription management.
 #RUN sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || \
+    (SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -62,7 +66,7 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach)
 
 ENV R_VERSION=4.4.1
 
@@ -169,7 +173,7 @@ COPY ${RSTUDIO_SOURCE_CODE}/utils utils/
 COPY ${RSTUDIO_SOURCE_CODE}/run-rstudio.sh ${RSTUDIO_SOURCE_CODE}/setup_rstudio.py ${RSTUDIO_SOURCE_CODE}/rsession.sh ${RSTUDIO_SOURCE_CODE}/run-nginx.sh ./
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
 
 USER 1001
 

ci/secrets/pull-secret.txt

@@ -41,8 +41,12 @@ ARG BASEURL_DEFAULT=""
 USER 0
 WORKDIR /opt/app-root/bin
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# Check if we are already entitled
+RUN subscription-manager status || touch /var/tmp/.subscription-initially-not-present
+
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || \
+    (SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -52,7 +56,7 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach)
 
 ENV NVARCH=x86_64
 ENV NVIDIA_REQUIRE_CUDA="cuda>=12.4 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526 brand=tesla,driver>=535,driver<536 brand=unknown,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=geforce,driver>=535,driver<536 brand=geforcertx,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=titan,driver>=535,driver<536 brand=titanrtx,driver>=535,driver<536"
@@ -165,7 +169,7 @@ RUN yum -y install cuda-toolkit-12-4 && \
     yum -y clean all --enablerepo="*"
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
 
 # Restore notebook user workspace
 USER 1001
@@ -194,11 +198,15 @@ LABEL name="odh-notebook-rstudio-server-rhel9-python-3.11" \
 
 USER 0
 
+# Check if we are already entitled
+RUN subscription-manager status || touch /var/tmp/.subscription-initially-not-present
+
 # uncomment the below line if you fall on this error: subscription-manager is disabled when running inside a container. Please refer to your host system for subscription management.
 #RUN sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || \
+    (SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -208,7 +216,7 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach)
 
 ENV R_VERSION=4.4.1
 
@@ -315,7 +323,7 @@ COPY ${RSTUDIO_SOURCE_CODE}/utils utils/
 COPY ${RSTUDIO_SOURCE_CODE}/run-rstudio.sh ${RSTUDIO_SOURCE_CODE}/setup_rstudio.py ${RSTUDIO_SOURCE_CODE}/rsession.sh ${RSTUDIO_SOURCE_CODE}/run-nginx.sh ./
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN [ ! -f '/var/tmp/.subscription-initially-not-present' ] || subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
 
 USER 1001