RHOAIENG-21668: chore(gha): run GitHub Actions CI for our RHEL-based workbenches

.gitattributes

@@ -0,0 +1 @@
+ci/secrets/** filter=git-crypt diff=git-crypt

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -15,6 +15,11 @@ name: Build & Publish Notebook Servers (TEMPLATE)
         required: true
         description: "top workflow's `github`"
         type: string
+      subscription:
+        required: false
+        default: false
+        description: "add RHEL subscription from github secret"
+        type: boolean
 
 jobs:
   build:
@@ -49,9 +54,40 @@ jobs:
           echo "CACHE=${CACHE,,}" >>${GITHUB_ENV}
 
       - uses: actions/checkout@v4
+        if: ${{ fromJson(inputs.github).event_name != 'pull_request_target' }}
+      # we need to checkout the pr branch, not pr target (the default for pull_request_target)
+      # user access check is done in calling workflow
+      - uses: actions/checkout@v4
+        if: ${{ fromJson(inputs.github).event_name == 'pull_request_target' }}
+        with:
+          ref: "refs/pull/${{ fromJson(inputs.github).event.number }}/merge"
 
       - run: mkdir -p $TMPDIR
 
+      # do this early because it's fast and why not
+      - name: Unlock encrypted secrets with git-crypt
+        if: ${{ inputs.subscription }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install git-crypt
+          echo "${GIT_CRYPT_KEY}" | base64 --decode > ./git-crypt-key
+          git-crypt unlock ./git-crypt-key
+          rm ./git-crypt-key
+        env:
+          GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }}
+
+      - name: Add subscriptions from GitHub secret
+        if: ${{ inputs.subscription }}
+        run: |
+          sudo mkdir -p /etc/pki/
+          sudo cp -R ${PWD}/ci/secrets/pki/* /etc/pki/
+          # https://access.redhat.com/solutions/5870841
+          # https://github.com/containers/common/issues/1735
+          printf "${PWD}/ci/secrets/run/secrets/rhsm:/etc/rhsm\n${PWD}/ci/secrets/run/secrets/etc-pki-entitlement:/etc/pki/entitlement\n${PWD}/ci/secrets/pki/consumer:/etc/pki/consumer\n" | sudo tee /usr/share/containers/mounts.conf
+
+          mkdir -p $HOME/.config/containers/
+          sudo cp ${PWD}/ci/secrets/pull-secret.json $HOME/.config/containers/auth.json
+
       # for bin/buildinputs in scripts/sandbox.py
       - uses: actions/setup-go@v5
         with:
@@ -257,7 +293,8 @@ jobs:
       - name: "pull_request: make ${{ inputs.target }}"
         run: |
           make ${{ inputs.target }}
-        if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
+        if: "${{ fromJson(inputs.github).event_name == 'pull_request' ||
+          fromJson(inputs.github).event_name == 'pull_request_target' }}"
         env:
           IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"

.github/workflows/build-notebooks-pr-rhel.yaml

@@ -0,0 +1,69 @@
+---
+"name": "Build Notebooks (pr, RHEL images)"
+"on":
+  "pull_request_target":
+
+# BEWARE: This GitHub Actions workflow runs on pull_request_target, meaning it has access to our secrets
+# see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-secrets
+# and https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+permissions:
+  contents: read
+  packages: read
+
+env:
+  # language=json
+  contributors: |
+    ["atheo89", "andyatmiami", "caponetto", "daniellutz", "dibryant", "harshad16", "jesuino", "jiridanek", "jstourac", "paulovmr"]
+
+jobs:
+  gen:
+    name: Generate job matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.gen.outputs.matrix }}
+      has_jobs: ${{ steps.gen.outputs.has_jobs }}
+    steps:
+
+      - name: Check permissions and deny untrusted users (this must be done FIRST, for security, before we checkout)
+        if: ${{ !contains(fromJSON(env.contributors), github.actor) }}
+        run: |
+          echo "GitHub user ${{ github.actor }} is not a registered project contributor, not allowed to run actions on RHEL!"
+          exit 1
+
+      # Here we are checking out the pull request, so that we can build from the new code
+      # We can do this because we already checked that the submitting user is a contributor
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name != 'pull_request_target' }}
+
+      - name: Determine targets to build based on changed files
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          set -x
+          git fetch --no-tags origin 'pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}'
+          git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
+          python3 ci/cached-builds/gen_gha_matrix_jobs.py \
+            --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
+            --to-ref '${{ github.event.pull_request.head.ref }}' \
+            --rhel-images include-only
+        id: gen
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+
+  build:
+    needs: ["gen"]
+    strategy:
+      fail-fast: false
+      matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
+    uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
+    if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
+    with:
+      target: "${{ matrix.target }}"
+      github: "${{ toJSON(github) }}"
+      subscription: "${{ matrix.subscription }}"
+    secrets: inherit

.github/workflows/build-notebooks-pr.yaml

@@ -33,7 +33,8 @@ jobs:
           git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
           python3 ci/cached-builds/gen_gha_matrix_jobs.py \
             --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
-            --to-ref '${{ github.event.pull_request.head.ref }}'
+            --to-ref '${{ github.event.pull_request.head.ref }}' \
+            --rhel-images exclude
         id: gen
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -49,4 +50,5 @@ jobs:
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"
+      subscription: "${{ matrix.subscription }}"
     secrets: inherit

.github/workflows/build-notebooks-push.yaml

@@ -39,4 +39,5 @@ jobs:
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"
+      subscription: "${{ matrix.subscription }}"
     secrets: inherit

.github/workflows/code-quality.yaml

@@ -80,6 +80,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Do not check secrets, they are encrypted
+        run: rm -rf ./ci/secrets
+
       - name: Validate YAML files (best code practices check included)
         id: validate-yaml-files
         run: |

Makefile

@@ -172,6 +172,16 @@ rstudio-c9s-python-$(RELEASE_PYTHON_VERSION):
 cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION):
 	$(call image,$@,rstudio/c9s-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
 
+####################################### Buildchain for Python using rhel9 #######################################
+
+.PHONY: rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION)
+rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION):
+	$(call image,$@,rstudio/rhel9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cpu)
+
+.PHONY: cuda-rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION)
+cuda-rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION):
+	$(call image,$@,rstudio/rhel9-python-$(RELEASE_PYTHON_VERSION)/Dockerfile.cuda)
+
 ####################################### Buildchain for AMD Python using UBI9 #######################################
 .PHONY: rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
 rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION):
@@ -438,6 +448,8 @@ all-images: jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	codeserver-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rstudio-c9s-python-$(RELEASE_PYTHON_VERSION) \
 	cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION) \
+	rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION) \
+	cuda-rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -1,12 +1,14 @@
+#!/usr/bin/env python3
+
 import argparse
+import enum
 import json
 import logging
 import platform
 import subprocess
 import os
 import pathlib
 import re
-import string
 import sys
 import unittest
 
@@ -21,7 +23,7 @@
 project_dir = pathlib.Path(__file__).parent.parent.parent.absolute()
 
 
-def parse_makefile(target: str, makefile_dir: str) -> str:
+def parse_makefile(target: str, makefile_dir: pathlib.Path | str) -> str:
     # Check if the operating system is macOS
     if platform.system() == 'Darwin':
         make_command = 'gmake'
@@ -30,7 +32,9 @@ def parse_makefile(target: str, makefile_dir: str) -> str:
 
     try:
         # Run the make (or gmake) command and capture the output
-        result = subprocess.run([make_command, '-nps', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, cwd=makefile_dir)
+        result = subprocess.run([make_command, '-nps', target],
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+                                check=True, cwd=makefile_dir)
     except subprocess.CalledProcessError as e:
         # Handle errors if the make command fails
         print(f'{make_command} failed with return code: {e.returncode}:\n{e.stderr}', file=sys.stderr)
@@ -43,7 +47,7 @@ def parse_makefile(target: str, makefile_dir: str) -> str:
     return result.stdout
 
 
-def extract_image_targets(makefile_dir: str = os.getcwd()) -> list[str]:
+def extract_image_targets(makefile_dir: pathlib.Path | str = os.getcwd()) -> list[str]:
     makefile_all_target = 'all-images'
 
     output = parse_makefile(target=makefile_all_target, makefile_dir=makefile_dir)
@@ -60,6 +64,12 @@ def extract_image_targets(makefile_dir: str = os.getcwd()) -> list[str]:
     return all_images
 
 
+class RhelImages(enum.Enum):
+    EXCLUDE = "exclude"
+    INCLUDE = "include"
+    INCLUDE_ONLY = "include-only"
+
+
 def main() -> None:
     logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
 
@@ -68,19 +78,36 @@ def main() -> None:
                            help="Git ref of the base branch (to determine changed files)")
     argparser.add_argument("--to-ref", type=str, required=False,
                            help="Git ref of the PR branch (to determine changed files)")
+    argparser.add_argument("--rhel-images", type=RhelImages, required=False, default=RhelImages.INCLUDE, nargs='?',
+                           help="Whether to `include` rhel images or `exclude` them or `include-only` them")
     args = argparser.parse_args()
 
-
     targets = extract_image_targets()
 
     if args.from_ref:
         logging.info(f"Skipping targets not modified in the PR")
         changed_files = gha_pr_changed_files.list_changed_files(args.from_ref, args.to_ref)
         targets = gha_pr_changed_files.filter_out_unchanged(targets, changed_files)
 
+    if args.rhel_images == RhelImages.INCLUDE:
+        pass
+    elif args.rhel_images == RhelImages.EXCLUDE:
+        targets = [target for target in targets if "rhel" not in target]
+    elif args.rhel_images == RhelImages.INCLUDE_ONLY:
+        targets = [target for target in targets if "rhel" in target]
+    else:
+        raise Exception(f"Unknown value for --rhel-images: {args.rhel_images}")
+
+    # https://stackoverflow.com/questions/66025220/paired-values-in-github-actions-matrix
     output = [
-        f"matrix={json.dumps({"target": targets}, separators=(',', ':'))}",
-        f"has_jobs={json.dumps(len(targets) > 0, separators=(',', ':'))}"
+        "matrix=" + json.dumps({
+            "include": [
+                {"target": target, "subscription": "rhel" in target} for target in targets
+            ],
+        }, separators=(',', ':')),
+        "has_jobs=" + json.dumps(
+            len(targets) > 0, separators=(',', ':')
+        ),
     ]
 
     print("targets", targets)

ci/cached-builds/mounts.conf

@@ -0,0 +1,5 @@
+# /usr/share/containers/mounts.conf
+# https://github.com/containers/common/issues/1735
+
+/etc/pki/consumer:/etc/pki/consumer
+/etc/pki/entitlement:/etc/pki/entitlement

rstudio/rhel9-python-3.11/Dockerfile.cpu

@@ -51,8 +51,9 @@ USER 0
 # uncomment the below line if you fall on this error: subscription-manager is disabled when running inside a container. Please refer to your host system for subscription management.
 #RUN sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -62,7 +63,8 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach; \
+  fi
 
 ENV R_VERSION=4.4.1
 
@@ -169,7 +171,9 @@ COPY ${RSTUDIO_SOURCE_CODE}/utils utils/
 COPY ${RSTUDIO_SOURCE_CODE}/run-rstudio.sh ${RSTUDIO_SOURCE_CODE}/setup_rstudio.py ${RSTUDIO_SOURCE_CODE}/rsession.sh ${RSTUDIO_SOURCE_CODE}/run-nginx.sh ./
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    subscription-manager remove --all && subscription-manager unregister && subscription-manager clean; \
+  fi
 
 USER 1001
 

rstudio/rhel9-python-3.11/Dockerfile.cuda

@@ -41,8 +41,9 @@ ARG BASEURL_DEFAULT=""
 USER 0
 WORKDIR /opt/app-root/bin
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -52,7 +53,8 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach; \
+  fi
 
 ENV NVARCH=x86_64
 ENV NVIDIA_REQUIRE_CUDA="cuda>=12.4 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526 brand=tesla,driver>=535,driver<536 brand=unknown,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=geforce,driver>=535,driver<536 brand=geforcertx,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=titan,driver>=535,driver<536 brand=titanrtx,driver>=535,driver<536"
@@ -165,7 +167,9 @@ RUN yum -y install cuda-toolkit-12-4 && \
     yum -y clean all --enablerepo="*"
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    subscription-manager remove --all && subscription-manager unregister && subscription-manager clean; \
+  fi
 
 # Restore notebook user workspace
 USER 1001
@@ -197,8 +201,9 @@ USER 0
 # uncomment the below line if you fall on this error: subscription-manager is disabled when running inside a container. Please refer to your host system for subscription management.
 #RUN sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
+# If necessary, run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
     BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
     USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
     PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
@@ -208,7 +213,8 @@ RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFA
     --username=$USERNAME \
     --password=$PASSWORD \
     --force \
-    --auto-attach
+    --auto-attach; \
+  fi
 
 ENV R_VERSION=4.4.1
 
@@ -315,7 +321,9 @@ COPY ${RSTUDIO_SOURCE_CODE}/utils utils/
 COPY ${RSTUDIO_SOURCE_CODE}/run-rstudio.sh ${RSTUDIO_SOURCE_CODE}/setup_rstudio.py ${RSTUDIO_SOURCE_CODE}/rsession.sh ${RSTUDIO_SOURCE_CODE}/run-nginx.sh ./
 
 # Unregister the system
-RUN subscription-manager remove --all && subscription-manager unregister && subscription-manager clean
+RUN if [ -d "${SECRET_DIR}" ]; then \
+    subscription-manager remove --all && subscription-manager unregister && subscription-manager clean; \
+  fi
 
 USER 1001