RHOAIENG-28514, RHOAIENG-28513: incorporate python-3.12 workbench images into project's Makefile in opendatahub-io/notebooks (#1247)

* RHOAIENG-28514, RHOAIENG-28513: incorporate python-3.12 workbench images into project's Makefile in opendatahub-io/notebooks

* building images on github actions
* updating Pipfile.lock
* running (at least an initial part of) our tests

* RHOAIENG-28513: add Python version matrix to GitHub Actions workflows for building notebooks

* fixup, reformat Python code in CI scripts for improved readability

* fixup, enable additional Python 3.12 Jupyter image builds in Makefile and fix bad syntax

* fixup, fix indentation issue in Makefile for Python 3.12 Jupyter image builds

Author: jiridanek

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -11,6 +11,10 @@ name: Build & Publish Notebook Servers (TEMPLATE)
         required: true
         description: "make target to build"
         type: string
+      python:
+        required: true
+        description: "python version"
+        type: string
       github:
         required: true
         description: "top workflow's `github`"
@@ -43,7 +47,9 @@ jobs:
       TRIVY_VULNDB: "/home/runner/.local/share/containers/trivy_db"
       # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
       TRIVY_SCAN_FS_JSON: '{}'
+      # Makefile variables
       BUILD_ARCH: ${{ inputs.platform }}
+      RELEASE_PYTHON_VERSION: ${{ inputs.python }}
 
     steps:
 

.github/workflows/build-notebooks-pr-rhel.yaml

@@ -70,6 +70,7 @@ jobs:
     if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
+      python: "${{ matrix.python }}"
       github: "${{ toJSON(github) }}"
       platform: "${{ matrix.platform }}"
       subscription: "${{ matrix.subscription }}"

.github/workflows/build-notebooks-pr.yaml

@@ -56,6 +56,7 @@ jobs:
     if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
+      python: "${{ matrix.python }}"
       github: "${{ toJSON(github) }}"
       platform: "${{ matrix.platform }}"
       subscription: "${{ matrix.subscription }}"

.github/workflows/build-notebooks-push.yaml

@@ -45,6 +45,7 @@ jobs:
     if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
+      python: "${{ matrix.python }}"
       github: "${{ toJSON(github) }}"
       platform: "${{ matrix.platform }}"
       subscription: "${{ matrix.subscription }}"

.github/workflows/piplock-renewal.yaml

@@ -18,6 +18,7 @@ on:  # yamllint disable-line rule:truthy
         default: '3.11'
         type: choice
         options:
+          - '3.12'
           - '3.11'
           - '3.9'
           - '3.8'

Makefile

@@ -392,7 +392,9 @@ validate-rstudio-image: bin/kubectl
 # Default Python version
 PYTHON_VERSION ?= 3.11
 ROOT_DIR := $(shell pwd)
-BASE_DIRS := jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
+ifeq ($(PYTHON_VERSION), 3.11)
+	BASE_DIRS := \
+	    jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
 		jupyter/datascience/ubi9-python-$(PYTHON_VERSION) \
 		jupyter/pytorch/ubi9-python-$(PYTHON_VERSION) \
 		jupyter/tensorflow/ubi9-python-$(PYTHON_VERSION) \
@@ -408,6 +410,25 @@ BASE_DIRS := jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
 		runtimes/tensorflow/ubi9-python-$(PYTHON_VERSION) \
 		runtimes/rocm-tensorflow/ubi9-python-$(PYTHON_VERSION) \
 		runtimes/rocm-pytorch/ubi9-python-$(PYTHON_VERSION)
+else ifeq ($(PYTHON_VERSION), 3.12)
+	BASE_DIRS := \
+	    jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \
+		jupyter/datascience/ubi9-python-$(PYTHON_VERSION) \
+		jupyter/pytorch/ubi9-python-$(PYTHON_VERSION) \
+		jupyter/tensorflow/ubi9-python-$(PYTHON_VERSION)
+		# jupyter/trustyai/ubi9-python-$(PYTHON_VERSION)
+		# jupyter/rocm/tensorflow/ubi9-python-$(PYTHON_VERSION)
+		# jupyter/rocm/pytorch/ubi9-python-$(PYTHON_VERSION)
+		# codeserver/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/minimal/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/datascience/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/pytorch/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/tensorflow/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/rocm-tensorflow/ubi9-python-$(PYTHON_VERSION)
+		# runtimes/rocm-pytorch/ubi9-python-$(PYTHON_VERSION)
+else
+	$(error Invalid Python version $(PYTHON_VERSION))
+endif
 
 # Default value is false, can be overiden
 # The below directories are not supported on tier-1
@@ -452,9 +473,11 @@ refresh-pipfilelock-files:
 scan-image-vulnerabilities:
 	python ci/security-scan/quay_security_analysis.py
 
-# This is used primarly for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build
+# This is used primarily for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build
 .PHONY: all-images
-all-images: jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
+ifeq ($(RELEASE_PYTHON_VERSION), 3.11)
+all-images: \
+	jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	jupyter-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	cuda-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	cuda-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
@@ -474,8 +497,33 @@ all-images: jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-runtime-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION) \
 	rocm-runtime-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
+else ifeq ($(RELEASE_PYTHON_VERSION), 3.12)
+all-images: \
+	jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
+	jupyter-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION) \
+	cuda-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION) \
+	cuda-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION) \
+	cuda-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# jupyter-trustyai-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# runtime-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# runtime-datascience-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# runtime-cuda-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# runtime-cuda-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# codeserver-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# rstudio-c9s-python-$(RELEASE_PYTHON_VERSION)
+# cuda-rstudio-c9s-python-$(RELEASE_PYTHON_VERSION)
+# rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION)
+# cuda-rstudio-rhel9-python-$(RELEASE_PYTHON_VERSION)
+# rocm-jupyter-minimal-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# rocm-jupyter-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# rocm-jupyter-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# rocm-runtime-pytorch-ubi9-python-$(RELEASE_PYTHON_VERSION)
+# rocm-runtime-tensorflow-ubi9-python-$(RELEASE_PYTHON_VERSION)
+else
+	$(error Invalid Python version $(RELEASE_PYTHON_VERSION))
+endif
 
-# This is used primarly for konflux_generate_component_build_pipelines.py to we know the build release version
+# This is used primarily for `konflux_generate_component_build_pipelines.py` to we know the build release version
 .PHONY: print-release
 print-release:
 	@echo "$(RELEASE)"

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -27,13 +27,17 @@
 }
 
 
-def extract_image_targets(makefile_dir: pathlib.Path | str | None = None) -> list[str]:
+def extract_image_targets(
+    makefile_dir: pathlib.Path | str | None = None, env: dict[str, str] | None = None
+) -> list[str]:
     if makefile_dir is None:
         makefile_dir = os.getcwd()
+    if env is None:
+        env = {}
 
     makefile_all_target = "all-images"
 
-    output = makefile_helper.dry_run_makefile(target=makefile_all_target, makefile_dir=makefile_dir)
+    output = makefile_helper.dry_run_makefile(target=makefile_all_target, makefile_dir=makefile_dir, env=env)
 
     # Extract the 'all-images' entry and its values
     all_images = []
@@ -89,7 +93,9 @@ def main() -> None:
     )
     args = argparser.parse_args()
 
-    targets = extract_image_targets()
+    targets = extract_image_targets(env={"RELEASE_PYTHON_VERSION": "3.11"}) + extract_image_targets(
+        env={"RELEASE_PYTHON_VERSION": "3.12"}
+    )
 
     if args.from_ref:
         logging.info("Skipping targets not modified in the PR")
@@ -122,6 +128,11 @@ def main() -> None:
                 "include": [
                     {
                         "target": target,
+                        "python": "3.11"
+                        if "-python-3.11" in target
+                        else "3.12"
+                        if "-python-3.12" in target
+                        else "invalid-python-version",
                         "platform": platform,
                         "subscription": "rhel" in target,
                     }

ci/cached-builds/gha_pr_changed_files.py

@@ -29,14 +29,21 @@ def list_changed_files(from_ref: str, to_ref: str) -> list[str]:
     return files
 
 
-def _query_build(make_target: str, query: str) -> str:
+def _query_build(make_target: str, query: str, env: dict[str, str] | None = None) -> str:
     results = []
 
+    if env is None:
+        env = {}
+
+    envs = []
+    for k, v in env.items():
+        envs.extend(("-e", f"{k}={v}"))
+
     pattern = re.compile(r"#\*# " + query + r": <(?P<result>[^>]+)> #\(MACHINE-PARSED LINE\)#\*#\.\.\.")
     try:
         logging.debug(f"Running make in --just-print mode for target {make_target}")
         for line in subprocess.check_output(
-            [MAKE, make_target, "--just-print"], encoding="utf-8", cwd=PROJECT_ROOT
+            [MAKE, make_target, "--just-print", *envs], encoding="utf-8", cwd=PROJECT_ROOT
         ).splitlines():
             if m := pattern.match(line):
                 results.append(m["result"])
@@ -51,12 +58,12 @@ def _query_build(make_target: str, query: str) -> str:
     return results[0]
 
 
-def get_build_directory(make_target) -> str:
-    return _query_build(make_target, "Image build directory")
+def get_build_directory(make_target, env: dict[str, str] | None = None) -> str:
+    return _query_build(make_target, "Image build directory", env=env)
 
 
-def get_build_dockerfile(make_target) -> str:
-    return _query_build(make_target, "Image build Dockerfile")
+def get_build_dockerfile(make_target: str, env: dict[str, str] | None = None) -> str:
+    return _query_build(make_target, "Image build Dockerfile", env=env)
 
 
 def find_dockerfiles(directory: str) -> list:
@@ -97,7 +104,10 @@ def should_build_target(changed_files: list[str], target_directory: str) -> str:
 def filter_out_unchanged(targets: list[str], changed_files: list[str]) -> list[str]:
     changed = []
     for target in targets:
-        build_directory = get_build_directory(target)
+        python_version = (
+            "3.11" if "-python-3.11" in target else "3.12" if "-python-3.12" in target else "invalid-python-version"
+        )
+        build_directory = get_build_directory(target, env={"RELEASE_PYTHON_VERSION": python_version})
         if reason := should_build_target(changed_files, build_directory):
             logging.info(f"✅ Will build {target} because file {reason} has been changed")
             changed.append(target)

ci/cached-builds/has_tests.py

@@ -38,6 +38,9 @@ def check_tests(target: str) -> bool:
     if target.startswith(("rocm-jupyter-minimal-", "rocm-jupyter-datascience-")):
         return False  # we don't have specific tests for -minimal-, ... in ci-operator/config
 
+    if target.endswith("-python-3.12"):
+        return False  # we haven't yet enabled on-cluster testing because that needs valid manifests
+
     build_directory = gha_pr_changed_files.get_build_directory(target)
     kustomization = (
         pathlib.Path(gha_pr_changed_files.PROJECT_ROOT) / build_directory / "kustomize/base/kustomization.yaml"

ci/cached-builds/makefile_helper.py

@@ -34,7 +34,14 @@ def exec_makefile(target: str, makefile_dir: pathlib.Path | str, options: list[s
     return result.stdout
 
 
-def dry_run_makefile(target: str, makefile_dir: pathlib.Path | str) -> str:
+def dry_run_makefile(target: str, makefile_dir: pathlib.Path | str, env: dict[str, str] | None = None) -> str:
+    if env is None:
+        env = {}
+
+    envs = []
+    for k, v in env.items():
+        envs.extend(("-e", f"{k}={v}"))
+
     return exec_makefile(
-        target=target, makefile_dir=makefile_dir, options=["--dry-run", "--print-data-base", "--quiet"]
+        target=target, makefile_dir=makefile_dir, options=["--dry-run", "--print-data-base", "--quiet", *envs]
     )