chore(ci, build): add support for Dockerfile.konflux.* builds in Makefile and tests

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -28,6 +28,11 @@ name: Build & Publish Notebook Servers (TEMPLATE)
         default: false
         description: "add RHEL subscription from github secret"
         type: boolean
+      konflux:
+        required: false
+        default: false
+        description: "build from Dockerfile.konflux.* instead of Dockerfile.*"
+        type: boolean
       PLATFORM_RUNNERS:
         required: false
         # language=json
@@ -127,7 +132,11 @@ jobs:
           printf "${PWD}/entitlement:/etc/pki/entitlement\n${PWD}/consumer:/etc/pki/consumer\n" | sudo tee /usr/share/containers/mounts.conf
 
           mkdir -p $HOME/.config/containers/
-          sudo cp ${PWD}/ci/secrets/pull-secret.json $HOME/.config/containers/auth.json
+
+          # Don't use sudo here.
+          # With CONTAINER_HOST set, podman is a remote client that forwards auth to the
+          # rootful server during builds. Credentials must be readable by the runner user.
+          cp ${PWD}/ci/secrets/pull-secret.json $HOME/.config/containers/auth.json
         env:
           SUBSCRIPTION_ORG: ${{ secrets.SUBSCRIPTION_ORG }}
           SUBSCRIPTION_ACTIVATION_KEY: ${{ secrets.SUBSCRIPTION_ACTIVATION_KEY }}
@@ -260,6 +269,7 @@ jobs:
           CONTAINER_BUILD_CACHE_ARGS: "${{ steps.extra-container-build-args.outputs.CONTAINER_BUILD_CACHE_ARGS }}"
           # We don't have access to image registry for PRs, so disable pushing
           PUSH_IMAGES: "${{ (fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'pull_request_target') && 'no' || 'yes' }}"
+          KONFLUX: "${{ inputs.konflux && 'yes' || 'no' }}"
 
       - name: "Show podman images information"
         run: podman images --digests

.github/workflows/build-notebooks-pr-aipcc.yaml

@@ -95,4 +95,5 @@ jobs:
       platform: "${{ matrix.platform }}"
       # rhds/notebooks builds from AIPCC base images that are RHEL-based
       subscription: ${{ true }}
+      konflux: ${{ true }}
     secrets: inherit

.github/workflows/build-notebooks-push.yaml

@@ -9,6 +9,15 @@
       # In fact, skip the build if there are only changes in manifests and nowhere else
       - manifests/**
   "workflow_dispatch":
+    inputs:
+      "odh":
+        description: "Build Dockerfile.*"
+        type: boolean
+        default: false
+      "rhds":
+        description: "Build Dockerfile.konflux.* (requires subscription)"
+        type: boolean
+        default: false
   "schedule":
     - "cron": "0 2 * * *"
 
@@ -52,12 +61,28 @@ jobs:
       fail-fast: false
       matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
     uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
-    if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
+    if: ${{ (inputs.odh || (github.event_name != 'workflow_dispatch' && github.repository != 'red-hat-data-services/notebooks')) && fromJson(needs.gen.outputs.has_jobs) }}
+    with:
+      target: "${{ matrix.target }}"
+      python: "${{ matrix.python }}"
+      github: "${{ toJSON(github) }}"
+      platform: "${{ matrix.platform }}"
+      subscription: "${{ matrix.subscription }}"
+    secrets: inherit
+
+  build-aipcc:
+    needs: ["gen"]
+    strategy:
+      fail-fast: false
+      matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
+    uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
+    if: ${{ (inputs.rhds || (github.event_name != 'workflow_dispatch' && github.repository == 'red-hat-data-services/notebooks')) && fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
       python: "${{ matrix.python }}"
       github: "${{ toJSON(github) }}"
       platform: "${{ matrix.platform }}"
       # rhds/notebooks builds from AIPCC base images that are RHEL-based
-      subscription: "${{ matrix.subscription || (github.repository == 'red-hat-data-services/notebooks') }}"
+      subscription: ${{ true }}
+      konflux: ${{ true }}
     secrets: inherit

Makefile

@@ -27,6 +27,8 @@ CONTAINER_BUILD_CACHE_ARGS ?= --no-cache
 PUSH_IMAGES ?= yes
 # INDEX_MODE: auto (default), public-index, or rh-index - controls lock file generation
 INDEX_MODE ?= auto
+# KONFLUX: whether to build images from Dockerfile.konflux.* (default: no)
+KONFLUX ?= no
 
 # OS dependant: Generate date, select appropriate cmd to locate container engine
 ifdef OS
@@ -67,13 +69,14 @@ endif
 # Build function for the notebook image:
 #   ARG 1: Image tag name.
 #   ARG 2: Path of Dockerfile we want to build.
+#   ARG 3: Path of the build-args conf file to use.
 define build_image
 	$(eval IMAGE_NAME := $(IMAGE_REGISTRY):$(1)-$(IMAGE_TAG))
 
 	# Checks if there’s a build-args/*.conf matching the Dockerfile
 	$(eval BUILD_DIR := $(dir $(2)))
 	$(eval DOCKERFILE_NAME := $(notdir $(2)))
-	$(eval CONF_FILE := $(BUILD_DIR)build-args/$(shell echo $(DOCKERFILE_NAME) | cut -d. -f2).conf)
+	$(eval CONF_FILE := $(3))
 
 	# if the conf file exists, transform it into --build-arg KEY=VALUE flags
 	$(eval BUILD_ARGS := $(shell \
@@ -101,11 +104,14 @@ endef
 #
 # PUSH_IMAGES: allows skipping podman push
 define image
-	$(info #*# Image build Dockerfile: <$(2)> #(MACHINE-PARSED LINE)#*#...)
 	$(eval BUILD_DIRECTORY := $(shell echo $(2) | sed 's/\/Dockerfile.*//'))
+	$(eval VARIANT := $(shell echo $(notdir $(2)) | cut -d. -f2))
+	$(eval DOCKERFILE := $(BUILD_DIRECTORY)/Dockerfile$(if $(KONFLUX:no=),.konflux,$()).$(VARIANT))
+	$(eval CONF_FILE := $(BUILD_DIRECTORY)/build-args/$(if $(KONFLUX:no=),konflux.,$())$(shell echo $(VARIANT)).conf)
+	$(info #*# Image build Dockerfile: <$(DOCKERFILE)> #(MACHINE-PARSED LINE)#*#...)
 	$(info #*# Image build directory: <$(BUILD_DIRECTORY)> #(MACHINE-PARSED LINE)#*#...)
 
-	$(call build_image,$(1),$(2))
+	$(call build_image,$(1),$(DOCKERFILE),$(CONF_FILE))
 
 	$(if $(PUSH_IMAGES:no=),
 		$(call push_image,$(1))

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -214,15 +214,18 @@ def test_select_changed_targets_dockerfile(self):
     def test_select_changed_targets_shared_file(self):
         targets = extract_image_targets(makefile_dir=project_dir)
 
-        changed_files = ["cuda/ubi9-python-3.12/NGC-DL-CONTAINER-LICENSE"]
+        changed_files = ["jupyter/utils/addons/dist/pf.css"]
 
         targets = gha_pr_changed_files.filter_out_unchanged(targets, changed_files)
         assert set(targets) == {
-            "jupyter-minimal-ubi9-python-3.12",
             "cuda-jupyter-minimal-ubi9-python-3.12",
+            "cuda-jupyter-pytorch-llmcompressor-ubi9-python-3.12",
             "cuda-jupyter-pytorch-ubi9-python-3.12",
-            "runtime-cuda-pytorch-ubi9-python-3.12",
             "cuda-jupyter-tensorflow-ubi9-python-3.12",
+            "jupyter-datascience-ubi9-python-3.12",
+            "jupyter-minimal-ubi9-python-3.12",
+            "jupyter-trustyai-ubi9-python-3.12",
             "rocm-jupyter-minimal-ubi9-python-3.12",
-            "runtime-cuda-tensorflow-ubi9-python-3.12",
+            "rocm-jupyter-pytorch-ubi9-python-3.12",
+            "rocm-jupyter-tensorflow-ubi9-python-3.12",
         }

ci/cached-builds/makefile_helper.py

@@ -3,15 +3,20 @@
 from __future__ import annotations
 
 import platform
+import string
 import subprocess
 import sys
 from typing import TYPE_CHECKING
 
+from gha_pr_changed_files import PROJECT_ROOT
+
 if TYPE_CHECKING:
     import pathlib
+    from collections.abc import Sequence
 
 
-def exec_makefile(target: str, makefile_dir: pathlib.Path | str, options: list[str] = []) -> str:
+def exec_makefile(target: str, makefile_dir: pathlib.Path | str, options: Sequence[str] = ()) -> str:
+    """Returns the stdout of the make command."""
     # Check if the operating system is macOS
     if platform.system() == "Darwin":
         make_command = "gmake"
@@ -50,3 +55,54 @@ def dry_run_makefile(target: str, makefile_dir: pathlib.Path | str, env: dict[st
     return exec_makefile(
         target=target, makefile_dir=makefile_dir, options=["--dry-run", "--print-data-base", "--quiet", *envs]
     )
+
+
+class TestMakefile:
+    MINIMAL_IMAGE = "jupyter-minimal-ubi9-python-3.12"
+
+    def test_makefile__build_image__konflux(self):
+        konflux_default = dry_run_makefile(target=self.MINIMAL_IMAGE, makefile_dir=PROJECT_ROOT)
+        konflux_yes = dry_run_makefile(target=self.MINIMAL_IMAGE, makefile_dir=PROJECT_ROOT, env={"KONFLUX": "yes"})
+
+        _assert_subdict(
+            {
+                "VARIANT": "cpu",
+                "DOCKERFILE_NAME": "Dockerfile.cpu",
+                "CONF_FILE": "jupyter/minimal/ubi9-python-3.12/build-args/cpu.conf",
+            },
+            _extract_assignments(konflux_default),
+        )
+
+        _assert_subdict(
+            {
+                "VARIANT": "cpu",
+                "DOCKERFILE_NAME": "Dockerfile.konflux.cpu",
+                "CONF_FILE": "jupyter/minimal/ubi9-python-3.12/build-args/konflux.cpu.conf",
+            },
+            _extract_assignments(konflux_yes),
+        )
+
+        assert "--file 'jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu'" in konflux_default
+        assert "--file 'jupyter/minimal/ubi9-python-3.12/Dockerfile.konflux.cpu'" in konflux_yes
+
+
+def _extract_assignments(makefile_output: str) -> dict[str, str]:
+    assignments = {}
+    for line in makefile_output.splitlines():
+        if not line.strip():
+            continue
+        if line[0] not in string.ascii_letters:
+            continue
+
+        if ":=" not in line:
+            continue
+        key, value = line.split(":=", 1)
+        assignments[key.strip()] = value.strip()
+    return assignments
+
+
+# see also https://pypi.org/project/pytest-assert-utils/
+def _assert_subdict(subdict: dict[str, str], superdict: dict[str, str]):
+    """Filter subdict to only keys in superdict, then compare the remaining items."""
+    __tracebackhide__ = True
+    assert subdict == {k: superdict[k] for k in subdict if k in superdict}