RHOAIENG-9822: chore(Makefile): allow not pushing built images in Makefile and allow skipping building dependent images (#657)

Co-authored-by: Guilherme Caponetto <[email protected]>

Author: jiridanek

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -123,21 +123,20 @@ jobs:
         env:
           IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
+          # dependent images were already built and pushed, so just let podman pull it
+          BUILD_DEPENDENT_IMAGES: "no"
 
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
       - name: "pull_request: make ${{ inputs.target }}"
         run: |
-          # start a black hole container registry as make target always does a push
-          mkdir -p $HOME/.config/containers/registries.conf.d/
-          cp ci/cached-builds/insecure_localhost_registry.conf $HOME/.config/containers/registries.conf.d/insecure_localhost_registry.conf
-          go run ci/cached-builds/dev_null_container_registry.go &
-          # build and push the image
           make ${{ inputs.target }}
         if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
         env:
           IMAGE_TAG: "${{ github.sha }}"
           IMAGE_REGISTRY: "localhost:5000/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"
+          # We don't have access to image registry, so disable pushing
+          PUSH_IMAGES: "no"
 
       - name: "Show podman images information"
         run: podman images

Makefile

@@ -2,6 +2,10 @@ IMAGE_REGISTRY   ?= quay.io/opendatahub/workbench-images
 RELEASE	 		 ?= 2024a
 # additional user-specified caching parameters for $(CONTAINER_ENGINE) build
 CONTAINER_BUILD_CACHE_ARGS ?= --no-cache
+# whether to build all dependent images or just the one specified
+BUILD_DEPENDENT_IMAGES ?= yes
+# whether to push the images to a registry as they are built
+PUSH_IMAGES ?= yes
 
 # OS dependant: Generate date, select appropriate cmd to locate container engine
 ifeq ($(OS), Windows_NT)
@@ -58,10 +62,19 @@ endef
 #   ARG 1: Image tag name.
 #   ARG 2: Path of image context we want to build.
 #   ARG 3: Base image tag name (optional).
+#
+# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line
+# PUSH_IMAGES: allows skipping podman push
 define image
 	$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
-	$(call build_image,$(1),$(2),$(3))
-	$(call push_image,$(1))
+
+	$(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))),
+		$(call build_image,$(1),$(2),$(3))
+
+		$(if $(PUSH_IMAGES:no=),
+			$(call push_image,$(1))
+		)
+	)
 endef
 
 ####################################### Buildchain for Python 3.8 using ubi8 #######################################

README.md

@@ -37,7 +37,9 @@ To build a workbench image, you can execute the following command:
 make ${WORKBENCH_NAME} -e  IMAGE_REGISTRY=quay.io/${YOUR_USER}/workbench-images  -e  RELEASE=2023x
 ```
 
-Using  `IMAGE_REGISTRY` and `RELEASE` variables  you can overwrite the default values and use a different registry or release tag
+Using  `IMAGE_REGISTRY` and `RELEASE` variables you can overwrite the default values and use a different registry or release tag
+
+Using `CONTAINER_BUILD_CACHE_ARGS` (default: `--no-cache`), `BUILD_DEPENDENT_IMAGES`, and `PUSH_IMAGES` variables you can further customize the build process.
 
 ### Local Execution
 

ci/cached-builds/dev_null_container_registry.go

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
+import os
+import logging
 import pathlib
+import subprocess
 import tomllib
 from typing import TYPE_CHECKING
 
@@ -32,3 +35,36 @@ def test_files_that_should_be_same_are_same(subtests: pytest_subtests.plugin.Sub
         with subtests.test(msg=f"Checking {group_name}"):
             for file in rest:
                 assert first_file.read_text() == file.read_text(), f"The files {first_file} and {file} do not match"
+
+
+def test_make_building_only_specified_images(subtests: pytest_subtests.plugin.SubTests):
+    for goals in (["rocm-jupyter-tensorflow-ubi9-python-3.9"], ["rocm-jupyter-tensorflow-ubi9-python-3.9", "base-ubi9-python-3.9"]):
+        with subtests.test(msg="Running goals", goals=goals):
+            lines = dryrun_make(goals, env={"BUILD_DEPENDENT_IMAGES": "no"})
+            builds_number = 0
+            for line in lines:
+                if "podman build" in line:
+                    builds_number += 1
+            assert builds_number == len(goals)
+
+
+def test_make_disable_pushing():
+    lines = dryrun_make(["rocm-jupyter-tensorflow-ubi9-python-3.9"], env={"PUSH_IMAGES": ""})
+    for line in lines:
+        assert "podman push" not in line
+
+
+def dryrun_make(make_args: list[str], env: dict[str, str] | None = None) -> list[str]:
+    env = env or {}
+
+    try:
+        logging.info(f"Running make in --just-print mode for target(s) {make_args} with env {env}")
+        lines = subprocess.check_output(["make", "--just-print", *make_args], encoding="utf-8",
+                                        env={**os.environ, **env},
+                                        cwd=PROJECT_ROOT).splitlines()
+        for line in lines:
+            logging.debug(line)
+        return lines
+    except subprocess.CalledProcessError as e:
+        print(e.stderr, e.stdout)
+        raise