Merge pull request opendatahub-io#570 from jstourac/sync-main

Sync odh/main into rhds/main

Author: harshad16

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -209,7 +209,10 @@ jobs:
       - name: Calculate image name and tag
         id: calculated_vars
         run: |
-          SANITIZED_REF_NAME=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g')
+          # Need for sanitization explained in https://github.com/opendatahub-io/notebooks/issues/631
+          # For length, Docker image tags have 128-character limit, and we form them as <inputs.target>-<ref_name>_<sha>
+          # therefore since sha is 40 characters, and our target names are <40 chars, we should cut ref_name at 40
+          SANITIZED_REF_NAME=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') | cut -c 1-40
           IMAGE_TAG="${SANITIZED_REF_NAME}_${{ github.sha }}"
 
           echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"

ci/check-software-versions.py

@@ -14,7 +14,6 @@
 import json
 import logging
 import os
-import re
 import subprocess
 import sys
 import uuid
@@ -65,10 +64,9 @@ def load_yaml(filepath):
 
 
 def extract_variable(reference):
-    """Extracts a variable name from a string (e.g.: '$(odh-rstudio-notebook-image-commit-n-1)') using regex."""
+    """Extracts a variable name from a string (e.g.: 'odh-rstudio-notebook-image-commit-n-1_PLACEHOLDER') using regex."""
 
-    match = re.search(r"\((.*?)\)", reference)
-    return match.group(1) if match else None
+    return reference.replace("_PLACEHOLDER", "")
 
 
 def get_variable_value(variable_name, params_file_path=PARAMS_ENV_PATH):

ci/kustomize.sh

@@ -14,6 +14,8 @@
 # https://github.com/red-hat-data-services/rhods-operator/blob/7ccc405135f99c014982d7e297b8949e970dd750/go.mod#L28-L29
 # and then to match appropriate kustomize release https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize%2Fv5.0.3
 DEFAULT_KUSTOMIZE_VERSION=5.0.3
+# The latest kustomize version we want to check with to be sure we're prepared for the future
+THE_LATEST_KUSTOMIZE=5.6.0
 
 KUSTOMIZE_VERSION="${KUSTOMIZE_VERSION:-$DEFAULT_KUSTOMIZE_VERSION}"
 
@@ -27,7 +29,30 @@ function download_kustomize() {
     echo "---------------------------------------------------------------------------------"
     echo "Download kustomize '${kustomize_version}'"
     echo "---------------------------------------------------------------------------------"
-    wget --output-document="${kustomize_tar}" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${kustomize_version}/kustomize_v${kustomize_version}_linux_amd64.tar.gz"
+
+    # Detect OS
+    local uname_out
+    uname_out="$(uname -s)"
+    case "${uname_out}" in
+        Linux*)     os=linux;;
+        Darwin*)    os=darwin;;
+        *)          echo "Unsupported OS: ${uname_out}" && return 1;;
+    esac
+
+    # Detect architecture
+    local arch
+    arch="$(uname -m)"
+    case "${arch}" in
+        x86_64)   arch=amd64;;
+        arm64)    arch=arm64;;
+        aarch64)  arch=arm64;;
+        *)        echo "Unsupported architecture: ${arch}" && return 1;;
+    esac
+
+    local download_url="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${kustomize_version}/kustomize_v${kustomize_version}_${os}_${arch}.tar.gz"
+    echo "Downloading from: ${download_url}"
+
+    wget --output-document="${kustomize_tar}" "${download_url}"
     tar -C "${tmp_dir}" -xvf "${kustomize_tar}"
     mv "${tmp_dir}/kustomize" "${kustomize_bin}"
 
@@ -46,22 +71,72 @@ function execute_kustomize() {
     echo "Starting to run kustomize '${kustomize_version}' for each kustomization.yaml file except components"
     echo "---------------------------------------------------------------------------------------------------"
     # We don't want to execute kustomization on the components part as it's not intended to be used that way.
-    find . -name "kustomization.yaml" | xargs dirname | grep -v "components" | xargs -t -I {} "${kustomize_bin}" build {} >"${kustomize_stdout}" 2>"${kustomize_stderr}"
+    # This first run is for the actual execution to get the generated output and eventual errors/warnings.
+    find . -name "kustomization.yaml" | xargs dirname | grep -v "components" | xargs -I {} "${kustomize_bin}" build {} >"${kustomize_stdout}" 2>"${kustomize_stderr}"
+    # This second run is with verbose output to see eventual errors/warnings together with which command they are present for easier debugging.
+    find . -name "kustomization.yaml" | xargs dirname | grep -v "components" | xargs --verbose -I {} "${kustomize_bin}" build {} >/dev/null
 
     echo "Let's print the STDERR:"
     cat "${kustomize_stderr}"
 }
 
+function check_the_results() {
+    local tmp_dir="${1}"
+    local kustomize_version_1="${2}"
+    local kustomize_version_2="${3}"
+
+    local kustomize_stdout_1="${tmp_dir}/kustomize-${kustomize_version_1}-stdout.yaml"
+    local kustomize_stderr_1="${tmp_dir}/kustomize-${kustomize_version_1}-stderr.txt"
+    local kustomize_stdout_2="${tmp_dir}/kustomize-${kustomize_version_2}-stdout.yaml"
+    local kustomize_stderr_2="${tmp_dir}/kustomize-${kustomize_version_2}-stderr.txt"
+
+    echo "---------------------------------------------------------------------------------"
+    echo "Checking the generated outputs - should be identical:"
+    echo "  - ${kustomize_stdout_1}"
+    echo "  - ${kustomize_stdout_2}"
+    echo "---------------------------------------------------------------------------------"
+    diff -u "${kustomize_stdout_1}" "${kustomize_stdout_2}" || {
+        echo "Generated files from kustomize differs between kustomize version ${kustomize_version_1} and ${kustomize_version_2}. Please check above!"
+        return 1
+    }
+
+    echo "---------------------------------------------------------------------------------"
+    echo "No log in STDERR outputs should be printed:"
+    echo "  - ${kustomize_stderr_1}"
+    echo "  - ${kustomize_stderr_2}"
+    echo "---------------------------------------------------------------------------------"
+    if [ -s "${kustomize_stderr_1}" ] || [ -s "${kustomize_stderr_2}" ]; then
+        echo "There were some logs generated to STDERR during the kustomize build. Please check the log above!"
+        return 1
+    fi
+}
+
+function run_check() {
+    local tmp_dir="${1}"
+    local kustomize_version="${2}"
+
+    download_kustomize "${tmp_dir}" "${kustomize_version}" || return 1
+    execute_kustomize "${tmp_dir}" "${kustomize_version}" || return 1
+}
+
 function main() {
+    local ret_code=0
+
     local tmp_dir
     tmp_dir=$(mktemp --directory -t kustomize-XXXXXXXXXX)
     echo "Running in the following temporary directory: '${tmp_dir}'"
 
-    download_kustomize "${tmp_dir}" "${KUSTOMIZE_VERSION}" || return 1
-    execute_kustomize "${tmp_dir}" "${KUSTOMIZE_VERSION}" || return 1
+    run_check "${tmp_dir}" "${KUSTOMIZE_VERSION}" || return 1
+    run_check "${tmp_dir}" "${THE_LATEST_KUSTOMIZE}" || return 1
+
+    # --------------------------------------------------------------------------------------
+
+    check_the_results "${tmp_dir}" "${KUSTOMIZE_VERSION}" "${THE_LATEST_KUSTOMIZE}" || return 1
+
+    exit "${ret_code}"
 }
 
 # allows sourcing the script into interactive session without executing it
 if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then
-    main $@
+    main "$@"
 fi

ci/package_versions_selftestdata.py

@@ -30,7 +30,7 @@
         openshift.io/imported-from: quay.io/opendatahub/workbench-images
         opendatahub.io/workbench-image-recommended: 'true'
         opendatahub.io/default-image: "true"
-        opendatahub.io/notebook-build-commit: $(odh-minimal-notebook-image-commit-n)
+        opendatahub.io/notebook-build-commit: odh-minimal-notebook-image-commit-n_PLACEHOLDER
       from:
         kind: DockerImage
         name: $(odh-minimal-notebook-image-n)
@@ -53,7 +53,7 @@
         openshift.io/imported-from: quay.io/opendatahub/workbench-images
         opendatahub.io/workbench-image-recommended: 'false'
         opendatahub.io/default-image: "true"
-        opendatahub.io/notebook-build-commit: $(odh-minimal-notebook-image-commit-n-1)
+        opendatahub.io/notebook-build-commit: odh-minimal-notebook-image-commit-n-1_PLACEHOLDER
       from:
         kind: DockerImage
         name: $(odh-minimal-notebook-image-n-1)