RHAIENG-2645: add loop to retry package installation when it fails

daniellutz · daniellutz · commit 83c426252f14 · 2026-02-10T17:56:45.000-03:00
diff --git a/codeserver/ubi9-python-3.12/Dockerfile.cpu b/codeserver/ubi9-python-3.12/Dockerfile.cpu
@@ -26,6 +26,7 @@ ARG NODE_VERSION=22.18.0
 ARG CODESERVER_VERSION=v4.104.0
 
 COPY ${CODESERVER_SOURCE_CODE}/get_code_server_rpm.sh .
+COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
 
 # create dummy file to ensure this stage is awaited before installing rpm
 RUN ./get_code_server_rpm.sh && touch /tmp/control
@@ -66,6 +67,8 @@ RUN touch /tmp/control
 ####################
 FROM ${BASE_IMAGE} AS cpu-base
 
+ARG CODESERVER_SOURCE_CODE=codeserver/ubi9-python-3.12
+
 WORKDIR /opt/app-root/bin
 
 # OS Packages needs to be installed as root
@@ -85,7 +88,8 @@ RUN dnf -y upgrade --refresh --nobest --skip-broken --nodocs --noplugins --setop
 # upgrade first to avoid fixable vulnerabilities end
 
 # Install useful OS packages
-RUN dnf install -y perl mesa-libGL skopeo && dnf clean all && rm -rf /var/cache/yum
+COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
+RUN --mount=type=cache,target=/var/cache/dnf ./utils/install_with_retry.sh dnf-install perl mesa-libGL skopeo
 
 # (ARCH-ppc64le): since wheels are compiled from source, we need shared libs available at runtime
 RUN --mount=type=cache,from=whl-cache,source=/root/OpenBLAS,target=/OpenBlas,rw \
@@ -136,7 +140,7 @@ USER 0
 WORKDIR /opt/app-root/bin
 
 # Install useful OS packages
-RUN dnf install -y jq git-lfs libsndfile && dnf clean all && rm -rf /var/cache/yum
+RUN --mount=type=cache,target=/var/cache/dnf ./utils/install_with_retry.sh dnf-install jq git-lfs libsndfile
 
 # wait for rpm-base stage (rpm builds for ppc64le)
 COPY --from=rpm-base /tmp/control /dev/null
@@ -152,8 +156,6 @@ RUN --mount=type=cache,from=rpm-base,source=/tmp/,target=/code-server-rpm/,rw \
     dnf install -y cpio && dnf -y clean all && \
     cd / && rpm2cpio "/code-server-rpm/code-server-${CODESERVER_VERSION/v/}-${TARGETARCH}.rpm" | cpio -idmv
 
-COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
-
 # Create and intall the extensions though build-time on a temporary directory. Later this directory will copied on the `/opt/app-root/src/.local/share/code-server/extensions` via run-code-server.sh file when it starts up.
 # https://coder.com/docs/code-server/FAQ#how-do-i-install-an-extension
 RUN mkdir -p /opt/app-root/extensions-temp && \
@@ -175,10 +177,10 @@ ENV NGINX_CONFIGURATION_PATH=${APP_ROOT}/etc/nginx.d \
     NGINX_PERL_MODULE_PATH=${APP_ROOT}/etc/perl
 
 # Modules does not exist
-RUN INSTALL_PKGS="bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd" && \
-    dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    dnf -y clean all --enablerepo='*'
+RUN --mount=type=cache,target=/var/cache/dnf \
+    DNF_EXTRA_OPTS="--setopt=tsflags=nodocs" ./utils/install_with_retry.sh dnf-install bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd \
+    && rpm -V bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd \
+    && dnf -y clean all --enablerepo='*'
 
 # Configure httpd for CGI processing
 COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/httpd/httpd.conf /etc/httpd/conf/httpd.conf
diff --git a/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu b/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu
@@ -26,6 +26,7 @@ ARG NODE_VERSION=22.18.0
 ARG CODESERVER_VERSION=v4.104.0
 
 COPY ${CODESERVER_SOURCE_CODE}/get_code_server_rpm.sh .
+COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
 
 # create dummy file to ensure this stage is awaited before installing rpm
 RUN ./get_code_server_rpm.sh && touch /tmp/control
@@ -66,6 +67,8 @@ RUN touch /tmp/control
 ####################
 FROM ${BASE_IMAGE} AS cpu-base
 
+ARG CODESERVER_SOURCE_CODE=codeserver/ubi9-python-3.12
+
 WORKDIR /opt/app-root/bin
 
 # OS Packages needs to be installed as root
@@ -85,7 +88,8 @@ RUN dnf -y upgrade --refresh --nobest --skip-broken --nodocs --noplugins --setop
 # upgrade first to avoid fixable vulnerabilities end
 
 # Install useful OS packages
-RUN dnf install -y perl mesa-libGL skopeo && dnf clean all && rm -rf /var/cache/yum
+COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
+RUN --mount=type=cache,target=/var/cache/dnf ./utils/install_with_retry.sh dnf-install perl mesa-libGL skopeo
 
 # (ARCH-ppc64le): since wheels are compiled from source, we need shared libs available at runtime
 RUN --mount=type=cache,from=whl-cache,source=/root/OpenBLAS,target=/OpenBlas,rw \
@@ -134,7 +138,7 @@ USER 0
 WORKDIR /opt/app-root/bin
 
 # Install useful OS packages
-RUN dnf install -y jq git-lfs libsndfile && dnf clean all && rm -rf /var/cache/yum
+RUN --mount=type=cache,target=/var/cache/dnf ./utils/install_with_retry.sh dnf-install jq git-lfs libsndfile
 
 # wait for rpm-base stage (rpm builds for ppc64le)
 COPY --from=rpm-base /tmp/control /dev/null
@@ -171,10 +175,10 @@ ENV APP_ROOT=/opt/app-root \
     NGINX_PERL_MODULE_PATH=${APP_ROOT}/etc/perl
 
 # Modules does not exist
-RUN INSTALL_PKGS="bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd" && \
-    dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
-    rpm -V $INSTALL_PKGS && \
-    dnf -y clean all --enablerepo='*'
+RUN --mount=type=cache,target=/var/cache/dnf \
+    DNF_EXTRA_OPTS="--setopt=tsflags=nodocs" ./utils/install_with_retry.sh dnf-install bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd \
+    && rpm -V bind-utils nginx nginx-mod-stream nginx-mod-http-perl httpd \
+    && dnf -y clean all --enablerepo='*'
 
 # Configure httpd for CGI processing
 COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/httpd/httpd.conf /etc/httpd/conf/httpd.conf
diff --git a/codeserver/ubi9-python-3.12/get_code_server_rpm.sh b/codeserver/ubi9-python-3.12/get_code_server_rpm.sh
@@ -31,7 +31,7 @@ if [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ||"$ARCH" == "ppc64le" ]]; then
 	# install build dependencies
 	# https://access.redhat.com/support/policy/updates/rhel-app-streams-life-cycle
 	# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/developing_c_and_cpp_applications_in_rhel_9/assembly_additional-toolsets-for-development-rhel-9_developing-applications#cpp-compatibility-in-gcc-toolset-14_gcc-toolset-14
-	dnf install -y jq patch libtool rsync gettext gcc-toolset-14 krb5-devel libX11-devel
+	./utils/install_with_retry.sh dnf-install jq patch libtool rsync gettext gcc-toolset-14 krb5-devel libX11-devel
 
 	# starting with node-22, c++20 is required
 	. /opt/rh/gcc-toolset-14/enable
diff --git a/codeserver/ubi9-python-3.12/run-code-server.sh b/codeserver/ubi9-python-3.12/run-code-server.sh
@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# Load bash libraries
+# Load bash libraries (only process.sh is needed at runtime; install_with_retry.sh is for build-only)
 SCRIPT_DIR=$(dirname -- "$0")
-source ${SCRIPT_DIR}/utils/*.sh
+source "${SCRIPT_DIR}/utils/process.sh"
 
 # Start nginx and httpd
 run-nginx.sh &
diff --git a/codeserver/ubi9-python-3.12/utils/install_with_retry.sh b/codeserver/ubi9-python-3.12/utils/install_with_retry.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Shared install script with retry logic for dnf and texlive (install_pdf_deps).
+# Usage:
+#   ./install_with_retry.sh dnf-install <package> [package ...]
+#   ./install_with_retry.sh texlive-install
+
+set -Eeuxo pipefail
+
+readonly MAX_RETRIES="${MAX_RETRIES:-3}"
+readonly RETRY_DELAY="${RETRY_DELAY:-30}"
+
+# Runs a command with retry logic.
+# Optional: set CLEANUP_CMD to a shell command (e.g. "dnf clean metadata") to run between retries.
+# Returns 0 on success, exits 1 after MAX_RETRIES failures.
+run_with_retry() {
+    local retry_count=0
+    while true; do
+        if "$@"; then
+            return 0
+        fi
+        retry_count=$((retry_count + 1))
+        if [ "$retry_count" -ge "$MAX_RETRIES" ]; then
+            echo "ERROR: Command failed after $MAX_RETRIES attempts" >&2
+            exit 1
+        fi
+        echo "Command failed (attempt $retry_count/$MAX_RETRIES), retrying in ${RETRY_DELAY} seconds..."
+        if [ -n "${CLEANUP_CMD:-}" ]; then
+            $CLEANUP_CMD || true
+        fi
+        sleep "$RETRY_DELAY"
+    done
+}
+
+dnf_install() {
+    local packages=("$@")
+    if [ ${#packages[@]} -eq 0 ]; then
+        echo "Usage: $0 dnf-install <package> [package ...]" >&2
+        exit 1
+    fi
+    CLEANUP_CMD="dnf clean metadata" run_with_retry dnf install -y ${DNF_EXTRA_OPTS:-} "${packages[@]}"
+    dnf clean all
+    rm -rf /var/cache/yum
+}
+
+texlive_install() {
+    local script_dir
+    script_dir="$(cd "$(dirname "$0")" && pwd)"
+    CLEANUP_CMD= run_with_retry "$script_dir/install_pdf_deps.sh"
+}
+
+main() {
+    case "${1:-}" in
+        dnf-install)
+            shift
+            dnf_install "$@"
+            ;;
+        texlive-install)
+            texlive_install
+            ;;
+        *)
+            echo "Usage: $0 {dnf-install|texlive-install} [args...]" >&2
+            echo "  dnf-install <package> [package ...]  Install RPM packages with retry" >&2
+            echo "  texlive-install                       Run install_pdf_deps.sh with retry" >&2
+            exit 1
+            ;;
+    esac
+}
+
+# Only run main when this script is executed directly (not when sourced by run-code-server.sh)
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main "$@"
+fi
diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu
@@ -55,16 +55,16 @@ RUN dnf -y upgrade --refresh --nobest --skip-broken --nodocs --noplugins --setop
 # upgrade first to avoid fixable vulnerabilities end
 
 # Install useful OS packages
-RUN --mount=type=cache,target=/var/cache/dnf \
-    echo "Building for architecture: ${TARGETARCH}" && \
-    if [ "$TARGETARCH" = "s390x" ]; then \
-        PACKAGES="perl mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel"; \
-    else \
-        PACKAGES="perl mesa-libGL skopeo"; \
-    fi && \
-    echo "Installing: $PACKAGES" && \
-    dnf install -y $PACKAGES && \
-    dnf clean all && rm -rf /var/cache/yum
+COPY jupyter/utils utils/
+RUN --mount=type=cache,target=/var/cache/dnf /bin/bash <<'EOF'
+set -Eeuxo pipefail
+echo "Building for architecture: ${TARGETARCH}"
+if [ "$TARGETARCH" = "s390x" ]; then
+  ./utils/install_with_retry.sh dnf-install perl mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel
+else
+  ./utils/install_with_retry.sh dnf-install perl mesa-libGL skopeo
+fi
+EOF
 
 RUN if [ "$TARGETARCH" = "s390x" ]; then \
     # Install Rust and set up environment
@@ -114,51 +114,58 @@ WORKDIR /tmp/build-wheels
 
 # Build pyarrow on ppc64le and s390x
 RUN --mount=type=cache,target=/root/.cache/pip \
-    --mount=type=cache,target=/root/.cache/dnf \
-    if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \
-        # Install build dependencies (shared for pyarrow and onnx)
-        dnf install -y cmake make gcc-c++ pybind11-devel wget && \
-        dnf clean all && \
-        # Build and collect pyarrow wheel
-        git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git && \
-        cd arrow/cpp && \
-        mkdir release && cd release && \
-        cmake -DCMAKE_BUILD_TYPE=Release \
-              -DCMAKE_INSTALL_PREFIX=/usr/local \
-              -DARROW_PYTHON=ON \
-              -DARROW_PARQUET=ON \
-              -DARROW_ORC=ON \
-              -DARROW_FILESYSTEM=ON \
-              -DARROW_JSON=ON \
-              -DARROW_CSV=ON \
-              -DARROW_DATASET=ON \
-              -DARROW_DEPENDENCY_SOURCE=BUNDLED \
-              -DARROW_WITH_LZ4=OFF \
-              -DARROW_WITH_ZSTD=OFF \
-              -DARROW_WITH_SNAPPY=OFF \
-              -DARROW_BUILD_TESTS=OFF \
-              -DARROW_BUILD_BENCHMARKS=OFF \
-              .. && \
-        make -j$(nproc) VERBOSE=1 && \
-        make install -j$(nproc) && \
-        cd ../../python && \
-        pip install --no-cache-dir -r requirements-build.txt && \
-        PYARROW_WITH_PARQUET=1 \
-        PYARROW_WITH_DATASET=1 \
-        PYARROW_WITH_FILESYSTEM=1 \
-        PYARROW_WITH_JSON=1 \
-        PYARROW_WITH_CSV=1 \
-        PYARROW_PARALLEL=$(nproc) \
-        python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
-        mkdir -p /tmp/wheels && \
-        cp dist/pyarrow-*.whl /tmp/wheels/ && \
-        chmod -R 777 /tmp/wheels && \
-        # Ensure wheels directory exists and has content
-        ls -la /tmp/wheels/; \
-    else \
-        # Create empty wheels directory for non-s390x
-        mkdir -p /tmp/wheels; \
-    fi
+    --mount=type=cache,target=/root/.cache/dnf /bin/bash <<'EOF'
+set -Eeuxo pipefail
+
+if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then
+    /opt/app-root/bin/utils/install_with_retry.sh dnf-install cmake make gcc-c++ pybind11-devel wget
+
+    # Build and collect pyarrow wheel
+    git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git
+    cd arrow/cpp && mkdir release && cd release
+
+    cmake -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=/usr/local \
+          -DARROW_PYTHON=ON \
+          -DARROW_PARQUET=ON \
+          -DARROW_ORC=ON \
+          -DARROW_FILESYSTEM=ON \
+          -DARROW_JSON=ON \
+          -DARROW_CSV=ON \
+          -DARROW_DATASET=ON \
+          -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+          -DARROW_WITH_LZ4=OFF \
+          -DARROW_WITH_ZSTD=OFF \
+          -DARROW_WITH_SNAPPY=OFF \
+          -DARROW_BUILD_TESTS=OFF \
+          -DARROW_BUILD_BENCHMARKS=OFF \
+          ..
+
+    make -j$(nproc) VERBOSE=1
+    make install -j$(nproc)
+
+    cd ../../python
+
+    pip install --no-cache-dir -r requirements-build.txt
+
+    PYARROW_WITH_PARQUET=1 \
+    PYARROW_WITH_DATASET=1 \
+    PYARROW_WITH_FILESYSTEM=1 \
+    PYARROW_WITH_JSON=1 \
+    PYARROW_WITH_CSV=1 \
+    PYARROW_PARALLEL=$(nproc) \
+    python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel
+
+    mkdir -p /tmp/wheels
+    cp dist/pyarrow-*.whl /tmp/wheels/
+    chmod -R 777 /tmp/wheels
+    # Ensure wheels directory exists and has content
+    ls -la /tmp/wheels/;
+else
+    # Create empty wheels directory for non-s390x
+    mkdir -p /tmp/wheels;
+fi
+EOF
 
 #######################################################
 # common-builder (for Power-only)
@@ -169,8 +176,7 @@ USER root
 RUN <<'EOF'
 set -Eeuxo pipefail
 if [ "${TARGETARCH}" = "ppc64le" ]; then
-    dnf install -y gcc-toolset-13 cmake ninja-build git wget unzip
-    dnf clean all
+    /opt/app-root/bin/utils/install_with_retry.sh dnf-install gcc-toolset-13 cmake ninja-build git wget unzip
 else
     echo "Skipping common-builder package install on non-Power"
 fi
@@ -235,7 +241,7 @@ COPY ${MINIMAL_SOURCE_CODE}/start-notebook.sh ./
 USER 0
 
 # Dependencies for PDF export begin
-RUN ./utils/install_pdf_deps.sh
+RUN ./utils/install_with_retry.sh texlive-install
 ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH"
 # Dependencies for PDF export end
 
@@ -272,8 +278,7 @@ WORKDIR /opt/app-root/bin
 USER root
 
 # Install useful OS packages
-RUN dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat && \
-    dnf clean all && rm -rf /var/cache/yum
+RUN ./utils/install_with_retry.sh dnf-install jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat
 
 # Copy dynamically-linked mongocli built in earlier build stage
 COPY --from=mongocli-builder /tmp/mongocli /opt/app-root/bin/
diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu
diff --git a/jupyter/utils/install_with_retry.sh b/jupyter/utils/install_with_retry.sh
