Add retry mechanism for registry push commands

There are 2 types of commands that are handled by this commit.

1. Bash commands executed in the Tekton task
2. Commands executed in the Python script

Both of these uses re-tries mechanism now. The bash commands use a
generic script that controls the execution and retries if needed.

This commit "fixes" the tasks and commands that failed the most often
and improves stability.

JIRA: ISV-6114

Signed-off-by: Ales Raszka <[email protected]>

Author: Allda

ansible/roles/index_signature_verification/files/tasks/verify-index-signatures.yml

@@ -10,15 +10,9 @@ spec:
   params:
     - name: organization
       description: Index organization
-    - name: jq_image
-      description: Operator pipeline default image
+    - name: pipeline_image
+      description: Pipeline image
       default: "quay.io/redhat-isv/operator-pipelines-images:released"
-    - name: podman_image
-      description: Podman image
-      default: "registry.redhat.io/ubi9/podman:9.5"
-    - name: skopeo_image
-      description: Skopeo image
-      default: "registry.redhat.io/ubi9/skopeo:9.5"
     - name: indices_api
       description: Pyxis API endpoint for Operator index images
       default: https://catalog.redhat.com/api/containers/v1/operators/indices
@@ -50,7 +44,7 @@ spec:
 
   steps:
     - name: get-index-images
-      image: "$(params.jq_image)"
+      image: "$(params.pipeline_image)"
       script: |
         #!/usr/bin/env bash
         set +x -e -o pipefail
@@ -65,19 +59,19 @@ spec:
           | tee image-tags.txt
 
     - name: inspect-images
-      image: "$(params.skopeo_image)"
+      image: "$(params.pipeline_image)"
       script: |
         #!/usr/bin/env bash
         set +x -e -o pipefail
 
         for image in $(cat image-tags.txt); do
           dest=$(echo -n $image | base64)
           echo "Inspecting $image"
-          skopeo inspect --retry-times 5 --raw docker://$image > $dest.json
+          retry 5 skopeo inspect --raw docker://$image > $dest.json
         done
 
     - name: get-digest-pull-specs
-      image: "$(params.jq_image)"
+      image: "$(params.pipeline_image)"
       script: |
         #!/usr/bin/env bash
         set +x -e -o pipefail
@@ -93,7 +87,7 @@ spec:
     - name: pull-images-by-digest
       securityContext:
         privileged: true
-      image: "$(params.podman_image)"
+      image: "$(params.pipeline_image)"
       env:
         - name: STORAGE_DRIVER
           value: vfs
@@ -104,22 +98,6 @@ spec:
         cp /etc/containers/policy.json /tmp/
         podman image trust set --policypath=/tmp/policy.json -f /mnt/keys/pub.gpg registry.redhat.io
 
-        max_retries=5
-
         for pull_spec in $(cat image-digests.txt); do
-            wait_time=1
-            for ((i=1; i<=max_retries; i++)); do
-                podman pull --retry 0 --signature-policy=/tmp/policy.json $pull_spec
-                if [ $? -eq 0 ]; then
-                    break
-                fi
-
-                sleep $wait_time
-                wait_time=$((wait_time * 2))
-
-                if [ $i -eq $max_retries ]; then
-                    echo "ERROR: Podman pull failed after $max_retries attempts."
-                    exit 1
-                fi
-            done
+          retry 5 podman pull --retry 0 --signature-policy=/tmp/policy.json $pull_spec
         done

ansible/roles/operator-pipeline/templates/openshift/tasks/buildah.yml

@@ -110,6 +110,23 @@ spec:
           EXTRA_ARGS+=" --authfile $(workspaces.dockerconfig.path)/.dockerconfigjson"
         fi
 
+        retry() {
+            local max_retries=$1
+            shift
+
+            local attempt=1
+
+            until "$@"; do
+                if [ $attempt -ge $max_retries ]; then
+                    echo "Command failed after $max_retries attempts" >&2
+                    return 1
+                fi
+                attempt=$((attempt + 1))
+                echo "Attempt $attempt failed. Retrying..." >&2
+                sleep 1
+            done
+        }
+
         echo "Building $(params.IMAGE)"
         BUILD_ARGS=()
         for buildarg in "$@"
@@ -125,7 +142,7 @@ spec:
         [ "${PARAM_SKIP_PUSH}" = "true" ] && echo "Push skipped" && exit 0
         # push the image (CERT_DIR_FLAG should be omitted if empty and PUSH_EXTRA_ARGS can contain multiple args)
         # shellcheck disable=SC2046,SC2086
-        buildah ${CERT_DIR_FLAG} "--storage-driver=${PARAM_STORAGE_DRIVER}" push \
+        retry 5 buildah ${CERT_DIR_FLAG} "--storage-driver=${PARAM_STORAGE_DRIVER}" push \
           "--tls-verify=${PARAM_TLSVERIFY}" --digestfile /tmp/image-digest ${PARAM_PUSH_EXTRA_ARGS} ${EXTRA_ARGS} \
           "${PARAM_IMAGE}" "docker://${PARAM_IMAGE}"
         tee "$(results.IMAGE_DIGEST.path)" < /tmp/image-digest

ansible/roles/operator-pipeline/templates/openshift/tasks/copy-image.yml

@@ -58,22 +58,22 @@ spec:
         DEST_AUTHFILE=$(workspaces.dest-registry-credentials.path)/.dockerconfigjson
         CONNECT_REPO_PATH="$(params.connect_registry)/$(params.vendor_label)/$(params.repo_name)"
 
-        skopeo copy \
+        retry 5 skopeo copy \
           --retry-times 5 \
           --src-authfile $SRC_AUTHFILE \
           --dest-authfile $DEST_AUTHFILE \
           docker://$(params.src_image) \
           docker://"$(params.dest_image_registry_namespace_certproject):$(params.dest_image_tag)"
 
 
-        skopeo copy \
+        retry 5 skopeo copy \
           --retry-times 5 \
           --src-authfile $SRC_AUTHFILE \
           --dest-authfile $DEST_AUTHFILE \
           docker://$(params.src_image) \
           docker://"$(params.dest_image_registry_namespace_certproject):latest"
 
-        DIGEST=$(skopeo inspect --retry-times 5 --authfile $DEST_AUTHFILE docker://$(params.dest_image_registry_namespace_certproject):$(params.dest_image_tag) | jq -r .Digest)
+        DIGEST=$(retry 5 skopeo inspect --retry-times 5 --authfile $DEST_AUTHFILE docker://$(params.dest_image_registry_namespace_certproject):$(params.dest_image_tag) | jq -r .Digest)
         echo -n $DIGEST | tee $(results.container_digest.path)
         echo "- $CONNECT_REPO_PATH:$(params.dest_image_tag)" | tee "$RELEASE_INFO_DIR_PATH/released_bundle.txt"
         echo -n "$CONNECT_REPO_PATH@${DIGEST}" > $(results.image_pullspec.path)

ansible/roles/operator-pipeline/templates/openshift/tasks/publish-to-index.yml

@@ -116,7 +116,7 @@ spec:
           echo " - $DEST_IMAGE_PERMANENT_TAG"
 
           # Add version tag to an index
-          skopeo \
+          retry 5 skopeo \
             copy \
             --retry-times 5 \
             --format v2s2 --all \
@@ -126,7 +126,7 @@ spec:
             docker://$DEST_IMAGE_VERSION_TAG
 
           # Add permanent tag to an index
-          skopeo \
+          retry 5 skopeo \
             copy \
             --retry-times 5 \
             --format v2s2 --all \

operator-pipeline-images/Dockerfile

@@ -44,6 +44,7 @@ RUN dnf update -y && \
     dnf clean all
 
 COPY operator-pipeline-images/config/krb5.conf /etc/krb5.conf
+COPY operator-pipeline-images/hacks/retry-command.sh /usr/local/bin/retry
 
 # Install oc, opm and operator-sdk CLI
 RUN curl -LO https://github.com/operator-framework/operator-registry/releases/download/v1.46.0/linux-${ARCH}-opm && \

operator-pipeline-images/hacks/retry-command.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+max_retries=$1
+shift
+
+if [ -z "$1" ]; then
+    echo "Usage: retry <max_retries> <command> [args...]" >&2
+    exit 1
+fi
+
+attempt=1
+
+until "$@"; do
+    if [ "$attempt" -ge "$max_retries" ]; then
+        echo "Command failed after $max_retries attempts" >&2
+        exit 1
+    fi
+    attempt=$((attempt + 1))
+    echo "Attempt $attempt failed. Retrying..." >&2
+
+done

operator-pipeline-images/operatorcert/buildah.py

@@ -33,7 +33,7 @@ def build_image(dockerfile_path: str, context: str, output_image: str) -> Any:
         context,
     ]
     LOGGER.info("Building image: %s", output_image)
-    return run_command(cmd)
+    return run_command(cmd, retries=2)
 
 
 def push_image(image: str, authfile: str) -> Any:
@@ -58,4 +58,4 @@ def push_image(image: str, authfile: str) -> Any:
     ]
 
     LOGGER.info("Pushing image: %s", image)
-    return run_command(cmd)
+    return run_command(cmd, retries=5)

operator-pipeline-images/operatorcert/bundle.py

@@ -116,7 +116,7 @@ def _copy_image(self) -> None:
         if self.auth_file_path:
             command.extend(["--authfile", self.auth_file_path])
 
-        utils.run_command(command)
+        utils.run_command(command, retries=5)
 
     def _extract_content(self) -> None:
         """

operator-pipeline-images/operatorcert/utils.py

@@ -134,33 +134,43 @@ def __call__(
 
 
 def run_command(
-    cmd: List[str], check: bool = True, cwd: Optional[str] = None
+    cmd: List[str], check: bool = True, cwd: Optional[str] = None, retries: int = 1
 ) -> subprocess.CompletedProcess[bytes]:
     """
     Run a shell command and return its output.
 
     Args:
         cmd (List[str]): Command to run
+        check (bool): Whether to check the command exit code
+        cwd (Optional[str]): Current working directory
+        retries (int): Number of retries before failing
 
     Returns:
         CompletedProcess: Command output
     """
     LOGGER.debug("Running command: %s", cmd)
-    try:
-        output = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=check,
-            cwd=cwd,
-        )
-    except subprocess.CalledProcessError as e:
-        LOGGER.error(
-            "Error running command: \nstdout: %s\nstderr: %s",
-            e.stdout,
-            e.stderr,
-        )
-        raise e
+    for attempt in range(1, retries + 1):
+        try:
+            output = subprocess.run(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=check,
+                cwd=cwd,
+            )
+            break
+        except subprocess.CalledProcessError as e:
+            LOGGER.error(
+                "Error running command: \nstdout: %s\nstderr: %s",
+                e.stdout,
+                e.stderr,
+            )
+            if attempt >= retries:
+                raise e
+            LOGGER.warning(
+                "Command failed, retrying... (attempt %d of %d)", attempt, retries
+            )
+
     LOGGER.debug("Command output: %s", output.stdout.decode("utf-8"))
     return output
 
@@ -227,7 +237,8 @@ def copy_images_to_destination(
             cmd.extend(["--authfile", auth_file])
 
         LOGGER.info("Copying image to destination: %s", cmd)
-        subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
+
+        run_command(cmd, retries=5)
 
 
 def sort_versions(version_list: list[Any]) -> list[Any]:

operator-pipeline-images/tests/test_buildah.py

@@ -19,7 +19,8 @@ def test_build_image(mock_run_command: MagicMock) -> None:
             "-t",
             "image",
             "context",
-        ]
+        ],
+        retries=2,
     )
 
 
@@ -29,5 +30,6 @@ def test_push_image(mock_run_command: MagicMock) -> None:
     assert result == mock_run_command.return_value
 
     mock_run_command.assert_called_once_with(
-        ["buildah", "push", "--authfile", "authfile", "image", "docker://image"]
+        ["buildah", "push", "--authfile", "authfile", "image", "docker://image"],
+        retries=5,
     )