Implement our own package sync function with exponential backoff.

Author: gizmoguy

entrypoint.sh

@@ -7,16 +7,18 @@ CLOUDSMITH_REPO="${2}"
 CLOUDSMITH_USERNAME="${3}"
 export CLOUDSMITH_API_KEY="${4}"
 
-cloudsmith_default_args=(--error-retry-max 30 --republish)
+cloudsmith_default_args=(--no-wait-for-sync --republish)
 
 # required to make python 3 work with cloudsmith script
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
+# redirect fd 5 to stdout
+exec 5>&1
+
 function upload_rpm {
-    sync=$1
-    distro=$2
-    pkg_fullpath=$3
+    distro=$1
+    pkg_fullpath=$2
     pkg_filename="$(basename "${pkg_fullpath}")"
     rev_filename=$(echo "${pkg_filename}" | rev)
 
@@ -26,40 +28,53 @@ function upload_rpm {
     pkg_rel=$(echo "${rev_filename}" | cut -d '.' -f3 | rev)
     release_ver="${pkg_rel:2}"
 
-    extra_args=("${cloudsmith_default_args[@]}")
-    if [ "${sync}" == "nosync" ]; then
-        extra_args+=(--no-wait-for-sync)
-    fi
-
-    cloudsmith push rpm "${extra_args[@]}" "${CLOUDSMITH_REPO}/${distro}/${release_ver}" "${pkg_fullpath}"
+    output=$(cloudsmith push rpm "${cloudsmith_default_args[@]}" "${CLOUDSMITH_REPO}/${distro}/${release_ver}" "${pkg_fullpath}" | tee /dev/fd/5)
+    pkg_slug=$(echo "${output}" | grep "Created: ${CLOUDSMITH_REPO}" | awk '{print $2}')
+    cloudsmith_sync "${pkg_slug}"
 }
 
 function upload_deb {
-    sync=$1
-    distro=$2
-    release=$3
-    pkg_fullpath=$4
-
-    extra_args=("${cloudsmith_default_args[@]}")
-    if [ "${sync}" == "nosync" ]; then
-        extra_args+=(--no-wait-for-sync)
-    fi
+    distro=$1
+    release=$2
+    pkg_fullpath=$3
 
-    cloudsmith push deb "${extra_args[@]}" "${CLOUDSMITH_REPO}/${distro}/${release}" "${pkg_fullpath}"
+    output=$(cloudsmith push deb "${cloudsmith_default_args[@]}" "${CLOUDSMITH_REPO}/${distro}/${release}" "${pkg_fullpath}" | tee /dev/fd/5)
+    pkg_slug=$(echo "${output}" | grep "Created: ${CLOUDSMITH_REPO}" | awk '{print $2}')
+    cloudsmith_sync "${pkg_slug}"
+}
+
+function cloudsmith_sync {
+    pkg_slug=$1
+
+    retry_count=1
+    timeout=5
+    backoff=1.2
+    while true; do
+        if [ "${retry_count}" -gt 20 ]; then
+            echo "Exceeded retry attempts for package synchronisation"
+            exit 1
+        fi
+        output=$(cloudsmith status "${pkg_slug}" | tee /dev/fd/5)
+        if echo "${output}" | grep "Completed / Fully Synchronised"; then
+            break
+        fi
+        sleep ${timeout}
+        retry_count=$((retry_count+1))
+        timeout=$(python3 -c "print(round(${timeout}*${backoff}))")
+    done
 }
 
 function cloudsmith_upload {
-    sync=$1
-    distro=$2
-    release=$3
-    pkg_fullpath=$4
+    distro=$1
+    release=$2
+    pkg_fullpath=$3
 
     if [[ ${distro} =~ centos ]]; then
-        upload_rpm "${sync}" "centos" "${pkg_fullpath}"
+        upload_rpm "centos" "${pkg_fullpath}"
     elif [[ ${distro} =~ fedora ]]; then
-        upload_rpm "${sync}" "fedora" "${pkg_fullpath}"
+        upload_rpm "fedora" "${pkg_fullpath}"
     else
-        upload_deb "${sync}" "${distro}" "${release}" "${pkg_fullpath}"
+        upload_deb "${distro}" "${release}" "${pkg_fullpath}"
     fi
 }
 
@@ -69,21 +84,7 @@ pip3 install --upgrade cloudsmith-cli
 while IFS= read -r -d '' path; do
     IFS=_ read -r distro release <<< "$(basename "${path}")"
 
-    pkgs=()
-
     while IFS= read -r -d '' pkg; do
-        pkgs+=("${pkg}")
+        cloudsmith_upload "${distro}" "${release}" "${pkg}"
     done <    <(find "${path}" -maxdepth 1 -type f -print0)
-
-    i=0
-    last=$((${#pkgs[@]}-1))
-    for pkg in "${pkgs[@]}"; do
-        if [ ${i} -eq ${last} ]; then
-            # wait for final package upload for each distro release to synchronise
-            cloudsmith_upload "sync" "${distro}" "${release}" "${pkg}"
-        else
-            cloudsmith_upload "nosync" "${distro}" "${release}" "${pkg}"
-        fi
-        i=$((i+1))
-    done
 done <   <(find "${PACKAGE_LOCATION}" -mindepth 1 -maxdepth 1 -type d -print0)