Add apt-retry wrapper for transient apt mirror failures

.github/workflows/internal-build.yml

@@ -39,7 +39,7 @@ on:
       cache_id:
         description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing"
         type: "string"
-        default: 18
+        default: 19
       cache_prefix:
         description: "A prefix added to all cache keys generated by this workflow"
         type: "string"

Dockerfile

@@ -40,12 +40,13 @@ COPY --from=stellar-xdr-builder /usr/local/cargo/bin/stellar-xdr /stellar-xdr
 FROM ubuntu:24.04 AS stellar-core-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
+COPY apt-retry /usr/local/bin/
+RUN apt-retry sh -c 'apt-get update && \
     apt-get -y install iproute2 procps lsb-release \
                        git build-essential pkg-config autoconf automake libtool \
                        bison flex sed perl libpq-dev parallel \
                        clang-20 libc++abi-20-dev libc++-20-dev \
-                       postgresql curl jq
+                       postgresql curl jq'
 
 ARG CORE_REPO
 ARG CORE_REF
@@ -96,7 +97,8 @@ ENV RUSTUP_HOME=/rust/.rust
 ENV PATH="/usr/local/go/bin:$CARGO_HOME/bin:${PATH}"
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y build-essential jq && apt-get clean
+COPY apt-retry /usr/local/bin/
+RUN apt-retry sh -c 'apt-get update && apt-get install -y build-essential jq' && apt-get clean
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION
 
 RUN make build-stellar-rpc
@@ -110,7 +112,8 @@ COPY --from=stellar-rpc-builder /go/src/github.com/stellar/stellar-rpc/stellar-r
 FROM golang:1.24-trixie AS stellar-horizon-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install jq
+COPY apt-retry /usr/local/bin/
+RUN apt-retry sh -c 'apt-get update && apt-get -y install jq'
 
 ARG HORIZON_REPO
 ARG HORIZON_REF
@@ -134,7 +137,8 @@ COPY --from=stellar-horizon-builder /stellar-horizon /stellar-horizon
 FROM golang:1.24-trixie AS stellar-friendbot-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install jq
+COPY apt-retry /usr/local/bin/
+RUN apt-retry sh -c 'apt-get update && apt-get -y install jq'
 
 ARG FRIENDBOT_REPO
 ARG FRIENDBOT_REF
@@ -206,6 +210,7 @@ EXPOSE 8100
 EXPOSE 11625
 EXPOSE 11626
 
+COPY apt-retry /usr/local/bin/
 ADD dependencies /
 RUN /dependencies
 

apt-retry

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+
+# Retry wrapper for apt commands to handle transient mirror failures.
+# Uses decorrelated jitter: delay = rand(base, 3 * prev_delay), capped at max_delay
+# Usage: apt-retry apt-get update && apt-retry apt-get install -y <packages>
+
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+max_attempts=5
+base_delay=2
+max_delay=20
+
+# Decorrelated jitter: random value between base_delay and 3 * previous delay, capped
+next_delay() {
+    local prev=$1
+    local range=$((3 * prev - base_delay + 1))
+    local next=$((base_delay + RANDOM % range))
+    if [ $next -gt $max_delay ]; then
+        next=$max_delay
+    fi
+    echo $next
+}
+
+delay=$base_delay
+attempt=1
+
+while [ $attempt -le $max_attempts ]; do
+    if "$@"; then
+        exit 0
+    fi
+    if [ $attempt -lt $max_attempts ]; then
+        echo -e "${RED}apt command failed (attempt $attempt/$max_attempts), retrying in ${delay}s...${NC}"
+        sleep $delay
+    fi
+    attempt=$((attempt + 1))
+    delay=$(next_delay $delay)
+done
+
+echo -e "${RED}apt command failed after $max_attempts attempts${NC}"
+exit 1

dependencies

@@ -5,19 +5,17 @@ set -e
 export DEBIAN_FRONTEND=noninteractive
 
 # Add PostgreSQL APT repository for PostgreSQL 14
-apt-get update
-apt-get install -y curl ca-certificates lsb-release
+apt-retry sh -c 'apt-get update && apt-get install -y curl ca-certificates lsb-release'
 install -d /usr/share/postgresql-common/pgdg
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
 . /etc/os-release
 sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main' > /etc/apt/sources.list.d/pgdg.list"
 
-apt-get update
-apt-get install -y curl apt-transport-https \
+apt-retry sh -c 'apt-get update && apt-get install -y curl apt-transport-https \
                    postgresql-client-14 postgresql-14 postgresql-contrib \
                    sudo supervisor psmisc \
                    nginx rsync jq golang-github-pelletier-go-toml netcat-openbsd \
-                   libunwind8 sqlite3 libc++abi1-20 libc++1-20
+                   libunwind8 sqlite3 libc++abi1-20 libc++1-20'
 apt-get clean
 rm -rf /var/lib/apt/lists/*