From cd7495e91d15188735cad8fa83bfa44dcad6ec06 Mon Sep 17 00:00:00 2001
From: Leigh <351529+leighmcculloch@users.noreply.github.com>
Date: Thu, 8 Jan 2026 17:11:18 +1000
Subject: [PATCH 1/4] add apt-retry wrapper for transient mirror failures

---
 Dockerfile   | 15 ++++++++++-----
 apt-retry    | 40 ++++++++++++++++++++++++++++++++++++++++
 dependencies |  8 ++++----
 3 files changed, 54 insertions(+), 9 deletions(-)
 create mode 100755 apt-retry

diff --git a/Dockerfile b/Dockerfile
index 6dbc5064..0b9f0ed4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,8 +40,9 @@ COPY --from=stellar-xdr-builder /usr/local/cargo/bin/stellar-xdr /stellar-xdr
 FROM ubuntu:24.04 AS stellar-core-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get -y install iproute2 procps lsb-release \
+COPY apt-retry /usr/local/bin/
+RUN apt-retry apt-get update && \
+    apt-retry apt-get -y install iproute2 procps lsb-release \
                        git build-essential pkg-config autoconf automake libtool \
                        bison flex sed perl libpq-dev parallel \
                        clang-20 libc++abi-20-dev libc++-20-dev \
@@ -96,7 +97,8 @@ ENV RUSTUP_HOME=/rust/.rust
 ENV PATH="/usr/local/go/bin:$CARGO_HOME/bin:${PATH}"
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y build-essential jq && apt-get clean
+COPY apt-retry /usr/local/bin/
+RUN apt-retry apt-get update && apt-retry apt-get install -y build-essential jq && apt-get clean
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION
 
 RUN make build-stellar-rpc
@@ -110,7 +112,8 @@ COPY --from=stellar-rpc-builder /go/src/github.com/stellar/stellar-rpc/stellar-r
 FROM golang:1.24-trixie AS stellar-horizon-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install jq
+COPY apt-retry /usr/local/bin/
+RUN apt-retry apt-get update && apt-retry apt-get -y install jq
 
 ARG HORIZON_REPO
 ARG HORIZON_REF
@@ -134,7 +137,8 @@ COPY --from=stellar-horizon-builder /stellar-horizon /stellar-horizon
 FROM golang:1.24-trixie AS stellar-friendbot-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install jq
+COPY apt-retry /usr/local/bin/
+RUN apt-retry apt-get update && apt-retry apt-get -y install jq
 
 ARG FRIENDBOT_REPO
 ARG FRIENDBOT_REF
@@ -206,6 +210,7 @@ EXPOSE 8100
 EXPOSE 11625
 EXPOSE 11626
 
+COPY apt-retry /usr/local/bin/
 ADD dependencies /
 RUN /dependencies
 
diff --git a/apt-retry b/apt-retry
new file mode 100755
index 00000000..7b3118d8
--- /dev/null
+++ b/apt-retry
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -e
+
+# Retry wrapper for apt commands to handle transient mirror failures.
+# Uses decorrelated jitter: delay = rand(base, 3 * prev_delay), capped at max_delay
+# Usage: apt-retry apt-get update && apt-retry apt-get install -y <packages>
+
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+max_attempts=5
+base_delay=2
+max_delay=20
+
+# Decorrelated jitter: random value between base_delay and 3 * previous delay, capped
+next_delay() {
+    local prev=$1
+    local range=$((3 * prev - base_delay + 1))
+    local next=$((base_delay + RANDOM % range))
+    if [ $next -gt $max_delay ]; then
+        next=$max_delay
+    fi
+    echo $next
+}
+
+delay=$base_delay
+attempt=1
+
+while [ $attempt -le $max_attempts ]; do
+    if "$@"; then
+        exit 0
+    fi
+    echo -e "${RED}apt command failed (attempt $attempt/$max_attempts), retrying in ${delay}s...${NC}"
+    sleep $delay
+    attempt=$((attempt + 1))
+    delay=$(next_delay $delay)
+done
+
+echo -e "${RED}apt command failed after $max_attempts attempts${NC}"
+exit 1
diff --git a/dependencies b/dependencies
index 634e2c73..d1ed89c9 100755
--- a/dependencies
+++ b/dependencies
@@ -5,15 +5,15 @@ set -e
 export DEBIAN_FRONTEND=noninteractive
 
 # Add PostgreSQL APT repository for PostgreSQL 14
-apt-get update
-apt-get install -y curl ca-certificates lsb-release
+apt-retry apt-get update
+apt-retry apt-get install -y curl ca-certificates lsb-release
 install -d /usr/share/postgresql-common/pgdg
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
 . /etc/os-release
 sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main' > /etc/apt/sources.list.d/pgdg.list"
 
-apt-get update
-apt-get install -y curl apt-transport-https \
+apt-retry apt-get update
+apt-retry apt-get install -y curl apt-transport-https \
                    postgresql-client-14 postgresql-14 postgresql-contrib \
                    sudo supervisor psmisc \
                    nginx rsync jq golang-github-pelletier-go-toml netcat-openbsd \

From 01bc11b5584226d848548e43bb783b4b282cb20e Mon Sep 17 00:00:00 2001
From: Leigh <351529+leighmcculloch@users.noreply.github.com>
Date: Thu, 8 Jan 2026 17:15:15 +1000
Subject: [PATCH 2/4] wrap apt-retry commands with sh -c for proper execution

---
 Dockerfile   | 12 ++++++------
 dependencies |  8 +++-----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0b9f0ed4..4587eec0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,12 +41,12 @@ FROM ubuntu:24.04 AS stellar-core-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 COPY apt-retry /usr/local/bin/
-RUN apt-retry apt-get update && \
-    apt-retry apt-get -y install iproute2 procps lsb-release \
+RUN apt-retry sh -c 'apt-get update && \
+    apt-get -y install iproute2 procps lsb-release \
                        git build-essential pkg-config autoconf automake libtool \
                        bison flex sed perl libpq-dev parallel \
                        clang-20 libc++abi-20-dev libc++-20-dev \
-                       postgresql curl jq
+                       postgresql curl jq'
 
 ARG CORE_REPO
 ARG CORE_REF
@@ -98,7 +98,7 @@ ENV PATH="/usr/local/go/bin:$CARGO_HOME/bin:${PATH}"
 ENV DEBIAN_FRONTEND=noninteractive
 
 COPY apt-retry /usr/local/bin/
-RUN apt-retry apt-get update && apt-retry apt-get install -y build-essential jq && apt-get clean
+RUN apt-retry sh -c 'apt-get update && apt-get install -y build-essential jq' && apt-get clean
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION
 
 RUN make build-stellar-rpc
@@ -113,7 +113,7 @@ FROM golang:1.24-trixie AS stellar-horizon-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 COPY apt-retry /usr/local/bin/
-RUN apt-retry apt-get update && apt-retry apt-get -y install jq
+RUN apt-retry sh -c 'apt-get update && apt-get -y install jq'
 
 ARG HORIZON_REPO
 ARG HORIZON_REF
@@ -138,7 +138,7 @@ FROM golang:1.24-trixie AS stellar-friendbot-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 COPY apt-retry /usr/local/bin/
-RUN apt-retry apt-get update && apt-retry apt-get -y install jq
+RUN apt-retry sh -c 'apt-get update && apt-get -y install jq'
 
 ARG FRIENDBOT_REPO
 ARG FRIENDBOT_REF
diff --git a/dependencies b/dependencies
index d1ed89c9..98d480e1 100755
--- a/dependencies
+++ b/dependencies
@@ -5,19 +5,17 @@ set -e
 export DEBIAN_FRONTEND=noninteractive
 
 # Add PostgreSQL APT repository for PostgreSQL 14
-apt-retry apt-get update
-apt-retry apt-get install -y curl ca-certificates lsb-release
+apt-retry sh -c 'apt-get update && apt-get install -y curl ca-certificates lsb-release'
 install -d /usr/share/postgresql-common/pgdg
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
 . /etc/os-release
 sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main' > /etc/apt/sources.list.d/pgdg.list"
 
-apt-retry apt-get update
-apt-retry apt-get install -y curl apt-transport-https \
+apt-retry sh -c 'apt-get update && apt-get install -y curl apt-transport-https \
                    postgresql-client-14 postgresql-14 postgresql-contrib \
                    sudo supervisor psmisc \
                    nginx rsync jq golang-github-pelletier-go-toml netcat-openbsd \
-                   libunwind8 sqlite3 libc++abi1-20 libc++1-20
+                   libunwind8 sqlite3 libc++abi1-20 libc++1-20'
 apt-get clean
 rm -rf /var/lib/apt/lists/*
 

From 277d690d3dba8caacfb054cb98d50e1dfa02cc99 Mon Sep 17 00:00:00 2001
From: Leigh <351529+leighmcculloch@users.noreply.github.com>
Date: Thu, 8 Jan 2026 17:16:59 +1000
Subject: [PATCH 3/4] increment cache_id default from 18 to 19

---
 .github/workflows/internal-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/internal-build.yml b/.github/workflows/internal-build.yml
index 8bac9c10..d6fd2a80 100644
--- a/.github/workflows/internal-build.yml
+++ b/.github/workflows/internal-build.yml
@@ -39,7 +39,7 @@ on:
       cache_id:
         description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing"
         type: "string"
-        default: 18
+        default: 19
       cache_prefix:
         description: "A prefix added to all cache keys generated by this workflow"
         type: "string"

From b48f780aa2e1c0bbfb8c13d52bf4c7c99ea756d7 Mon Sep 17 00:00:00 2001
From: Leigh <351529+leighmcculloch@users.noreply.github.com>
Date: Thu, 8 Jan 2026 17:25:53 +1000
Subject: [PATCH 4/4] skip retry message on final attempt

---
 apt-retry | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/apt-retry b/apt-retry
index 7b3118d8..d01fd5e5 100755
--- a/apt-retry
+++ b/apt-retry
@@ -30,8 +30,10 @@ while [ $attempt -le $max_attempts ]; do
     if "$@"; then
         exit 0
     fi
-    echo -e "${RED}apt command failed (attempt $attempt/$max_attempts), retrying in ${delay}s...${NC}"
-    sleep $delay
+    if [ $attempt -lt $max_attempts ]; then
+        echo -e "${RED}apt command failed (attempt $attempt/$max_attempts), retrying in ${delay}s...${NC}"
+        sleep $delay
+    fi
     attempt=$((attempt + 1))
     delay=$(next_delay $delay)
 done