diff --git a/.github/workflows/internal-build.yml b/.github/workflows/internal-build.yml index 8bac9c10..d6fd2a80 100644 --- a/.github/workflows/internal-build.yml +++ b/.github/workflows/internal-build.yml @@ -39,7 +39,7 @@ on: cache_id: description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing" type: "string" - default: 18 + default: 19 cache_prefix: description: "A prefix added to all cache keys generated by this workflow" type: "string" diff --git a/Dockerfile b/Dockerfile index 6dbc5064..4587eec0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,12 +40,13 @@ COPY --from=stellar-xdr-builder /usr/local/cargo/bin/stellar-xdr /stellar-xdr FROM ubuntu:24.04 AS stellar-core-builder ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ +COPY apt-retry /usr/local/bin/ +RUN apt-retry sh -c 'apt-get update && \ apt-get -y install iproute2 procps lsb-release \ git build-essential pkg-config autoconf automake libtool \ bison flex sed perl libpq-dev parallel \ clang-20 libc++abi-20-dev libc++-20-dev \ - postgresql curl jq + postgresql curl jq' ARG CORE_REPO ARG CORE_REF @@ -96,7 +97,8 @@ ENV RUSTUP_HOME=/rust/.rust ENV PATH="/usr/local/go/bin:$CARGO_HOME/bin:${PATH}" ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y build-essential jq && apt-get clean +COPY apt-retry /usr/local/bin/ +RUN apt-retry sh -c 'apt-get update && apt-get install -y build-essential jq' && apt-get clean RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION RUN make build-stellar-rpc @@ -110,7 +112,8 @@ COPY --from=stellar-rpc-builder /go/src/github.com/stellar/stellar-rpc/stellar-r FROM golang:1.24-trixie AS stellar-horizon-builder ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y install jq +COPY apt-retry /usr/local/bin/ +RUN apt-retry sh -c 'apt-get update && apt-get -y install jq' ARG HORIZON_REPO ARG HORIZON_REF @@ -134,7 +137,8 @@ COPY --from=stellar-horizon-builder /stellar-horizon /stellar-horizon FROM golang:1.24-trixie AS stellar-friendbot-builder ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -y install jq +COPY apt-retry /usr/local/bin/ +RUN apt-retry sh -c 'apt-get update && apt-get -y install jq' ARG FRIENDBOT_REPO ARG FRIENDBOT_REF @@ -206,6 +210,7 @@ EXPOSE 8100 EXPOSE 11625 EXPOSE 11626 +COPY apt-retry /usr/local/bin/ ADD dependencies / RUN /dependencies diff --git a/apt-retry b/apt-retry new file mode 100755 index 00000000..d01fd5e5 --- /dev/null +++ b/apt-retry @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -e + +# Retry wrapper for apt commands to handle transient mirror failures. +# Uses decorrelated jitter: delay = rand(base, 3 * prev_delay), capped at max_delay +# Usage: apt-retry apt-get update && apt-retry apt-get install -y + +RED='\033[0;31m' +NC='\033[0m' # No Color + +max_attempts=5 +base_delay=2 +max_delay=20 + +# Decorrelated jitter: random value between base_delay and 3 * previous delay, capped +next_delay() { + local prev=$1 + local range=$((3 * prev - base_delay + 1)) + local next=$((base_delay + RANDOM % range)) + if [ $next -gt $max_delay ]; then + next=$max_delay + fi + echo $next +} + +delay=$base_delay +attempt=1 + +while [ $attempt -le $max_attempts ]; do + if "$@"; then + exit 0 + fi + if [ $attempt -lt $max_attempts ]; then + echo -e "${RED}apt command failed (attempt $attempt/$max_attempts), retrying in ${delay}s...${NC}" + sleep $delay + fi + attempt=$((attempt + 1)) + delay=$(next_delay $delay) +done + +echo -e "${RED}apt command failed after $max_attempts attempts${NC}" +exit 1 diff --git a/dependencies b/dependencies index 634e2c73..98d480e1 100755 --- a/dependencies +++ b/dependencies @@ -5,19 +5,17 @@ set -e export DEBIAN_FRONTEND=noninteractive # Add PostgreSQL APT repository for PostgreSQL 14 -apt-get update -apt-get install -y curl ca-certificates lsb-release +apt-retry sh -c 'apt-get update && apt-get install -y curl ca-certificates lsb-release' install -d /usr/share/postgresql-common/pgdg curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc . /etc/os-release sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main' > /etc/apt/sources.list.d/pgdg.list" -apt-get update -apt-get install -y curl apt-transport-https \ +apt-retry sh -c 'apt-get update && apt-get install -y curl apt-transport-https \ postgresql-client-14 postgresql-14 postgresql-contrib \ sudo supervisor psmisc \ nginx rsync jq golang-github-pelletier-go-toml netcat-openbsd \ - libunwind8 sqlite3 libc++abi1-20 libc++1-20 + libunwind8 sqlite3 libc++abi1-20 libc++1-20' apt-get clean rm -rf /var/lib/apt/lists/*