Skip to content

Commit c845268

Browse files
samrosesamrose
committed
feat: mirror fallbacks
1 parent 336f49e commit c845268

File tree

3 files changed

+181
-14
lines changed

3 files changed

+181
-14
lines changed

‎ansible/vars.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.064-orioledb"
14-
postgres17: "17.6.1.043"
15-
postgres15: "15.14.1.043"
13+
postgresorioledb-17: "17.5.1.064-orioledb-apt-1"
14+
postgres17: "17.6.1.043-apt-1"
15+
postgres15: "15.14.1.043-apt-1"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0

‎ebssurrogate/scripts/chroot-bootstrap-nix.sh‎

Lines changed: 82 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,95 @@ export APT_OPTIONS="-oAPT::Install-Recommends=false \
1414
-oAPT::Install-Suggests=false \
1515
-oAcquire::Languages=none"
1616

17-
if [ $(dpkg --print-architecture) = "amd64" ];
18-
then
17+
if [ $(dpkg --print-architecture) = "amd64" ];
18+
then
1919
ARCH="amd64";
2020
else
2121
ARCH="arm64";
2222
fi
2323

24+
# Mirror fallback function for resilient apt-get update
25+
function apt_update_with_fallback {
26+
local sources_file="/etc/apt/sources.list"
27+
local max_attempts=3
28+
local attempt=1
29+
30+
# Detect the current region from sources.list (it's already been substituted)
31+
# Extract the region from existing sources.list entries
32+
local current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "${sources_file}" | head -1 || echo "")
33+
34+
# Define mirror tiers (in priority order)
35+
local -a mirror_tiers=(
36+
"${current_region}.clouds.ports.ubuntu.com" # Tier 1: Regional CDN (as set in sources.list)
37+
"ports.ubuntu.com" # Tier 2: Global pool
38+
"mirror.aarnet.edu.au" # Tier 3: Australia mirror (close to ap-southeast-1)
39+
)
40+
41+
# If we couldn't detect current region, skip tier 1
42+
if [ -z "${current_region}" ]; then
43+
echo "Warning: Could not determine region from sources.list, skipping regional CDN"
44+
mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element
45+
fi
46+
47+
for mirror in "${mirror_tiers[@]}"; do
48+
echo "========================================="
49+
echo "Attempting apt-get update with mirror: ${mirror}"
50+
echo "Attempt ${attempt} of ${max_attempts}"
51+
echo "========================================="
52+
53+
# Update sources.list to use current mirror
54+
sed -i "s|http://[^/]*/ubuntu-ports/|http://${mirror}/ubuntu-ports/|g" "${sources_file}"
55+
56+
# Show what we're using
57+
echo "Current sources.list configuration:"
58+
grep -E '^deb ' "${sources_file}" | head -3
59+
60+
# Attempt update with timeout (5 minutes)
61+
if timeout 300 apt-get $APT_OPTIONS update 2>&1; then
62+
echo "========================================="
63+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"
64+
echo "========================================="
65+
return 0
66+
else
67+
local exit_code=$?
68+
echo "========================================="
69+
echo "✗ Failed to update using mirror: ${mirror}"
70+
echo "Exit code: ${exit_code}"
71+
echo "========================================="
72+
73+
# Clean partial downloads
74+
apt-get clean
75+
rm -rf /var/lib/apt/lists/*
76+
77+
# Exponential backoff before next attempt
78+
if [ ${attempt} -lt ${max_attempts} ]; then
79+
local sleep_time=$((attempt * 5))
80+
echo "Waiting ${sleep_time} seconds before trying next mirror..."
81+
sleep ${sleep_time}
82+
fi
83+
fi
84+
85+
attempt=$((attempt + 1))
86+
done
87+
88+
echo "========================================="
89+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"
90+
echo "========================================="
91+
return 1
92+
}
93+
2494

2595

2696
function update_install_packages {
2797
source /etc/os-release
2898

29-
# Update APT with new sources
99+
# Update APT with new sources (using fallback mechanism)
30100
cat /etc/apt/sources.list
31-
apt-get $APT_OPTIONS update && apt-get $APT_OPTIONS --yes dist-upgrade
101+
if ! apt_update_with_fallback; then
102+
echo "FATAL: Failed to update package lists with any mirror tier"
103+
exit 1
104+
fi
105+
apt-get $APT_OPTIONS --yes dist-upgrade
32106

33107
# Do not configure grub during package install
34108
if [ "${ARCH}" = "amd64" ]; then
@@ -59,7 +133,10 @@ function update_install_packages {
59133

60134
# Install OpenSSH and other packages
61135
sudo add-apt-repository --yes universe
62-
apt-get update
136+
if ! apt_update_with_fallback; then
137+
echo "FATAL: Failed to update package lists after adding universe repository"
138+
exit 1
139+
fi
63140
apt-get install -y --no-install-recommends \
64141
openssh-server \
65142
git \

‎ebssurrogate/scripts/surrogate-bootstrap-nix.sh‎

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,87 @@ set -o errexit
1010
set -o pipefail
1111
set -o xtrace
1212

13-
if [ $(dpkg --print-architecture) = "amd64" ];
14-
then
13+
if [ $(dpkg --print-architecture) = "amd64" ];
14+
then
1515
ARCH="amd64";
1616
else
17-
ARCH="arm64";
17+
ARCH="arm64";
1818
fi
1919

20+
# Mirror fallback function for resilient apt-get update
21+
function apt_update_with_fallback {
22+
local sources_file="/etc/apt/sources.list"
23+
local max_attempts=3
24+
local attempt=1
25+
26+
# Get EC2 region if not already set
27+
if [ -z "${REGION}" ]; then
28+
REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g' || echo "")
29+
fi
30+
31+
# Define mirror tiers (in priority order)
32+
local -a mirror_tiers=(
33+
"${REGION}.clouds.ports.ubuntu.com" # Tier 1: Regional CDN
34+
"ports.ubuntu.com" # Tier 2: Global pool
35+
"mirror.aarnet.edu.au" # Tier 3: Australia mirror (close to ap-southeast-1)
36+
)
37+
38+
# If we couldn't get REGION, skip tier 1
39+
if [ -z "${REGION}" ]; then
40+
echo "Warning: Could not determine EC2 region, skipping regional CDN"
41+
mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element
42+
fi
43+
44+
for mirror in "${mirror_tiers[@]}"; do
45+
echo "========================================="
46+
echo "Attempting apt-get update with mirror: ${mirror}"
47+
echo "Attempt ${attempt} of ${max_attempts}"
48+
echo "========================================="
49+
50+
# Update sources.list to use current mirror
51+
# Replace the region-specific mirror URL
52+
sed -i "s|http://[^/]*/ubuntu-ports/|http://${mirror}/ubuntu-ports/|g" "${sources_file}"
53+
# Also update any security sources
54+
sed -i "s|http://ports.ubuntu.com/ubuntu-ports|http://${mirror}/ubuntu-ports|g" "${sources_file}"
55+
56+
# Show what we're using
57+
echo "Current sources.list configuration:"
58+
grep -E '^deb ' "${sources_file}" | head -3
59+
60+
# Attempt update with timeout (5 minutes)
61+
if timeout 300 apt-get update 2>&1; then
62+
echo "========================================="
63+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"
64+
echo "========================================="
65+
return 0
66+
else
67+
local exit_code=$?
68+
echo "========================================="
69+
echo "✗ Failed to update using mirror: ${mirror}"
70+
echo "Exit code: ${exit_code}"
71+
echo "========================================="
72+
73+
# Clean partial downloads
74+
apt-get clean
75+
rm -rf /var/lib/apt/lists/*
76+
77+
# Exponential backoff before next attempt
78+
if [ ${attempt} -lt ${max_attempts} ]; then
79+
local sleep_time=$((attempt * 5))
80+
echo "Waiting ${sleep_time} seconds before trying next mirror..."
81+
sleep ${sleep_time}
82+
fi
83+
fi
84+
85+
attempt=$((attempt + 1))
86+
done
87+
88+
echo "========================================="
89+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"
90+
echo "========================================="
91+
return 1
92+
}
93+
2094
function waitfor_boot_finished {
2195
export DEBIAN_FRONTEND=noninteractive
2296

@@ -30,12 +104,28 @@ function waitfor_boot_finished {
30104

31105
function install_packages {
32106
# Setup Ansible on host VM
33-
apt-get update && sudo apt-get install software-properties-common -y
34-
add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y
107+
if ! apt_update_with_fallback; then
108+
echo "FATAL: Failed to update package lists on host VM"
109+
exit 1
110+
fi
111+
112+
sudo apt-get install software-properties-common -y
113+
add-apt-repository --yes --update ppa:ansible/ansible
114+
115+
if ! apt_update_with_fallback; then
116+
echo "FATAL: Failed to update package lists after adding Ansible PPA"
117+
exit 1
118+
fi
119+
120+
sudo apt-get install ansible -y
35121
ansible-galaxy collection install community.general
36122

37123
# Update apt and install required packages
38-
apt-get update
124+
if ! apt_update_with_fallback; then
125+
echo "FATAL: Failed to update package lists before installing tools"
126+
exit 1
127+
fi
128+
39129
apt-get install -y \
40130
gdisk \
41131
e2fsprogs \

0 commit comments

Comments
 (0)