Skip to content

Commit b17061e

Browse files
ShawnMcKeeShawnMcKee
authored
Merge pull request #72 from osg-htc/host-network-tuning-apply-flags
fix: replace Docker-specific auto-update grep with Podman digest comp…
2 parents dfea788 + b10b289 commit b17061e

9 files changed

+358
-195
lines changed

‎docs/perfsonar/tools_scripts/install-systemd-units.sh‎

Lines changed: 143 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
# Options:
1414
# --install-dir PATH Installation directory (default: /opt/perfsonar-tp)
1515
# --with-certbot Install certbot service alongside testpoint
16+
# --auto-update Install perfsonar-auto-update.sh and a daily systemd
17+
# timer that pulls new images and restarts services only
18+
# when an image digest has changed (Podman-compatible;
19+
# does not rely on Docker-specific output strings)
1620
# --help Show this help message
1721
#
1822
# Requirements:
@@ -21,14 +25,15 @@
2125
# - perfSONAR testpoint scripts in installation directory
2226
#
2327
# Author: OSG perfSONAR deployment tools
24-
# Version: 1.0.0
28+
# Version: 1.1.0
2529
# Acknowledgements: Supported by IRIS-HEP and OSG-LHC
2630

2731
set -e
2832

2933
# Default values
3034
INSTALL_DIR="/opt/perfsonar-tp"
3135
WITH_CERTBOT=false
36+
AUTO_UPDATE=false
3237

3338
# Parse arguments
3439
while [[ $# -gt 0 ]]; do
@@ -41,8 +46,12 @@ while [[ $# -gt 0 ]]; do
4146
WITH_CERTBOT=true
4247
shift
4348
;;
49+
--auto-update)
50+
AUTO_UPDATE=true
51+
shift
52+
;;
4453
--help)
45-
head -n 20 "$0" | grep "^#" | sed 's/^# \?//'
54+
head -n 25 "$0" | grep "^#" | sed 's/^# \?//'
4655
exit 0
4756
;;
4857
*)
@@ -85,7 +94,16 @@ fi
8594
echo "==> Installing systemd units for perfSONAR testpoint"
8695
echo " Installation directory: $INSTALL_DIR"
8796

88-
# Create perfsonar-testpoint service
97+
# When --auto-update is the only goal (service already exists), skip rewriting
98+
# the testpoint/certbot service units to avoid disrupting a running deployment.
99+
SKIP_SERVICE_UNITS=false
100+
if [[ "$AUTO_UPDATE" == "true" && -f "$TESTPOINT_SERVICE" ]]; then
101+
echo "==> Existing $TESTPOINT_SERVICE detected — skipping service unit rewrite (use without --auto-update to reinstall)"
102+
SKIP_SERVICE_UNITS=true
103+
fi
104+
105+
# Create perfsonar-testpoint service (skip if already present and only --auto-update was requested)
106+
if [[ "$SKIP_SERVICE_UNITS" == "false" ]]; then
89107
cat > "$TESTPOINT_SERVICE" << EOF
90108
[Unit]
91109
Description=perfSONAR Testpoint Container
@@ -158,43 +176,133 @@ EOF
158176
echo "==> ✓ Created $CERTBOT_SERVICE"
159177
fi
160178

179+
fi # end SKIP_SERVICE_UNITS guard
180+
161181
# Reload systemd
162182
echo "==> Reloading systemd daemon"
163183
systemctl daemon-reload
164184

165-
# Enable services
166-
echo "==> Enabling perfsonar-testpoint service"
167-
systemctl enable perfsonar-testpoint.service
185+
# Enable services (only if service units were written)
186+
if [[ "$SKIP_SERVICE_UNITS" == "false" ]]; then
187+
echo "==> Enabling perfsonar-testpoint service"
188+
systemctl enable perfsonar-testpoint.service
168189

169-
if [[ "$WITH_CERTBOT" == "true" ]]; then
170-
echo "==> Enabling perfsonar-certbot service"
171-
systemctl enable perfsonar-certbot.service
172-
fi
190+
if [[ "$WITH_CERTBOT" == "true" ]]; then
191+
echo "==> Enabling perfsonar-certbot service"
192+
systemctl enable perfsonar-certbot.service
193+
fi
173194

174-
echo ""
175-
echo "==> ✓ Systemd units installed and enabled successfully"
176-
echo ""
177-
echo "Useful commands:"
178-
echo " Start services: systemctl start perfsonar-testpoint.service"
179-
if [[ "$WITH_CERTBOT" == "true" ]]; then
180-
echo " systemctl start perfsonar-certbot.service"
195+
echo ""
196+
echo "==> ✓ Systemd units installed and enabled successfully"
197+
echo ""
198+
echo "Useful commands:"
199+
echo " Start services: systemctl start perfsonar-testpoint.service"
200+
if [[ "$WITH_CERTBOT" == "true" ]]; then
201+
echo " systemctl start perfsonar-certbot.service"
202+
fi
203+
echo " Stop services: systemctl stop perfsonar-testpoint.service"
204+
if [[ "$WITH_CERTBOT" == "true" ]]; then
205+
echo " systemctl stop perfsonar-certbot.service"
206+
fi
207+
echo " Check status: systemctl status perfsonar-testpoint.service"
208+
if [[ "$WITH_CERTBOT" == "true" ]]; then
209+
echo " systemctl status perfsonar-certbot.service"
210+
fi
211+
echo " View logs: journalctl -u perfsonar-testpoint.service -f"
212+
if [[ "$WITH_CERTBOT" == "true" ]]; then
213+
echo " journalctl -u perfsonar-certbot.service -f"
214+
fi
215+
echo " Check containers: podman ps"
216+
echo ""
217+
echo "The services will automatically start containers on boot."
218+
echo ""
219+
echo "Note: These units use 'podman run --systemd=always' for proper systemd"
220+
echo " support inside the container. This is required for the testpoint"
221+
echo " image which runs systemd internally."
181222
fi
182-
echo " Stop services: systemctl stop perfsonar-testpoint.service"
183-
if [[ "$WITH_CERTBOT" == "true" ]]; then
184-
echo " systemctl stop perfsonar-certbot.service"
185-
fi
186-
echo " Check status: systemctl status perfsonar-testpoint.service"
187-
if [[ "$WITH_CERTBOT" == "true" ]]; then
188-
echo " systemctl status perfsonar-certbot.service"
189-
fi
190-
echo " View logs: journalctl -u perfsonar-testpoint.service -f"
191-
if [[ "$WITH_CERTBOT" == "true" ]]; then
192-
echo " journalctl -u perfsonar-certbot.service -f"
223+
224+
# ── Optional: auto-update timer ────────────────────────────────────────────────
225+
if [[ "$AUTO_UPDATE" == "true" ]]; then
226+
AUTO_UPDATE_SCRIPT="$INSTALL_DIR/tools_scripts/perfSONAR-auto-update.sh"
227+
AUTO_UPDATE_BIN="/usr/local/bin/perfsonar-auto-update.sh"
228+
AUTO_UPDATE_SVC="/etc/systemd/system/perfsonar-auto-update.service"
229+
AUTO_UPDATE_TIMER="/etc/systemd/system/perfsonar-auto-update.timer"
230+
231+
echo ""
232+
echo "==> Installing auto-update timer"
233+
234+
# Use the versioned script from tools_scripts if present, else fall back to a
235+
# minimal inline version.
236+
if [[ -f "$AUTO_UPDATE_SCRIPT" ]]; then
237+
cp "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_BIN"
238+
else
239+
echo "WARNING: $AUTO_UPDATE_SCRIPT not found; writing minimal inline script."
240+
cat > "$AUTO_UPDATE_BIN" << 'AUTOUPDATE_EOF'
241+
#!/bin/bash
242+
# perfsonar-auto-update.sh (minimal inline fallback)
243+
# For the full versioned script, re-run bootstrap (install_tools_scripts.sh).
244+
set -euo pipefail
245+
LOGFILE="/var/log/perfsonar-auto-update.log"
246+
TESTPOINT_IMAGE="hub.opensciencegrid.org/osg-htc/perfsonar-testpoint:production"
247+
CERTBOT_IMAGE="docker.io/certbot/certbot:latest"
248+
log() { echo "$(date -Iseconds) $*" | tee -a "$LOGFILE"; }
249+
get_id() { podman image inspect "$1" --format '{{.Id}}' 2>/dev/null || echo none; }
250+
check_pull() {
251+
local img=$1 before after
252+
before=$(get_id "$img")
253+
podman pull "$img" >> "$LOGFILE" 2>&1 || { log "WARNING: pull failed for $img"; echo unchanged; return; }
254+
after=$(get_id "$img")
255+
[[ "$before" == "none" || "$before" != "$after" ]] && echo updated || echo unchanged
256+
}
257+
log '=== perfSONAR auto-update check ==='
258+
ANY=false
259+
[[ $(check_pull "$TESTPOINT_IMAGE") == updated ]] && ANY=true
260+
podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^certbot$' && \
261+
[[ $(check_pull "$CERTBOT_IMAGE") == updated ]] && ANY=true
262+
$ANY && systemctl restart perfsonar-testpoint.service && log 'Restarted testpoint.service' || log 'No updates'
263+
log '=== done ==='
264+
AUTOUPDATE_EOF
265+
fi
266+
chmod 0755 "$AUTO_UPDATE_BIN"
267+
echo "==> ✓ Installed $AUTO_UPDATE_BIN"
268+
269+
cat > "$AUTO_UPDATE_SVC" << 'EOF'
270+
[Unit]
271+
Description=perfSONAR Container Auto-Update
272+
After=network-online.target
273+
274+
[Service]
275+
Type=oneshot
276+
ExecStart=/usr/local/bin/perfsonar-auto-update.sh
277+
StandardOutput=journal
278+
StandardError=journal
279+
280+
[Install]
281+
WantedBy=multi-user.target
282+
EOF
283+
echo "==> ✓ Created $AUTO_UPDATE_SVC"
284+
285+
cat > "$AUTO_UPDATE_TIMER" << 'EOF'
286+
[Unit]
287+
Description=perfSONAR Container Auto-Update Timer
288+
289+
[Timer]
290+
OnCalendar=*-*-* 03:00:00
291+
RandomizedDelaySec=1h
292+
Persistent=true
293+
294+
[Install]
295+
WantedBy=timers.target
296+
EOF
297+
echo "==> ✓ Created $AUTO_UPDATE_TIMER"
298+
299+
systemctl daemon-reload
300+
systemctl enable --now perfsonar-auto-update.timer
301+
echo "==> ✓ Enabled perfsonar-auto-update.timer (runs daily at 03:00 + up to 1h random delay)"
302+
echo ""
303+
echo "Useful auto-update commands:"
304+
echo " Check timer: systemctl list-timers perfsonar-auto-update.timer"
305+
echo " Run now (test): systemctl start perfsonar-auto-update.service"
306+
echo " View log: journalctl -u perfsonar-auto-update.service -f"
307+
echo " Update log file: tail -f /var/log/perfsonar-auto-update.log"
193308
fi
194-
echo " Check containers: podman ps"
195-
echo ""
196-
echo "The services will automatically start containers on boot."
197-
echo ""
198-
echo "Note: These units use 'podman run --systemd=always' for proper systemd"
199-
echo " support inside the container. This is required for the testpoint"
200-
echo " image which runs systemd internally."

‎docs/perfsonar/tools_scripts/install-systemd-units.sh.sha256‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
4b2b91593a3ceb3c8159cc2d6ca749aeeb7a558a766227c46493feb84655f04a
1+
ac05461f8a9745c95e0630f3e92de099961d129e910c5d87da53381b366cfa3c install-systemd-units.sh

‎docs/perfsonar/tools_scripts/install_tools_scripts.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ set -euo pipefail
55
# Purpose: Ensure the perfSONAR testpoint repository is cloned and the tools_scripts
66
# directory is present under /opt/perfsonar-tp/tools_scripts.
77
#
8-
# Version: 1.0.1 - 2025-12-16
8+
# Version: 1.0.2 - 2026-02-24
99
# Author: Shawn McKee, University of Michigan
1010
# Acknowledgements: Supported by IRIS-HEP and OSG-LHC
1111

12-
VERSION="1.0.1"
12+
VERSION="1.0.2"
1313
PROG_NAME="$(basename "$0")"
1414

1515
# Check for --version or --help flags
@@ -84,6 +84,7 @@ files=(
8484
perfSONAR-install-nftables.sh
8585
perfSONAR-update-lsregistration.sh
8686
perfSONAR-auto-enroll-psconfig.sh
87+
perfSONAR-auto-update.sh
8788
seed_testpoint_host_dirs.sh
8889
perfSONAR-orchestrator.sh
8990
install_tools_scripts.sh

‎docs/perfsonar/tools_scripts/install_tools_scripts.sh.sha256‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
b213dc73e0c95973e8695cd490e5910636011426a88387f21c361c54ef5e14e5 install_tools_scripts.sh
1+
81b7c4892da2f11b27803fd36d03ef70fd8cdc41a6fddb1e669d03a28f69e905 install_tools_scripts.sh

‎docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/bin/bash
2+
# perfSONAR-auto-update.sh
3+
# ------------------------
4+
# Purpose:
5+
# Check for updated container images and restart services if new images are
6+
# found. Uses image digest comparison (Podman-compatible). Does NOT rely on
7+
# Docker-specific output strings like "Downloaded newer image" which Podman
8+
# never emits.
9+
#
10+
# Usage:
11+
# Run as root, typically via the perfsonar-auto-update.timer systemd timer.
12+
# Can also be invoked manually: bash /usr/local/bin/perfsonar-auto-update.sh
13+
#
14+
# What it does:
15+
# 1. Records the current local image digest for each managed image.
16+
# 2. Pulls each image from the registry.
17+
# 3. Compares the new digest to the old one.
18+
# 4. If any digest changed, restarts perfsonar-testpoint.service (which
19+
# manages both the testpoint and certbot containers via podman-compose or
20+
# direct podman run, depending on how the service was installed).
21+
#
22+
# Logs:
23+
# /var/log/perfsonar-auto-update.log (appended on every run)
24+
#
25+
# Author: OSG perfSONAR deployment tools
26+
# Version: 1.0.0
27+
# Acknowledgements: Supported by IRIS-HEP and OSG-LHC
28+
29+
set -euo pipefail
30+
31+
# ── Configuration ──────────────────────────────────────────────────────────────
32+
33+
LOGFILE="/var/log/perfsonar-auto-update.log"
34+
35+
# Images managed by this host. Edit if you have pinned a specific digest tag.
36+
TESTPOINT_IMAGE="hub.opensciencegrid.org/osg-htc/perfsonar-testpoint:production"
37+
CERTBOT_IMAGE="docker.io/certbot/certbot:latest"
38+
39+
# systemd service that starts/stops all perfSONAR containers on this host.
40+
TESTPOINT_SERVICE="perfsonar-testpoint.service"
41+
42+
# ── Helpers ────────────────────────────────────────────────────────────────────
43+
44+
log() { echo "$(date -Iseconds) $*" | tee -a "$LOGFILE"; }
45+
46+
# Return the local image ID, or "none" if image is not present.
47+
get_image_id() {
48+
podman image inspect "$1" --format "{{.Id}}" 2>/dev/null || echo "none"
49+
}
50+
51+
# Return the image ID of a running container by name, or "none" if the
52+
# container is not running.
53+
get_container_image_id() {
54+
podman inspect "$1" --format "{{.Image}}" 2>/dev/null || echo "none"
55+
}
56+
57+
# Pull an image and return "updated" or "unchanged".
58+
# Writes pull output to the log file.
59+
pull_and_check() {
60+
local image="$1"
61+
local before after
62+
before=$(get_image_id "$image")
63+
64+
log "Pulling: $image (current: ${before:0:12})"
65+
if ! podman pull "$image" >> "$LOGFILE" 2>&1; then
66+
log "WARNING: pull failed for $image — skipping (network issue?)"
67+
echo "unchanged"
68+
return
69+
fi
70+
71+
after=$(get_image_id "$image")
72+
73+
if [[ "$after" == "none" ]]; then
74+
log "WARNING: could not verify image digest after pull: $image"
75+
echo "unchanged"
76+
elif [[ "$before" == "none" || "$before" != "$after" ]]; then
77+
log "UPDATED: $image ${before:0:12} -> ${after:0:12}"
78+
echo "updated"
79+
else
80+
log "Up to date: $image"
81+
echo "unchanged"
82+
fi
83+
}
84+
85+
# ── Main ───────────────────────────────────────────────────────────────────────
86+
87+
# Must run as root
88+
if [[ $EUID -ne 0 ]]; then
89+
echo "ERROR: must be run as root" >&2
90+
exit 1
91+
fi
92+
93+
log "=== perfSONAR auto-update check started ==="
94+
95+
ANY_UPDATED=false
96+
97+
# ── Testpoint image ────────────────────────────────────────────────────────────
98+
result=$(pull_and_check "$TESTPOINT_IMAGE")
99+
[[ "$result" == "updated" ]] && ANY_UPDATED=true
100+
101+
# ── Certbot image (only if a certbot container exists on this host) ────────────
102+
if podman ps -a --format "{{.Names}}" 2>/dev/null | grep -q "^certbot$"; then
103+
result=$(pull_and_check "$CERTBOT_IMAGE")
104+
[[ "$result" == "updated" ]] && ANY_UPDATED=true
105+
fi
106+
107+
# ── Stale-container check ──────────────────────────────────────────────────────
108+
# Handle the case where the local image tag was already updated by a prior manual
109+
# pull (pull says "up to date") but the running container is still using the old
110+
# image digest. Compare running container's image ID vs the current local tag.
111+
if [[ "$ANY_UPDATED" == "false" ]]; then
112+
LATEST_TESTPOINT_ID=$(get_image_id "$TESTPOINT_IMAGE")
113+
RUNNING_TESTPOINT_ID=$(get_container_image_id "perfsonar-testpoint")
114+
if [[ "$RUNNING_TESTPOINT_ID" != "none" && "$LATEST_TESTPOINT_ID" != "none" \
115+
&& "$RUNNING_TESTPOINT_ID" != "$LATEST_TESTPOINT_ID" ]]; then
116+
log "Running container uses stale image ${RUNNING_TESTPOINT_ID:0:12} (latest: ${LATEST_TESTPOINT_ID:0:12}) — forcing restart"
117+
ANY_UPDATED=true
118+
fi
119+
fi
120+
121+
# ── Restart if any image changed ───────────────────────────────────────────────
122+
if [[ "$ANY_UPDATED" == "true" ]]; then
123+
log "New image(s) found — restarting $TESTPOINT_SERVICE"
124+
if systemctl restart "$TESTPOINT_SERVICE"; then
125+
log "Restarted $TESTPOINT_SERVICE successfully"
126+
else
127+
log "ERROR: failed to restart $TESTPOINT_SERVICE (exit $?)"
128+
exit 1
129+
fi
130+
else
131+
log "All images up to date — no restart needed"
132+
fi
133+
134+
log "=== perfSONAR auto-update check complete ==="

‎docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh.sha256‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
f7e14a1cc2744e9f653ed5ace910a2f016b5772a1f4df1a23e3a2c30bbf0e7ab perfSONAR-auto-update.sh

0 commit comments

Comments
 (0)