osg-htc · ShawnMcKee · Feb 24, 2026 · Feb 24, 2026
diff --git a/docs/perfsonar/tools_scripts/install-systemd-units.sh b/docs/perfsonar/tools_scripts/install-systemd-units.sh
@@ -13,6 +13,10 @@
 # Options:
 #   --install-dir PATH    Installation directory (default: /opt/perfsonar-tp)
 #   --with-certbot        Install certbot service alongside testpoint
+#   --auto-update         Install perfsonar-auto-update.sh and a daily systemd
+#                         timer that pulls new images and restarts services only
+#                         when an image digest has changed (Podman-compatible;
+#                         does not rely on Docker-specific output strings)
 #   --help                Show this help message
 #
 # Requirements:
@@ -21,14 +25,15 @@
 #   - perfSONAR testpoint scripts in installation directory
 #
 # Author: OSG perfSONAR deployment tools
-# Version: 1.0.0
+# Version: 1.1.0
 # Acknowledgements: Supported by IRIS-HEP and OSG-LHC
 
 set -e
 
 # Default values
 INSTALL_DIR="/opt/perfsonar-tp"
 WITH_CERTBOT=false
+AUTO_UPDATE=false
 
 # Parse arguments
 while [[ $# -gt 0 ]]; do
@@ -41,8 +46,12 @@ while [[ $# -gt 0 ]]; do
             WITH_CERTBOT=true
             shift
             ;;
+        --auto-update)
+            AUTO_UPDATE=true
+            shift
+            ;;
         --help)
-            head -n 20 "$0" | grep "^#" | sed 's/^# \?//'
+            head -n 25 "$0" | grep "^#" | sed 's/^# \?//'
             exit 0
             ;;
         *)
@@ -85,7 +94,16 @@ fi
 echo "==> Installing systemd units for perfSONAR testpoint"
 echo "    Installation directory: $INSTALL_DIR"
 
-# Create perfsonar-testpoint service
+# When --auto-update is the only goal (service already exists), skip rewriting
+# the testpoint/certbot service units to avoid disrupting a running deployment.
+SKIP_SERVICE_UNITS=false
+if [[ "$AUTO_UPDATE" == "true" && -f "$TESTPOINT_SERVICE" ]]; then
+    echo "==> Existing $TESTPOINT_SERVICE detected — skipping service unit rewrite (use without --auto-update to reinstall)"
+    SKIP_SERVICE_UNITS=true
+fi
+
+# Create perfsonar-testpoint service (skip if already present and only --auto-update was requested)
+if [[ "$SKIP_SERVICE_UNITS" == "false" ]]; then
 cat > "$TESTPOINT_SERVICE" << EOF
 [Unit]
 Description=perfSONAR Testpoint Container
@@ -158,43 +176,133 @@ EOF
     echo "==> ✓ Created $CERTBOT_SERVICE"
 fi
 
+fi  # end SKIP_SERVICE_UNITS guard
+
 # Reload systemd
 echo "==> Reloading systemd daemon"
 systemctl daemon-reload
 
-# Enable services
-echo "==> Enabling perfsonar-testpoint service"
-systemctl enable perfsonar-testpoint.service
+# Enable services (only if service units were written)
+if [[ "$SKIP_SERVICE_UNITS" == "false" ]]; then
+    echo "==> Enabling perfsonar-testpoint service"
+    systemctl enable perfsonar-testpoint.service
 
-if [[ "$WITH_CERTBOT" == "true" ]]; then
-    echo "==> Enabling perfsonar-certbot service"
-    systemctl enable perfsonar-certbot.service
-fi
+    if [[ "$WITH_CERTBOT" == "true" ]]; then
+        echo "==> Enabling perfsonar-certbot service"
+        systemctl enable perfsonar-certbot.service
+    fi
 
-echo ""
-echo "==> ✓ Systemd units installed and enabled successfully"
-echo ""
-echo "Useful commands:"
-echo "  Start services:   systemctl start perfsonar-testpoint.service"
-if [[ "$WITH_CERTBOT" == "true" ]]; then
-    echo "                    systemctl start perfsonar-certbot.service"
+    echo ""
+    echo "==> ✓ Systemd units installed and enabled successfully"
+    echo ""
+    echo "Useful commands:"
+    echo "  Start services:   systemctl start perfsonar-testpoint.service"
+    if [[ "$WITH_CERTBOT" == "true" ]]; then
+        echo "                    systemctl start perfsonar-certbot.service"
+    fi
+    echo "  Stop services:    systemctl stop perfsonar-testpoint.service"
+    if [[ "$WITH_CERTBOT" == "true" ]]; then
+        echo "                    systemctl stop perfsonar-certbot.service"
+    fi
+    echo "  Check status:     systemctl status perfsonar-testpoint.service"
+    if [[ "$WITH_CERTBOT" == "true" ]]; then
+        echo "                    systemctl status perfsonar-certbot.service"
+    fi
+    echo "  View logs:        journalctl -u perfsonar-testpoint.service -f"
+    if [[ "$WITH_CERTBOT" == "true" ]]; then
+        echo "                    journalctl -u perfsonar-certbot.service -f"
+    fi
+    echo "  Check containers: podman ps"
+    echo ""
+    echo "The services will automatically start containers on boot."
+    echo ""
+    echo "Note: These units use 'podman run --systemd=always' for proper systemd"
+    echo "      support inside the container. This is required for the testpoint"
+    echo "      image which runs systemd internally."
 fi
-echo "  Stop services:    systemctl stop perfsonar-testpoint.service"
-if [[ "$WITH_CERTBOT" == "true" ]]; then
-    echo "                    systemctl stop perfsonar-certbot.service"
-fi
-echo "  Check status:     systemctl status perfsonar-testpoint.service"
-if [[ "$WITH_CERTBOT" == "true" ]]; then
-    echo "                    systemctl status perfsonar-certbot.service"
-fi
-echo "  View logs:        journalctl -u perfsonar-testpoint.service -f"
-if [[ "$WITH_CERTBOT" == "true" ]]; then
-    echo "                    journalctl -u perfsonar-certbot.service -f"
+
+# ── Optional: auto-update timer ────────────────────────────────────────────────
+if [[ "$AUTO_UPDATE" == "true" ]]; then
+    AUTO_UPDATE_SCRIPT="$INSTALL_DIR/tools_scripts/perfSONAR-auto-update.sh"
+    AUTO_UPDATE_BIN="/usr/local/bin/perfsonar-auto-update.sh"
+    AUTO_UPDATE_SVC="/etc/systemd/system/perfsonar-auto-update.service"
+    AUTO_UPDATE_TIMER="/etc/systemd/system/perfsonar-auto-update.timer"
+
+    echo ""
+    echo "==> Installing auto-update timer"
+
+    # Use the versioned script from tools_scripts if present, else fall back to a
+    # minimal inline version.
+    if [[ -f "$AUTO_UPDATE_SCRIPT" ]]; then
+        cp "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_BIN"
+    else
+        echo "WARNING: $AUTO_UPDATE_SCRIPT not found; writing minimal inline script."
+        cat > "$AUTO_UPDATE_BIN" << 'AUTOUPDATE_EOF'
+#!/bin/bash
+# perfsonar-auto-update.sh (minimal inline fallback)
+# For the full versioned script, re-run bootstrap (install_tools_scripts.sh).
+set -euo pipefail
+LOGFILE="/var/log/perfsonar-auto-update.log"
+TESTPOINT_IMAGE="hub.opensciencegrid.org/osg-htc/perfsonar-testpoint:production"
+CERTBOT_IMAGE="docker.io/certbot/certbot:latest"
+log() { echo "$(date -Iseconds) $*" | tee -a "$LOGFILE"; }
+get_id() { podman image inspect "$1" --format '{{.Id}}' 2>/dev/null || echo none; }
+check_pull() {
+    local img=$1 before after
+    before=$(get_id "$img")
+    podman pull "$img" >> "$LOGFILE" 2>&1 || { log "WARNING: pull failed for $img"; echo unchanged; return; }
+    after=$(get_id "$img")
+    [[ "$before" == "none" || "$before" != "$after" ]] && echo updated || echo unchanged
+}
+log '=== perfSONAR auto-update check ==='
+ANY=false
+[[ $(check_pull "$TESTPOINT_IMAGE") == updated ]] && ANY=true
+podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^certbot$' && \
+    [[ $(check_pull "$CERTBOT_IMAGE") == updated ]] && ANY=true
+$ANY && systemctl restart perfsonar-testpoint.service && log 'Restarted testpoint.service' || log 'No updates'
+log '=== done ==='
+AUTOUPDATE_EOF
+    fi
+    chmod 0755 "$AUTO_UPDATE_BIN"
+    echo "==> ✓ Installed $AUTO_UPDATE_BIN"
+
+    cat > "$AUTO_UPDATE_SVC" << 'EOF'
+[Unit]
+Description=perfSONAR Container Auto-Update
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/perfsonar-auto-update.sh
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
+EOF
+    echo "==> ✓ Created $AUTO_UPDATE_SVC"
+
+    cat > "$AUTO_UPDATE_TIMER" << 'EOF'
+[Unit]
+Description=perfSONAR Container Auto-Update Timer
+
+[Timer]
+OnCalendar=*-*-* 03:00:00
+RandomizedDelaySec=1h
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+    echo "==> ✓ Created $AUTO_UPDATE_TIMER"
+
+    systemctl daemon-reload
+    systemctl enable --now perfsonar-auto-update.timer
+    echo "==> ✓ Enabled perfsonar-auto-update.timer (runs daily at 03:00 + up to 1h random delay)"
+    echo ""
+    echo "Useful auto-update commands:"
+    echo "  Check timer:      systemctl list-timers perfsonar-auto-update.timer"
+    echo "  Run now (test):   systemctl start perfsonar-auto-update.service"
+    echo "  View log:         journalctl -u perfsonar-auto-update.service -f"
+    echo "  Update log file:  tail -f /var/log/perfsonar-auto-update.log"
 fi
-echo "  Check containers: podman ps"
-echo ""
-echo "The services will automatically start containers on boot."
-echo ""
-echo "Note: These units use 'podman run --systemd=always' for proper systemd"
-echo "      support inside the container. This is required for the testpoint"
-echo "      image which runs systemd internally."
diff --git a/docs/perfsonar/tools_scripts/install-systemd-units.sh.sha256 b/docs/perfsonar/tools_scripts/install-systemd-units.sh.sha256
@@ -1 +1 @@
-4b2b91593a3ceb3c8159cc2d6ca749aeeb7a558a766227c46493feb84655f04a
+ac05461f8a9745c95e0630f3e92de099961d129e910c5d87da53381b366cfa3c  install-systemd-units.sh
diff --git a/docs/perfsonar/tools_scripts/install_tools_scripts.sh b/docs/perfsonar/tools_scripts/install_tools_scripts.sh
@@ -5,11 +5,11 @@ set -euo pipefail
 # Purpose: Ensure the perfSONAR testpoint repository is cloned and the tools_scripts
 #          directory is present under /opt/perfsonar-tp/tools_scripts.
 #
-# Version: 1.0.1 - 2025-12-16
+# Version: 1.0.2 - 2026-02-24
 # Author: Shawn McKee, University of Michigan
 # Acknowledgements: Supported by IRIS-HEP and OSG-LHC
 
-VERSION="1.0.1"
+VERSION="1.0.2"
 PROG_NAME="$(basename "$0")"
 
 # Check for --version or --help flags
@@ -84,6 +84,7 @@ files=(
     perfSONAR-install-nftables.sh
     perfSONAR-update-lsregistration.sh
     perfSONAR-auto-enroll-psconfig.sh
+    perfSONAR-auto-update.sh
     seed_testpoint_host_dirs.sh
     perfSONAR-orchestrator.sh
     install_tools_scripts.sh

diff --git a/docs/perfsonar/tools_scripts/install_tools_scripts.sh.sha256 b/docs/perfsonar/tools_scripts/install_tools_scripts.sh.sha256
@@ -1 +1 @@
-b213dc73e0c95973e8695cd490e5910636011426a88387f21c361c54ef5e14e5  install_tools_scripts.sh
+81b7c4892da2f11b27803fd36d03ef70fd8cdc41a6fddb1e669d03a28f69e905  install_tools_scripts.sh
diff --git a/docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh b/docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh
@@ -0,0 +1,134 @@
+#!/bin/bash
+# perfSONAR-auto-update.sh
+# ------------------------
+# Purpose:
+#   Check for updated container images and restart services if new images are
+#   found. Uses image digest comparison (Podman-compatible). Does NOT rely on
+#   Docker-specific output strings like "Downloaded newer image" which Podman
+#   never emits.
+#
+# Usage:
+#   Run as root, typically via the perfsonar-auto-update.timer systemd timer.
+#   Can also be invoked manually: bash /usr/local/bin/perfsonar-auto-update.sh
+#
+# What it does:
+#   1. Records the current local image digest for each managed image.
+#   2. Pulls each image from the registry.
+#   3. Compares the new digest to the old one.
+#   4. If any digest changed, restarts perfsonar-testpoint.service (which
+#      manages both the testpoint and certbot containers via podman-compose or
+#      direct podman run, depending on how the service was installed).
+#
+# Logs:
+#   /var/log/perfsonar-auto-update.log  (appended on every run)
+#
+# Author: OSG perfSONAR deployment tools
+# Version: 1.0.0
+# Acknowledgements: Supported by IRIS-HEP and OSG-LHC
+
+set -euo pipefail
+
+# ── Configuration ──────────────────────────────────────────────────────────────
+
+LOGFILE="/var/log/perfsonar-auto-update.log"
+
+# Images managed by this host. Edit if you have pinned a specific digest tag.
+TESTPOINT_IMAGE="hub.opensciencegrid.org/osg-htc/perfsonar-testpoint:production"
+CERTBOT_IMAGE="docker.io/certbot/certbot:latest"
+
+# systemd service that starts/stops all perfSONAR containers on this host.
+TESTPOINT_SERVICE="perfsonar-testpoint.service"
+
+# ── Helpers ────────────────────────────────────────────────────────────────────
+
+log() { echo "$(date -Iseconds) $*" | tee -a "$LOGFILE"; }
+
+# Return the local image ID, or "none" if image is not present.
+get_image_id() {
+    podman image inspect "$1" --format "{{.Id}}" 2>/dev/null || echo "none"
+}
+
+# Return the image ID of a running container by name, or "none" if the
+# container is not running.
+get_container_image_id() {
+    podman inspect "$1" --format "{{.Image}}" 2>/dev/null || echo "none"
+}
+
+# Pull an image and return "updated" or "unchanged".
+# Writes pull output to the log file.
+pull_and_check() {
+    local image="$1"
+    local before after
+    before=$(get_image_id "$image")
+
+    log "Pulling: $image (current: ${before:0:12})"
+    if ! podman pull "$image" >> "$LOGFILE" 2>&1; then
+        log "WARNING: pull failed for $image — skipping (network issue?)"
+        echo "unchanged"
+        return
+    fi
+
+    after=$(get_image_id "$image")
+
+    if [[ "$after" == "none" ]]; then
+        log "WARNING: could not verify image digest after pull: $image"
+        echo "unchanged"
+    elif [[ "$before" == "none" || "$before" != "$after" ]]; then
+        log "UPDATED: $image  ${before:0:12} -> ${after:0:12}"
+        echo "updated"
+    else
+        log "Up to date: $image"
+        echo "unchanged"
+    fi
+}
+
+# ── Main ───────────────────────────────────────────────────────────────────────
+
+# Must run as root
+if [[ $EUID -ne 0 ]]; then
+    echo "ERROR: must be run as root" >&2
+    exit 1
+fi
+
+log "=== perfSONAR auto-update check started ==="
+
+ANY_UPDATED=false
+
+# ── Testpoint image ────────────────────────────────────────────────────────────
+result=$(pull_and_check "$TESTPOINT_IMAGE")
+[[ "$result" == "updated" ]] && ANY_UPDATED=true
+
+# ── Certbot image (only if a certbot container exists on this host) ────────────
+if podman ps -a --format "{{.Names}}" 2>/dev/null | grep -q "^certbot$"; then
+    result=$(pull_and_check "$CERTBOT_IMAGE")
+    [[ "$result" == "updated" ]] && ANY_UPDATED=true
+fi
+
+# ── Stale-container check ──────────────────────────────────────────────────────
+# Handle the case where the local image tag was already updated by a prior manual
+# pull (pull says "up to date") but the running container is still using the old
+# image digest. Compare running container's image ID vs the current local tag.
+if [[ "$ANY_UPDATED" == "false" ]]; then
+    LATEST_TESTPOINT_ID=$(get_image_id "$TESTPOINT_IMAGE")
+    RUNNING_TESTPOINT_ID=$(get_container_image_id "perfsonar-testpoint")
+    if [[ "$RUNNING_TESTPOINT_ID" != "none" && "$LATEST_TESTPOINT_ID" != "none" \
+          && "$RUNNING_TESTPOINT_ID" != "$LATEST_TESTPOINT_ID" ]]; then
+        log "Running container uses stale image ${RUNNING_TESTPOINT_ID:0:12} (latest: ${LATEST_TESTPOINT_ID:0:12}) — forcing restart"
+        ANY_UPDATED=true
+    fi
+fi
+
+# ── Restart if any image changed ───────────────────────────────────────────────
+if [[ "$ANY_UPDATED" == "true" ]]; then
+    log "New image(s) found — restarting $TESTPOINT_SERVICE"
+    if systemctl restart "$TESTPOINT_SERVICE"; then
+        log "Restarted $TESTPOINT_SERVICE successfully"
+    else
+        log "ERROR: failed to restart $TESTPOINT_SERVICE (exit $?)"
+        exit 1
+    fi
+else
+    log "All images up to date — no restart needed"
+fi
+
+log "=== perfSONAR auto-update check complete ==="
diff --git a/docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh.sha256 b/docs/perfsonar/tools_scripts/perfSONAR-auto-update.sh.sha256
@@ -0,0 +1 @@
+f7e14a1cc2744e9f653ed5ace910a2f016b5772a1f4df1a23e3a2c30bbf0e7ab  perfSONAR-auto-update.sh
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		4b2b91593a3ceb3c8159cc2d6ca749aeeb7a558a766227c46493feb84655f04a
		ac05461f8a9745c95e0630f3e92de099961d129e910c5d87da53381b366cfa3c install-systemd-units.sh
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		b213dc73e0c95973e8695cd490e5910636011426a88387f21c361c54ef5e14e5 install_tools_scripts.sh
		81b7c4892da2f11b27803fd36d03ef70fd8cdc41a6fddb1e669d03a28f69e905 install_tools_scripts.sh
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		f7e14a1cc2744e9f653ed5ace910a2f016b5772a1f4df1a23e3a2c30bbf0e7ab perfSONAR-auto-update.sh