From d9f5b563f199f0991aa13aa6db910fec404021ac Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Tue, 15 Jul 2025 11:11:18 -0700 Subject: [PATCH 1/2] #5553: add data path file purge after verify-range container finishes --- services/horizon/docker/verify-range/start | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index a3fe33f5d4..8b69e4210a 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -8,6 +8,19 @@ set -e CONTAINER_WORKING_DIR="/data/job_${AWS_BATCH_JOB_ARRAY_INDEX:-0}" echo "Job working directory path on data volume: $CONTAINER_WORKING_DIR" +cleanup() { + # need to purge these files at end as the db and the downloaed datastore + # files can be significatn GB's in size and in a cloud batch environment + # we don't want to keep them around after the job is done, i.e. pay for that space still. + sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/14/bin/pg_ctl stop -D "$PGDATA" + sudo chattr -R -i -a "$CONTAINER_WORKING_DIR" + if [ -d "$CONTAINER_WORKING_DIR" ]; then + sudo rm -rf "$CONTAINER_WORKING_DIR" + echo "Purged working data files from $CONTAINER_WORKING_DIR" + fi +} +trap cleanup EXIT + # Ensure CONTAINER_WORKING_DIR exists and is empty rm -rf "$CONTAINER_WORKING_DIR"/* mkdir -p "$CONTAINER_WORKING_DIR" From 2acd31d7f9528d116fad3fd1cbc68b22b65b21d0 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Tue, 15 Jul 2025 21:43:35 -0700 Subject: [PATCH 2/2] #5553: don't do file attribute reset before file data file purge, not needed --- services/horizon/docker/verify-range/start | 1 - 1 file changed, 1 deletion(-) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index 8b69e4210a..21e8327860 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -13,7 +13,6 @@ cleanup() { # files can be significatn GB's in size and in a cloud batch environment # we don't want to keep them around after the job is done, i.e. pay for that space still. sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/14/bin/pg_ctl stop -D "$PGDATA" - sudo chattr -R -i -a "$CONTAINER_WORKING_DIR" if [ -d "$CONTAINER_WORKING_DIR" ]; then sudo rm -rf "$CONTAINER_WORKING_DIR" echo "Purged working data files from $CONTAINER_WORKING_DIR"