From 964f8b1ebb0d7318423609bdc5868ae70fbeae14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 31 Oct 2025 14:39:35 +0100 Subject: [PATCH 1/4] RHDH EKS Mapt unlock --- ...per-rhdh-eks-mapt-orphaned-get-commands.sh | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) mode change 100644 => 100755 ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh diff --git a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh old mode 100644 new mode 100755 index 081c0345ab4bf..2fdf8bc5e1d96 --- a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh +++ b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh @@ -25,4 +25,37 @@ if [ -f "${SHARED_DIR}/s3_top_level_folders.txt" ]; then else echo "Error: Failed to create S3 object list file" exit 1 -fi \ No newline at end of file +fi + +# Check if input file is empty +if [ ! -s "${SHARED_DIR}/s3_top_level_folders.txt" ]; then + echo "WARNING: Input file ${SHARED_DIR}/s3_top_level_folders.txt is empty" + echo "No S3 folders to process" + exit 0 +fi + +mapfile -t CORRELATE_MAPT_ARRAY < "${SHARED_DIR}/s3_top_level_folders.txt" + +total=${#CORRELATE_MAPT_ARRAY[@]} +current=0 +echo "Found ${total} S3 top-level folders to process" + +echo "Deleting .pulumi/locks/ folders from each top-level prefix..." +for S3_TOP_LEVEL_FOLDER in "${CORRELATE_MAPT_ARRAY[@]}"; do + current=$((current + 1)) + echo "Processing folder: ${S3_TOP_LEVEL_FOLDER} ($current/$total)" + + [ -z "$S3_TOP_LEVEL_FOLDER" ] && echo "Skipping empty folder name" && continue + + echo "Checking for .pulumi/locks/ in ${S3_TOP_LEVEL_FOLDER}..." + # Check if .pulumi/locks/ prefix exists before attempting deletion + if aws s3 ls "s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/" >/dev/null 2>&1; then + echo "Deleting s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/..." + aws s3 rm "s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/" --recursive + echo "Successfully deleted .pulumi/locks/ from ${S3_TOP_LEVEL_FOLDER}" + else + echo "No .pulumi/locks/ folder found in ${S3_TOP_LEVEL_FOLDER}, skipping" + fi +done + +echo "Finished processing all ${total} S3 folders" From f799f0d2069d46f05cfe3f921d4c86cb14bc09ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 31 Oct 2025 15:04:46 +0100 Subject: [PATCH 2/4] Batch processing --- ...per-rhdh-eks-mapt-orphaned-get-commands.sh | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh index 2fdf8bc5e1d96..6549588532f35 100755 --- a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh +++ b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/get/redhat-developer-rhdh-eks-mapt-orphaned-get-commands.sh @@ -27,35 +27,31 @@ else exit 1 fi -# Check if input file is empty -if [ ! -s "${SHARED_DIR}/s3_top_level_folders.txt" ]; then - echo "WARNING: Input file ${SHARED_DIR}/s3_top_level_folders.txt is empty" - echo "No S3 folders to process" +echo "Finding all .pulumi/locks/ directories in S3 bucket ${AWS_S3_BUCKET}..." + +# Get unique top-level folders that have .pulumi/locks/ +aws s3api list-objects-v2 \ + --bucket "${AWS_S3_BUCKET}" \ + --output json \ + --query 'Contents[?contains(Key, `.pulumi/locks/`)].Key' \ + | jq -r '.[]?' \ + | sed 's|/\.pulumi/locks/.*||' \ + | sort -u > "${SHARED_DIR}/folders_with_locks.txt" + +if [ ! -s "${SHARED_DIR}/folders_with_locks.txt" ]; then + echo "No .pulumi/locks/ directories found in bucket" exit 0 fi -mapfile -t CORRELATE_MAPT_ARRAY < "${SHARED_DIR}/s3_top_level_folders.txt" - -total=${#CORRELATE_MAPT_ARRAY[@]} -current=0 -echo "Found ${total} S3 top-level folders to process" - -echo "Deleting .pulumi/locks/ folders from each top-level prefix..." -for S3_TOP_LEVEL_FOLDER in "${CORRELATE_MAPT_ARRAY[@]}"; do - current=$((current + 1)) - echo "Processing folder: ${S3_TOP_LEVEL_FOLDER} ($current/$total)" - - [ -z "$S3_TOP_LEVEL_FOLDER" ] && echo "Skipping empty folder name" && continue +folder_count=$(wc -l < "${SHARED_DIR}/folders_with_locks.txt") +echo "Found ${folder_count} folders with .pulumi/locks/ to clean" - echo "Checking for .pulumi/locks/ in ${S3_TOP_LEVEL_FOLDER}..." - # Check if .pulumi/locks/ prefix exists before attempting deletion - if aws s3 ls "s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/" >/dev/null 2>&1; then - echo "Deleting s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/..." - aws s3 rm "s3://${AWS_S3_BUCKET}/${S3_TOP_LEVEL_FOLDER}/.pulumi/locks/" --recursive - echo "Successfully deleted .pulumi/locks/ from ${S3_TOP_LEVEL_FOLDER}" - else - echo "No .pulumi/locks/ folder found in ${S3_TOP_LEVEL_FOLDER}, skipping" - fi -done +# Delete all lock files in one efficient command +echo "Deleting all .pulumi/locks/ files across all folders..." +aws s3 rm "s3://${AWS_S3_BUCKET}/" \ + --recursive \ + --exclude "*" \ + --include "*/.pulumi/locks/*" +cp "${SHARED_DIR}/folders_with_locks.txt" "${ARTIFACT_DIR}/folders_cleaned.txt" -echo "Finished processing all ${total} S3 folders" +echo "Successfully deleted .pulumi/locks/ from ${folder_count} folders" From de5b1be3f56b0acb38d3b874d094cf8a5031414b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 31 Oct 2025 15:05:14 +0100 Subject: [PATCH 3/4] AKS unlock --- ...per-rhdh-aks-mapt-orphaned-get-commands.sh | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/get/redhat-developer-rhdh-aks-mapt-orphaned-get-commands.sh b/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/get/redhat-developer-rhdh-aks-mapt-orphaned-get-commands.sh index e330f048ff643..d5953f35eeeb6 100644 --- a/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/get/redhat-developer-rhdh-aks-mapt-orphaned-get-commands.sh +++ b/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/get/redhat-developer-rhdh-aks-mapt-orphaned-get-commands.sh @@ -36,4 +36,34 @@ if [ -f "${SHARED_DIR}/blob_top_level_folders.txt" ]; then else echo "Error: Failed to create blob list file" exit 1 -fi \ No newline at end of file +fi + +echo "Finding all .pulumi/locks/ blobs in container ${AZURE_STORAGE_BLOB}..." + +# Get unique top-level folders that have .pulumi/locks/ +az storage blob list \ + --container-name "${AZURE_STORAGE_BLOB}" \ + --account-name "${AZURE_STORAGE_ACCOUNT}" \ + --account-key "${AZURE_STORAGE_KEY}" \ + --output json | \ + jq -r '.[].name | select(contains("/.pulumi/locks/")) | split("/.pulumi/locks/")[0]' | \ + sort -u > "${SHARED_DIR}/folders_with_locks.txt" + +if [ ! -s "${SHARED_DIR}/folders_with_locks.txt" ]; then + echo "No .pulumi/locks/ directories found in container" + exit 0 +fi + +folder_count=$(wc -l < "${SHARED_DIR}/folders_with_locks.txt") +echo "Found ${folder_count} folders with .pulumi/locks/ to clean" + +# Delete all lock blobs in one efficient command +echo "Deleting all .pulumi/locks/ blobs across all folders..." +az storage blob delete-batch \ + --source "${AZURE_STORAGE_BLOB}" \ + --account-name "${AZURE_STORAGE_ACCOUNT}" \ + --account-key "${AZURE_STORAGE_KEY}" \ + --pattern "*/.pulumi/locks/*" +cp "${SHARED_DIR}/folders_with_locks.txt" "${ARTIFACT_DIR}/folders_cleaned.txt" + +echo "Successfully deleted .pulumi/locks/ from ${folder_count} folders" From 57c7eb3a6beafd5ea9d8d7f2c7426a33c24166d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Sat, 1 Nov 2025 12:06:50 +0100 Subject: [PATCH 4/4] Grace period --- .../redhat-developer-rhdh-aks-mapt-orphaned-destroy-ref.yaml | 2 +- .../redhat-developer-rhdh-eks-mapt-orphaned-destroy-ref.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/destroy/redhat-developer-rhdh-aks-mapt-orphaned-destroy-ref.yaml b/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/destroy/redhat-developer-rhdh-aks-mapt-orphaned-destroy-ref.yaml index 1de95cd17e8f2..2fcc9bebf9a22 100644 --- a/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/destroy/redhat-developer-rhdh-aks-mapt-orphaned-destroy-ref.yaml +++ b/ci-operator/step-registry/redhat-developer/rhdh/aks/mapt/orphaned/destroy/redhat-developer-rhdh-aks-mapt-orphaned-destroy-ref.yaml @@ -9,7 +9,7 @@ ref: name: mapt namespace: ci tag: v0.9.6 - grace_period: 15m + grace_period: 2h resources: limits: cpu: "2" diff --git a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/destroy/redhat-developer-rhdh-eks-mapt-orphaned-destroy-ref.yaml b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/destroy/redhat-developer-rhdh-eks-mapt-orphaned-destroy-ref.yaml index f601d89a0ac3a..babfdb56f4efe 100644 --- a/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/destroy/redhat-developer-rhdh-eks-mapt-orphaned-destroy-ref.yaml +++ b/ci-operator/step-registry/redhat-developer/rhdh/eks/mapt/orphaned/destroy/redhat-developer-rhdh-eks-mapt-orphaned-destroy-ref.yaml @@ -9,7 +9,7 @@ ref: name: mapt namespace: ci tag: v0.9.6 - grace_period: 15m + grace_period: 2h resources: limits: cpu: "2"