diff --git a/.github/workflows/cleanup-caches.yml b/.github/workflows/cleanup-caches.yml
index 2f666007a..c268da8e6 100644
--- a/.github/workflows/cleanup-caches.yml
+++ b/.github/workflows/cleanup-caches.yml
@@ -13,15 +13,13 @@ jobs:
         run: |
           echo "Fetching list of cache keys"
           cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id')
-
-          ## Setting this to not fail the workflow while deleting cache keys.
-          set +e
-          echo "Deleting caches..."
-          for cacheKey in $cacheKeysForPR
-          do
-              gh cache delete $cacheKey
+          while [ "$cacheKeysForPR" ]; do
+            for cacheKey in $cacheKeysForPR; do
+                echo gh cache delete $cacheKey
+                gh cache delete $cacheKey
+            done
+            cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id')
           done
-          echo "Done"
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GH_REPO: ${{ github.repository }}