Merge pull request #2 from stackhpc/break-condition

Misc fixes

Author: markgoddard

.github/workflows/multinode.yml

@@ -362,16 +362,28 @@ jobs:
 
       - name: Break on failure
         run: |
-          sleep ${{ env.break_duration }}m
-        if: ${{ failure() && steps.config_ach.outcome == 'success' && contains(fromJSON('["failure", "always"]'), env.break_on) }}
+          sleep ${{ inputs.break_duration }}m
+        if: ${{ failure() && steps.config_ach.outcome == 'success' && contains(fromJSON('["failure", "always"]'), inputs.break_on) }}
 
       - name: Break on success
         run: |
-          sleep ${{ env.break_duration }}m
-        if: ${{ steps.config_ach.outcome == 'success' && contains(fromJSON('["success", "always"]'), env.break_on) }}
+          sleep ${{ inputs.break_duration }}m
+        if: ${{ steps.config_ach.outcome == 'success' && contains(fromJSON('["success", "always"]'), inputs.break_on) }}
 
       - name: Destroy
-        run: terraform destroy -auto-approve -input=false
+        run: |
+          for attempt in $(seq 5); do
+              if terraform destroy -auto-approve -input=false -lock-timeout=200s; then
+                  echo "Destroyed infrastructure on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to destroy infrastructure on attempt $attempt"
+              sleep 120
+          done
+          echo "Failed to destroy infrastructure after $attempt attempts"
+          echo "Forcefully destroying infrastructure"
+          terraform destroy -auto-approve -input=false -lock=false
+          exit 1
         working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
         env:
           OS_CLOUD: ${{ vars.OS_CLOUD }}