updated the failure logic for destroy job

Author: harshavemula-ua

.github/workflows/infra_deploy_val.yaml

@@ -436,10 +436,10 @@ jobs:
       - name: Get State Machine ARN
         id: get_sm_arn
         run: |
-          # Get test state machine ARN
-          ARN=$(aws stepfunctions describe-state-machine \
-            --state-machine-arn "arn:aws:states:${{ env.AWS_REGION }}:$(aws sts get-caller-identity --query Account --output text):stateMachine:nrds_test_sm" \
-            --query 'stateMachineArn' \
+          # Get state machine ARN from SSM parameter (set by Terraform)
+          ARN=$(aws ssm get-parameter \
+            --name "/datastream/state-machine-arn" \
+            --query 'Parameter.Value' \
             --output text \
             --region ${{ env.AWS_REGION }})
           echo "arn=$ARN" >> "$GITHUB_OUTPUT"
@@ -724,7 +724,7 @@ jobs:
     name: Terraform Destroy Test Resources
     needs: [test-short-range-vpus]
     runs-on: ubuntu-latest
-    if: success()  # Only destroy on success, leave resources for debugging on failure
+    if: always()  # Always cleanup resources, even on failure
     defaults:
       run:
         working-directory: infra/aws/terraform
@@ -800,12 +800,15 @@ jobs:
         run: |
           echo "Stopping test Step Functions executions..."
 
-          # Use test state machine
-          AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-          STATE_MACHINE_ARN="arn:aws:states:${{ env.AWS_REGION }}:${AWS_ACCOUNT_ID}:stateMachine:nrds_test_sm"
+          # Get state machine ARN from SSM parameter (set by Terraform)
+          STATE_MACHINE_ARN=$(aws ssm get-parameter \
+            --name "/datastream/state-machine-arn" \
+            --query 'Parameter.Value' \
+            --output text \
+            --region ${{ env.AWS_REGION }} 2>/dev/null) || true
 
           # Check if state machine exists
-          if aws stepfunctions describe-state-machine --state-machine-arn "$STATE_MACHINE_ARN" --region ${{ env.AWS_REGION }} >/dev/null 2>&1; then
+          if [ -n "$STATE_MACHINE_ARN" ] && aws stepfunctions describe-state-machine --state-machine-arn "$STATE_MACHINE_ARN" --region ${{ env.AWS_REGION }} >/dev/null 2>&1; then
             # Stop all running executions matching test pattern (sr=short_range, mr=medium_range, aae=analysis_assim_extend)
             aws stepfunctions list-executions \
               --state-machine-arn "$STATE_MACHINE_ARN" \