CI: Add cleanup of DOCA builder instances

Author: priteau

.github/workflows/package-build-ofed.yml

@@ -97,10 +97,12 @@ jobs:
           aio_vm_network = "stackhpc-ci"
           aio_vm_subnet = "stackhpc-ci"
           aio_vm_interface = "ens3"
+          aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
         working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
         env:
           VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
+          VM_TAGS: '["skc-ci-doca-builder", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
         run: terraform plan

.github/workflows/stackhpc-ci-cleanup.yml

@@ -60,6 +60,26 @@ jobs:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
+      - name: Clean up DOCA builder instances over 5 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -5hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-ci-doca-builder --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
       - name: Clean up host image builder instances over 5 hours old
         run: |
           result=0