Switch AMI build back to simple approach with runner cleanup

joseph-isaacs · joseph-isaacs · commit 8431dfbd579a · 2025-12-19T13:23:27.000Z
- Revert from Packer to simple create-image approach
- Build jobs now run on runs-on instances directly
- Added runner state cleanup before AMI creation to fix registration issues
- Removes .runner, .credentials files so new instances register fresh

Signed-off-by: Joe Isaacs &lt;joe@spiraldb.com&gt;
diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml
@@ -0,0 +1,128 @@
+name: "Build Vortex CI AMI"
+description: "Build a custom Amazon Machine Image for Vortex CI runners"
+
+inputs:
+  arch:
+    description: "Target architecture: x64 or arm64"
+    required: true
+  ami-prefix:
+    description: "Prefix for AMI name"
+    required: false
+    default: "vortex-ci"
+  retention-days:
+    description: "Number of days before AMI is deprecated"
+    required: false
+    default: "30"
+
+outputs:
+  ami-id:
+    description: "The ID of the built AMI"
+    value: ${{ steps.create-ami.outputs.ami_id }}
+  ami-name:
+    description: "The name of the built AMI"
+    value: ${{ steps.create-ami.outputs.ami_name }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Rust
+      uses: ./.github/actions/setup-rust
+
+    - name: Setup flatc
+      uses: ./.github/actions/setup-flatc
+
+    - name: Install extra dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          cmake \
+          ninja-build \
+          clang \
+          lld \
+          llvm
+
+    - name: Install nightly toolchain
+      shell: bash
+      run: |
+        rustup toolchain install nightly
+        rustup component add --toolchain nightly rustfmt clippy rust-src miri llvm-tools-preview
+
+    - name: Install cargo tools
+      shell: bash
+      run: |
+        cargo install cargo-nextest --locked
+        cargo install cargo-hack --locked
+        cargo install grcov --locked
+
+    - name: Clean runner for AMI
+      shell: bash
+      run: |
+        echo "=== Cleaning runner state for clean AMI ==="
+
+        # The runner service name
+        RUNNER_SVC=$(systemctl list-units --type=service | grep actions.runner | awk '{print $1}' | head -1)
+
+        if [ -n "$RUNNER_SVC" ]; then
+          echo "Stopping runner service: $RUNNER_SVC"
+          sudo systemctl stop "$RUNNER_SVC" || true
+        fi
+
+        # Remove runner registration (this is the key!)
+        RUNNER_DIR="/home/runner/actions-runner"
+        if [ -d "$RUNNER_DIR" ]; then
+          echo "Cleaning runner directory..."
+          sudo rm -f "$RUNNER_DIR/.runner" || true
+          sudo rm -f "$RUNNER_DIR/.credentials" || true
+          sudo rm -f "$RUNNER_DIR/.credentials_rsaparams" || true
+          sudo rm -rf "$RUNNER_DIR/_diag" || true
+          sudo rm -rf "$RUNNER_DIR/_work" || true
+        fi
+
+        # Clear temp files
+        sudo rm -rf /tmp/* || true
+        sudo rm -rf /var/tmp/* || true
+
+        # Clear apt cache to reduce AMI size
+        sudo apt-get clean
+        sudo rm -rf /var/lib/apt/lists/*
+
+        echo "=== Runner state cleaned ==="
+
+    - name: Create AMI
+      id: create-ami
+      shell: bash
+      env:
+        AMI_PREFIX: ${{ inputs.ami-prefix }}
+        ARCH: ${{ inputs.arch }}
+        RETENTION_DAYS: ${{ inputs.retention-days }}
+      run: |
+        export AWS_REGION="$RUNS_ON_AWS_REGION"
+        export AWS_DEFAULT_REGION="$RUNS_ON_AWS_REGION"
+
+        TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+        AMI_NAME="${AMI_PREFIX}-${ARCH}-${TIMESTAMP}"
+        DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)
+
+        echo "Creating AMI: $AMI_NAME from instance $RUNS_ON_INSTANCE_ID in region $AWS_REGION"
+
+        AMI_ID=$(aws ec2 create-image \
+          --instance-id "$RUNS_ON_INSTANCE_ID" \
+          --name "$AMI_NAME" \
+          --description "Vortex CI runner image for ${ARCH}" \
+          --no-reboot \
+          --tag-specifications "ResourceType=image,Tags=[{Key=Name,Value=$AMI_NAME},{Key=Environment,Value=ci},{Key=Arch,Value=$ARCH},{Key=ManagedBy,Value=github-actions}]" \
+          --query 'ImageId' \
+          --output text)
+
+        echo "Waiting for AMI to be available..."
+        aws ec2 wait image-available --image-ids "$AMI_ID"
+
+        echo "Setting deprecation time to $DEPRECATION_TIME"
+        aws ec2 enable-image-deprecation \
+          --image-id "$AMI_ID" \
+          --deprecate-at "$DEPRECATION_TIME"
+
+        echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT
+        echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT
+        echo "AMI created: $AMI_ID ($AMI_NAME)"
diff --git a/.github/workflows/ami-prebuild.yml b/.github/workflows/ami-prebuild.yml
@@ -40,14 +40,19 @@ env:
   AWS_REGION: eu-west-1
 
 jobs:
+  # Build AMI by running on a runs-on instance and creating an image from it
   build-x64:
     name: "Build AMI (x64)"
     if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'x64' }}
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=2cpu-linux-x64
+      - family=m7i+m7i-flex+m7a
+      - tag=ami-build-x64
     timeout-minutes: 60
     outputs:
-      ami-id: ${{ steps.build.outputs.ami_id }}
-      ami-name: ${{ steps.build.outputs.ami_name }}
+      ami-id: ${{ steps.build-ami.outputs.ami-id }}
+      ami-name: ${{ steps.build-ami.outputs.ami-name }}
 
     steps:
       - name: Checkout
@@ -59,58 +64,34 @@ jobs:
           role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Setup Packer
-        uses: hashicorp/setup-packer@main
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
         with:
-          version: "1.11.2"
-
-      - name: Packer Init
-        working-directory: .github/packer
-        run: packer init vortex-ci.pkr.hcl
-
-      - name: Packer Build
-        id: build
-        working-directory: .github/packer
-        run: |
-          packer build \
-            -var "arch=x64" \
-            -var "aws_region=${{ env.AWS_REGION }}" \
-            -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \
-            -machine-readable \
-            vortex-ci.pkr.hcl | tee packer-output.log
-
-          # Extract AMI ID from Packer output
-          AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2)
-          AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-x64")
-
-          echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT
-          echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT
-          echo "Built AMI: $AMI_ID"
-
-      - name: Set AMI Deprecation
-        run: |
-          RETENTION_DAYS=${{ inputs.retention-days || '30' }}
-          DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)
-          aws ec2 enable-image-deprecation \
-            --image-id "${{ steps.build.outputs.ami_id }}" \
-            --deprecate-at "$DEPRECATION_TIME"
-          echo "AMI will be deprecated at $DEPRECATION_TIME"
+          arch: x64
+          ami-prefix: vortex-ci
+          retention-days: ${{ inputs.retention-days || '30' }}
 
       - name: Summary
         run: |
           echo "## AMI Build Complete (x64)" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY
           echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY
 
   build-arm64:
     name: "Build AMI (arm64)"
     if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.arch == '' || github.event.inputs.arch == 'arm64' }}
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=2cpu-linux-arm64
+      - family=m7g
+      - tag=ami-build-arm64
     timeout-minutes: 60
     outputs:
-      ami-id: ${{ steps.build.outputs.ami_id }}
-      ami-name: ${{ steps.build.outputs.ami_name }}
+      ami-id: ${{ steps.build-ami.outputs.ami-id }}
+      ami-name: ${{ steps.build-ami.outputs.ami-name }}
 
     steps:
       - name: Checkout
@@ -122,48 +103,20 @@ jobs:
           role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Setup Packer
-        uses: hashicorp/setup-packer@main
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
         with:
-          version: "1.11.2"
-
-      - name: Packer Init
-        working-directory: .github/packer
-        run: packer init vortex-ci.pkr.hcl
-
-      - name: Packer Build
-        id: build
-        working-directory: .github/packer
-        run: |
-          packer build \
-            -var "arch=arm64" \
-            -var "aws_region=${{ env.AWS_REGION }}" \
-            -var "subnet_id=${{ secrets.AWS_SUBNET_ID }}" \
-            -machine-readable \
-            vortex-ci.pkr.hcl | tee packer-output.log
-
-          # Extract AMI ID from Packer output
-          AMI_ID=$(grep 'artifact,0,id' packer-output.log | tail -1 | cut -d',' -f6 | cut -d':' -f2)
-          AMI_NAME=$(grep 'artifact,0,string' packer-output.log | tail -1 | grep -oP 'AMIs were created:.*' | sed 's/AMIs were created:\\n\\n//' | head -1 || echo "vortex-ci-arm64")
-
-          echo "ami_id=$AMI_ID" >> $GITHUB_OUTPUT
-          echo "ami_name=$AMI_NAME" >> $GITHUB_OUTPUT
-          echo "Built AMI: $AMI_ID"
-
-      - name: Set AMI Deprecation
-        run: |
-          RETENTION_DAYS=${{ inputs.retention-days || '30' }}
-          DEPRECATION_TIME=$(date -u -d "+${RETENTION_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)
-          aws ec2 enable-image-deprecation \
-            --image-id "${{ steps.build.outputs.ami_id }}" \
-            --deprecate-at "$DEPRECATION_TIME"
-          echo "AMI will be deprecated at $DEPRECATION_TIME"
+          arch: arm64
+          ami-prefix: vortex-ci
+          retention-days: ${{ inputs.retention-days || '30' }}
 
       - name: Summary
         run: |
           echo "## AMI Build Complete (arm64)" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "- **AMI ID:** ${{ steps.build.outputs.ami_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **AMI ID:** ${{ steps.build-ami.outputs.ami-id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **AMI Name:** ${{ steps.build-ami.outputs.ami-name }}" >> $GITHUB_STEP_SUMMARY
           echo "- **Deprecation:** ${{ inputs.retention-days || '30' }} days" >> $GITHUB_STEP_SUMMARY
 
   # Test the newly built AMI
