Automate image build process

.github/workflows/build-images-ssh.yml

@@ -0,0 +1,88 @@
+name: Build VM Images (SSH)
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: build-vm-images
+  cancel-in-progress: false
+
+jobs:
+  build-images:
+    name: Build ${{ matrix.image-type }} Image
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        image-type: [cpu, gpu]
+      max-parallel: 1
+      fail-fast: false
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -p ${{ secrets.SSH_PORT }} -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts
+          cat >> ~/.ssh/config << EOF
+          Host gpuserver
+            HostName ${{ secrets.SSH_HOST }}
+            Port ${{ secrets.SSH_PORT }}
+            User ${{ secrets.SSH_USER }}
+            IdentityFile ~/.ssh/id_rsa
+            StrictHostKeyChecking no
+          EOF
+          chmod 600 ~/.ssh/config
+
+      - name: Sync repository to server
+        run: |
+          ssh gpuserver "mkdir -p /tmp/repo-${{ github.run_id }}"
+          scp -r ./* gpuserver:/tmp/repo-${{ github.run_id }}/
+
+      - name: Install uv on server
+        run: |
+          ssh gpuserver "mkdir -p /tmp/uv-${{ github.run_id }} && curl -LsSf https://astral.sh/uv/install.sh | env INSTALLER_NO_MODIFY_PATH=1 sh && mv ~/.local/bin/uv ~/.local/bin/uvx /tmp/uv-${{ github.run_id }}/"
+
+      - name: Build ${{ matrix.image-type }} image on remote server
+        id: build
+        run: |
+          ssh -t gpuserver "cd /tmp/repo-${{ github.run_id }} && make build-${{ matrix.image-type }} BUILD_DIR=/tmp/vm-images-build-${{ github.run_id }} UV=/tmp/uv-${{ github.run_id }}/uv"
+          IMAGE_NAME=$(ssh gpuserver "cd /tmp/vm-images-build-${{ github.run_id }}-${{ matrix.image-type }} && ls -1 *.qcow2 2>/dev/null | head -1 | sed 's/.qcow2$//'")
+          echo "image_name=$IMAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Upload image to OpenStack from server
+        id: upload
+        if: success() && github.ref == 'refs/heads/main'
+        run: |
+          IMAGE_TAGS="--tag name=${{ steps.build.outputs.image_name }} --tag environment=ci --tag project=conda-forge-gpu-ci --tag image-type=${{ matrix.image-type }} --tag build-job-url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} --tag commit-hash=${{ github.sha }} --tag branch=${{ github.ref_name }} --tag build-timestamp=$(date -u +%Y%m%d-%H%M%S)"
+          ssh gpuserver "cd /tmp/repo-${{ github.run_id }} && make upload-${{ matrix.image-type }} BUILD_DIR=/tmp/vm-images-build-${{ github.run_id }} UV=/tmp/uv-${{ github.run_id }}/uv IMAGE_TAGS='$IMAGE_TAGS'"
+
+      - name: Cleanup temp directories on server
+        if: always()
+        run: |
+          ssh gpuserver "cd /tmp/repo-${{ github.run_id }} && make clean BUILD_DIR=/tmp/vm-images-build-${{ github.run_id }} && cd / && rm -rf /tmp/repo-${{ github.run_id }} /tmp/uv-${{ github.run_id }}" || true
+
+      - name: Add job summary
+        if: always()
+        run: |
+          echo "## 🖼️ ${{ matrix.image-type }} Image Build Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ "${{ steps.build.outcome }}" == "success" ]; then
+            echo "✅ **Build Status:** Success" >> $GITHUB_STEP_SUMMARY
+            echo "📦 **Image Name:** \`${{ steps.build.outputs.image_name }}\`" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            if [ "${{ steps.upload.outcome }}" == "success" ]; then
+              echo "☁️ **OpenStack Upload:** ✅ Success" >> $GITHUB_STEP_SUMMARY
+            elif [ "${{ steps.upload.outcome }}" == "failure" ]; then
+              echo "☁️ **OpenStack Upload:** ❌ Failed" >> $GITHUB_STEP_SUMMARY
+            elif [ "${{ steps.upload.outcome }}" == "skipped" ]; then
+              echo "☁️ **OpenStack Upload:** ⏭️ Skipped (not main branch)" >> $GITHUB_STEP_SUMMARY
+            fi
+          else
+            echo "❌ **Build Status:** Failed" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -0,0 +1,9 @@
+# Build artifacts
+vm-images/dib.log
+vm-images/build-output.log
+vm-images/*.qcow2
+vm-images/*.d/
+
+# UV environment
+vm-images/.venv/
+vm_images.egg-info/

Makefile

@@ -0,0 +1,115 @@
+.PHONY: help build-cpu build-gpu build-all upload-cpu upload-gpu upload-all clean show-image
+
+# Default variables
+BUILD_DIR ?= /tmp/vm-images-build-$(shell date +%s)
+IMAGE_TYPE ?= cpu
+TIMESTAMP := $(shell date +%Y%m%d-%H%M%S)
+IMAGE_NAME := ubuntu-2404-$(IMAGE_TYPE)-$(TIMESTAMP)
+
+# Path configuration
+REPO_DIR := $(shell pwd)
+VM_IMAGES_DIR := $(REPO_DIR)/vm-images
+OPENSTACK_RC ?= /etc/kolla/admin-openrc.sh
+UV ?= uv
+IMAGE_TAGS ?=
+
+help:
+	@echo "VM Image Build Makefile"
+	@echo ""
+	@echo "Requirements:"
+	@echo "  - uv (https://docs.astral.sh/uv/)"
+	@echo ""
+	@echo "Targets:"
+	@echo "  build-cpu          - Build CPU image"
+	@echo "  build-gpu          - Build GPU image"
+	@echo "  build-all          - Build both CPU and GPU images"
+	@echo "  upload-cpu         - Upload CPU image to OpenStack"
+	@echo "  upload-gpu         - Upload GPU image to OpenStack"
+	@echo "  upload-all         - Upload both images to OpenStack"
+	@echo "  show-image         - Show built image information"
+	@echo "  clean              - Clean up build directory"
+	@echo ""
+	@echo "Variables:"
+	@echo "  BUILD_DIR=$(BUILD_DIR)"
+	@echo "  IMAGE_TYPE=$(IMAGE_TYPE)"
+	@echo "  IMAGE_NAME=$(IMAGE_NAME)"
+	@echo "  UV=$(UV)"
+	@echo "  IMAGE_TAGS=$(IMAGE_TAGS)  # OpenStack image tags (e.g., '--tag key=value --tag key2=value2')"
+
+build-cpu:
+	@$(MAKE) _build IMAGE_TYPE=cpu
+
+build-gpu:
+	@$(MAKE) _build IMAGE_TYPE=gpu
+
+build-all:
+	@$(MAKE) build-cpu
+	@$(MAKE) build-gpu
+
+_build:
+	@echo "======================================"
+	@echo "Building $(IMAGE_TYPE) image: $(IMAGE_NAME)"
+	@echo "======================================"
+	mkdir -p $(BUILD_DIR)-$(IMAGE_TYPE)
+	cp -r $(VM_IMAGES_DIR)/* $(BUILD_DIR)-$(IMAGE_TYPE)/
+	cd $(BUILD_DIR)-$(IMAGE_TYPE) && \
+		sed -i 's/imagename: $(IMAGE_TYPE)-image.qcow2/imagename: $(IMAGE_NAME).qcow2/' $(IMAGE_TYPE)-image.yaml
+	cd $(BUILD_DIR)-$(IMAGE_TYPE) && \
+		$(UV) sync && \
+		IMAGE_YAML=$(IMAGE_TYPE)-image.yaml UV=$(UV) $(UV) run bash build-image.sh
+	@echo ""
+	@$(MAKE) show-image IMAGE_TYPE=$(IMAGE_TYPE) BUILD_DIR=$(BUILD_DIR)
+
+show-image:
+	@echo "======================================"
+	@echo "Built Image Information"
+	@echo "======================================"
+	@cd $(BUILD_DIR)-$(IMAGE_TYPE) && \
+		IMAGE_FILE=$$(ls -1 *.qcow2 2>/dev/null | head -1) && \
+		if [ -n "$$IMAGE_FILE" ]; then \
+			echo "Image Path: $(BUILD_DIR)-$(IMAGE_TYPE)/$$IMAGE_FILE"; \
+			echo "Image Size: $$(ls -lh $$IMAGE_FILE | awk '{print $$5}')"; \
+			echo "Full Details:"; \
+			ls -lh $$IMAGE_FILE; \
+			echo ""; \
+			if command -v qemu-img &> /dev/null; then \
+				echo "QEMU Image Info:"; \
+				qemu-img info $$IMAGE_FILE; \
+			fi; \
+		else \
+			echo "No .qcow2 image file found!"; \
+		fi
+	@echo "======================================"
+
+upload-cpu:
+	@$(MAKE) _upload IMAGE_TYPE=cpu
+
+upload-gpu:
+	@$(MAKE) _upload IMAGE_TYPE=gpu
+
+upload-all:
+	@$(MAKE) upload-cpu
+	@$(MAKE) upload-gpu
+
+_upload:
+	@echo "======================================"
+	@echo "Uploading $(IMAGE_TYPE) image to OpenStack"
+	@echo "======================================"
+	@cd $(BUILD_DIR)-$(IMAGE_TYPE) && \
+		. $(OPENSTACK_RC) && \
+		IMAGE_FILE=$$(ls -1 *.qcow2 2>/dev/null | head -1) && \
+		IMAGE_BASE=$$(basename $$IMAGE_FILE .qcow2) && \
+		echo "Uploading $$IMAGE_FILE..." && \
+		$(UV) run openstack image create $$IMAGE_BASE \
+			--public --disk-format qcow2 \
+			--container-format bare \
+			--file $$IMAGE_FILE \
+			$(IMAGE_TAGS) && \
+		echo "Image uploaded successfully!" && \
+		$(UV) run openstack image show $$IMAGE_BASE
+
+clean:
+	@echo "Cleaning up build directories..."
+	@if [ -d "$(BUILD_DIR)-cpu" ]; then sudo rm -rf $(BUILD_DIR)-cpu; fi
+	@if [ -d "$(BUILD_DIR)-gpu" ]; then sudo rm -rf $(BUILD_DIR)-gpu; fi
+	@echo "Cleanup complete"

vm-images/README.md

@@ -1,31 +1,78 @@
-# VM Images for OpenStack
+# VM Images
 
-Creates Ubuntu VM Images with following things installed:
+Builds Ubuntu 24.04 images for OpenStack with Docker and NVIDIA drivers (GPU only).
 
-- Nvidia drivers (GPU images only)
-- Docker
+## Quick Start
+
+From the repo root, run:
 
-The created image is uploaded to a GCS bucket, which is then retrieved by a self-hosted GHA runner running on our OpenStack instance.
+```bash
+# Build CPU image (can run anywhere)
+make build-cpu
 
-## GHA workflows
+# Build GPU image (MUST run on GPU server)
+make build-gpu
 
-These should automate the creation of the images:
+# Upload to OpenStack
+make upload-cpu
+make upload-gpu
+```
 
-- `.github/workflows/build-vm-images.yml` - Github Action to build the image
-- `.github/workflows/openstack.yml` - Github Action to upload the VM image to OpenStack
+## Requirements
 
-## Building manually on Linux
+- `uv` package manager ([install here](https://docs.astral.sh/uv/))
+- Linux (builds won't work on Mac)
+- Root access (diskimage-builder needs it)
+- OpenStack credentials from `/etc/kolla/admin-openrc.sh` on GPU server (for uploads only)
 
-1. Install `diskimage-builder` manually via `pip install -r requirements.txt`. A virtual environment is advised. Check if you have all the system dependencies in `ubuntu24-system-requirements.txt`)
-2. Run `scripts/build-image.sh` to build the image. This will create a `.qcow2` file. 
-    - Export `$IMAGE_YAML` to choose a different image to build (e.g. `cpu-image.yaml`, `gpu-image.yaml`).
-    - Export `$OUTPUT_IMAGE` to change the qcow2 output filename.
+## Build Directory
 
-## Add Image to OpenStack
+By default images build to `/tmp/vm-images-build-{timestamp}`. Override with:
 
 ```bash
-openstack image create ubuntu-2404-nvidia-docker \
+make build-cpu BUILD_DIR=/path/to/build
+```
+
+## Custom UV Path
+
+If uv isn't in your PATH:
+
+```bash
+make build-cpu UV=/path/to/uv
+```
+
+## Automated Builds
+
+GitHub Actions workflow (`.github/workflows/build-images-ssh.yml`) can be triggered manually via workflow_dispatch.
+
+The workflow SSHs into the GPU server to run both CPU and GPU builds. Uploads to OpenStack only happen on `main` branch. GPU images require actual GPU hardware - they'll fail without it.
+
+## Image Contents
+
+**Both images:**
+- Ubuntu 24.04 (Noble)
+- Docker
+- Node.js
+- cloud-init
+
+**GPU images only:**
+- NVIDIA drivers
+- CUDA
+
+## Manual Upload
+
+If you need to upload manually:
+
+```bash
+source /etc/kolla/admin-openrc.sh
+openstack image create my-image-name \
   --public --disk-format qcow2 \
   --container-format bare \
-  --file <created-image>.qcow2
+  --file path/to/image.qcow2
+```
+
+## Cleanup
+
+```bash
+make clean BUILD_DIR=/path/to/build
 ```