feat: Enhance runner management with cloud provider support and monitoring scripts

Author: Fernando Korndorfer

README.md

@@ -86,40 +86,60 @@ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
 sudo systemctl start docker
 sudo systemctl enable docker
 
-# Get the runner startup and stop scripts and make them executable:
+# Get the runner management scripts and make them executable:
 sudo curl -sO https://raw.githubusercontent.com/fok666/gitlab-selfhosted-runner/main/run.sh
 sudo curl -sO https://raw.githubusercontent.com/fok666/gitlab-selfhosted-runner/main/stop.sh
+sudo curl -sO https://raw.githubusercontent.com/fok666/gitlab-selfhosted-runner/main/vmss_monitor.sh
+sudo curl -sO https://raw.githubusercontent.com/fok666/gitlab-selfhosted-runner/main/ec2_monitor.sh
 sudo chmod +x *.sh
 
 # Set the parameters from GitLab:
 export GITLAB_URL="https://gitlab.com"
 export GITLAB_TOKEN="xxxxxxxxxxxxxxxxxxxxxxxxxxx"
-export RUNNER_EXECUTOR="docker"
 export RUNNER_TAGS="docker,linux"
-export RUNNER_DOCKER_IMAGE="ubuntu:24.04"
-
-# Start the runner in privileged mode using the parameters above:
-sudo docker run -d --privileged \
-  -e GITLAB_URL="${GITLAB_URL}" \
-  -e GITLAB_TOKEN="${GITLAB_TOKEN}" \
-  -e RUNNER_EXECUTOR="${RUNNER_EXECUTOR}" \
-  -e RUNNER_TAGS="${RUNNER_TAGS}" \
-  -e RUNNER_DOCKER_IMAGE="${RUNNER_DOCKER_IMAGE}" \
-  -v /var/run/docker.sock:/var/run/docker.sock \
-  fok666/gitlab-runner:latest
+
+# Start the runners in privileged mode, one runner for each vCPU (default), using the parameters above:
+sudo ./run.sh fok666/gitlab-runner:latest $GITLAB_URL $GITLAB_TOKEN $RUNNER_TAGS
+
+# Or specify a custom number of runners (e.g., 4 runners):
+sudo ./run.sh fok666/gitlab-runner:latest $GITLAB_URL $GITLAB_TOKEN $RUNNER_TAGS 4
 ```
 
 
-## Azure VMSS support
+## Cloud Provider Support
+
+This project supports graceful shutdown for multiple cloud providers:
+
+### Azure VMSS (Virtual Machine Scale Sets)
+- `vmss_monitor.sh`: Monitor Azure VMSS scheduled events for termination notices
+- Reference: https://learn.microsoft.com/en-us/azure/virtual-machines/linux/scheduled-events
+
+### AWS EC2 Spot Instances
+- `ec2_monitor.sh`: Monitor EC2 spot instance termination notices
+- Reference: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-instance-termination-notices.html
 
-This project is designed to use Azure Virtual Machine Scale Sets, but can be used with different settings.
+### Setup
+
+```bash
+# Copy stop script to /opt for monitor scripts to use:
+sudo cp stop.sh /opt/stop.sh
+sudo chmod +x /opt/stop.sh
+
+# For Azure VMSS - check every minute:
+(crontab -l 2>/dev/null; echo "* * * * * /opt/vmss_monitor.sh >> /var/log/vmss_monitor.log 2>&1") | crontab -
+
+# For AWS EC2 Spot - check every 5 seconds:
+(crontab -l 2>/dev/null; echo "* * * * * /opt/ec2_monitor.sh >> /var/log/ec2_monitor.log 2>&1") | crontab -
+(crontab -l 2>/dev/null; echo "* * * * * sleep 5; /opt/ec2_monitor.sh >> /var/log/ec2_monitor.log 2>&1") | crontab -
+(crontab -l 2>/dev/null; echo "* * * * * sleep 10; /opt/ec2_monitor.sh >> /var/log/ec2_monitor.log 2>&1") | crontab -
+(crontab -l 2>/dev/null; echo "* * * * * sleep 15; /opt/ec2_monitor.sh >> /var/log/ec2_monitor.log 2>&1") | crontab -
+```
 
-- `stop.sh`: Add this script to `/opt/stop.sh` to enable graceful Runner shutdown. Requires SUDO.
+Both monitor scripts require `curl` and `jq` to be installed.
 
 
 # TO DO
 
 - Add Google Compute Cloud (GCP) CLI bundles
 - Add GKE auth support
-- Improve support for Spot/Preemptive VM instances
-- Improve support for other Cloud providers (AWS, GCP...)
+- Add GCP Preemptible VM support

ec2_monitor.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -e
+
+# AWS EC2 Spot Instance Termination Monitor
+# Monitors EC2 spot instance termination notices
+# Reference: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-instance-termination-notices.html
+
+METADATA_ENDPOINT='http://169.254.169.254/latest/meta-data/spot/instance-action'
+STOP_SCRIPT='/opt/stop.sh'
+
+echo "Checking for EC2 spot instance termination notice..."
+
+# Query EC2 Instance Metadata Service for spot termination notice
+# Returns HTTP 404 if no termination notice is present
+RESPONSE=$(curl -s -f "$METADATA_ENDPOINT" 2>/dev/null || echo "")
+
+# Check if termination notice exists
+if [ -n "$RESPONSE" ]; then
+  echo "Spot instance termination notice detected!"
+  echo "Termination details: $RESPONSE"
+  
+  # Parse termination time
+  TERMINATION_TIME=$(echo "$RESPONSE" | jq -r '.time' 2>/dev/null || echo "Unknown")
+  ACTION=$(echo "$RESPONSE" | jq -r '.action' 2>/dev/null || echo "terminate")
+  
+  echo "Action: $ACTION"
+  echo "Termination time: $TERMINATION_TIME"
+  echo "Initiating graceful shutdown..."
+  
+  # Start agent shutdown, if script exists and is executable
+  if [ -x "$STOP_SCRIPT" ]; then
+    echo "Executing stop script: $STOP_SCRIPT"
+    "$STOP_SCRIPT"
+  else
+    echo "Warning: Stop script not found or not executable: $STOP_SCRIPT"
+  fi
+  
+  echo "Agents shutdown complete. Instance will terminate at: $TERMINATION_TIME"
+else
+  echo "No spot instance termination notice"
+fi
+
+exit 0

run.sh

@@ -1,57 +1,138 @@
 #!/bin/bash
+set -e
 
-AZP_IMAGE="$1"
-AZP_URL="$2"
-AZP_TOKEN="$3"
-AZP_POOL="$4"
+# GitLab Self-Hosted Runner Script
+# Reference: https://docs.gitlab.com/runner/
 
-USAGE_HELP="Usage: $0 <AZP_IMAGE> <AZP_URL> <AZP_TOKEN> <AZP_POOL>"
+RUNNER_IMAGE="$1"
+GITLAB_URL="$2"
+GITLAB_TOKEN="$3"
+RUNNER_TAGS="${4:-docker,linux}"
+RUNNER_COUNT="${5}"
 
-# Test if AZP_IMAGE is empty, exit with error if true
-if [ -z "$AZP_IMAGE" ]; then
-  echo $USAGE_HELP
+USAGE_HELP="Usage: $0 <RUNNER_IMAGE> <GITLAB_URL> <GITLAB_TOKEN> [RUNNER_TAGS] [RUNNER_COUNT]
+
+Parameters:
+  RUNNER_IMAGE    - Docker image name for GitLab runner
+  GITLAB_URL      - GitLab instance URL
+                    Examples:
+                      - GitLab.com: https://gitlab.com
+                      - Self-hosted: https://gitlab.example.com
+  GITLAB_TOKEN    - GitLab runner registration token
+                    Can be found in:
+                      - Project: Settings > CI/CD > Runners
+                      - Group: Settings > CI/CD > Runners
+                      - Instance: Admin Area > CI/CD > Runners
+  RUNNER_TAGS     - Comma-separated tags for runner (default: 'docker,linux')
+  RUNNER_COUNT    - Number of runner instances (default: auto-detect from CPU count)
+
+Example:
+  $0 gitlab-runner:18.7.1 https://gitlab.com glrt-xxxxxxxxxxxx \"docker,linux,production\" 4
+"
+
+# Validate required parameters
+if [ -z "$RUNNER_IMAGE" ]; then
+  echo "Error: RUNNER_IMAGE is required"
+  echo "$USAGE_HELP"
   exit 1
 fi
 
-# Test if AZP_URL is empty, exit with error if true
-if [ -z "$AZP_URL" ]; then
-  echo $USAGE_HELP
+if [ -z "$GITLAB_URL" ]; then
+  echo "Error: GITLAB_URL is required"
+  echo "$USAGE_HELP"
   exit 1
 fi
 
-# Test if AZP_TOKEN is empty, exit with error if true
-if [ -z "$AZP_TOKEN" ]; then
-  echo $USAGE_HELP
+if [ -z "$GITLAB_TOKEN" ]; then
+  echo "Error: GITLAB_TOKEN is required"
+  echo "$USAGE_HELP"
   exit 1
 fi
 
-# Test if AZP_POOL is empty, exit with error if true
-if [ -z "$AZP_POOL" ]; then
-  echo $USAGE_HELP
+# Validate GitLab URL format
+if [[ ! "$GITLAB_URL" =~ ^https?://[^/]+/?$ ]]; then
+  echo "Error: Invalid GITLAB_URL format. Must be https://gitlab.com or https://gitlab.example.com"
   exit 1
 fi
 
 # Get total CPU count from the system
-CPU_COUNT=$(lscpu -p=CPU | grep -v "^#" | wc -l)
-# Limit the number of vCPU count per agent to 2 when there are more than 1 vCPU is available, cap it to 1 vCPU otherwise
-MAX_CPU=$(($CPU_COUNT>1 ? 2 : 1))
+CPU_COUNT=$(lscpu -p=CPU | grep -v "^#" | wc -l 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "2")
+
+# Set runner count (use provided value or default to CPU count)
+RUNNER_COUNT=${RUNNER_COUNT:-$CPU_COUNT}
+
+# Limit the number of vCPU count per runner to 2 when there are more than 1 vCPU available, cap it to 1 vCPU otherwise
+MAX_CPU=$((CPU_COUNT > 1 ? 2 : 1))
+
 # Get the Docker socket endpoint from current context
-DOCKER_SOCK_ENDPOINT=$(docker context inspect | jq -r '.[]|.Endpoints.docker.Host')
-
-for R in `seq 1 $CPU_COUNT`; do
-  sudo mkdir -p /mnt/agent${R}/w
-  sudo docker run \
-  --privileged \
-  --tty \
-  --detach \
-  --cpus="${MAX_CPU}" \
-  -e AZP_URL="$AZP_URL" \
-  -e AZP_TOKEN="$AZP_TOKEN" \
-  -e AZP_POOL="$AZP_POOL" \
-  -e AZP_AGENT_NAME=sha-`hostname`-$R \
-  -v ${DOCKER_SOCK_ENDPOINT#unix://*}:/var/run/docker.sock \
-  -v /mnt/agent${R}/w:/_work \
-  --restart always \
-  --name agent$R \
-  $AZP_IMAGE
+DOCKER_SOCK_ENDPOINT=$(docker context inspect 2>/dev/null | jq -r '.[]|.Endpoints.docker.Host' || echo "unix:///var/run/docker.sock")
+
+# Extract socket path
+DOCKER_SOCK_PATH=${DOCKER_SOCK_ENDPOINT#unix://}
+DOCKER_SOCK_PATH=${DOCKER_SOCK_PATH:-/var/run/docker.sock}
+
+echo "Starting $RUNNER_COUNT GitLab runner(s)..."
+echo "Image: $RUNNER_IMAGE"
+echo "GitLab URL: $GITLAB_URL"
+echo "Tags: $RUNNER_TAGS"
+echo "CPUs per runner: $MAX_CPU"
+echo ""
+
+# Launch runners
+for R in $(seq 1 $RUNNER_COUNT); do
+  RUNNER_NAME="gitlab-runner-$(hostname)-$R"
+  CONFIG_DIR="/mnt/gitlab-runner${R}/config"
+  DATA_DIR="/mnt/gitlab-runner${R}/data"
+  CONTAINER_NAME="gitlab-runner-$R"
+  
+  # Create config and data directories
+  sudo mkdir -p "$CONFIG_DIR"
+  sudo mkdir -p "$DATA_DIR"
+  
+  echo "Starting runner $R/$RUNNER_COUNT: $RUNNER_NAME"
+  
+  # Check if container already exists
+  if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    echo "  Removing existing container: $CONTAINER_NAME"
+    docker rm -f "$CONTAINER_NAME" > /dev/null 2>&1 || true
+  fi
+  
+  # Run GitLab runner container
+  docker run \
+    --privileged \
+    --tty \
+    --detach \
+    --cpus="${MAX_CPU}" \
+    -e GITLAB_URL="$GITLAB_URL" \
+    -e GITLAB_TOKEN="$GITLAB_TOKEN" \
+    -e RUNNER_NAME="$RUNNER_NAME" \
+    -e RUNNER_TAGS="$RUNNER_TAGS" \
+    -e RUNNER_EXECUTOR="docker" \
+    -e RUNNER_DOCKER_IMAGE="alpine:latest" \
+    -e RUNNER_RUN_UNTAGGED="false" \
+    -e RUNNER_LOCKED="false" \
+    -e RUNNER_ACCESS_LEVEL="not_protected" \
+    -v "$CONFIG_DIR":/etc/gitlab-runner \
+    -v "$DATA_DIR":/runner \
+    -v "$DOCKER_SOCK_PATH":/var/run/docker.sock \
+    --restart unless-stopped \
+    --name "$CONTAINER_NAME" \
+    "$RUNNER_IMAGE"
+  
+  echo "  Container $CONTAINER_NAME started successfully"
 done
+
+echo ""
+echo "All runners started successfully!"
+echo ""
+echo "To check runner status:"
+echo "  docker ps --filter name=gitlab-runner"
+echo ""
+echo "To view runner logs:"
+echo "  docker logs -f gitlab-runner-1"
+echo ""
+echo "To verify runner registration in GitLab:"
+echo "  Go to your project/group/instance Settings > CI/CD > Runners"
+echo ""
+echo "To stop all runners:"
+echo "  ./stop.sh"

stop.sh

@@ -1,23 +1,37 @@
 #!/bin/bash
+set -e
 
-CPU_COUNT=$(lscpu -p=CPU | grep -v "^#" | wc -l)
-MAX_CPU=$(($CPU_COUNT>1 ? 2 : 1))
+# GitLab Runner Stop Script
+# Gracefully stops and removes all GitLab runner containers
 
-# Get URL and PAT from first running instance:
-eval $(sudo docker inspect agent1 | jq -r '. [] | . | .Config.Env[]' | grep "AZP_TOKEN\|AZP_URL")
+echo "Stopping GitLab runners..."
+echo ""
 
-# Graceful agent shutdown
-for R in `seq 1 $MAX_CPU`; do
-  sudo docker exec -ti \
-    -e VSTS_AGENT_INPUT_AUTH="pat" \
-    -e VSTS_AGENT_INPUT_URL="$AZP_URL" \
-    -e VSTS_AGENT_INPUT_TOKEN="$AZP_TOKEN" \
-    agent$R \
-    ./config.sh remove --unattended \
-  && sudo docker stop agent$R \
-  && sudo docker rm agent$R &
-done
+# Get list of running runner containers
+RUNNER_CONTAINERS=$(docker ps --filter "name=gitlab-runner-" --format "{{.Names}}" | sort)
+
+if [ -z "$RUNNER_CONTAINERS" ]; then
+  echo "No running GitLab runner containers found."
+  exit 0
+fi
 
-wait
+CONTAINER_COUNT=$(echo "$RUNNER_CONTAINERS" | wc -l | tr -d ' ')
+echo "Found $CONTAINER_COUNT runner container(s)"
+echo ""
+
+# Stop each runner gracefully
+for CONTAINER_NAME in $RUNNER_CONTAINERS; do
+  echo "Stopping $CONTAINER_NAME..."
+  
+  # Try graceful shutdown first (runner will unregister itself via start.sh cleanup)
+  docker stop -t 30 "$CONTAINER_NAME" > /dev/null 2>&1 || true
+  
+  # Remove the container
+  docker rm "$CONTAINER_NAME" > /dev/null 2>&1 || true
+  
+  echo "  $CONTAINER_NAME stopped and removed"
+done
 
+echo ""
+echo "All GitLab runners stopped successfully!"
 exit 0

test-tools.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+# Azure VMSS Scheduled Events Monitor
+# Monitors Azure VM Scale Set scheduled events for termination signals
+# Reference: https://learn.microsoft.com/en-us/azure/virtual-machines/linux/scheduled-events
+
+METADATA_ENDPOINT='http://169.254.169.254/metadata/scheduledevents?api-version=2020-07-01'
+EVENTS_FILE='/tmp/scheduledevents.json'
+STOP_SCRIPT='/opt/stop.sh'
+
+echo "Checking for VMSS scheduled events..."
+
+# Query Azure Instance Metadata Service for scheduled events
+curl -s "$METADATA_ENDPOINT" -H 'Metadata: true' > "$EVENTS_FILE"
+
+# Check if termination event is scheduled
+if grep -q "Terminate" "$EVENTS_FILE"; then
+  echo "Terminate event detected! Initiating graceful shutdown..."
+  
+  # Start agent shutdown, if script exists and is executable
+  if [ -x "$STOP_SCRIPT" ]; then
+    echo "Executing stop script: $STOP_SCRIPT"
+    "$STOP_SCRIPT"
+  else
+    echo "Warning: Stop script not found or not executable: $STOP_SCRIPT"
+  fi
+
+  # Acknowledge the event to Azure
+  EventId=$(jq -r '.Events[] | select(.EventType == "Terminate") | .EventId' "$EVENTS_FILE")
+  if [ -n "$EventId" ]; then
+    echo "Acknowledging event: $EventId"
+    curl -s -X POST "$METADATA_ENDPOINT" \
+      -H 'Metadata: true' \
+      -d "{\"StartRequests\": [{\"EventId\": \"${EventId}\"}]}"
+    echo "Event acknowledged successfully"
+  fi
+else
+  echo "No termination events scheduled"
+fi
+
+exit 0