hack: scripts and helpers

bnallapeta · bnallapeta · commit f7dfd16f69c3 · 2025-08-03T09:35:02.000+05:30
Signed-off-by: Bharath Nallapeta &lt;bnallapeta@mirantis.com&gt;
diff --git a/.gopls.yaml b/.gopls.yaml
@@ -0,0 +1,5 @@
+build:
+  buildFlags:
+    - "-tags=e2e"
+  env:
+    CGO_ENABLED: "1" 
diff --git a/cleanup-hcp-test.sh b/cleanup-hcp-test.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Cleanup script for failed HCP test resources
+
+# Set OpenStack environment
+export OS_CLOUD=openstack
+export OS_CLOUD_YAML_FILE=/Users/bnr/work/openstack/clouds.yaml
+
+echo "🧹 Cleaning up HCP test resources..."
+
+# Delete the specific instances from your screenshot
+echo "Deleting OpenStack instances..."
+openstack server delete hcp-mgmt-hcp-mgmt-hcp-1752250951-al4m23-bastion
+openstack server delete hcp-mgmt-hcp-mgmt-hcp-1752250951-al4m23-control-plane-sinpp5-bastion
+
+# Clean up any floating IPs that might be allocated
+echo "Cleaning up floating IPs..."
+openstack floating ip list --status DOWN -f value -c ID | xargs -r openstack floating ip delete
+
+# Clean up security groups (if any were created)
+echo "Cleaning up security groups..."
+openstack security group list --project $(openstack token issue -c project_id -f value) | grep "hcp-mgmt\|cluster-api" | awk '{print $2}' | xargs -r openstack security group delete
+
+# Clean up keypairs (if any were created)
+echo "Cleaning up keypairs..."
+openstack keypair list | grep "cluster-api-provider-openstack-sigs-k8s-io" | awk '{print $2}' | xargs -r openstack keypair delete
+
+# Clean up networks and subnets (if any were created)
+echo "Cleaning up networks..."
+openstack network list | grep "hcp-mgmt\|cluster-api" | awk '{print $2}' | xargs -r openstack network delete
+
+# Clean up any volumes
+echo "Cleaning up volumes..."
+openstack volume list --status available | grep "hcp-mgmt\|cluster-api" | awk '{print $2}' | xargs -r openstack volume delete
+
+echo "✅ Cleanup completed!"
+echo ""
+echo "Manual verification commands:"
+echo "openstack server list"
+echo "openstack floating ip list"
+echo "openstack security group list"
+echo "openstack network list" 
diff --git a/cleanup-proper-order.sh b/cleanup-proper-order.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+# Proper OpenStack cleanup script - deletes in correct dependency order
+
+export OS_CLOUD=openstack
+export OS_CLOUD_YAML_FILE=/Users/bnr/work/openstack/clouds.yaml
+
+echo "🧹 Starting proper OpenStack cleanup (dependency order)..."
+
+# 1. First, delete servers (instances)
+echo "=== Step 1: Deleting Servers ==="
+openstack server list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting server: $name ($id)"
+    openstack server delete "$id" || true
+done
+
+# Wait a bit for servers to be deleted
+echo "Waiting for servers to be deleted..."
+sleep 10
+
+# 2. Delete floating IPs
+echo "=== Step 2: Deleting Floating IPs ==="
+openstack floating ip list -f value -c ID | xargs -r -I {} bash -c 'echo "Deleting floating IP: {}"; openstack floating ip delete {} || true'
+
+# 3. Delete load balancers (if any)
+echo "=== Step 3: Deleting Load Balancers ==="
+openstack loadbalancer list -f value -c id -c name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting loadbalancer: $name ($id)"
+    openstack loadbalancer delete "$id" --cascade || true
+done
+
+# Wait for load balancers to be deleted
+echo "Waiting for load balancers to be deleted..."
+sleep 15
+
+# 4. Delete router interfaces and routers
+echo "=== Step 4: Deleting Routers ==="
+openstack router list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Processing router: $name ($id)"
+    
+    # First remove all interfaces from the router
+    echo "  Removing interfaces from router $name"
+    openstack port list --router "$id" -f value -c ID | while read port_id; do
+        echo "    Removing interface $port_id"
+        openstack router remove port "$id" "$port_id" || true
+    done
+    
+    # Then delete the router
+    echo "  Deleting router $name"
+    openstack router delete "$id" || true
+done
+
+# 5. Delete ports
+echo "=== Step 5: Deleting Ports ==="
+openstack port list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting port: $name ($id)"
+    openstack port delete "$id" || true
+done
+
+# 6. Delete subnets
+echo "=== Step 6: Deleting Subnets ==="
+openstack subnet list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting subnet: $name ($id)"
+    openstack subnet delete "$id" || true
+done
+
+# 7. Finally, delete networks
+echo "=== Step 7: Deleting Networks ==="
+openstack network list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting network: $name ($id)"
+    openstack network delete "$id" || true
+done
+
+# 8. Delete security groups
+echo "=== Step 8: Deleting Security Groups ==="
+openstack security group list -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting security group: $name ($id)"
+    openstack security group delete "$id" || true
+done
+
+# 9. Delete keypairs
+echo "=== Step 9: Deleting Keypairs ==="
+openstack keypair list -f value -c Name | grep -E "(hcp|e2e|cluster-api)" | while read name; do
+    echo "Deleting keypair: $name"
+    openstack keypair delete "$name" || true
+done
+
+# 10. Delete volumes
+echo "=== Step 10: Deleting Volumes ==="
+openstack volume list --status available -f value -c ID -c Name | grep -E "(hcp|e2e|cluster-api)" | while read id name; do
+    echo "Deleting volume: $name ($id)"
+    openstack volume delete "$id" || true
+done
+
+echo "✅ Cleanup completed!"
+echo ""
+echo "Verification commands:"
+echo "openstack server list"
+echo "openstack network list | grep -E '(hcp|e2e|cluster-api)'"
+echo "openstack router list | grep -E '(hcp|e2e|cluster-api)'"
+echo "openstack security group list | grep -E '(hcp|e2e|cluster-api)'" 
diff --git a/cleanup_openstack.sh b/cleanup_openstack.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+echo "=== Cleaning up Security Groups ==="
+openstack security group list -f value -c ID -c Name | grep -E "(hcp|k8s-cluster.*e2e)" | awk '{print $1}' | xargs -I {} openstack security group delete {}
+
+echo "=== Cleaning up Load Balancers ==="
+openstack loadbalancer list -f value -c id -c name | grep -E "(hcp|e2e)" | awk '{print $1}' | xargs -I {} openstack loadbalancer delete {} || true
+
+echo "=== Cleaning up Routers ==="
+openstack router list -f value -c ID -c Name | grep -E "(hcp|e2e)" | while read router_id router_name; do
+  echo "Cleaning router: $router_name ($router_id)"
+  # Remove external gateway
+  openstack router unset --external-gateway $router_id 2>/dev/null || true
+  # Remove all ports
+  openstack port list --router $router_id -f value -c ID | xargs -I {} openstack router remove port $router_id {} 2>/dev/null || true
+  # Delete router
+  openstack router delete $router_id
+done
+
+echo "=== Cleaning up Subnets ==="
+openstack subnet list -f value -c ID -c Name | grep -E "(hcp|e2e)" | awk '{print $1}' | xargs -I {} openstack subnet delete {}
+
+echo "=== Cleaning up Networks ==="
+openstack network list -f value -c ID -c Name | grep -E "(hcp|e2e)" | awk '{print $1}' | xargs -I {} openstack network delete {}
+
+echo "=== Cleaning up Floating IPs ==="
+openstack floating ip list -f value -c ID -c Description | grep -E "(hcp|e2e|cluster)" | awk '{print $1}' | xargs -I {} openstack floating ip delete {} || true
+
+echo "=== Cleanup Complete ==="
diff --git a/hcp-test-vars.sh b/hcp-test-vars.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Complete environment variables for CAPO HCP testing
+# Usage: source hcp-test-vars.sh && make test-hcp
+
+# CRITICAL: Kubernetes version must match your image
+# Check your ubuntu-22.04 image and set accordingly (probably v1.28.x, v1.29.x, or v1.30.x)
+export KUBERNETES_VERSION="v1.30.2"  # CHANGE THIS to match your image's k8s version
+
+# OpenStack Configuration
+export OPENSTACK_CLOUD="openstack"
+export OPENSTACK_CLOUD_ADMIN="openstack"  # Same as OPENSTACK_CLOUD for most setups
+export OPENSTACK_CLOUD_YAML_FILE="/Users/bnr/work/openstack/clouds.yaml"
+
+# Image Configuration - Use existing image with dummy URLs to pass validation
+export OPENSTACK_IMAGE_NAME="ubuntu-2404-kube-v1.33.1"
+export OPENSTACK_IMAGE_URL="file:///dev/null"  # Dummy URL - image already exists
+export OPENSTACK_BASTION_IMAGE_NAME="ubuntu-2404-kube-v1.33.1"  # Use same image for bastion
+export OPENSTACK_BASTION_IMAGE_URL="file:///dev/null"  # Dummy URL - image already exists
+
+# Flavor Configuration - ADJUST based on your OpenStack
+export OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR="m1.medium"
+export OPENSTACK_NODE_MACHINE_FLAVOR="m1.small"
+export OPENSTACK_BASTION_MACHINE_FLAVOR="m1.small"
+
+# Network Configuration - ADJUST based on your OpenStack
+export OPENSTACK_EXTERNAL_NETWORK_NAME="public"  # Check: openstack network list --external
+export OPENSTACK_DNS_NAMESERVERS="8.8.8.8"  # or your preferred DNS
+export OPENSTACK_FAILURE_DOMAIN="nova"  # Check: openstack availability zone list
+
+# SSH Key - CRITICAL
+export OPENSTACK_SSH_KEY_NAME="cluster-api-provider-openstack-sigs-k8s-io"  # Must exist in OpenStack
+
+# HCP Specific Configuration
+export KAMAJI_VERSION="v0.15.3"  # Stable version instead of edge
+export KAMAJI_NAMESPACE="kamaji-system"
+export CLUSTER_DATASTORE="default"
+export HCP_SERVICE_TYPE="LoadBalancer"
+export HCP_CPU_LIMIT="1000m"
+export HCP_MEMORY_LIMIT="1Gi"
+export HCP_CPU_REQUEST="100m"
+export HCP_MEMORY_REQUEST="300Mi"
+
+# Test Configuration
+export E2E_GINKGO_FOCUS="Management cluster verification"
+
+# Timeout adjustments for slower environments
+export GINKGO_ARGS="-v --progress --timeout=45m"
+
+echo "✅ Environment variables set for HCP testing"
+echo "🔍 Key settings:"
+echo "   Kubernetes Version: $KUBERNETES_VERSION"
+echo "   Image Name: $OPENSTACK_IMAGE_NAME"
+echo "   Control Plane Flavor: $OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR"
+echo "   External Network: $OPENSTACK_EXTERNAL_NETWORK_NAME"
+echo "   SSH Key: $OPENSTACK_SSH_KEY_NAME"
+echo ""
+echo "⚠️  VERIFY these match your OpenStack environment:"
+echo "   1. Check image exists: openstack image show $OPENSTACK_IMAGE_NAME"
+echo "   2. Check flavors exist: openstack flavor show $OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR"
+echo "   3. Check SSH key exists: openstack keypair show $OPENSTACK_SSH_KEY_NAME"
+echo "   4. Check external network: openstack network show $OPENSTACK_EXTERNAL_NETWORK_NAME"
+echo ""
+echo "🚀 Run: make test-hcp" 
diff --git a/test/e2e/suites/hcp/README.md b/test/e2e/suites/hcp/README.md
@@ -0,0 +1,104 @@
+# HCP (Hosted Control Plane) E2E Tests
+
+This directory contains end-to-end tests for Hosted Control Plane (HCP) functionality using Kamaji as the control plane provider.
+
+## Overview
+
+The HCP tests verify that:
+
+1. **Management Cluster**: A traditional CAPO cluster can be configured to host control planes for other clusters
+2. **Workload Cluster**: Worker-only clusters can be created with external control planes managed by Kamaji
+3. **Graceful Failures**: Broken configurations fail gracefully without panics or nil pointer exceptions
+
+## Test Structure
+
+### Test Suites
+
+- **Management Cluster Test**: Creates a CAPO cluster with HCP hosting capabilities
+- **Workload Cluster Test**: Creates worker-only clusters using KamajiControlPlane  
+- **Graceful Failure Tests**: Validates error handling for broken configurations
+
+### Test Flow
+
+1. **Suite Setup**: Creates shared management cluster with Kamaji installed
+2. **Workload Tests**: Creates multiple workload clusters using the shared management cluster
+3. **Failure Tests**: Tests broken scenarios to ensure graceful error handling
+4. **Suite Cleanup**: Cleans up shared resources
+
+## Running Tests
+
+### Prerequisites
+
+- OpenStack environment configured
+- `OPENSTACK_CLOUD_YAML_FILE` environment variable set
+- Docker installed ([[memory:2673423]])
+
+### Run HCP Tests
+
+```bash
+# Run all HCP tests
+make test-hcp
+
+# Run with specific focus
+E2E_GINKGO_FOCUS="Management cluster" make test-hcp
+
+# Run with existing cluster
+E2E_ARGS="-use-existing-cluster=true" make test-hcp
+```
+
+### Test Configuration
+
+Tests use the same configuration as other e2e tests:
+- Config: `test/e2e/data/e2e_conf.yaml` 
+- Templates: `test/e2e/data/kustomize/hcp-*`
+- Artifacts: `_artifacts/` directory
+
+## Template Structure
+
+### HCP Management (`hcp-management`)
+- Traditional CAPO cluster with larger worker nodes
+- Additional security rules for hosting control planes
+- Kamaji installation and configuration
+
+### HCP Workload (`hcp-workload`) 
+- Worker-only cluster configuration
+- Uses `KamajiControlPlane` instead of `KubeadmControlPlane`
+- Different network CIDRs to avoid conflicts
+
+### HCP Broken (`hcp-broken`)
+- Intentionally broken networking configuration
+- Used to test graceful failure scenarios
+
+## Test Intervals
+
+HCP tests use dedicated intervals defined in `e2e_conf.yaml`:
+
+```yaml
+intervals:
+  hcp/wait-kamaji-install: ["10m", "30s"]
+  hcp/wait-kamaji-control-plane: ["15m", "30s"] 
+  hcp/wait-cluster: ["25m", "10s"]
+  hcp/wait-control-plane: ["30m", "10s"]
+  hcp/wait-worker-nodes: ["30m", "10s"]
+```
+
+## Debugging
+
+### Log Collection
+Logs are automatically collected in `_artifacts/clusters/` for failed tests.
+
+### Manual Debug
+Use the `skip-cleanup` flag to preserve resources for investigation:
+
+```bash
+E2E_ARGS="-skip-cleanup=true" make test-hcp
+```
+
+### Kamaji Resources
+Check Kamaji-specific resources:
+
+```bash
+kubectl get kamajicontrolplane -A
+kubectl get datastore -A  
+kubectl get -n kamaji-system pods
+``` 
