e2e: implement e2e suite for hcp using kamaji

Signed-off-by: Bharath Nallapeta <[email protected]>

Author: bnallapeta

Makefile

@@ -236,6 +236,16 @@ test-conformance: $(GINKGO) e2e-prerequisites ## Run clusterctl based conformanc
 test-conformance-fast: ## Run clusterctl based conformance test on workload cluster (requires Docker) using a subset of the conformance suite in parallel.
 	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)"
 
+# Run HCP tests - Kamaji v1.0.0 is installed via Helm during test setup
+HCP_E2E_ARGS ?= $(E2E_ARGS)
+.PHONY: test-hcp
+test-hcp: $(GINKGO) e2e-prerequisites ## Run hosted control plane (HCP) tests using Kamaji
+	time $(GINKGO) -fail-fast -trace -timeout=4h -show-node-events -v -tags=e2e \
+		--output-dir="$(ARTIFACTS)" --junit-report="junit.hcp_suite.1.xml" \
+		-focus="Hosted Control Plane tests" $(E2E_GINKGO_ARGS) ./test/e2e/suites/hcp/... -- \
+			-config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" \
+			-data-folder="$(E2E_DATA_DIR)" $(HCP_E2E_ARGS)
+
 APIDIFF_OLD_COMMIT ?= $(shell git rev-parse origin/main)
 
 .PHONY: apidiff

docs/book/src/development/development.md

@@ -562,3 +562,84 @@ kubectl get openstackservers
 ```
 
 This object is immutable and is created by the controller when a machine or a bastion is created. The `OpenStackServer` object is deleted when the machine or the bastion is deleted.
+
+## Hosted Control Plane (HCP) Testing
+
+CAPO supports testing Hosted Control Planes using Kamaji as the control plane provider. This allows testing scenarios where the Kubernetes control plane runs as pods in a management cluster while worker nodes run on OpenStack infrastructure.
+
+### Prerequisites
+
+* A working OpenStack environment (same requirements as regular e2e tests)
+* Helm 3.x installed locally
+* Management cluster with sufficient resources for hosting control planes
+
+### Running HCP Tests
+
+To run the hosted control plane e2e tests:
+
+```bash
+make test-hcp OPENSTACK_CLOUD_YAML_FILE=/path/to/clouds.yaml OPENSTACK_CLOUD=my_cloud
+```
+
+This will:
+1. Create a management cluster using kind
+2. Install CAPO in the management cluster
+3. Install Kamaji v1.0.0 using Helm
+4. Install the Kamaji Control Plane Provider v0.15.3
+5. Run HCP-specific test suites that validate:
+   - Network configuration fixes for HCP scenarios
+   - Security group precedence between machine and cluster levels
+   - Worker node connectivity to hosted control planes via Konnectivity
+   - Machine spec validation without explicit networks
+
+### Architecture
+
+The HCP testing follows this architecture:
+
+```
+Management Cluster (kind)
+├── CAPO Controller
+├── Kamaji v1.0.0 
+├── Kamaji Control Plane Provider v0.15.3
+└── TenantControlPlanes (running as pods)
+    └── Connected to OpenStack worker nodes via Konnectivity
+```
+
+### Test Flavors
+
+HCP tests use the `hcp` flavor, which:
+- Uses `KamajiControlPlane` instead of `KubeadmControlPlane`
+- Configures Konnectivity for worker node communication
+- Sets up proper security groups for HCP networking
+- Validates the network configuration fixes from the `hcp-2380` branch
+
+### Manual Installation
+
+If you need to install Kamaji manually for development:
+
+```bash
+# Install Kamaji using Helm
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install kamaji clastix/kamaji --version v1.0.0 --namespace kamaji-system --create-namespace --wait
+
+# Or use the convenience script
+./hack/install-kamaji.sh
+```
+
+### Troubleshooting
+
+**Worker nodes can't connect to control plane:**
+- Check Konnectivity service is running
+- Verify security groups allow traffic on port 8132
+- Ensure LoadBalancer service is properly configured
+
+**TenantControlPlane fails to start:**
+- Check Kamaji controller logs: `kubectl logs -n kamaji-system -l app.kubernetes.io/name=kamaji`
+- Verify DataStore is ready: `kubectl get datastore -n kamaji-system`
+- Check etcd connectivity in the management cluster
+
+**Network validation tests fail:**
+- Ensure OpenStack cloud has proper networking configuration
+- Verify security groups are properly configured
+- Check that external network is accessible for LoadBalancer services

hack/install-kamaji.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# Simple Kamaji installation using Helm
+# This script installs Kamaji v1.0.0 in the management cluster
+
+KAMAJI_VERSION="v1.0.0"
+KAMAJI_NAMESPACE="kamaji-system"
+
+echo "Installing Kamaji ${KAMAJI_VERSION}..."
+
+# Add Clastix Helm repository
+echo "Adding Clastix Helm repository..."
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+
+# Install Kamaji
+echo "Installing Kamaji in namespace ${KAMAJI_NAMESPACE}..."
+helm install kamaji clastix/kamaji \
+  --version ${KAMAJI_VERSION} \
+  --namespace ${KAMAJI_NAMESPACE} \
+  --create-namespace \
+  --wait
+
+# Verify installation
+echo "Verifying Kamaji installation..."
+kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=kamaji -n ${KAMAJI_NAMESPACE} --timeout=300s
+
+# Create default datastore
+echo "Creating default datastore..."
+cat <<EOF | kubectl apply -f -
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: default
+  namespace: ${KAMAJI_NAMESPACE}
+spec:
+  driver: etcd
+  endpoints:
+  - kamaji-etcd.${KAMAJI_NAMESPACE}.svc.cluster.local:2379
+EOF
+
+echo "Kamaji installation completed successfully!"
+echo "Ready to create TenantControlPlanes with the 'default' datastore." 

templates/cluster-template-hcp.yaml

@@ -0,0 +1,149 @@
+apiVersion: v1
+data:
+  cacert: ${OPENSTACK_CLOUD_CACERT_B64}
+  clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64}
+kind: Secret
+metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+  name: ${CLUSTER_NAME}-cloud-config
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      files: []
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+            cloud-provider: external
+            provider-id: openstack:///'{{ instance_id }}'
+          name: '{{ local_hostname }}'
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 192.168.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+    kind: KamajiControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: OpenStackCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels: null
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      clusterName: ${CLUSTER_NAME}
+      failureDomain: ${OPENSTACK_FAILURE_DOMAIN}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: OpenStackMachineTemplate
+        name: ${CLUSTER_NAME}-md-0
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: KamajiControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  dataStoreName: ${CLUSTER_DATASTORE:-default}
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      enabled: true
+      proxyMode: HTTPConnect
+  kubelet:
+    cgroupfs: systemd
+    preferredAddressTypes:
+    - InternalIP
+    - ExternalIP
+  network:
+    serviceType: LoadBalancer
+    serviceAnnotations:
+      service.beta.kubernetes.io/openstack-internal-load-balancer: "false"
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT:-1}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: OpenStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  apiServerLoadBalancer:
+    enabled: false  # Disabled since Kamaji provides its own LoadBalancer service
+  externalNetwork:
+    id: ${OPENSTACK_EXTERNAL_NETWORK_ID}
+  identityRef:
+    cloudName: ${OPENSTACK_CLOUD}
+    name: ${CLUSTER_NAME}-cloud-config
+  managedSecurityGroups:
+    allNodesSecurityGroupRules:
+    - description: Created by cluster-api-provider-openstack - BGP (calico)
+      direction: ingress
+      etherType: IPv4
+      name: BGP (Calico)
+      portRangeMax: 179
+      portRangeMin: 179
+      protocol: tcp
+      remoteManagedGroups:
+      - worker
+    - description: Created by cluster-api-provider-openstack - IP-in-IP (calico)
+      direction: ingress
+      etherType: IPv4
+      name: IP-in-IP (calico)
+      protocol: "4"
+      remoteManagedGroups:
+      - worker
+    # Konnectivity for Kamaji hosted control plane communication
+    - description: Created by cluster-api-provider-openstack - Konnectivity (Kamaji HCP)
+      direction: ingress
+      etherType: IPv4
+      name: Konnectivity (Kamaji HCP)
+      portRangeMax: 8132
+      portRangeMin: 8132
+      protocol: tcp
+      remoteManagedGroups:
+      - worker
+  managedSubnets:
+  - cidr: 10.6.0.0/24
+    dnsNameservers:
+    - ${OPENSTACK_DNS_NAMESERVERS}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: OpenStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR}
+      image:
+        filter:
+          name: ${OPENSTACK_IMAGE_NAME}
+      sshKeyName: ${OPENSTACK_SSH_KEY_NAME} 

test/e2e/data/e2e_conf.yaml

@@ -92,6 +92,20 @@ providers:
       new: "imagePullPolicy: IfNotPresent"
     - old: "--leader-elect"
       new: "--leader-elect=false\n        - --sync-period=1m"
+- name: kamaji
+  type: ControlPlaneProvider
+  versions:
+  - name: v0.15.3
+    value: "https://github.com/clastix/cluster-api-control-plane-provider-kamaji/releases/download/v0.15.3/control-plane-components.yaml"
+    type: url
+    contract: v1beta1
+    files:
+    - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+    replacements:
+    - old: "imagePullPolicy: Always"
+      new: "imagePullPolicy: IfNotPresent"
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
 - name: openstack
   type: InfrastructureProvider
   versions:
@@ -134,6 +148,7 @@ providers:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
     - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
     - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template-without-lb.yaml"
+    - sourcePath: "../../../../templates/cluster-template-hcp.yaml"
     replacements:
     - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev
       new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
@@ -233,3 +248,6 @@ intervals:
   default/wait-machine-remediation: ["10m", "10s"]
   default/wait-image-create: ["15m", "10s"]
   default/wait-image-delete: ["2m", "10s"]
+  hcp/wait-cluster: ["30m", "10s"]
+  hcp/wait-control-plane: ["45m", "10s"]
+  hcp/wait-worker-nodes: ["45m", "10s"]

test/e2e/data/kustomize/hcp/kamaji-controlplane.yaml

@@ -0,0 +1,41 @@
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: KamajiControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+  labels:
+    clusterctl.cluster.x-k8s.io/move: ""
+spec:
+  dataStore: default
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+  kubelet:
+    cgroupfs: systemd
+    preferredAddressTypes:
+    - ExternalIP
+    - InternalIP
+  network:
+    serviceType: LoadBalancer
+    ingress:
+      hostname: ${CLUSTER_NAME}-lb.${OPENSTACK_DNS_NAMESERVERS}
+    certSANs:
+    - ${CLUSTER_NAME}-lb.${OPENSTACK_DNS_NAMESERVERS}
+  deployment:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 250m
+        memory: 512Mi
+      limits:
+        cpu: 500m
+        memory: 1Gi
+    extraArgs:
+      # Enable Konnectivity for worker node communication
+      - --enable-aggregator-routing=true
+      - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
+      - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
+    # Add annotations for OpenStack cloud provider
+    annotations:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+      cluster.x-k8s.io/cluster-namespace: ${NAMESPACE}
+  version: ${KUBERNETES_VERSION}