kubernetes-sigs · k8s-ci-robot · Mar 20, 2025 · Jan 30, 2025 · Jan 10, 2025
diff --git a/Makefile b/Makefile
@@ -974,6 +974,10 @@ test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests
 kind-cluster: ## Create a new kind cluster designed for development with Tilt
 	hack/kind-install-for-capd.sh
 
+.PHONY: kind-cluster-kubevirt
+kind-cluster-kubevirt: ## Create a new kind cluster with KubeVirt designed for development with Tilt
+	hack/kind-install-for-capk.sh
+
 .PHONY: tilt-e2e-prerequisites
 tilt-e2e-prerequisites: ## Build the corresponding kindest/node images required for e2e testing and generate the e2e templates
 	scripts/build-kind.sh

diff --git a/docs/book/src/developer/core/tilt.md b/docs/book/src/developer/core/tilt.md
@@ -20,14 +20,73 @@ workflow that offers easy deployments and rapid iterative builds.
 ## Getting started
 
 ### Create a kind cluster
-A script to create a KIND cluster along with a local Docker registry and the correct mounts to run CAPD is included in the hack/ folder.
 
-To create a pre-configured cluster run:
+This guide offers instructions for using the following CAPI infrastructure providers for running a
+development environment without using real machines or cloud resources:
+
+- [CAPD](https://github.com/kubernetes-sigs/cluster-api/blob/main/test/infrastructure/docker/README.md) - uses Docker containers as workload cluster nodes
+- [CAPK](https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt) - uses KubeVirt VMs as workload cluster nodes
+
+CAPD is the default as it's more lightweight and requires less setup. KubeVirt is useful when
+Docker isn't suitable for whatever reason. Other infrastructure providers may be enabled as well
+(see [below](#create-a-tilt-settings-file)).
+
+{{#tabs name:"tab-management-cluster-creation" tabs:"Docker,KubeVirt"}}
+{{#tab Docker}}
+
+To create a kind cluster along with a local Docker registry and the correct mounts to run CAPD, run
+the following:
+
+```bash
+make kind-cluster
+```
+
+{{#/tab }}
+{{#tab KubeVirt}}
+
+To create a kind cluster with CAPK, run the following:
+
+```bash
+make kind-cluster-kubevirt
+```
+
+<aside class="note">
+
+KubeVirt uses *container disks* to create VMs inside pods. These are special container images which
+need to be pulled from a registry. To support pulling container disks from private registries as
+well as avoid getting rate-limited by Docker Hub (if used), the CAPK script mounts your Docker
+config file inside the kind cluster to let the Kubelet access your credentials.
+
+The script looks for the Docker config file at `$HOME/.docker/config.json` by default. To specify
+a different path, set the following variable before running Make above:
 
 ```bash
-./hack/kind-install-for-capd.sh
+export DOCKER_CONFIG_FILE="/foo/config.json"
 ```
 
+</aside>
+
+<aside class="note">
+
+The CAPK script uses [MetalLB](https://metallb.org/) to expose the API servers of workload clusters
+on the local machine. The API servers are exposed as LoadBalancer services handled by MetalLB. For
+this to work, MetalLB needs to figure out your container runtime IP prefix. The script assumes
+Docker is used and figures the IP prefix out automatically. In case a different runtime is used,
+specify your container runtime's IP prefix manually (the first two octets only):
+
+```bash
+export CAPI_METALLB_IP_PREFIX="172.20"
+```
+
+The script uses 255.200-255.250 in the last two octets to set the range MetalLB should use to
+allocate IPs to LoadBalancer services. For example, for `172.20` the resulting IP range is
+`172.20.255.200-172.20.255.250`.
+
+</aside>
+
+{{#/tab }}
+{{#/tabs }}
+
 You can see the status of the cluster with:
 
 ```bash
@@ -36,7 +95,12 @@ kubectl cluster-info --context kind-capi-test
 
 ### Create a tilt-settings file
 
-Next, create a `tilt-settings.yaml` file and place it in your local copy of `cluster-api`. Here is an example that uses the components from the CAPI repo:
+Next, create a `tilt-settings.yaml` file and place it in your local copy of `cluster-api`.
+
+Here are some examples:
+
+{{#tabs name:"tab-tilt-settings" tabs:"Docker,KubeVirt"}}
+{{#tab Docker}}
 
 ```yaml
 default_registry: gcr.io/your-project-name-here
@@ -46,7 +110,33 @@ enable_providers:
 - kubeadm-control-plane
 ```
 
-To use tilt to launch a provider with its own repo, using Cluster API Provider AWS here, `tilt-settings.yaml` should look like:
+{{#/tab }}
+{{#tab KubeVirt}}
+
+```yaml
+enable_providers:
+- kubevirt
+- kubeadm-bootstrap
+- kubeadm-control-plane
+provider_repos:
+# Path to a local clone of CAPK (replace with actual path)
+- ../cluster-api-provider-kubevirt
+kustomize_substitutions:
+  # CAPK needs access to the containerd socket (replace with actual path)
+  CRI_PATH: "/var/run/containerd/containerd.sock"
+  KUBERNETES_VERSION: "v1.30.1"
+  # An example - replace with an appropriate container disk image for the desired k8s version
+  NODE_VM_IMAGE_TEMPLATE: "quay.io/capk/ubuntu-2204-container-disk:v1.30.1"
+# Allow deploying CAPK workload clusters from the Tilt UI (optional)
+template_dirs:
+  kubevirt:
+  - ../cluster-api-provider-kubevirt/templates
+```
+
+{{#/tab }}
+{{#/tabs }}
+
+Other infrastructure providers may be added to the cluster using local clones and a configuration similar to the following:
 
 ```yaml
 default_registry: gcr.io/your-project-name-here

diff --git a/hack/kind-install-for-capd.sh b/hack/kind-install-for-capd.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright 2021 The Kubernetes Authors.
+# Copyright 2025 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,33 +30,10 @@ if [[ "${TRACE-0}" == "1" ]]; then
     set -o xtrace
 fi
 
-KIND_CLUSTER_NAME=${CAPI_KIND_CLUSTER_NAME:-"capi-test"}
 # See: https://kind.sigs.k8s.io/docs/user/configuration/#ip-family
-KIND_NETWORK_IPFAMILY=${KIND_NETWORK_IPFAMILY:-"dual"}
+KIND_NETWORK_IPFAMILY=${CAPI_KIND_NETWORK_IPFAMILY:-"dual"}
 
-# 1. If kind cluster already exists exit.
-if [[ "$(kind get clusters)" =~ .*"${KIND_CLUSTER_NAME}".* ]]; then
-  echo "kind cluster already exists, moving on"
-  exit 0
-fi
-
-# 2. Create registry container unless it already exists
-reg_name='kind-registry'
-reg_port='5000'
-if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
-  docker run \
-    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
-    registry:2
-fi
-
-# 3. Create kind cluster with containerd registry config dir enabled.
-# TODO(killianmuldoon): kind will eventually enable this by default and this patch will be unnecessary.
-#
-# See:
-# https://github.com/kubernetes-sigs/kind/issues/2875
-# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
-# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
-cat <<EOF | kind create cluster --name="$KIND_CLUSTER_NAME"  --config=-
+KIND_CLUSTER_CONFIG="$(cat <<EOF
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
@@ -71,39 +48,7 @@ containerdConfigPatches:
   [plugins."io.containerd.grpc.v1.cri".registry]
     config_path = "/etc/containerd/certs.d"
 EOF
+)"
+export KIND_CLUSTER_CONFIG
 
-# 4. Add the registry config to the nodes
-#
-# This is necessary because localhost resolves to loopback addresses that are
-# network-namespace local.
-# In other words: localhost in the container is not localhost on the host.
-#
-# We want a consistent name that works from both ends, so we tell containerd to
-# alias localhost:${reg_port} to the registry container when pulling images
-REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
-for node in $(kind get nodes --name "$KIND_CLUSTER_NAME"); do
-  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
-  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
-[host."http://${reg_name}:5000"]
-EOF
-done
-
-# 5. Connect the registry to the cluster network if not already connected
-# This allows kind to bootstrap the network but ensures they're on the same network
-if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
-  docker network connect "kind" "${reg_name}"
-fi
-
-# 6. Document the local registry
-# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: local-registry-hosting
-  namespace: kube-public
-data:
-  localRegistryHosting.v1: |
-    host: "localhost:${reg_port}"
-    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
-EOF
+"$(dirname "${BASH_SOURCE[0]}")/kind-install.sh"
diff --git a/hack/kind-install-for-capk.sh b/hack/kind-install-for-capk.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script installs a local Kind cluster with a local container registry and the correct files
+# mounted for using CAPK to test Cluster API.
+# The default Kind CNI is disabled because it doesn't work CAPK. Instead, Calico is used.
+# MetalLB is used as a local layer 2 load balancer to support exposing the API servers of workload
+# clusters on the local machine that's running this script.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${TRACE-0}" == "1" ]]; then
+    set -o xtrace
+fi
+
+CALICO_VERSION=${CAPI_CALICO_VERSION:-"v3.29.1"}
+METALLB_VERSION=${CAPI_METALLB_VERSION:-""}
+# Set manually for non-Docker runtimes - example: "172.20"
+METALLB_IP_PREFIX=${CAPI_METALLB_IP_PREFIX:-""}
+KUBEVIRT_VERSION=${CAPI_KUBEVIRT_VERSION:-""}
+# Required to support pulling KubeVirt container disk images from private registries as well as
+# avoid Docker Hub rate limiting
+KIND_DOCKER_CONFIG_PATH=${CAPI_KIND_DOCKER_CONFIG_PATH:-"$HOME/.docker/config.json"}
+
+# Deploy Kind cluster
+KIND_CLUSTER_CONFIG="$(cat <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  disableDefaultCNI: true
+nodes:
+- role: control-plane
+  extraMounts:
+  - containerPath: /var/lib/kubelet/config.json
+    hostPath: ${KIND_DOCKER_CONFIG_PATH}
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+EOF
+)"
+export KIND_CLUSTER_CONFIG
+
+"$(dirname "${BASH_SOURCE[0]}")/kind-install.sh"
+
+# Deploy Calico
+kubectl apply -f \
+  "https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml"
+
+# Deploy MetalLB
+if [[ -z "$METALLB_VERSION" ]]; then
+  METALLB_VERSION=$(curl "https://api.github.com/repos/metallb/metallb/releases/latest" \
+    | jq -r ".tag_name")
+fi
+
+kubectl apply -f \
+  "https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/config/manifests/metallb-native.yaml"
+
+echo "Waiting for MetalLB controller pod to be created..."
+kubectl wait -n metallb-system deployment controller --for condition=available --timeout 5m
+echo "MetalLB controller pod created!"
+
+echo "Waiting for all MetalLB pods to become ready..."
+kubectl wait pods -n metallb-system -l app=metallb,component=controller --for condition=Ready --timeout 5m
+kubectl wait pods -n metallb-system -l app=metallb,component=speaker --for condition=Ready --timeout 5m
+echo "MetalLB pods ready!"
+
+if [[ -z "$METALLB_IP_PREFIX" ]]; then
+  SUBNET=$(docker network inspect \
+    -f '{{range .IPAM.Config}}{{if .Gateway}}{{.Subnet}}{{end}}{{end}}' kind)
+  METALLB_IP_PREFIX=$(echo "$SUBNET" | sed -E 's|^([0-9]+\.[0-9]+)\..*$|\1|g')
+fi
+
+cat <<EOF | kubectl apply -f -
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: capi-ip-pool
+  namespace: metallb-system
+spec:
+  addresses:
+  - ${METALLB_IP_PREFIX}.255.200-${METALLB_IP_PREFIX}.255.250
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: empty
+  namespace: metallb-system
+EOF
+
+# Deploy KubeVirt
+if [[ -z "$KUBEVIRT_VERSION" ]]; then
+  KUBEVIRT_VERSION=$(curl "https://api.github.com/repos/kubevirt/kubevirt/releases/latest" \
+    | jq -r ".tag_name")
+fi
+kubectl apply -f \
+  "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml"
+kubectl apply -f \
+  "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml"
+echo "Waiting for KubeVirt to become ready..."
+kubectl wait -n kubevirt kv kubevirt --for=condition=Available --timeout=10m
+echo "KubeVirt ready!"