Skip to content

Commit 23be555

Browse files
toschnecktoschneck
authored
Merge pull request #155 from kubermatic/example/kubermatic-virtualization
add kubermatic-virtualization example
2 parents e7f9cef + 9fa7ae3 commit 23be555

26 files changed

+1392
-3
lines changed

‎README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ Dedicated components for customer purposes.
3131
| [vmware-exporter](components/vmware-exporter) | Helm chart for VMware Exporter and Dashboard for Prometheus and Grafana for monitoring of vSphere environments in the KKP MLA stack. |
3232
| [nutanix-exporter](components/nutanix-exporter) | Helm chart for [nutanix-exporter](https://github.com/claranet/nutnix-exporter) - exporter for Prometheus that can be used for monitoring of Nutanix-based environments. |
3333
| [user-cluster-alertmanager-alerts](components/user-cluster-alertmanager-alerts) | Set of user-cluster alert rules for usage with User-Cluster MLA. See cluster-mamangement-by-api to deploy Alertrules programatically. |
34-
| [user-cluster-grafana-dashboards](components/user-cluster-grafana-dashboards) | Set of user-cluster grafana dashboards for usage with User-Cluster MLA. |
35-
[kubevirt](components/kubevirt) | Help Components for e.g. installing kvm quemu packages on baremetal. |
34+
| [user-cluster-grafana-dashboards](components/user-cluster-grafana-dashboards) | Set of user-cluster grafana dashboards for usage with User-Cluster MLA. |
35+
| [kubevirt](components/kubevirt) | Help Components for e.g. installing kvm quemu packages on baremetal. |
3636

3737
## Kubermatic Example Setups
3838

@@ -43,6 +43,7 @@ Dedicated components for customer purposes.
4343
| [Bare Metal - KubeOne Static Hosts](./examples/baremetal/kubeone/vsphere-static-machines) | Example how to managed static bare metal workers. The "bare metal" workers are simulated with vSphere by terraform automation |
4444
| [Bare Metal - KKP and kubeadm join implementation examples](examples/baremetal/kkp) | Example how to use [kubeadm](https://kubernetes.io/docs/setup/independent/install-kubeadm/) to join the KKP managed controlplan: [1 Manual Example](examples/baremetal/kkp/kubadm-manual), [2 SSH Multi Client join script](examples/baremetal/kkp/kubeadm-multi-client) |
4545
| [Baremetal node provisioning with OSM](examples/baremetal/kkp/kubeadm-osm/README.md) | This method allows you to provision a baremetal machine as a Kubernetes node, using the provisioning logic of OSM as provided by the specific OSP. |
46+
| [Kubermatic Virtualization](examples/kubermatic-virtualization/README.md) | Example config to run Kubermatic Virtualization setup with dedicated (protected) tooling container |
4647

4748
## Kubermatic Addons
4849

‎examples/kubermatic-virtualization/.env‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# KubeOne Offline Bootstrap Environment Configuration
2+
#
3+
# This file contains the environment variables needed to configure and run the bootstrap environment.
4+
# Copy this file to .env and fill in the appropriate values for your setup.
5+
6+
# --- Kubermatic Virtualization Settings ---
7+
DOCKER_REPO=quay.io/toschneck/kubermatic-virtualization-tooling
8+
TOOLING_CONTAINER_TAG=kubev-d12dc16_kkp-v2.29.4-2026-02-24

‎examples/kubermatic-virtualization/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*-kubeconfig
2+
*-kubeconfig.crypt.yaml

‎examples/kubermatic-virtualization/Justfile‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
2+
_default:
3+
@just --list
4+
set shell := ["bash", "-uc"]
5+
6+
KUBEV_FOLDER := source_directory()/"kubev"
7+
KUBEV_KUBECONFIG := KUBEV_FOLDER/"kubev-kubeconfig.crypt.yaml"
8+
9+
#### Local KubeV Tooling
10+
KUBEV_SSH_KEY := source_directory()/"../git-submodules/secrets/ssh/id_rsa"
11+
KUBEV_TF_INFRA := source_directory()/"./keepalived-lb-tf-infra"
12+
DOCKER_EXTRA_FLAG := "--detach"
13+
CONTAINER_NAME := "kubev"
14+
ENV_FILE := source_directory()/".env"
15+
16+
local-kubev-tooling-start:
17+
#!/usr/bin/env bash
18+
source "{{ ENV_FILE }}"
19+
docker pull --platform linux/amd64 ${DOCKER_REPO}:${TOOLING_CONTAINER_TAG}
20+
docker run -it --platform linux/amd64 --user 0 --rm \
21+
--name {{ CONTAINER_NAME }} {{ DOCKER_EXTRA_FLAG }} \
22+
-v {{ source_directory() }}/../../../:/data \
23+
${DOCKER_REPO}:${TOOLING_CONTAINER_TAG} bash
24+
just local-kubev-tooling-exec
25+
26+
local-kubev-tooling-rm:
27+
docker rm -f {{ CONTAINER_NAME }}
28+
29+
local-kubev-tooling-exec:
30+
docker exec -w /data -it {{ CONTAINER_NAME }} bash
31+
32+
load-env:
33+
@test -f {{ KUBEV_SSH_KEY }} && chmod 600 {{ KUBEV_SSH_KEY }} && ssh-add {{ KUBEV_SSH_KEY }} && echo "[ok] "|| (echo "ERROR: ssh key permission ..." && exit 2)
34+
35+
#### #### KubeV Cluster Management
36+
#### Terraform (K8s API LB)
37+
38+
kubev-lb-tf-init: load-env
39+
cd {{ KUBEV_TF_INFRA }} && terraform init
40+
41+
kubev-lb-tf-apply: load-env
42+
cd {{ KUBEV_TF_INFRA }} && terraform apply
43+
44+
kubev-lb-output: load-env
45+
cd {{ KUBEV_TF_INFRA }} && terraform output
46+
47+
kubev-apply:
48+
kubermatic-virtualization version > {{ KUBEV_FOLDER }}/kubermatic-virtualization.version.txt
49+
kubermatic-virtualization apply --verbose -f {{ KUBEV_FOLDER }}/kubev.yaml
50+
cp ./*-kubeconfig {{ KUBEV_KUBECONFIG }}
51+
just untaint-cp-nodes
52+
just kubev-apply-services
53+
54+
kubev-apply-services:
55+
cd {{ KUBEV_FOLDER }} && KUBECONFIG={{ KUBEV_KUBECONFIG }} helmfile sync
56+
57+
#### Kubernetes Helpers
58+
untaint-cp-nodes:
59+
kubectl --kubeconfig {{ KUBEV_KUBECONFIG }} taint nodes --all node-role.kubernetes.io/master- || echo "... untainted"
60+
kubectl --kubeconfig {{ KUBEV_KUBECONFIG }} taint nodes --all node-role.kubernetes.io/control-plane- || echo "... untainted"

‎examples/kubermatic-virtualization/README.md‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# KubeV Cluster Setup
2+
3+
This directory contains an example `Justfile` for managing a Kubermatic Virtualization (KubeV) cluster (example: `kubev`).
4+
5+
## Requirements
6+
7+
Before starting, verify your infrastructure meets the hardware and software requirements:
8+
- [KubeV Requirements](https://docs.kubermatic.com/kubermatic-virtualization/main/architecture/requirements/)
9+
10+
## Prepare Configuration
11+
12+
### Print Full Config Template
13+
14+
Generate a full example configuration to understand all available options:
15+
16+
```bash
17+
kubermatic-virtualization config print --full
18+
```
19+
20+
See [Declarative Installation Docs](https://docs.kubermatic.com/kubermatic-virtualization/main/installation/declarative-installation/) for details.
21+
22+
Edit your cluster config at `kubev/kubev.yaml`.
23+
24+
### Interactive Installation (alternative)
25+
26+
Alternatively, use the interactive installer to generate a config:
27+
28+
```bash
29+
kubermatic-virtualization install
30+
```
31+
32+
See [Interactive Installation Docs](https://docs.kubermatic.com/kubermatic-virtualization/main/installation/interactive-installation/).
33+
34+
## Setup Steps
35+
36+
### 1. Start Local KubeV Tooling Container
37+
38+
```bash
39+
just local-kubev-tooling-start
40+
```
41+
42+
Re-attach to a running container:
43+
44+
```bash
45+
just local-kubev-tooling-exec
46+
```
47+
48+
Remove the container:
49+
50+
```bash
51+
just local-kubev-tooling-rm
52+
```
53+
54+
### 2. Load SSH Key & Environment
55+
56+
```bash
57+
just load-env
58+
```
59+
60+
### 3. Provision Load Balancer (Terraform)
61+
62+
```bash
63+
just kubev-lb-tf-init
64+
just kubev-lb-tf-apply
65+
just kubev-lb-output
66+
```
67+
68+
### 4. Apply KubeV Cluster
69+
70+
Runs `kubermatic-virtualization apply`, copies the kubeconfig, untaints control-plane nodes, and syncs Helm releases:
71+
72+
```bash
73+
just kubev-apply
74+
```
75+
76+
Individual steps:
77+
78+
```bash
79+
just untaint-cp-nodes # remove control-plane taints
80+
just kubev-apply-services # helmfile sync
81+
```
82+
83+
## Key Files
84+
85+
| Path | Description |
86+
|---|---|
87+
| `kubev/kubev.yaml` | KubeV cluster configuration |
88+
| `kubev/kubev-kubeconfig.crypt.yaml` | Encrypted kubeconfig (written after apply) |
89+
| `../00_config-generator/.env` | Environment variables (Docker registry, tags, etc.) |
90+
| `../git-submodules/secrets/ssh/id_rsa` | SSH key for cluster access |

‎examples/kubermatic-virtualization/keepalived-lb-tf-infra/README.md‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# Keepalived Load Balancer for KubeV Control Plane
2+
3+
This Terraform module deploys keepalived on KubeV control plane nodes to provide a Virtual IP (VIP) for Kubernetes API high availability.
4+
5+
## Overview
6+
7+
Keepalived uses VRRP (Virtual Router Redundancy Protocol) to manage a floating Virtual IP address across multiple control plane nodes. When the master node fails, the VIP automatically moves to a backup node, ensuring continuous API server availability.
8+
9+
## Use Cases
10+
11+
This module is recommended for:
12+
- **On-premise / bare-metal deployments** where cloud load balancers are not available
13+
- **VMware vSphere** environments
14+
- **Private data centers** with direct L2 network connectivity
15+
16+
> **Note**: For GCP test deployments, use the [GCP Internal Load Balancer](../../infra-machines/gce/README.md) instead, as keepalived VIPs are not easily accessible from outside the VPC and GCP does not support multicast.
17+
18+
## Prerequisites
19+
20+
- Control plane nodes must be provisioned and accessible via SSH
21+
- All control plane nodes must be on the same L2 network segment
22+
- The VIP address must be unused and within the same subnet as control plane nodes
23+
- SSH key-based authentication configured
24+
25+
## Usage
26+
27+
### 1. Create terraform.tfvars
28+
29+
```hcl
30+
cluster_name = "kubev-cluster"
31+
32+
# Virtual IP for the Kubernetes API
33+
api_vip = "10.0.2.100"
34+
35+
# Control plane node IPs
36+
control_plane_hosts = [
37+
"10.0.2.10",
38+
"10.0.2.11",
39+
"10.0.2.12"
40+
]
41+
42+
# SSH configuration
43+
ssh_username = "ubuntu"
44+
ssh_private_key_file = "~/.ssh/id_rsa"
45+
46+
# Network interface for VRRP (check with: ip link show)
47+
vrrp_interface = "ens192"
48+
49+
# VRRP router ID (must be unique in your network, 1-255)
50+
vrrp_router_id = 42
51+
52+
# Optional: Bastion host for SSH jump
53+
# bastion_host = "bastion.example.com"
54+
# bastion_port = 22
55+
# bastion_username = "ubuntu"
56+
```
57+
58+
### 2. Initialize and Apply
59+
60+
```bash
61+
terraform init
62+
terraform plan
63+
terraform apply
64+
```
65+
66+
### 3. Verify Installation
67+
68+
SSH into any control plane node and check:
69+
70+
```bash
71+
# Check keepalived status
72+
sudo systemctl status keepalived
73+
74+
# Check which node holds the VIP
75+
ip addr show | grep <VIP>
76+
77+
# Test API server via VIP
78+
curl -k https://<VIP>:6443/healthz
79+
```
80+
81+
## Configuration Variables
82+
83+
| Variable | Description | Default | Required |
84+
|----------|-------------|---------|:--------:|
85+
| `cluster_name` | Name of the cluster | - | yes |
86+
| `api_vip` | Virtual IP address for Kubernetes API | - | yes |
87+
| `control_plane_hosts` | List of control plane node IPs | - | yes |
88+
| `ssh_username` | SSH user for provisioning | `root` | no |
89+
| `ssh_private_key_file` | Path to SSH private key | - | yes |
90+
| `vrrp_interface` | Network interface for VRRP | `ens192` | no |
91+
| `vrrp_router_id` | VRRP router ID (1-255, must be unique) | `42` | no |
92+
| `bastion_host` | SSH bastion/jump host | `""` | no |
93+
| `bastion_port` | SSH bastion port | `22` | no |
94+
| `bastion_username` | SSH bastion user | `""` | no |
95+
96+
## Outputs
97+
98+
| Output | Description |
99+
|--------|-------------|
100+
| `kubev_api` | API endpoint configuration with VIP |
101+
| `kubev_hosts` | Control plane host information |
102+
103+
## How It Works
104+
105+
1. **Installation**: The module installs keepalived on all control plane nodes
106+
2. **Configuration**: Each node is configured with VRRP:
107+
- First node becomes MASTER (priority 101)
108+
- Other nodes become BACKUP (priority 100)
109+
3. **Health Check**: A script checks the local API server every 3 seconds
110+
4. **Failover**: If the master fails the health check, the VIP moves to a backup node
111+
112+
## Architecture
113+
114+
```
115+
┌─────────────────┐
116+
│ VIP: api_vip │
117+
│ (Floating IP) │
118+
└────────┬────────┘
119+
│
120+
┌─────────────────┼─────────────────┐
121+
│ │ │
122+
▼ ▼ ▼
123+
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
124+
│ CP Node 1 │ │ CP Node 2 │ │ CP Node 3 │
125+
│ (MASTER) │ │ (BACKUP) │ │ (BACKUP) │
126+
│ Priority:101│ │ Priority:100│ │ Priority:100│
127+
│ kube-api:6443│ │ kube-api:6443│ │ kube-api:6443│
128+
└──────────────┘ └──────────────┘ └──────────────┘
129+
```
130+
131+
## Troubleshooting
132+
133+
### Check keepalived logs
134+
```bash
135+
sudo journalctl -u keepalived -f
136+
```
137+
138+
### Verify VRRP communication
139+
```bash
140+
sudo tcpdump -i <interface> vrrp
141+
```
142+
143+
### Manual failover test
144+
```bash
145+
# On master node, stop keepalived
146+
sudo systemctl stop keepalived
147+
148+
# Check VIP moved to another node
149+
ip addr show | grep <VIP>
150+
```
151+
152+
### Common issues
153+
154+
1. **VIP not assigned**: Check firewall allows VRRP protocol (IP protocol 112)
155+
2. **Split-brain**: Ensure all nodes can communicate via VRRP
156+
3. **Health check failing**: Verify API server is running on localhost:6443
157+
158+
## Integration with KubeV
159+
160+
When using this module with KubeV, configure the cluster to use the VIP as the API endpoint:
161+
162+
```yaml
163+
# kubev.yaml
164+
apiVersion: kubermatic.k8c.io/v1
165+
kind: KubevCluster
166+
spec:
167+
controlPlaneEndpoint:
168+
host: "<api_vip>"
169+
port: 6443
170+
```

‎examples/kubermatic-virtualization/keepalived-lb-tf-infra/etc_keepalived_check_apiserver_sh.tpl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/sh
2+
3+
errorExit() {
4+
echo "*** $*" 1>&2
5+
exit 1
6+
}
7+
8+
curl --silent --max-time 2 --insecure https://localhost:6443/healthz -o /dev/null || errorExit "Error GET https://localhost:6443/healthz"
9+
if ip addr | grep -q ${APISERVER_VIP}; then
10+
curl --silent --max-time 2 --insecure https://${APISERVER_VIP}:6443/healthz -o /dev/null || errorExit "Error GET https://${APISERVER_VIP}:6443/healthz"
11+
fi

‎examples/kubermatic-virtualization/keepalived-lb-tf-infra/etc_keepalived_keepalived_conf.tpl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
global_defs {
2+
router_id LVS_DEVEL
3+
}
4+
vrrp_script check_apiserver {
5+
script "/etc/keepalived/check_apiserver.sh"
6+
interval 3
7+
weight -2
8+
fall 10
9+
rise 2
10+
}
11+
12+
vrrp_instance VI_1 {
13+
state ${STATE}
14+
interface ${INTERFACE}
15+
virtual_router_id ${ROUTER_ID}
16+
priority ${PRIORITY}
17+
authentication {
18+
auth_type PASS
19+
auth_pass ${AUTH_PASS}
20+
}
21+
virtual_ipaddress {
22+
${APISERVER_VIP}
23+
}
24+
track_script {
25+
check_apiserver
26+
}
27+
}

0 commit comments

Comments
 (0)