diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 24a2d2d1..0a3c2e5c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -18,8 +18,7 @@ jobs: name: Build & Run E2E Images runs-on: [self-hosted, linux, X64, jammy, xlarge] steps: - - - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: # We run into rate limiting issues if we don't authenticate @@ -40,28 +39,16 @@ jobs: sudo snap install kubectl --classic --channel=1.32/stable - name: Build provider images run: sudo env "PATH=$PATH" make docker-build-e2e - - name: Build k8s-snap images - working-directory: hack/ - run: | - ./build-e2e-images.sh - name: Save provider image run: | sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev sudo chmod 775 provider-images.tar - - name: Save k8s-snap image - run: | - sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old - sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new - sudo chmod 775 k8s-snap-image-old.tar - sudo chmod 775 k8s-snap-image-new.tar - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: e2e-images path: | provider-images.tar - k8s-snap-image-old.tar - k8s-snap-image-new.tar run-e2e-tests: name: Run E2E Tests @@ -80,8 +67,7 @@ jobs: # TODO(ben): Remove once all tests are running stable. fail-fast: false steps: - - - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: # We run into rate limiting issues if we don't authenticate @@ -94,6 +80,13 @@ jobs: uses: actions/setup-go@v5 with: go-version-file: go.mod + - name: Setup LXD + uses: canonical/setup-lxd@v0.1.3 + with: + bridges: "lxdbr0" + - name: Configure LXD + run: | + sudo ./hack/setup-lxd.sh - name: Install requirements run: | sudo apt update @@ -105,23 +98,12 @@ jobs: with: name: e2e-images path: . - - name: Load provider image - run: sudo docker load -i provider-images.tar - - name: Load k8s-snap old image - run: | - sudo docker load -i k8s-snap-image-old.tar - - name: Load k8s-snap new image - if: matrix.ginkgo_focus == 'Workload cluster upgrade' + - name: Setup bootstrap cluster run: | - sudo docker load -i k8s-snap-image-new.tar + sudo ./hack/setup-bootstrap-cluster.sh bootstrap-cluster 1.32 ./provider-images.tar - name: Create docker network run: | sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true - - name: Increase inotify watches - run: | - # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files - sudo sysctl fs.inotify.max_user_watches=1048576 - sudo sysctl fs.inotify.max_user_instances=8192 - name: Setup tmate session uses: canonical/action-tmate@main if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }} @@ -129,7 +111,10 @@ jobs: detached: true - name: Run e2e tests run: | - sudo env "PATH=$PATH" GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e + sudo env "PATH=$PATH" GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" \ + SKIP_RESOURCE_CLEANUP=true \ + USE_EXISTING_CLUSTER=true \ + make test-e2e - name: Change artifact permissions if: always() run: | diff --git a/Makefile b/Makefile index 6186a093..209712de 100644 --- a/Makefile +++ b/Makefile @@ -99,10 +99,13 @@ GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m -E2E_INFRA ?= docker +E2E_INFRA ?= incus E2E_CONF_FILE ?= $(TEST_DIR)/e2e/config/ck8s-$(E2E_INFRA).yaml SKIP_RESOURCE_CLEANUP ?= false +SKIP_BOOTSTRAP_CLUSTER_INITIALIZATION ?= false USE_EXISTING_CLUSTER ?= false +# EXISTING_CLUSTER_KUBECONFIG_PATH ?= $(HOME)/.kube/config +# PROVIDER_IMAGES_TAR_PATH ?= $(shell pwd)/provider-images.tar GINKGO_NOCOLOR ?= false # to set multiple ginkgo skip flags, if any @@ -287,7 +290,11 @@ test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(TEST_DIR)/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE)" \ - -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \ + -e2e.skip-bootstrap-cluster-initialization=$(SKIP_BOOTSTRAP_CLUSTER_INITIALIZATION) +# -e2e.existing-cluster-kubeconfig-path=$(EXISTING_CLUSTER_KUBECONFIG_PATH) \ +# -e2e.provider-images-tar-path=$(PROVIDER_IMAGES_TAR_PATH) # Build manager binary manager-controlplane: generate-controlplane diff --git a/bootstrap/controllers/ck8sconfig_controller.go b/bootstrap/controllers/ck8sconfig_controller.go index a702ddf2..43ddcf36 100644 --- a/bootstrap/controllers/ck8sconfig_controller.go +++ b/bootstrap/controllers/ck8sconfig_controller.go @@ -221,11 +221,6 @@ func (r *CK8sConfigReconciler) joinControlplane(ctx context.Context, scope *Scop // injects into config.Version values from top level object r.reconcileTopLevelObjectSettings(scope.Cluster, machine, scope.Config) - nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) - if err != nil { - return fmt.Errorf("failed to generate node token: %w", err) - } - microclusterPort := scope.Config.Spec.ControlPlaneConfig.GetMicroclusterPort() workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(scope.Cluster), microclusterPort) if err != nil { @@ -270,13 +265,19 @@ func (r *CK8sConfigReconciler) joinControlplane(ctx context.Context, scope *Scop // If the machine has an in-place upgrade annotation, use it to set the snap install data inPlaceInstallData := r.resolveInPlaceUpgradeRelease(machine) if inPlaceInstallData != nil { + scope.Info("Using in-place upgrade snap install data from machine annotation") snapInstallData = inPlaceInstallData } // log snapinstalldata - scope.Info("SnapInstallData Spec", "Option", scope.Config.Spec.Channel, "Value", scope.Config.Spec.Revision, "LocalPath", scope.Config.Spec.LocalPath) + scope.Info("Snap install options from spec", "Channel", scope.Config.Spec.Channel, "Revision", scope.Config.Spec.Revision, "LocalPath", scope.Config.Spec.LocalPath) scope.Info("SnapInstallData", "Option", snapInstallData.Option, "Value", snapInstallData.Value) + nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) + if err != nil { + return fmt.Errorf("failed to generate node token: %w", err) + } + input := cloudinit.JoinControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ BootCommands: scope.Config.Spec.BootCommands, @@ -336,11 +337,6 @@ func (r *CK8sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) err return fmt.Errorf("auth token not yet generated") } - nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) - if err != nil { - return fmt.Errorf("failed to generate node token: %w", err) - } - microclusterPort := scope.Config.Spec.ControlPlaneConfig.GetMicroclusterPort() workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(scope.Cluster), microclusterPort) if err != nil { @@ -377,9 +373,19 @@ func (r *CK8sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) err // If the machine has an in-place upgrade annotation, use it to set the snap install data inPlaceInstallData := r.resolveInPlaceUpgradeRelease(machine) if inPlaceInstallData != nil { + scope.Info("Using in-place upgrade snap install data from machine annotation") snapInstallData = inPlaceInstallData } + // log snapinstalldata + scope.Info("Snap install options from spec", "Channel", scope.Config.Spec.Channel, "Revision", scope.Config.Spec.Revision, "LocalPath", scope.Config.Spec.LocalPath) + scope.Info("SnapInstallData", "Option", snapInstallData.Option, "Value", snapInstallData.Value) + + nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) + if err != nil { + return fmt.Errorf("failed to generate node token: %w", err) + } + input := cloudinit.JoinWorkerInput{ BaseUserData: cloudinit.BaseUserData{ BootCommands: scope.Config.Spec.BootCommands, @@ -634,16 +640,6 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, } conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition) - authToken, err := token.Lookup(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster)) - if err != nil { - return ctrl.Result{}, err - } - - nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to generate node token: %w", err) - } - clusterInitConfig := ck8s.InitControlPlaneConfig{ ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, ControlPlaneConfig: scope.Config.Spec.ControlPlaneConfig, @@ -702,6 +698,20 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, return ctrl.Result{Requeue: true}, fmt.Errorf("failed to get snap install data from spec: %w", err) } + // log snapinstalldata + scope.Info("Snap install options from spec", "Channel", scope.Config.Spec.Channel, "Revision", scope.Config.Spec.Revision, "LocalPath", scope.Config.Spec.LocalPath) + scope.Info("SnapInstallData", "Option", snapInstallData.Option, "Value", snapInstallData.Value) + + nodeToken, err := token.GenerateAndStoreNodeToken(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster), machine.Name) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to generate node token: %w", err) + } + + authToken, err := token.Lookup(ctx, r.Client, client.ObjectKeyFromObject(scope.Cluster)) + if err != nil { + return ctrl.Result{}, err + } + cpinput := cloudinit.InitControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ BootCommands: scope.Config.Spec.BootCommands, @@ -780,7 +790,7 @@ func (r *CK8sConfigReconciler) storeBootstrapData(ctx context.Context, scope *Sc Kind: "CK8sConfig", Name: scope.Config.Name, UID: scope.Config.UID, - Controller: ptr.To[bool](true), + Controller: ptr.To(true), }, }, }, diff --git a/hack/setup-bootstrap-cluster.sh b/hack/setup-bootstrap-cluster.sh new file mode 100755 index 00000000..eee4fc35 --- /dev/null +++ b/hack/setup-bootstrap-cluster.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2022 The Tinkerbell Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Retry function +# Usage: retry +retry() { + local max_attempts=$1 + local delay=$2 + shift 2 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + else + if [ $attempt -lt $max_attempts ]; then + echo " Attempt $attempt failed. Retrying in ${delay}s..." + sleep $delay + attempt=$((attempt + 1)) + else + echo " All $max_attempts attempts failed." + return 1 + fi + fi + done +} + +# Description: +# Setup the bootstrap cluster in a LXD container +# Usage: +# ./hack/setup-management-cluster.sh +# Example: +# ./hack/setup-management-cluster.sh bootstrap-cluster 1.32 provider-images.tar + +if [ -z "${1:-}" ]; then + echo "Error: bootstrap-cluster-name is required" + echo "Usage: $0 " + exit 1 +fi + +if [ -z "${2:-}" ]; then + echo "Error: k8s-version is required" + echo "Usage: $0 " + exit 1 +fi + +if [ -z "${3:-}" ]; then + echo "Error: provider-images-path is required" + echo "Usage: $0 " + exit 1 +fi + +bootstrap_cluster_name=${1} +bootstrap_cluster_version=${2} +provider_images_path=${3} + +echo "==> Launching LXD container '$bootstrap_cluster_name' with Ubuntu 24.04..." +sudo lxc -p default -p k8s-integration launch ubuntu:24.04 $bootstrap_cluster_name + +echo "==> Installing k8s snap (version $bootstrap_cluster_version)..." +retry 5 5 sudo lxc exec $bootstrap_cluster_name -- snap install k8s --classic --channel=$bootstrap_cluster_version-classic/stable + +echo "==> Bootstrapping k8s cluster..." +retry 5 5 sudo lxc exec $bootstrap_cluster_name -- k8s bootstrap + +echo "==> Pushing provider images to container..." +sudo lxc file push $provider_images_path $bootstrap_cluster_name/root/provider-images.tar + +echo "==> Loading provider images into containerd..." +sudo lxc exec $bootstrap_cluster_name -- /snap/k8s/current/bin/ctr -n k8s.io images import /root/provider-images.tar + +echo "==> Getting bootstrap cluster IP address..." +bootstrap_cluster_ip=$(sudo lxc exec $bootstrap_cluster_name -- bash -c "ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'") +echo " Bootstrap cluster IP: $bootstrap_cluster_ip" + +echo "==> Creating cluster-info configmap..." +kubeconfig="apiVersion: v1 +clusters: +- cluster: + server: https://${bootstrap_cluster_ip}:6443 + name: "" +contexts: null +current-context: "" +kind: Config +users: null" + +echo " Creating temporary kubeconfig file..." +temp_kubeconfig=$(mktemp) +echo "$kubeconfig" > "$temp_kubeconfig" + +echo " Pushing kubeconfig $temp_kubeconfig to container at /tmp/$bootstrap_cluster_name-cluster-info.yaml..." +sudo lxc file push "$temp_kubeconfig" "$bootstrap_cluster_name/tmp/$bootstrap_cluster_name-cluster-info.yaml" + +echo " Creating cluster-info configmap in kube-public namespace..." +sudo lxc exec $bootstrap_cluster_name -- k8s kubectl create configmap cluster-info -n kube-public --from-file=kubeconfig=/tmp/$bootstrap_cluster_name-cluster-info.yaml + +echo " Cleaning up temporary files..." +rm "$temp_kubeconfig" +sudo lxc exec $bootstrap_cluster_name -- rm /tmp/$bootstrap_cluster_name-cluster-info.yaml + +echo "==> Setting up kubeconfig..." +sudo lxc exec $bootstrap_cluster_name -- mkdir -p /root/.kube +sudo lxc exec $bootstrap_cluster_name -- bash -c "k8s config > /root/.kube/config" + +echo "==> Pulling kubeconfig from $bootstrap_cluster_name to ~/.kube/config..." +mkdir -p ~/.kube +sudo lxc file pull $bootstrap_cluster_name/root/.kube/config ~/.kube/config + +echo "==> Setup complete! Bootstrap cluster '$bootstrap_cluster_name' is ready." + + + + diff --git a/hack/setup-lxd.sh b/hack/setup-lxd.sh new file mode 100755 index 00000000..844def06 --- /dev/null +++ b/hack/setup-lxd.sh @@ -0,0 +1,16 @@ +#!/usr/bin/bash + +# https://capn.linuxcontainers.org/tutorial/quick-start.html +ip_address="$(ip -o route get to 1.1.1.1 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')" + +sudo lxd init --auto --network-address "$ip_address" +sudo lxc network set lxdbr0 ipv6.address=none +sudo lxc cluster enable "$ip_address" + +token="$(sudo lxc config trust add --name client | tail -1)" +sudo lxc remote add local-https --token "$token" "https://$(sudo lxc config get core.https_address)" +sudo lxc remote set-default local-https + +wget https://raw.githubusercontent.com/canonical/k8s-snap/refs/heads/main/tests/integration/lxd-profile.yaml + +sudo lxc profile create k8s-integration < lxd-profile.yaml diff --git a/pkg/cloudinit/scripts/install.sh b/pkg/cloudinit/scripts/install.sh index c1e106f8..1f89b4e6 100644 --- a/pkg/cloudinit/scripts/install.sh +++ b/pkg/cloudinit/scripts/install.sh @@ -6,12 +6,38 @@ ## - /capi/etc/snap-local-path contains the path to the local snap file to be installed (e.g. /path/to/k8s.snap), ## or the path to a folder containing the local snap files to be installed (e.g. /path/to) +# Function to retry snap installation with a maximum number of attempts +# and a delay between attempts. This is useful in case of transient errors +retry_snap_install() { + local max_attempts=5 + local delay=3 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + echo "Attempt $attempt of $max_attempts to install snap..." + if "$@"; then + echo "Snap installation succeeded" + return 0 + else + echo "Snap installation failed" + if [ $attempt -lt $max_attempts ]; then + echo "Retrying in $delay seconds..." + sleep $delay + fi + fi + attempt=$((attempt + 1)) + done + + echo "Failed to install snap after $max_attempts attempts" + return 1 +} + if [ -f "/capi/etc/snap-channel" ]; then snap_channel="$(cat /capi/etc/snap-channel)" - snap install k8s --classic --channel "${snap_channel}" + retry_snap_install snap install k8s --classic --channel "${snap_channel}" elif [ -f "/capi/etc/snap-revision" ]; then snap_revision="$(cat /capi/etc/snap-revision)" - snap install k8s --classic --revision "${snap_revision}" + retry_snap_install snap install k8s --classic --revision "${snap_revision}" elif [ -f "/capi/etc/snap-local-path" ]; then snap_local_path="$(cat /capi/etc/snap-local-path)" snap_local_paths=( "${snap_local_path}" ) @@ -20,7 +46,7 @@ elif [ -f "/capi/etc/snap-local-path" ]; then if [[ -d "${snap_local_path}" ]]; then snap_local_paths=($(ls ${snap_local_path}/*.snap)) fi - snap install --classic --dangerous "${snap_local_paths[@]}" + retry_snap_install snap install --classic --dangerous "${snap_local_paths[@]}" else echo "No snap installation option found" exit 1 diff --git a/templates/lxd/cluster-template.yaml b/templates/lxd/cluster-template.yaml new file mode 100644 index 00000000..183eaeb7 --- /dev/null +++ b/templates/lxd/cluster-template.yaml @@ -0,0 +1,120 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" diff --git a/templates/lxd/template-variables.rc b/templates/lxd/template-variables.rc new file mode 100644 index 00000000..b3982103 --- /dev/null +++ b/templates/lxd/template-variables.rc @@ -0,0 +1,45 @@ +## Cluster version and size +export KUBERNETES_VERSION=v1.34.0 +export CONTROL_PLANE_MACHINE_COUNT=1 +export WORKER_MACHINE_COUNT=1 + +## [required] Name of secret with server credentials +export LXC_SECRET_NAME=lxc-secret + +## [required] Load Balancer configuration +export LOAD_BALANCER="lxc: {profiles: [default,k8s-integration], flavor: c1-m1}" +#export LOAD_BALANCER="oci: {profiles: [default], flavor: c1-m1}" +#export LOAD_BALANCER="kube-vip: {host: 10.0.42.1}" +#export LOAD_BALANCER="ovn: {host: 10.100.42.1, networkName: default}" + +## [optional] Deploy kube-flannel on the cluster. +#export DEPLOY_KUBE_FLANNEL=true + +## [optional] Use unprivileged containers. +#export PRIVILEGED=false + +## [optional] Base image to use. This must be set if there are no base images for the target Kubernetes version. +## +## See [1] for a list of pre-built kubeadm images that are available. +## See [2] for a list of well-known image prefixes that are supported, e.g. "ubuntu:24.04" or "debian:13" +## +## Set INSTALL_KUBEADM=true to inject preKubeadmCommands to install kubeadm for the target Kubernetes version. +## +## [1]: https://capn.linuxcontainers.org/reference/default-simplestreams-server.html#provided-images +## [2]: https://capn.linuxcontainers.org/reference/templates/default.html#lxc_image_name-and-install_kubeadm +export LXC_IMAGE_NAME="ubuntu:24.04" +#export INSTALL_KUBEADM="true" + +# Control plane machine configuration +export CONTROL_PLANE_MACHINE_TYPE=container # 'container' or 'virtual-machine' +export CONTROL_PLANE_MACHINE_FLAVOR=c2-m4 # instance type for control plane nodes +export CONTROL_PLANE_MACHINE_PROFILES=[default,k8s-integration] # profiles for control plane nodes +export CONTROL_PLANE_MACHINE_DEVICES=[] # override devices for control plane nodes +export CONTROL_PLANE_MACHINE_TARGET="" # override target for control plane nodes (e.g. "@default") + +# Worker machine configuration +export WORKER_MACHINE_TYPE=container # 'container' or 'virtual-machine' +export WORKER_MACHINE_FLAVOR=c2-m4 # instance type for worker nodes +export WORKER_MACHINE_PROFILES=[default,k8s-integration] # profiles for worker nodes +export WORKER_MACHINE_DEVICES=[] # override devices for worker nodes +export WORKER_MACHINE_TARGET="" # override target for worker nodes (e.g. "@default") diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index c70f31be..a0d3c38f 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -28,7 +28,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -129,6 +128,8 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + createLXCSecretForIncus(ctx, input.BootstrapClusterProxy, input.E2EConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()) @@ -178,8 +179,8 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp ControlPlane: result.ControlPlane, MaxControlPlaneMachineCount: maxControlPlaneMachineCount, KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), - UpgradeMachineTemplate: ptr.To(fmt.Sprintf("%s-control-plane-old", clusterName)), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + InfrastructureProviders: input.E2EConfig.InfrastructureProviders(), }) By("Upgrading the machine deployment") @@ -188,7 +189,6 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp Cluster: result.Cluster, UpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), MachineDeployments: result.MachineDeployments, - UpgradeMachineTemplate: ptr.To(fmt.Sprintf("%s-md-new-0", clusterName)), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), }) diff --git a/test/e2e/common.go b/test/e2e/common.go index dfe26fc2..baafb77f 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -21,11 +21,15 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" + "slices" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -133,3 +137,161 @@ func localLoadE2EConfig(configPath string) *clusterctl.E2EConfig { return config } + +// createLXCSecretForIncus creates the LXC secret for Incus provider if needed +func createLXCSecretForIncus(ctx context.Context, clusterProxy framework.ClusterProxy, e2eConfig *clusterctl.E2EConfig, namespace string) { + // Check if using incus provider + if !slices.Contains(e2eConfig.InfrastructureProviders(), "incus") { + return + } + + By("Creating LXC secret for Incus provider") + + // Get values from environment variables with defaults + homeDir, err := os.UserHomeDir() + Expect(err).ToNot(HaveOccurred(), "Failed to get user home directory") + + // Environment variables for LXD configuration + lxdAddress, err := getLXDDefaultAddress() + Expect(err).ToNot(HaveOccurred(), "Failed to get LXD default address") + + remote := getEnvWithDefault("LXD_REMOTE", lxdAddress) + serverCertPath := getEnvWithDefault("LXD_SERVER_CERT", filepath.Join(homeDir, "snap/lxd/common/config/servercerts/local-https.crt")) + clientCertPath := getEnvWithDefault("LXD_CLIENT_CERT", filepath.Join(homeDir, "snap/lxd/common/config/client.crt")) + clientKeyPath := getEnvWithDefault("LXD_CLIENT_KEY", filepath.Join(homeDir, "snap/lxd/common/config/client.key")) + project := getEnvWithDefault("LXD_PROJECT", "default") + + createLXCSecretInNamespace(ctx, clusterProxy, namespace, remote, serverCertPath, clientCertPath, clientKeyPath, project) +} + +func getEnvWithDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// getLXDDefaultAddress gets the LXD HTTPS address from lxc config +func getLXDDefaultAddress() (string, error) { + cmd := exec.Command("lxc", "config", "get", "core.https_address") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get LXD address: %w", err) + } + + address := strings.TrimSpace(string(output)) + if address == "" { + return "", fmt.Errorf("LXD core.https_address is empty") + } + + // Ensure it has https:// prefix + if !strings.HasPrefix(address, "https://") { + address = "https://" + address + } + + return address, nil +} + +func createLXCSecretInNamespace(ctx context.Context, clusterProxy framework.ClusterProxy, namespace, remote, serverCertPath, clientCertPath, clientKeyPath, project string) { + clientset := clusterProxy.GetClientSet() + + // Read certificate files + serverCrt, err := os.ReadFile(serverCertPath) + if err != nil { + fmt.Printf("Warning: Failed to read server certificate from %s: %v\n", serverCertPath, err) + serverCrt = []byte{} + } + + clientCrt, err := os.ReadFile(clientCertPath) + if err != nil { + fmt.Printf("Warning: Failed to read client certificate from %s: %v\n", clientCertPath, err) + clientCrt = []byte{} + } + + clientKey, err := os.ReadFile(clientKeyPath) + if err != nil { + fmt.Printf("Warning: Failed to read client key from %s: %v\n", clientKeyPath, err) + clientKey = []byte{} + } + + // Create the secret + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "lxc-secret", + Namespace: namespace, + }, + StringData: map[string]string{ + "server": remote, + "server-crt": string(serverCrt), + "client-crt": string(clientCrt), + "client-key": string(clientKey), + "project": project, + }, + } + + _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + // If secret already exists, update it + _, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred(), "Failed to create or update LXC secret") + } + + fmt.Printf("Created LXC secret with server: %s in namespace: %s\n", remote, namespace) +} + +// TODO: future work -- support creating bootstrap cluster with Incus provider +// // loadLXDProfileForIncus loads the LXD profile from k8s-snap if Incus provider is used +// func loadLXDProfileForIncus() { +// // Define the profile URL +// profileURL := "https://raw.githubusercontent.com/canonical/k8s-snap/refs/heads/main/tests/integration/lxd-profile.yaml" + +// // Fetch the profile content +// resp, err := http.Get(profileURL) +// Expect(err).ToNot(HaveOccurred(), "Failed to fetch LXD profile from URL") +// defer resp.Body.Close() + +// Expect(resp.StatusCode).To(Equal(http.StatusOK), "Failed to fetch LXD profile: HTTP %d", resp.StatusCode) + +// profileContent, err := io.ReadAll(resp.Body) +// Expect(err).ToNot(HaveOccurred(), "Failed to read LXD profile content") + +// // Create a temporary file to store the profile +// tmpFile, err := os.CreateTemp("", "lxd-profile-*.yaml") +// Expect(err).ToNot(HaveOccurred(), "Failed to create temporary file for LXD profile") +// defer os.Remove(tmpFile.Name()) +// defer tmpFile.Close() + +// // Write the profile content to the temporary file +// _, err = tmpFile.Write(profileContent) +// Expect(err).ToNot(HaveOccurred(), "Failed to write LXD profile to temporary file") + +// // Close the file before using it with lxc commands +// tmpFile.Close() + +// // // Check if profile already exists +// checkCmd := exec.Command("lxc", "profile", "show", "k8s-integration") +// if err := checkCmd.Run(); err == nil { +// By("LXD profile 'k8s-integration' already exists, updating it") +// // Profile exists, edit it +// editCmd := exec.Command("lxc", "profile", "edit", "k8s-integration") +// editCmd.Stdin, err = os.Open(tmpFile.Name()) +// Expect(err).ToNot(HaveOccurred(), "Failed to open profile file") +// output, err := editCmd.CombinedOutput() +// Expect(err).ToNot(HaveOccurred(), "Failed to update LXD profile: %s", string(output)) +// } else { +// By("Creating new LXD profile 'k8s-integration'") +// // Profile doesn't exist, create it +// createCmd := exec.Command("lxc", "profile", "create", "k8s-integration") +// output, err := createCmd.CombinedOutput() +// Expect(err).ToNot(HaveOccurred(), "Failed to create LXD profile: %s", string(output)) + +// // Edit the created profile +// editCmd := exec.Command("lxc", "profile", "edit", "k8s-integration") +// editCmd.Stdin, err = os.Open(tmpFile.Name()) +// Expect(err).ToNot(HaveOccurred(), "Failed to open profile file") +// output, err = editCmd.CombinedOutput() +// Expect(err).ToNot(HaveOccurred(), "Failed to edit LXD profile: %s", string(output)) +// } + +// fmt.Println("Successfully loaded LXD profile 'k8s-integration' for Incus provider") +// } diff --git a/test/e2e/config/ck8s-incus.yaml b/test/e2e/config/ck8s-incus.yaml new file mode 100644 index 00000000..ccf6083a --- /dev/null +++ b/test/e2e/config/ck8s-incus.yaml @@ -0,0 +1,116 @@ +--- +# E2E test scenario using local dev images and manifests built from the source tree for following providers: +# - cluster-api +# - bootstrap ck8s +# - control-plane ck8s +# - incus +images: + # Use local dev images built source tree; + - name: ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev + loadBehavior: mustLoad + - name: ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev + loadBehavior: mustLoad + +providers: + - name: cluster-api + type: CoreProvider + versions: + - name: v1.9.6 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.9.6/core-components.yaml + type: url + files: + - sourcePath: "../data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - name: incus + type: InfrastructureProvider + versions: + # Latest stable version + - name: v0.7.0 + value: https://github.com/lxc/cluster-api-provider-incus/releases/download/v0.7.0/infrastructure-components.yaml + type: url + files: + - sourcePath: "../data/shared/v1beta1_incus/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + files: + - sourcePath: "../data/infrastructure-incus/cluster-template-kcp-remediation.yaml" + - sourcePath: "../data/infrastructure-incus/cluster-template-md-remediation.yaml" + - sourcePath: "../data/infrastructure-incus/cluster-template-upgrades.yaml" + - sourcePath: "../data/infrastructure-incus/cluster-template-upgrades-max-surge-0.yaml" + - sourcePath: "../data/infrastructure-incus/cluster-template.yaml" + - name: ck8s + type: BootstrapProvider + versions: + # Could add older release version for upgrading test, but + # by default, will only use the latest version defined in + # ${ProjectRoot}/metadata.yaml to init the management cluster + # this version should be updated when ${ProjectRoot}/metadata.yaml + # is modified + - name: v0.4.99 # next; use manifest from source files + value: "../../../bootstrap/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev" + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + - name: ck8s + type: ControlPlaneProvider + versions: + - name: v0.4.99 # next; use manifest from source files + value: "../../../controlplane/config/default" + replacements: + - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest" + new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev" + files: + - sourcePath: "../../../metadata.yaml" + targetName: "metadata.yaml" + +variables: + KUBERNETES_VERSION_MANAGEMENT: "v1.32.9" + KUBERNETES_VERSION: "v1.32.9" + KUBERNETES_VERSION_UPGRADE_TO: "v1.33.5" + IP_FAMILY: "IPv4" + IN_PLACE_UPGRADE_OPTION: "channel=1.33-classic/stable" + + # LXD specific variables + LXC_SECRET_NAME: "lxc-secret" + LXC_IMAGE_NAME: "ubuntu:24.04" + CONTROL_PLANE_MACHINE_TYPE: "container" + CONTROL_PLANE_MACHINE_FLAVOR: "c2-m4" + CONTROL_PLANE_MACHINE_PROFILES: '["default", "k8s-integration"]' + WORKER_MACHINE_TYPE: "container" + WORKER_MACHINE_FLAVOR: "c2-m4" + WORKER_MACHINE_PROFILES: '["default", "k8s-integration"]' + LOAD_BALANCER_MACHINE_FLAVOR: "c1-m1" + # Enabling the feature flags by setting the env variables. + CLUSTER_TOPOLOGY: "true" + +intervals: + # The array is defined as [timeout, polling interval] + # copied from https://github.com/kubernetes-sigs/cluster-api/blob/main/test/e2e/config/docker.yaml + default/wait-controllers: ["3m", "10s"] + default/wait-cluster: ["5m", "10s"] + default/wait-control-plane: ["10m", "10s"] + default/wait-worker-nodes: ["10m", "10s"] + default/wait-machine-pool-nodes: ["5m", "10s"] + default/wait-delete-cluster: ["3m", "10s"] + default/wait-machine-upgrade: ["30m", "10s"] + default/wait-machine-pool-upgrade: ["5m", "10s"] + default/wait-nodes-ready: ["10m", "10s"] + default/wait-machine-remediation: ["5m", "10s"] + default/wait-autoscaler: ["5m", "10s"] + default/wait-machine-refresh: ["5m", "10s"] + node-drain/wait-deployment-available: ["3m", "10s"] + node-drain/wait-control-plane: ["15m", "10s"] + node-drain/wait-machine-deleted: ["2m", "10s"] + kcp-remediation/wait-machines: ["5m", "10s"] + kcp-remediation/check-machines-stable: ["30s", "5s"] + kcp-remediation/wait-machine-provisioned: ["5m", "10s"] + # Giving a bit more time during scale tests, we analyze independently if everything works quickly enough. + scale/wait-cluster: ["10m", "10s"] + scale/wait-control-plane: ["20m", "10s"] + scale/wait-worker-nodes: ["20m", "10s"] diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 41c0e762..e26fe559 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -53,6 +53,9 @@ var _ = Describe("Workload cluster creation", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) diff --git a/test/e2e/data/infrastructure-incus/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-incus/cluster-template-kcp-remediation.yaml new file mode 100644 index 00000000..2da05343 --- /dev/null +++ b/test/e2e/data/infrastructure-incus/cluster-template-kcp-remediation.yaml @@ -0,0 +1,168 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" + files: + - path: /wait-signal.sh + content: | + #!/bin/bash + + set -o errexit + set -o pipefail + + echo "Waiting for signal..." + + TOKEN=$1 + SERVER=$2 + NAMESPACE=$3 + + while true; + do + sleep 1s + + signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) + echo "signal $signal" + + if [ "$signal" == "pass" ]; then + curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test + exit 0 + fi + done + permissions: "0777" + owner: root:root + preRunCommands: + - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + nodeStartupTimeout: 30s + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + mhc-test: fail + unhealthyConditions: + - status: "False" + timeout: 10s + type: e2e.remediation.condition diff --git a/test/e2e/data/infrastructure-incus/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-incus/cluster-template-md-remediation.yaml new file mode 100644 index 00000000..322c0701 --- /dev/null +++ b/test/e2e/data/infrastructure-incus/cluster-template-md-remediation.yaml @@ -0,0 +1,139 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + e2e.remediation.label: "" + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +# MachineHealthCheck object with +# - a selector that targets all the machines with label e2e.remediation.label="" +# - unhealthyConditions triggering remediation after 10s the condition is set +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + e2e.remediation.label: "" + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 10s diff --git a/test/e2e/data/infrastructure-incus/cluster-template-upgrades-max-surge-0.yaml b/test/e2e/data/infrastructure-incus/cluster-template-upgrades-max-surge-0.yaml new file mode 100644 index 00000000..0f249516 --- /dev/null +++ b/test/e2e/data/infrastructure-incus/cluster-template-upgrades-max-surge-0.yaml @@ -0,0 +1,123 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + rolloutStrategy: + rollingUpdate: + maxSurge: 0 + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" diff --git a/test/e2e/data/infrastructure-incus/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-incus/cluster-template-upgrades.yaml new file mode 100644 index 00000000..183eaeb7 --- /dev/null +++ b/test/e2e/data/infrastructure-incus/cluster-template-upgrades.yaml @@ -0,0 +1,120 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" diff --git a/test/e2e/data/infrastructure-incus/cluster-template.yaml b/test/e2e/data/infrastructure-incus/cluster-template.yaml new file mode 100644 index 00000000..183eaeb7 --- /dev/null +++ b/test/e2e/data/infrastructure-incus/cluster-template.yaml @@ -0,0 +1,120 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: ${POD_CIDR:=[10.1.0.0/16]} + services: + cidrBlocks: ${SERVICE_CIDR:=[10.152.0.0/16]} + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCCluster +metadata: + name: ${CLUSTER_NAME} +spec: + secretRef: + name: ${LXC_SECRET_NAME} + loadBalancer: + lxc: + instanceSpec: + flavor: ${LOAD_BALANCER_MACHINE_FLAVOR:=""} + profiles: ${LOAD_BALANCER_MACHINE_PROFILES:=[default]} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} + spec: + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + instanceType: ${CONTROL_PLANE_MACHINE_TYPE} + flavor: ${CONTROL_PLANE_MACHINE_FLAVOR} + profiles: ${CONTROL_PLANE_MACHINE_PROFILES:=[default]} + devices: ${CONTROL_PLANE_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LXCMachineTemplate + name: ${CLUSTER_NAME}-md-0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LXCMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + instanceType: ${WORKER_MACHINE_TYPE} + flavor: ${WORKER_MACHINE_FLAVOR} + profiles: ${WORKER_MACHINE_PROFILES:=[default]} + devices: ${WORKER_MACHINE_DEVICES:=[]} + image: + name: ${LXC_IMAGE_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraKubeletArgs: + --provider-id: "lxc:///{{ v1.local_hostname }}" diff --git a/test/e2e/data/shared/v1beta1_incus/metadata.yaml b/test/e2e/data/shared/v1beta1_incus/metadata.yaml new file mode 100644 index 00000000..0b08ee54 --- /dev/null +++ b/test/e2e/data/shared/v1beta1_incus/metadata.yaml @@ -0,0 +1,24 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 + - major: 0 + minor: 2 + contract: v1beta1 + - major: 0 + minor: 3 + contract: v1beta1 + - major: 0 + minor: 4 + contract: v1beta1 + - major: 0 + minor: 5 + contract: v1beta1 + - major: 0 + minor: 6 + contract: v1beta1 + - major: 0 + minor: 7 + contract: v1beta1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9dd78e93..b3cc6529 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -25,6 +25,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "testing" @@ -32,6 +33,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" @@ -66,6 +68,12 @@ var ( // skipCleanup prevents cleanup of test resources e.g. for debug purposes. skipCleanup bool + + // existingClusterKubeconfigPath is the kubeconfig path to be used when using an existing cluster. + existingClusterKubeconfigPath string + + // skipBootstrapClusterInitialization skips the bootstrap cluster initialization step. + skipBootstrapClusterInitialization bool ) // Test suite global vars. @@ -88,6 +96,10 @@ var ( // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. bootstrapClusterProxy framework.ClusterProxy + + // providerImagesTarPath is the path to the tar file containing provider images + // to be loaded into the bootstrap cluster when using the Incus provider. + providerImagesTarPath string ) func init() { @@ -97,6 +109,10 @@ func init() { flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.StringVar(&clusterctlConfig, "e2e.clusterctl-config", "", "file which tests will use as a clusterctl config. If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically.") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") + flag.BoolVar(&skipBootstrapClusterInitialization, "e2e.skip-bootstrap-cluster-initialization", false, "if true, the test will skip the bootstrap cluster initialization step") + flag.StringVar(&existingClusterKubeconfigPath, "e2e.existing-cluster-kubeconfig-path", "", "kubeconfig path to be used when using an existing cluster. Should be set if using an existing cluster.") + flag.StringVar(&providerImagesTarPath, "e2e.provider-images-tar-path", "", "path to the tar file containing provider images to be loaded into the bootstrap cluster when using the Incus provider.") + } func TestE2E(t *testing.T) { @@ -145,10 +161,14 @@ var _ = SynchronizedBeforeSuite(func() []byte { } By("Setting up the bootstrap cluster") - bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) + bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster, existingClusterKubeconfigPath, providerImagesTarPath) - By("Initializing the bootstrap cluster") - initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + if !skipBootstrapClusterInitialization { + By("Initializing the bootstrap cluster") + initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + } else { + By("Skipping bootstrap cluster initialization") + } return []byte( strings.Join([]string{ @@ -223,25 +243,36 @@ func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFol return clusterctlConfig } -func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { +func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool, kubeconfigPath string, providerImagesTarPath string) (bootstrap.ClusterProvider, framework.ClusterProxy) { var clusterProvider bootstrap.ClusterProvider - kubeconfigPath := "" if !useExistingCluster { - By("Creating the bootstrap cluster") - clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - KubernetesVersion: config.GetVariable(KubernetesVersionManagement), - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, - IPFamily: config.GetVariable(IPFamily), - LogFolder: filepath.Join(artifactFolder, "kind"), - }) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") - - kubeconfigPath = clusterProvider.GetKubeconfigPath() - Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") + if slices.Contains(config.InfrastructureProviders(), "docker") { + By("Creating the bootstrap cluster using Docker") + clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ + Name: config.ManagementClusterName, + KubernetesVersion: config.GetVariable(KubernetesVersionManagement), + RequiresDockerSock: config.HasDockerProvider(), + Images: config.Images, + IPFamily: config.GetVariable(IPFamily), + LogFolder: filepath.Join(artifactFolder, "kind"), + }) + Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") + + kubeconfigPath = clusterProvider.GetKubeconfigPath() + Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") + } else if slices.Contains(config.InfrastructureProviders(), "incus") { + // TODO: future work -- support creating bootstrap cluster with Incus provider + // By("Creating the bootstrap cluster using LXD") + // clusterProvider = createLXDBootstrapCluster(ctx, createLXDBootstrapClusterInput{ + // Name: config.ManagementClusterName, + // KubernetesVersion: config.GetVariable(KubernetesVersionManagement), + // ProviderImagesTarPath: providerImagesTarPath, + // }) + // Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") + Fail("LXD bootstrap cluster creation is not yet implemented") + } } else { - By("Using an existing bootstrap cluster") + Byf("Using an existing bootstrap cluster with kubeconfig %q", kubeconfigPath) } clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) @@ -310,3 +341,113 @@ func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClust bootstrapClusterProvider.Dispose(ctx) } } + +// TODO: future work -- support creating bootstrap cluster with Incus provider +// type createLXDBootstrapClusterInput struct { +// // Name is the name of the LXD container. +// Name string +// KubernetesVersion string +// // ProviderImagesTarPath is the path to the tar file containing +// // provider images to be loaded into the bootstrap cluster. +// ProviderImagesTarPath string +// } + +// func createLXDBootstrapCluster(ctx context.Context, input createLXDBootstrapClusterInput) bootstrap.ClusterProvider { +// By("Creating LXD profile for Incus provider") +// loadLXDProfileForIncus() + +// // create lxd container +// cmd := exec.CommandContext(ctx, "lxc", "-p", "default", "-p", "k8s-integration", "launch", "ubuntu:24.04", input.Name) +// output, err := cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to create LXD container: %s\n%s\n", err, string(output)) +// return nil +// } + +// // install k8s +// k8sTrack := "1.32-classic" // TODO: Change with e2eConfig variable +// cmd = exec.CommandContext(ctx, "lxc", "exec", input.Name, "--", "snap", "install", "k8s", "--classic", fmt.Sprintf("--channel=%s-classic/stable", k8sTrack)) +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to install k8s: %s\n%s\n", err, string(output)) +// return nil +// } + +// // bootstrap k8s +// cmd = exec.CommandContext(ctx, "lxc", "exec", input.Name, "--", "k8s", "bootstrap") +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to bootstrap k8s: %s\n%s\n", err, string(output)) +// return nil +// } + +// // load images +// destinationTarPath := "/root/provider-images.tar" +// cmd = exec.CommandContext(ctx, "lxc", "file", "push", input.ProviderImagesTarPath, fmt.Sprintf("%s%s", input.Name, destinationTarPath)) +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to push images %s: %s\n%s\n", input.ProviderImagesTarPath, err, string(output)) +// return nil +// } + +// cmd = exec.CommandContext(ctx, "lxc", "exec", input.Name, "--", "/snap/k8s/current/bin/ctr", "images", "load", destinationTarPath) +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to load image %s: %s\n%s\n", input.ProviderImagesTarPath, err, string(output)) +// return nil +// } + +// // // create cluster-config configmap +// // cfg := api.Config{ +// // Clusters: map[string]*api.Cluster{ +// // "": { +// // Server: "https://10.88.119.7:6443", +// // }, +// // }, +// // } + +// // buf := &bytes.Buffer{} +// // if err := clientcmdlatest.Codec.Encode(&cfg, buf); err != nil { +// // fmt.Printf("Failed to encode kubeconfig: %s\n", err) +// // return nil +// // } +// // data := buf.Bytes() + +// cmd = exec.CommandContext(ctx, "lxc", "exec", "management", "--", "bash", "-c", `"ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'"`) +// managementIP, err := cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to get management IP: %s\n%s\n", err, string(managementIP)) +// return nil +// } + +// // NOTE(Hue): This is a minimal kubeconfig that is sufficient for the tests to run. +// // So far, only KCP remediation depends on it. +// kubeconfig := fmt.Sprintf(`apiVersion: v1 +// clusters: +// - cluster: +// server: https://%s:6443 +// name: "" +// contexts: null +// current-context: "" +// kind: Config +// users: null +// `, managementIP) + +// cmd = exec.CommandContext(ctx, "lxc", "exec", "management", "--", "k8s", "kubectl", "create", "configmap", "cluster-info", "--from-literal=kubeconfig="+kubeconfig, "-n", "kube-public") +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to get management IP: %s\n%s\n", err, string(output)) +// return nil +// } + +// // get kubeconfig +// kubeconfigPath := filepath.Join(os.TempDir(), fmt.Sprintf("%s.kubeconfig", input.Name)) +// cmd = exec.CommandContext(ctx, "lxc", "exec", input.Name, "--", "k8s", "config", ">", kubeconfigPath) +// output, err = cmd.CombinedOutput() +// if err != nil { +// fmt.Printf("Failed to get kubeconfig: %s\n%s\n", err, string(output)) +// return nil +// } + +// return nil +// } diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 4a6e51e5..5190cb22 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -982,8 +982,8 @@ type UpgradeControlPlaneAndWaitForUpgradeInput struct { ControlPlane *controlplanev1.CK8sControlPlane MaxControlPlaneMachineCount int64 KubernetesUpgradeVersion string - UpgradeMachineTemplate *string - WaitForMachinesToBeUpgraded []interface{} + WaitForMachinesToBeUpgraded []any + InfrastructureProviders []string } // UpgradeControlPlaneAndWaitForUpgrade upgrades a KubeadmControlPlane and waits for it to be upgraded. @@ -1002,17 +1002,6 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont input.ControlPlane.Spec.Version = input.KubernetesUpgradeVersion - // Create a new ObjectReference for the infrastructure provider - newInfrastructureRef := corev1.ObjectReference{ - APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", - Kind: "DockerMachineTemplate", - Name: fmt.Sprintf("%s-control-plane-new", input.Cluster.Name), - Namespace: input.ControlPlane.Spec.MachineTemplate.InfrastructureRef.Namespace, - } - - // Update the infrastructureRef - input.ControlPlane.Spec.MachineTemplate.InfrastructureRef = newInfrastructureRef - Eventually(func() error { return patchHelper.Patch(ctx, input.ControlPlane) }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch the new kubernetes version to KCP %s", klog.KObj(input.ControlPlane)) diff --git a/test/e2e/in_place_upgrade_test.go b/test/e2e/in_place_upgrade_test.go index 3d63471f..b1d37a22 100644 --- a/test/e2e/in_place_upgrade_test.go +++ b/test/e2e/in_place_upgrade_test.go @@ -53,6 +53,9 @@ var _ = Describe("In place upgrade", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) diff --git a/test/e2e/intermediate_ca_test.go b/test/e2e/intermediate_ca_test.go index c0f892b5..e21d750e 100755 --- a/test/e2e/intermediate_ca_test.go +++ b/test/e2e/intermediate_ca_test.go @@ -58,6 +58,10 @@ var _ = Describe("Intermediate CA", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) }) diff --git a/test/e2e/kcp_remediation_test.go b/test/e2e/kcp_remediation_test.go index 3c3b220e..1d8f0b05 100644 --- a/test/e2e/kcp_remediation_test.go +++ b/test/e2e/kcp_remediation_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" "k8s.io/utils/ptr" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) @@ -41,6 +42,9 @@ var _ = Describe("When testing KCP remediation", func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, InfrastructureProvider: ptr.To(clusterctl.DefaultInfrastructureProvider), + PostNamespaceCreated: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) { + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, workloadClusterNamespace) + }, } }) }) diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go index 5f40620e..d83ed223 100644 --- a/test/e2e/md_remediation_test.go +++ b/test/e2e/md_remediation_test.go @@ -60,6 +60,9 @@ var _ = Describe("When testing MachineDeployment remediation", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) diff --git a/test/e2e/node_scale_test.go b/test/e2e/node_scale_test.go index ed7b1e29..34cdd3aa 100644 --- a/test/e2e/node_scale_test.go +++ b/test/e2e/node_scale_test.go @@ -27,7 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" ) @@ -53,6 +53,9 @@ var _ = Describe("Workload cluster scaling", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) @@ -86,22 +89,14 @@ var _ = Describe("Workload cluster scaling", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(1), - WorkerMachineCount: pointer.Int64Ptr(1), + ControlPlaneMachineCount: ptr.To(int64(1)), + WorkerMachineCount: ptr.To(int64(1)), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) - // Check that the number of control plane cluster members matches what we expect. - workloadCluster := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name) - workloadClusterClientset := workloadCluster.GetClientSet() - - clusterMembers, err := getK8sdClusterMembers(ctx, workloadClusterClientset) - Expect(err).NotTo(HaveOccurred()) - Expect(clusterMembers).To(HaveLen(1)) - By("Scaling up worker nodes to 3") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ @@ -114,8 +109,8 @@ var _ = Describe("Workload cluster scaling", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(1), - WorkerMachineCount: pointer.Int64Ptr(3), + ControlPlaneMachineCount: ptr.To(int64(1)), + WorkerMachineCount: ptr.To(int64(3)), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), @@ -134,18 +129,14 @@ var _ = Describe("Workload cluster scaling", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(4), - WorkerMachineCount: pointer.Int64Ptr(3), + ControlPlaneMachineCount: ptr.To(int64(4)), + WorkerMachineCount: ptr.To(int64(3)), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) - clusterMembers, err = getK8sdClusterMembers(ctx, workloadClusterClientset) - Expect(err).NotTo(HaveOccurred()) - Expect(clusterMembers).To(HaveLen(4)) - By("Scaling down control planes to 3") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ @@ -158,18 +149,14 @@ var _ = Describe("Workload cluster scaling", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), - WorkerMachineCount: pointer.Int64Ptr(3), + ControlPlaneMachineCount: ptr.To(int64(3)), + WorkerMachineCount: ptr.To(int64(3)), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) - clusterMembers, err = getK8sdClusterMembers(ctx, workloadClusterClientset) - Expect(err).NotTo(HaveOccurred()) - Expect(clusterMembers).To(HaveLen(3)) - By("Scaling down worker nodes to 1") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ @@ -182,8 +169,8 @@ var _ = Describe("Workload cluster scaling", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), - WorkerMachineCount: pointer.Int64Ptr(1), + ControlPlaneMachineCount: ptr.To(int64(3)), + WorkerMachineCount: ptr.To(int64(1)), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), diff --git a/test/e2e/orchestrated_cp_in_place_upgrade_test.go b/test/e2e/orchestrated_cp_in_place_upgrade_test.go index 88b6b198..db21d17f 100644 --- a/test/e2e/orchestrated_cp_in_place_upgrade_test.go +++ b/test/e2e/orchestrated_cp_in_place_upgrade_test.go @@ -50,6 +50,9 @@ var _ = Describe("CK8sControlPlane Orchestrated In place upgrades", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) diff --git a/test/e2e/orchestrated_md_in_place_upgrade_test.go b/test/e2e/orchestrated_md_in_place_upgrade_test.go index 2eaf4bda..75e0dc27 100644 --- a/test/e2e/orchestrated_md_in_place_upgrade_test.go +++ b/test/e2e/orchestrated_md_in_place_upgrade_test.go @@ -53,6 +53,9 @@ var _ = Describe("Machine Deployment Orchestrated In place upgrades", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) diff --git a/test/e2e/refresh_certs_test.go b/test/e2e/refresh_certs_test.go index d28160e3..b367b5e3 100644 --- a/test/e2e/refresh_certs_test.go +++ b/test/e2e/refresh_certs_test.go @@ -56,6 +56,10 @@ var _ = Describe("Certificate Refresh", func() { // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + + // Create LXC secret for Incus provider if needed + createLXCSecretForIncus(ctx, bootstrapClusterProxy, e2eConfig, namespace.Name) + result = new(ApplyClusterTemplateAndWaitResult) clusterctlLogFolder = filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()) })