diff --git a/Makefile b/Makefile index 6146693..0235da6 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ test: manifests generate fmt vet setup-envtest ## Run tests. # CertManager is installed by default; skip with: # - CERT_MANAGER_INSTALL_SKIP=true K3D_CLUSTER ?= rt-bootstrapper-test-e2e +K3D_IMAGE ?= rancher/k3s:v1.33.6-k3s1 +K3D_ARGS ?= --k3s-arg '--kube-apiserver-arg=feature-gates=ClusterTrustBundle=true,ClusterTrustBundleProjection=true@server:*' \ + --k3s-arg '--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1/clustertrustbundles=true@server:*' .PHONY: setup-test-e2e setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist @@ -77,7 +80,7 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist echo "K3D cluster '$(K3D_CLUSTER)' already exists. Skipping creation." ;; \ *) \ echo "Creating K3D cluster '$(K3D_CLUSTER)'..."; \ - $(K3D) cluster create $(K3D_CLUSTER) ;; \ + $(K3D) cluster create --image=$(K3D_IMAGE) $(K3D_ARGS) $(K3D_CLUSTER) ;; \ esac .PHONY: test-e2e diff --git a/config/config/rt-bootstrapper-config.yaml b/config/config/rt-bootstrapper-config.yaml index 6897711..a51477f 100644 --- a/config/config/rt-bootstrapper-config.yaml +++ b/config/config/rt-bootstrapper-config.yaml @@ -8,6 +8,12 @@ data: "overrides": { "replace.me": "ghcr.io", "example.com": "ghcr.io" + }, + "clusterTrustBundleMapping": { + "clusterTrustBundleName": "rt-bootstrapper-k3d.test:ctb:1", + "certWritePath": "kube-apiserver-serving.pem", + "volumeMountPath": "/etc/ssl/certs", + "volumeName": "rt-bootstrapper-certs" } } kind: ConfigMap diff --git a/config/k3d/cluster_trust_bundle.yaml b/config/k3d/cluster_trust_bundle.yaml new file mode 100644 index 0000000..33e3b90 --- /dev/null +++ b/config/k3d/cluster_trust_bundle.yaml @@ -0,0 +1,18 @@ +apiVersion: certificates.k8s.io/v1beta1 +kind: ClusterTrustBundle +metadata: + name: k3d.test:ctb:1 +spec: + signerName: rt-bootstrapper-k3d.test/ctb + trustBundle: | + -----BEGIN CERTIFICATE----- + MIIBdzCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy + dmVyLWNhQDE3NjcwMTMxNjUwHhcNMjUxMjI5MTI1OTI1WhcNMzUxMjI3MTI1OTI1 + WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3NjcwMTMxNjUwWTATBgcqhkjO + PQIBBggqhkjOPQMBBwNCAASDZGb8hHA4r7/tLECdLLLtOQpfA0W+5FXdc4xJI7Zi + dwXz4WiliqVIxi77ow+c39EOe29X8yuNtbOouWsqn1Vho0IwQDAOBgNVHQ8BAf8E + BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoqr1Zk9sh+WqFtLhFgUe + e0m5zGEwCgYIKoZIzj0EAwIDSAAwRQIhAK5eY2h5Ui8OivvqqpmWPx7rJYiEWR1g + +K3J/5+FXUv2AiBQUtMXc/FlAHWT3u4j98v4XukRZftEVbrVK6+zn6EaFQ== + -----END CERTIFICATE----- + diff --git a/config/k3d/kustomization.yaml b/config/k3d/kustomization.yaml index 46a2497..1a77386 100644 --- a/config/k3d/kustomization.yaml +++ b/config/k3d/kustomization.yaml @@ -7,6 +7,7 @@ resources: - ../manager - ../certmanager - metrics_service.yaml +- cluster_trust_bundle.yaml - ../config - ../webhook diff --git a/internal/webhook/k8s/utilz.go b/internal/webhook/k8s/utilz.go index dd0d0db..a409f2b 100644 --- a/internal/webhook/k8s/utilz.go +++ b/internal/webhook/k8s/utilz.go @@ -3,6 +3,8 @@ package k8s import ( "fmt" "strings" + + corev1 "k8s.io/api/core/v1" ) func fixRegistry(registry string, overrides map[string]string) string { @@ -38,3 +40,46 @@ func Contains(l map[string]string, r map[string]string) bool { } return true } + +type ClusterTrustBundleMapping struct { + ClusterTrustBundleName string `json:"clusterTrustBundleName" validate:"required"` + CertWritePath string `json:"certWritePath" validate:"required"` + VolumeMountPath string `json:"volumeMountPath" validate:"required"` + VolumeName string `json:"volumeName" validate:"required"` +} + +func (r ClusterTrustBundleMapping) ClusterTrustedBundle() corev1.Volume { + return corev1.Volume{ + Name: r.VolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ClusterTrustBundle: &corev1.ClusterTrustBundleProjection{ + Name: &r.ClusterTrustBundleName, + Path: r.CertWritePath, + }, + }, + }, + }, + }, + } + +} + +func (r ClusterTrustBundleMapping) VolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: r.VolumeName, + ReadOnly: true, + MountPath: r.VolumeMountPath, + } +} + +func (r ClusterTrustBundleMapping) KeysAndValues() []any { + return []any{ + "name", r.VolumeName, + "signer", r.ClusterTrustBundleName, + "certWritePath", r.CertWritePath, + "volumeMountPath", r.VolumeMountPath, + } +} diff --git a/internal/webhook/v1/pod_defaulters.go b/internal/webhook/v1/pod_defaulters.go index 66f73f7..f8467b6 100644 --- a/internal/webhook/v1/pod_defaulters.go +++ b/internal/webhook/v1/pod_defaulters.go @@ -2,6 +2,7 @@ package v1 import ( "log/slog" + "reflect" "slices" "github.com/kyma-project/rt-bootstrapper/internal/webhook/k8s" @@ -18,6 +19,9 @@ var ( annotationsSetPullSecret = map[string]string{ apiv1.AnnotationSetPullSecret: "false", } + annotationAddClusterTrustBundle = map[string]string{ + apiv1.AnnotationAddClusterTrustBundle: "false", + } ) func defaultPod(update func(*corev1.Pod) bool, features map[string]string) PodDefaulter { @@ -96,3 +100,74 @@ func BuildPodDefaulterAddImagePullSecrets(secretName string) PodDefaulter { return defaultPod(addImgPullSecret, annotationsSetPullSecret) } + +func BuildDefaulterAddClusterTrustBundle(mapping k8s.ClusterTrustBundleMapping) PodDefaulter { + slog.Debug("building volume", mapping.KeysAndValues()...) + + vol := mapping.ClusterTrustedBundle() + + handleVolumeMount := func(cs []corev1.Container) bool { + // stores information if any container was modified + var result bool + + for i, c := range cs { + index := slices.IndexFunc(c.VolumeMounts, func(vm corev1.VolumeMount) bool { + return vm.Name == mapping.VolumeName + }) + + if index == -1 { + vm := mapping.VolumeMount() + cs[i].VolumeMounts = append(c.VolumeMounts, vm) + result = true + slog.Debug("volume mount added") + continue + } + + if reflect.DeepEqual(c.VolumeMounts[index], vol) { + slog.Debug("volume already mounted, nothing to do") + continue + } + + vm := mapping.VolumeMount() + cs[i].VolumeMounts[index] = vm + slog.Debug("volume mount replaced") + result = true + } + + return result + } + + handleVolumeMounts := func(modified bool, p *corev1.Pod) bool { + for _, cs := range [][]corev1.Container{p.Spec.Containers, p.Spec.InitContainers} { + if handleVolumeMount(cs) { + modified = true + } + } + return modified + } + + handleClusterTrustBundle := func(p *corev1.Pod) bool { + index := slices.IndexFunc(p.Spec.Volumes, func(v corev1.Volume) bool { + return v.Name == mapping.VolumeName + }) + + if index == -1 { + // volume does not exist, add it + p.Spec.Volumes = append(p.Spec.Volumes, vol) + slog.Debug("volume added") + return handleVolumeMounts(true, p) + } + + if reflect.DeepEqual(p.Spec.Volumes[index], vol) { + slog.Debug("volume already added, nothing to do") + return handleVolumeMounts(false, p) + } + + p.Spec.Volumes[index] = vol + slog.Debug("volume replaced") + + return handleVolumeMounts(true, p) + } + + return defaultPod(handleClusterTrustBundle, annotationAddClusterTrustBundle) +} diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index cea778e..324c2f6 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -62,6 +62,13 @@ func SetupPodWebhookWithManager(mgr ctrl.Manager, cfg *apiv1.Config) error { GetNsAnnotations: getNamespace, } + // conditional defaulters + + if cfg.ClusterTrustBundleMapping != nil { + d3 := BuildDefaulterAddClusterTrustBundle(*cfg.ClusterTrustBundleMapping) + defaulter.defaulters = append(defaulter.defaulters, d3) + } + return ctrl.NewWebhookManagedBy(mgr).For(&corev1.Pod{}). WithDefaulter(&defaulter). Complete() diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index 959b5a6..e4751b1 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -7,20 +7,23 @@ import ( "time" "github.com/go-playground/validator/v10" + "github.com/kyma-project/rt-bootstrapper/internal/webhook/k8s" ) const ( - AnnotationAlterImgRegistry = "rt-cfg.kyma-project.io/alter-img-registry" - AnnotationSetPullSecret = "rt-cfg.kyma-project.io/add-img-pull-secret" - AnnotationDefaulted = "rt-bootstrapper.kyma-project.io/defaulted" - FiledManager = "rt-bootstrapper" + AnnotationAlterImgRegistry = "rt-cfg.kyma-project.io/alter-img-registry" + AnnotationSetPullSecret = "rt-cfg.kyma-project.io/add-img-pull-secret" + AnnotationAddClusterTrustBundle = "rt-cfg.kyma-project.io/add-add-cluster-trust-bundle" + AnnotationDefaulted = "rt-bootstrapper.kyma-project.io/defaulted" + FiledManager = "rt-bootstrapper" ) type Config struct { - Overrides map[string]string `json:"overrides" validate:"required"` - ImagePullSecretName string `json:"imagePullSecretName" validate:"required"` - ImagePullSecretNamespace string `json:"imagePullSecretNamespace" validate:"required"` - SecretSyncInterval Duration `json:"secretSyncInterval" validate:"required"` + Overrides map[string]string `json:"overrides" validate:"required"` + ImagePullSecretName string `json:"imagePullSecretName" validate:"required"` + ImagePullSecretNamespace string `json:"imagePullSecretNamespace" validate:"required"` + SecretSyncInterval Duration `json:"secretSyncInterval" validate:"required"` + ClusterTrustBundleMapping *k8s.ClusterTrustBundleMapping `json:"clusterTrustBundleMapping,omitempty"` } type Duration time.Duration diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c66e02d..9d04a5f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -29,6 +29,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" ) // namespace where the project is deployed in @@ -308,6 +309,17 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyCAInjection).Should(Succeed()) }) + It("should provisioned cluster-trust-bundle", func() { + By("checking rt-bootstrapper-k3d.test:ctb:1") + cmd := exec.Command("kubectl", "get", + "clustertrustbundles.certificates.k8s.io", + "rt-bootstrapper-k3d.test:ctb:1", + "-o", "go-template={{ .spec.signerName }}") + signerName, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(signerName).To(Equal("rt-bootstrapper-k3d.test/ctb")) + }) + It("should alter the image name and add imagePullSecret property", func() { testNamespace := "rt-bootstrapper-test1" @@ -418,7 +430,7 @@ var _ = Describe("Manager", Ordered, func() { })) }) - It("should not modify pod spec", func() { + It("should inject cluster-trust-bundle", func() { By("applying the deployment") cmd := exec.Command("kubectl", "apply", "-f", "./test/e2e/testdata/test3.yaml", @@ -443,6 +455,52 @@ var _ = Describe("Manager", Ordered, func() { output, err := utils.Run(cmd) Expect(err).ShouldNot(HaveOccurred()) + pod, err := utils.ToPod(output) + Expect(err).ShouldNot(HaveOccurred()) + Expect(pod.Spec.Containers[0].Image).Should(HavePrefix("k8s.gcr.io")) + Expect(pod.Annotations[apiv1.AnnotationDefaulted]).Should(Equal("true")) + Expect(pod.Spec.Containers[0].VolumeMounts).Should(ContainElement(corev1.VolumeMount{ + Name: "rt-bootstrapper-certs", + ReadOnly: true, + MountPath: "/etc/ssl/certs", + })) + Expect(pod.Spec.Volumes[1].VolumeSource.Projected.Sources).Should(ContainElement(corev1.VolumeProjection{ + ClusterTrustBundle: &corev1.ClusterTrustBundleProjection{ + Name: ptr.To("rt-bootstrapper-k3d.test:ctb:1"), + Path: "kube-apiserver-serving.pem", + }, + })) + + Expect(pod.Spec.ImagePullSecrets).ShouldNot(ContainElement(corev1.LocalObjectReference{ + Name: "registry-credentials", + })) + }) + + It("should not modify pod spec", func() { + By("applying the deployment") + cmd := exec.Command("kubectl", "apply", + "-f", "./test/e2e/testdata/test4.yaml", + "-n", testNamespace1) + + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "wait", "deployment.apps/pause-test4", + "--for", "condition=Available", + "--namespace", testNamespace1, + "--timeout", "20s", + ) + + _, err = utils.Run(cmd) + Expect(err).ShouldNot(HaveOccurred()) + + cmd = exec.Command("kubectl", "get", "pod", + "-l", "app=pause-test4", + "-n", testNamespace1, + "-o", "jsonpath={.items[0]}") + output, err := utils.Run(cmd) + Expect(err).ShouldNot(HaveOccurred()) + pod, err := utils.ToPod(output) Expect(err).ShouldNot(HaveOccurred()) Expect(pod.Spec.Containers[0].Image).Should(HavePrefix("k8s.gcr.io")) diff --git a/test/e2e/testdata/test4.yaml b/test/e2e/testdata/test4.yaml new file mode 100644 index 0000000..dfd26c0 --- /dev/null +++ b/test/e2e/testdata/test4.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pause-test4 + namespace: rt-bootstrapper-test1 + labels: + app: pause-test4 +spec: + replicas: 1 + selector: + matchLabels: + app: pause-test4 + template: + metadata: + annotations: + rt-cfg.kyma-project.io/add-img-pull-secret: "false" + rt-cfg.kyma-project.io/add-add-cluster-trust-bundle: "false" + labels: + app: pause-test4 + spec: + containers: + - name: pause + image: k8s.gcr.io/pause:latest + +