diff --git a/hack/run-e2e-tests.sh b/hack/run-e2e-tests.sh index 0dbf1081..c8f2c464 100755 --- a/hack/run-e2e-tests.sh +++ b/hack/run-e2e-tests.sh @@ -70,6 +70,7 @@ fi echo "Kubeconfig is in $KUBECONFIG." KUBECTL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kubectl)" +KUSTOMIZE="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kustomize)" HELM="$(UGET_PRINT_PATH=absolute make --no-print-directory install-helm)" PROTOKOL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-protokol)" @@ -100,15 +101,17 @@ echo "Deploying cert-manager..." "$KUBECTL" apply --filename hack/ci/testdata/clusterissuer.yaml -# build operator image and deploy it into kind -echo "Building and deploying kcp-operator..." -export IMG="ghcr.io/kcp-dev/kcp-operator:e2e" -make --no-print-directory docker-build kind-load deploy +# build operator image it into kind +echo "Building and loading kcp-operator..." +export IMG="ghcr.io/kcp-dev/kcp-operator:local" +make --no-print-directory docker-build kind-load + +echo "Deploying kcp-operator..." +"$KUSTOMIZE" build hack/ci/testdata | "$KUBECTL" apply --filename - "$PROTOKOL" --namespace 'e2e-*' --namespace kcp-operator-system --output "$DATA_DIR/kind-logs" 2>/dev/null & PROTOKOL_PID=$! - echo "Running e2e tests..." export HELM_BINARY="$HELM" diff --git a/internal/resources/frontproxy/certificates.go b/internal/resources/frontproxy/certificates.go index 54211b35..c5078849 100644 --- a/internal/resources/frontproxy/certificates.go +++ b/internal/resources/frontproxy/certificates.go @@ -114,6 +114,8 @@ func (r *reconciler) serverCertificateReconciler() reconciling.NamedCertificateR Usages: []certmanagerv1.KeyUsage{ certmanagerv1.UsageServerAuth, + certmanagerv1.UsageKeyEncipherment, + certmanagerv1.UsageDigitalSignature, }, DNSNames: dnsNames, diff --git a/internal/resources/frontproxy/deployment.go b/internal/resources/frontproxy/deployment.go index b9ed9b4d..9bba2ae1 100644 --- a/internal/resources/frontproxy/deployment.go +++ b/internal/resources/frontproxy/deployment.go @@ -81,6 +81,13 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, }, + ReadOnlyRootFilesystem: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + corev1.Capability("ALL"), + }, + }, }, Ports: []corev1.ContainerPort{ { @@ -97,7 +104,7 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile TimeoutSeconds: 10, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/livez", + Path: "/readyz", Port: intstr.FromString("https"), Scheme: corev1.URISchemeHTTPS, }, @@ -111,7 +118,7 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile TimeoutSeconds: 10, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", + Path: "/livez", Port: intstr.FromString("https"), Scheme: corev1.URISchemeHTTPS, }, diff --git a/internal/resources/frontproxy/deployment_test.go b/internal/resources/frontproxy/deployment_test.go index 1e6d8a89..9159f57c 100644 --- a/internal/resources/frontproxy/deployment_test.go +++ b/internal/resources/frontproxy/deployment_test.go @@ -92,12 +92,12 @@ func TestDeploymentReconciler(t *testing.T) { // Check readiness probe assert.NotNil(t, container.ReadinessProbe) - assert.Equal(t, "/livez", container.ReadinessProbe.HTTPGet.Path) + assert.Equal(t, "/readyz", container.ReadinessProbe.HTTPGet.Path) assert.Equal(t, "https", container.ReadinessProbe.HTTPGet.Port.StrVal) // Check liveness probe assert.NotNil(t, container.LivenessProbe) - assert.Equal(t, "/readyz", container.LivenessProbe.HTTPGet.Path) + assert.Equal(t, "/livez", container.LivenessProbe.HTTPGet.Path) assert.Equal(t, "https", container.LivenessProbe.HTTPGet.Port.StrVal) }, }, diff --git a/internal/resources/rootshard/certificates.go b/internal/resources/rootshard/certificates.go index f9506e20..3d599346 100644 --- a/internal/resources/rootshard/certificates.go +++ b/internal/resources/rootshard/certificates.go @@ -52,6 +52,8 @@ func ServerCertificateReconciler(rootShard *operatorv1alpha1.RootShard) reconcil Usages: []certmanagerv1.KeyUsage{ certmanagerv1.UsageServerAuth, + certmanagerv1.UsageKeyEncipherment, + certmanagerv1.UsageDigitalSignature, }, DNSNames: []string{ @@ -135,6 +137,11 @@ func ServiceAccountCertificateReconciler(rootShard *operatorv1alpha1.RootShard) Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, + Usages: []certmanagerv1.KeyUsage{ + certmanagerv1.UsageDigitalSignature, + certmanagerv1.UsageKeyEncipherment, + }, + PrivateKey: &certmanagerv1.CertificatePrivateKey{ Algorithm: certmanagerv1.RSAKeyAlgorithm, Size: 4096, @@ -162,8 +169,13 @@ func LogicalClusterAdminCertificateReconciler(rootShard *operatorv1alpha1.RootSh return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetRootShardResourceLabels(rootShard)) cert.Spec = certmanagerv1.CertificateSpec{ - CommonName: "logical-cluster-admin", - SecretName: name, + CommonName: "logical-cluster-admin", + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + }, + }, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -202,8 +214,13 @@ func ExternalLogicalClusterAdminCertificateReconciler(rootShard *operatorv1alpha return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetRootShardResourceLabels(rootShard)) cert.Spec = certmanagerv1.CertificateSpec{ - CommonName: "external-logical-cluster-admin", - SecretName: name, + CommonName: "external-logical-cluster-admin", + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + }, + }, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -242,8 +259,13 @@ func OperatorClientCertificateReconciler(rootShard *operatorv1alpha1.RootShard) return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetRootShardResourceLabels(rootShard)) cert.Spec = certmanagerv1.CertificateSpec{ - CommonName: resources.OperatorUsername, - SecretName: name, + CommonName: resources.OperatorUsername, + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + }, + }, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, diff --git a/internal/resources/rootshard/deployment.go b/internal/resources/rootshard/deployment.go index 3e176af8..f905a87d 100644 --- a/internal/resources/rootshard/deployment.go +++ b/internal/resources/rootshard/deployment.go @@ -147,13 +147,10 @@ func DeploymentReconciler(rootShard *operatorv1alpha1.RootShard) reconciling.Nam Args: args, VolumeMounts: volumeMounts, Resources: defaultResourceRequirements, - SecurityContext: &corev1.SecurityContext{ - ReadOnlyRootFilesystem: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(false), - }, }} dep.Spec.Template.Spec.Volumes = volumes + dep = utils.ApplyCommonShardDeploymentProperties(dep) dep = utils.ApplyCommonShardConfig(dep, &rootShard.Spec.CommonShardSpec) dep = utils.ApplyDeploymentTemplate(dep, rootShard.Spec.DeploymentTemplate) dep = utils.ApplyAuthConfiguration(dep, rootShard.Spec.Auth) diff --git a/internal/resources/shard/certificates.go b/internal/resources/shard/certificates.go index a459ac78..4e310fbb 100644 --- a/internal/resources/shard/certificates.go +++ b/internal/resources/shard/certificates.go @@ -38,7 +38,14 @@ func ServerCertificateReconciler(shard *operatorv1alpha1.Shard, rootShard *opera return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ - SecretName: name, + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -49,6 +56,8 @@ func ServerCertificateReconciler(shard *operatorv1alpha1.Shard, rootShard *opera Usages: []certmanagerv1.KeyUsage{ certmanagerv1.UsageServerAuth, + certmanagerv1.UsageKeyEncipherment, + certmanagerv1.UsageDigitalSignature, }, DNSNames: []string{ @@ -78,7 +87,14 @@ func VirtualWorkspacesCertificateReconciler(shard *operatorv1alpha1.Shard, rootS return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ - SecretName: name, + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -117,11 +133,23 @@ func ServiceAccountCertificateReconciler(shard *operatorv1alpha1.Shard, rootShar return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + CommonName: name, - SecretName: name, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, + Usages: []certmanagerv1.KeyUsage{ + certmanagerv1.UsageDigitalSignature, + certmanagerv1.UsageKeyEncipherment, + }, + PrivateKey: &certmanagerv1.CertificatePrivateKey{ Algorithm: certmanagerv1.RSAKeyAlgorithm, Size: 4096, @@ -149,8 +177,15 @@ func RootShardClientCertificateReconciler(shard *operatorv1alpha1.Shard, rootSha return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + CommonName: fmt.Sprintf("shard-%s", shard.Name), - SecretName: name, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -189,8 +224,15 @@ func LogicalClusterAdminCertificateReconciler(shard *operatorv1alpha1.Shard, roo return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + CommonName: fmt.Sprintf("logical-cluster-admin-shard-%s", shard.Name), - SecretName: name, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, @@ -229,8 +271,15 @@ func ExternalLogicalClusterAdminCertificateReconciler(shard *operatorv1alpha1.Sh return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) { cert.SetLabels(resources.GetShardResourceLabels(shard)) cert.Spec = certmanagerv1.CertificateSpec{ + SecretName: name, + SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ + Labels: map[string]string{ + resources.RootShardLabel: rootShard.Name, + resources.ShardLabel: shard.Name, + }, + }, + CommonName: fmt.Sprintf("external-logical-cluster-admin-shard-%s", shard.Name), - SecretName: name, Duration: &operatorv1alpha1.DefaultCertificateDuration, RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal, diff --git a/internal/resources/shard/deployment.go b/internal/resources/shard/deployment.go index bd686f12..16e19b6d 100644 --- a/internal/resources/shard/deployment.go +++ b/internal/resources/shard/deployment.go @@ -149,13 +149,10 @@ func DeploymentReconciler(shard *operatorv1alpha1.Shard, rootShard *operatorv1al Args: args, VolumeMounts: volumeMounts, Resources: defaultResourceRequirements, - SecurityContext: &corev1.SecurityContext{ - ReadOnlyRootFilesystem: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(false), - }, }} dep.Spec.Template.Spec.Volumes = volumes + dep = utils.ApplyCommonShardDeploymentProperties(dep) dep = utils.ApplyCommonShardConfig(dep, &shard.Spec.CommonShardSpec) dep = utils.ApplyDeploymentTemplate(dep, shard.Spec.DeploymentTemplate) dep = utils.ApplyAuthConfiguration(dep, shard.Spec.Auth) diff --git a/internal/resources/utils/secrets.go b/internal/resources/utils/secrets.go index 7e688fff..e98edfc1 100644 --- a/internal/resources/utils/secrets.go +++ b/internal/resources/utils/secrets.go @@ -39,6 +39,7 @@ func (sm SecretMount) Build() (corev1.Volume, corev1.VolumeMount) { volumeMount := corev1.VolumeMount{ Name: sm.VolumeName, MountPath: sm.MountPath, + ReadOnly: true, } return volume, volumeMount diff --git a/internal/resources/utils/shard.go b/internal/resources/utils/shard.go index 8daf505c..17f8c81b 100644 --- a/internal/resources/utils/shard.go +++ b/internal/resources/utils/shard.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "github.com/kcp-dev/kcp-operator/internal/resources" @@ -40,6 +41,84 @@ func GetRootShardBatteries(rootShard *operatorv1alpha1.RootShard) []string { return getCommonShardBatteries() } +func ApplyCommonShardDeploymentProperties(deployment *appsv1.Deployment) *appsv1.Deployment { + if len(deployment.Spec.Template.Spec.Containers) == 0 { + panic("Deployment does not contain any containers.") + } + + container := deployment.Spec.Template.Spec.Containers[0] + + container.Ports = []corev1.ContainerPort{ + { + Name: "https", + ContainerPort: 6443, + Protocol: corev1.ProtocolTCP, + }, + } + + container.SecurityContext = &corev1.SecurityContext{ + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + ReadOnlyRootFilesystem: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + corev1.Capability("ALL"), + }, + }, + } + + container.StartupProbe = &corev1.Probe{ + FailureThreshold: 60, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + SuccessThreshold: 1, + TimeoutSeconds: 10, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("https"), + Scheme: corev1.URISchemeHTTPS, + }, + }, + } + + container.ReadinessProbe = &corev1.Probe{ + FailureThreshold: 6, + InitialDelaySeconds: 10, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 10, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("https"), + Scheme: corev1.URISchemeHTTPS, + }, + }, + } + + container.LivenessProbe = &corev1.Probe{ + FailureThreshold: 6, + InitialDelaySeconds: 10, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 10, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromString("https"), + Scheme: corev1.URISchemeHTTPS, + }, + }, + } + + deployment.Spec.Template.Spec.Containers[0] = container + + return deployment +} + func ApplyCommonShardConfig(deployment *appsv1.Deployment, spec *operatorv1alpha1.CommonShardSpec) *appsv1.Deployment { if len(deployment.Spec.Template.Spec.Containers) == 0 { panic("Deployment does not contain any containers.")