Skip to content

Commit 829b625

Browse files
kcp-ci-botkcp-ci-bot
authored
Merge pull request #149 from xrstf/probes
Improve reconciled resources
2 parents fa4c3be + d6284e6 commit 829b625

File tree

10 files changed

+186
-29
lines changed

10 files changed

+186
-29
lines changed

‎hack/run-e2e-tests.sh‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ fi
7070
echo "Kubeconfig is in $KUBECONFIG."
7171

7272
KUBECTL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kubectl)"
73+
KUSTOMIZE="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kustomize)"
7374
HELM="$(UGET_PRINT_PATH=absolute make --no-print-directory install-helm)"
7475
PROTOKOL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-protokol)"
7576

@@ -100,15 +101,17 @@ echo "Deploying cert-manager..."
100101

101102
"$KUBECTL" apply --filename hack/ci/testdata/clusterissuer.yaml
102103

103-
# build operator image and deploy it into kind
104-
echo "Building and deploying kcp-operator..."
105-
export IMG="ghcr.io/kcp-dev/kcp-operator:e2e"
106-
make --no-print-directory docker-build kind-load deploy
104+
# build operator image it into kind
105+
echo "Building and loading kcp-operator..."
106+
export IMG="ghcr.io/kcp-dev/kcp-operator:local"
107+
make --no-print-directory docker-build kind-load
108+
109+
echo "Deploying kcp-operator..."
110+
"$KUSTOMIZE" build hack/ci/testdata | "$KUBECTL" apply --filename -
107111

108112
"$PROTOKOL" --namespace 'e2e-*' --namespace kcp-operator-system --output "$DATA_DIR/kind-logs" 2>/dev/null &
109113
PROTOKOL_PID=$!
110114

111-
112115
echo "Running e2e tests..."
113116

114117
export HELM_BINARY="$HELM"

‎internal/resources/frontproxy/certificates.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ func (r *reconciler) serverCertificateReconciler() reconciling.NamedCertificateR
114114

115115
Usages: []certmanagerv1.KeyUsage{
116116
certmanagerv1.UsageServerAuth,
117+
certmanagerv1.UsageKeyEncipherment,
118+
certmanagerv1.UsageDigitalSignature,
117119
},
118120

119121
DNSNames: dnsNames,

‎internal/resources/frontproxy/deployment.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile
8181
SeccompProfile: &corev1.SeccompProfile{
8282
Type: corev1.SeccompProfileTypeRuntimeDefault,
8383
},
84+
ReadOnlyRootFilesystem: ptr.To(true),
85+
AllowPrivilegeEscalation: ptr.To(false),
86+
Capabilities: &corev1.Capabilities{
87+
Drop: []corev1.Capability{
88+
corev1.Capability("ALL"),
89+
},
90+
},
8491
},
8592
Ports: []corev1.ContainerPort{
8693
{
@@ -97,7 +104,7 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile
97104
TimeoutSeconds: 10,
98105
ProbeHandler: corev1.ProbeHandler{
99106
HTTPGet: &corev1.HTTPGetAction{
100-
Path: "/livez",
107+
Path: "/readyz",
101108
Port: intstr.FromString("https"),
102109
Scheme: corev1.URISchemeHTTPS,
103110
},
@@ -111,7 +118,7 @@ func (r *reconciler) deploymentReconciler() reconciling.NamedDeploymentReconcile
111118
TimeoutSeconds: 10,
112119
ProbeHandler: corev1.ProbeHandler{
113120
HTTPGet: &corev1.HTTPGetAction{
114-
Path: "/readyz",
121+
Path: "/livez",
115122
Port: intstr.FromString("https"),
116123
Scheme: corev1.URISchemeHTTPS,
117124
},

‎internal/resources/frontproxy/deployment_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,12 @@ func TestDeploymentReconciler(t *testing.T) {
9292

9393
// Check readiness probe
9494
assert.NotNil(t, container.ReadinessProbe)
95-
assert.Equal(t, "/livez", container.ReadinessProbe.HTTPGet.Path)
95+
assert.Equal(t, "/readyz", container.ReadinessProbe.HTTPGet.Path)
9696
assert.Equal(t, "https", container.ReadinessProbe.HTTPGet.Port.StrVal)
9797

9898
// Check liveness probe
9999
assert.NotNil(t, container.LivenessProbe)
100-
assert.Equal(t, "/readyz", container.LivenessProbe.HTTPGet.Path)
100+
assert.Equal(t, "/livez", container.LivenessProbe.HTTPGet.Path)
101101
assert.Equal(t, "https", container.LivenessProbe.HTTPGet.Port.StrVal)
102102
},
103103
},

‎internal/resources/rootshard/certificates.go‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ func ServerCertificateReconciler(rootShard *operatorv1alpha1.RootShard) reconcil
5252

5353
Usages: []certmanagerv1.KeyUsage{
5454
certmanagerv1.UsageServerAuth,
55+
certmanagerv1.UsageKeyEncipherment,
56+
certmanagerv1.UsageDigitalSignature,
5557
},
5658

5759
DNSNames: []string{
@@ -135,6 +137,11 @@ func ServiceAccountCertificateReconciler(rootShard *operatorv1alpha1.RootShard)
135137
Duration: &operatorv1alpha1.DefaultCertificateDuration,
136138
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
137139

140+
Usages: []certmanagerv1.KeyUsage{
141+
certmanagerv1.UsageDigitalSignature,
142+
certmanagerv1.UsageKeyEncipherment,
143+
},
144+
138145
PrivateKey: &certmanagerv1.CertificatePrivateKey{
139146
Algorithm: certmanagerv1.RSAKeyAlgorithm,
140147
Size: 4096,
@@ -162,8 +169,13 @@ func LogicalClusterAdminCertificateReconciler(rootShard *operatorv1alpha1.RootSh
162169
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
163170
cert.SetLabels(resources.GetRootShardResourceLabels(rootShard))
164171
cert.Spec = certmanagerv1.CertificateSpec{
165-
CommonName: "logical-cluster-admin",
166-
SecretName: name,
172+
CommonName: "logical-cluster-admin",
173+
SecretName: name,
174+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
175+
Labels: map[string]string{
176+
resources.RootShardLabel: rootShard.Name,
177+
},
178+
},
167179
Duration: &operatorv1alpha1.DefaultCertificateDuration,
168180
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
169181

@@ -202,8 +214,13 @@ func ExternalLogicalClusterAdminCertificateReconciler(rootShard *operatorv1alpha
202214
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
203215
cert.SetLabels(resources.GetRootShardResourceLabels(rootShard))
204216
cert.Spec = certmanagerv1.CertificateSpec{
205-
CommonName: "external-logical-cluster-admin",
206-
SecretName: name,
217+
CommonName: "external-logical-cluster-admin",
218+
SecretName: name,
219+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
220+
Labels: map[string]string{
221+
resources.RootShardLabel: rootShard.Name,
222+
},
223+
},
207224
Duration: &operatorv1alpha1.DefaultCertificateDuration,
208225
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
209226

@@ -242,8 +259,13 @@ func OperatorClientCertificateReconciler(rootShard *operatorv1alpha1.RootShard)
242259
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
243260
cert.SetLabels(resources.GetRootShardResourceLabels(rootShard))
244261
cert.Spec = certmanagerv1.CertificateSpec{
245-
CommonName: resources.OperatorUsername,
246-
SecretName: name,
262+
CommonName: resources.OperatorUsername,
263+
SecretName: name,
264+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
265+
Labels: map[string]string{
266+
resources.RootShardLabel: rootShard.Name,
267+
},
268+
},
247269
Duration: &operatorv1alpha1.DefaultCertificateDuration,
248270
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
249271

‎internal/resources/rootshard/deployment.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,10 @@ func DeploymentReconciler(rootShard *operatorv1alpha1.RootShard) reconciling.Nam
147147
Args: args,
148148
VolumeMounts: volumeMounts,
149149
Resources: defaultResourceRequirements,
150-
SecurityContext: &corev1.SecurityContext{
151-
ReadOnlyRootFilesystem: ptr.To(true),
152-
AllowPrivilegeEscalation: ptr.To(false),
153-
},
154150
}}
155151
dep.Spec.Template.Spec.Volumes = volumes
156152

153+
dep = utils.ApplyCommonShardDeploymentProperties(dep)
157154
dep = utils.ApplyCommonShardConfig(dep, &rootShard.Spec.CommonShardSpec)
158155
dep = utils.ApplyDeploymentTemplate(dep, rootShard.Spec.DeploymentTemplate)
159156
dep = utils.ApplyAuthConfiguration(dep, rootShard.Spec.Auth)

‎internal/resources/shard/certificates.go‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,14 @@ func ServerCertificateReconciler(shard *operatorv1alpha1.Shard, rootShard *opera
3838
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
3939
cert.SetLabels(resources.GetShardResourceLabels(shard))
4040
cert.Spec = certmanagerv1.CertificateSpec{
41-
SecretName: name,
41+
SecretName: name,
42+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
43+
Labels: map[string]string{
44+
resources.RootShardLabel: rootShard.Name,
45+
resources.ShardLabel: shard.Name,
46+
},
47+
},
48+
4249
Duration: &operatorv1alpha1.DefaultCertificateDuration,
4350
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
4451

@@ -49,6 +56,8 @@ func ServerCertificateReconciler(shard *operatorv1alpha1.Shard, rootShard *opera
4956

5057
Usages: []certmanagerv1.KeyUsage{
5158
certmanagerv1.UsageServerAuth,
59+
certmanagerv1.UsageKeyEncipherment,
60+
certmanagerv1.UsageDigitalSignature,
5261
},
5362

5463
DNSNames: []string{
@@ -78,7 +87,14 @@ func VirtualWorkspacesCertificateReconciler(shard *operatorv1alpha1.Shard, rootS
7887
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
7988
cert.SetLabels(resources.GetShardResourceLabels(shard))
8089
cert.Spec = certmanagerv1.CertificateSpec{
81-
SecretName: name,
90+
SecretName: name,
91+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
92+
Labels: map[string]string{
93+
resources.RootShardLabel: rootShard.Name,
94+
resources.ShardLabel: shard.Name,
95+
},
96+
},
97+
8298
Duration: &operatorv1alpha1.DefaultCertificateDuration,
8399
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
84100

@@ -117,11 +133,23 @@ func ServiceAccountCertificateReconciler(shard *operatorv1alpha1.Shard, rootShar
117133
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
118134
cert.SetLabels(resources.GetShardResourceLabels(shard))
119135
cert.Spec = certmanagerv1.CertificateSpec{
136+
SecretName: name,
137+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
138+
Labels: map[string]string{
139+
resources.RootShardLabel: rootShard.Name,
140+
resources.ShardLabel: shard.Name,
141+
},
142+
},
143+
120144
CommonName: name,
121-
SecretName: name,
122145
Duration: &operatorv1alpha1.DefaultCertificateDuration,
123146
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
124147

148+
Usages: []certmanagerv1.KeyUsage{
149+
certmanagerv1.UsageDigitalSignature,
150+
certmanagerv1.UsageKeyEncipherment,
151+
},
152+
125153
PrivateKey: &certmanagerv1.CertificatePrivateKey{
126154
Algorithm: certmanagerv1.RSAKeyAlgorithm,
127155
Size: 4096,
@@ -149,8 +177,15 @@ func RootShardClientCertificateReconciler(shard *operatorv1alpha1.Shard, rootSha
149177
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
150178
cert.SetLabels(resources.GetShardResourceLabels(shard))
151179
cert.Spec = certmanagerv1.CertificateSpec{
180+
SecretName: name,
181+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
182+
Labels: map[string]string{
183+
resources.RootShardLabel: rootShard.Name,
184+
resources.ShardLabel: shard.Name,
185+
},
186+
},
187+
152188
CommonName: fmt.Sprintf("shard-%s", shard.Name),
153-
SecretName: name,
154189
Duration: &operatorv1alpha1.DefaultCertificateDuration,
155190
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
156191

@@ -189,8 +224,15 @@ func LogicalClusterAdminCertificateReconciler(shard *operatorv1alpha1.Shard, roo
189224
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
190225
cert.SetLabels(resources.GetShardResourceLabels(shard))
191226
cert.Spec = certmanagerv1.CertificateSpec{
227+
SecretName: name,
228+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
229+
Labels: map[string]string{
230+
resources.RootShardLabel: rootShard.Name,
231+
resources.ShardLabel: shard.Name,
232+
},
233+
},
234+
192235
CommonName: fmt.Sprintf("logical-cluster-admin-shard-%s", shard.Name),
193-
SecretName: name,
194236
Duration: &operatorv1alpha1.DefaultCertificateDuration,
195237
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
196238

@@ -229,8 +271,15 @@ func ExternalLogicalClusterAdminCertificateReconciler(shard *operatorv1alpha1.Sh
229271
return name, func(cert *certmanagerv1.Certificate) (*certmanagerv1.Certificate, error) {
230272
cert.SetLabels(resources.GetShardResourceLabels(shard))
231273
cert.Spec = certmanagerv1.CertificateSpec{
274+
SecretName: name,
275+
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
276+
Labels: map[string]string{
277+
resources.RootShardLabel: rootShard.Name,
278+
resources.ShardLabel: shard.Name,
279+
},
280+
},
281+
232282
CommonName: fmt.Sprintf("external-logical-cluster-admin-shard-%s", shard.Name),
233-
SecretName: name,
234283
Duration: &operatorv1alpha1.DefaultCertificateDuration,
235284
RenewBefore: &operatorv1alpha1.DefaultCertificateRenewal,
236285

‎internal/resources/shard/deployment.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,10 @@ func DeploymentReconciler(shard *operatorv1alpha1.Shard, rootShard *operatorv1al
149149
Args: args,
150150
VolumeMounts: volumeMounts,
151151
Resources: defaultResourceRequirements,
152-
SecurityContext: &corev1.SecurityContext{
153-
ReadOnlyRootFilesystem: ptr.To(true),
154-
AllowPrivilegeEscalation: ptr.To(false),
155-
},
156152
}}
157153
dep.Spec.Template.Spec.Volumes = volumes
158154

155+
dep = utils.ApplyCommonShardDeploymentProperties(dep)
159156
dep = utils.ApplyCommonShardConfig(dep, &shard.Spec.CommonShardSpec)
160157
dep = utils.ApplyDeploymentTemplate(dep, shard.Spec.DeploymentTemplate)
161158
dep = utils.ApplyAuthConfiguration(dep, shard.Spec.Auth)

‎internal/resources/utils/secrets.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func (sm SecretMount) Build() (corev1.Volume, corev1.VolumeMount) {
3939
volumeMount := corev1.VolumeMount{
4040
Name: sm.VolumeName,
4141
MountPath: sm.MountPath,
42+
ReadOnly: true,
4243
}
4344

4445
return volume, volumeMount

0 commit comments

Comments
 (0)