Skip to content

Commit 7724a75

Browse files
wkingwking
committed
test/extended/cli/adm_upgrade/recommend: Trust the ingress CA
Avoid [1]: : [Serial][sig-cli] oc adm upgrade recommend When the update service has conditional recommendations runs successfully with an accepted conditional recommendation to the --version target [Suite:openshift/conformance/serial] 20s { fail [github.com/openshift/origin/test/extended/cli/adm_upgrade/recommend.go:225]: Unexpected error: <*errors.errorString | 0xc008463cc0>: expected: Failed to check for at least some preconditions: failed to get alerts from Thanos: unable to get /api/v1/alerts from URI in the openshift-monitoring/thanos-querier Route: thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX->Get "https://thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX/api/v1/alerts": tls: failed to verify certificate: x509: certificate signed by unknown authority by retrieving the default (self-signed) ingress certificate and passing it to 'oc' via --certificate-authority and a temporary file. The default ingress certificate knobs I'm using are documented in [2]. The router-certs-default fallback may not be documented, but is backed by [3,4]. We need to keep trusting the Kube API server though, and while I'm not clear on the mechanism it uses to pick its external API server certificate, iterating over all the TLS secrets in the Kube API server namespace [5] and aggregating seems like it should pick up the one we need. [1]: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/30113/pull-ci-openshift-origin-main-e2e-aws-ovn-serial-2of2/1956090254670696448 [2]: https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/security_and_compliance/configuring-certificates#replacing-default-ingress_replacing-default-ingress [3]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/certificate/default_cert.go#L76-L83 [4]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/names.go#L152-L159 [5]: https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets
1 parent 533440d commit 7724a75

File tree

1 file changed

+88
-16
lines changed

1 file changed

+88
-16
lines changed

‎test/extended/cli/adm_upgrade/recommend.go‎

Lines changed: 88 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"net"
77
"net/url"
8+
"os"
89
"strconv"
910
"strings"
1011
"text/template"
@@ -31,6 +32,7 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
3132
oc := exutil.NewCLIWithFramework(f).AsAdmin()
3233
var cv *configv1.ClusterVersion
3334
var restoreChannel, restoreUpstream bool
35+
var caBundleFilePath string
3436

3537
g.BeforeAll(func() {
3638
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
@@ -51,6 +53,10 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
5153
if restoreUpstream {
5254
oc.Run("patch", "clusterversions.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/upstream", "value": "%s"}]`, cv.Spec.Upstream)).Execute()
5355
}
56+
57+
if caBundleFilePath != "" {
58+
os.Remove(caBundleFilePath)
59+
}
5460
})
5561

5662
g.It("runs successfully, even without upstream OpenShift Update Service customization", func() {
@@ -110,6 +116,7 @@ No updates available. You may still upgrade to a specific release image.*`)
110116

111117
g.Context("When the update service has conditional recommendations", func() {
112118
var currentVersion *semver.Version
119+
var token string
113120

114121
g.BeforeAll(func() {
115122
isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient())
@@ -175,16 +182,45 @@ No updates available. You may still upgrade to a specific release image.*`)
175182
}
176183
restoreUpstream = true
177184

185+
defaultIngressCert, err := getDefaultIngressCertificate(ctx, oc)
186+
o.Expect(err).NotTo(o.HaveOccurred())
187+
188+
kubeCerts, err := getKubernetesAPIServerCertificates(ctx, oc)
189+
o.Expect(err).NotTo(o.HaveOccurred())
190+
191+
caBundleFile, err := os.CreateTemp("", "ca-bundle")
192+
caBundleFilePath = caBundleFile.Name()
193+
_, err = caBundleFile.WriteString(fmt.Sprintf("%s\n%s", defaultIngressCert, kubeCerts))
194+
o.Expect(err).NotTo(o.HaveOccurred())
195+
196+
// alert retrieval requires a token-based kubeconfig to avoid:
197+
// Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
198+
o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
199+
o.Expect(oc.Run("create").Args("clusterrolebinding", fmt.Sprintf("%s-test", oc.Namespace()), "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
200+
token, err = oc.Run("create").Args("token", "test").Output()
201+
o.Expect(err).NotTo(o.HaveOccurred())
202+
178203
time.Sleep(16 * time.Second) // Give the CVO time to retrieve recommendations and push to status
179204
})
180205

206+
g.AfterAll(func() {
207+
// apparently ClusterRoleBindings are not automatically garbage-collected after the referenced service-account is removed (as part of namespace removal).
208+
oc.Run("delete").Args("clusterrolebinding", fmt.Sprintf("%s-test", oc.Namespace())).Execute()
209+
})
210+
181211
g.It("runs successfully when listing all updates", func() {
182-
out, err := oc.Run("adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
183-
o.Expect(err).NotTo(o.HaveOccurred())
184-
err = matchRegexp(out, `Upstream update service: http://.*
212+
oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
213+
o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
214+
o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
215+
216+
out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
217+
o.Expect(err).NotTo(o.HaveOccurred())
218+
err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
219+
220+
Upstream update service: http://.*
185221
Channel: test-channel [(]available channels: other-channel, test-channel[)]
186222
187-
Updates to 4.[0-9]*:
223+
Updates to 4[.][0-9]*:
188224
189225
Version: 4[.][0-9]*[.]0
190226
Image: example[.]com/test@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
@@ -195,23 +231,18 @@ Updates to 4[.][0-9]*:
195231
VERSION *ISSUES
196232
4[.][0-9]*[.]999 *no known issues relevant to this cluster
197233
4[.][0-9]*[.]998 *no known issues relevant to this cluster`)
198-
o.Expect(err).NotTo(o.HaveOccurred())
234+
o.Expect(err).NotTo(o.HaveOccurred())
235+
})
199236
})
200237

201238
g.It("runs successfully with an accepted conditional recommendation to the --version target", func() {
202-
// alert retrieval requires a token-based kubeconfig to avoid:
203-
// Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
204-
o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
205-
o.Expect(oc.Run("create").Args("clusterrolebinding", "test", "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
206-
token, err := oc.Run("create").Args("token", "test").Output()
207-
o.Expect(err).NotTo(o.HaveOccurred())
208-
209239
oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
210240
o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
211241
o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
212242

213-
out, err := oc.Run("adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
214-
o.Expect(err).To(o.HaveOccurred())
243+
out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
244+
245+
o.Expect(err).NotTo(o.HaveOccurred())
215246
err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
216247
217248
Upstream update service: http://.*
@@ -220,8 +251,9 @@ Channel: test-channel [(]available channels: other-channel, test-channel[)]
220251
Update to 4[.][0-9]*[.]0 Recommended=False:
221252
Image: example.com/test@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
222253
Release URL: https://example.com/release/4[.][0-9]*[.]0
223-
Reason: (TestRiskA|MultipleReasons)
224-
Message: (?s:.*)This is a test risk. https://example.com/testRiskA`)
254+
Reason: (accepted TestRiskA via ConditionalUpdateRisk|MultipleReasons)
255+
Message: (?s:.*)This is a test risk[.] https://example.com/testRiskA
256+
Update to 4[.][0-9]*[.]0 has no known issues relevant to this cluster other than the accepted ConditionalUpdateRisk.`)
225257
o.Expect(err).NotTo(o.HaveOccurred())
226258
})
227259
})
@@ -305,3 +337,43 @@ python3 -m http.server --bind ::
305337
Path: "graph",
306338
}, nil
307339
}
340+
341+
func getDefaultIngressCertificate(ctx context.Context, oc *exutil.CLI) (string, error) {
342+
defaultIngressSecretName, err := oc.Run("get").Args("--namespace=openshift-ingress-operator", "-o", "jsonpath={.spec.defaultCertificate.name}", "ingresscontroller.operator.openshift.io", "default").Output()
343+
if err != nil {
344+
return "", err
345+
}
346+
347+
if defaultIngressSecretName == "" {
348+
defaultIngressSecretName = "router-certs-default"
349+
}
350+
351+
ingressNamespace := "openshift-ingress"
352+
defaultIngressCert, err := oc.Run("extract").Args("--namespace", ingressNamespace, fmt.Sprintf("secret/%s", defaultIngressSecretName), "--keys=tls.crt", "--to=-").Output()
353+
if err != nil {
354+
return "", err
355+
}
356+
defaultIngressCert = fmt.Sprintf("%s\n", defaultIngressCert) // ensure a trailing newline, even if the earlier Output() stripped trailing newlines
357+
framework.Logf("default ingress certificate from the %s secret in the %s namespace: %q", defaultIngressSecretName, ingressNamespace, fmt.Sprintf("%s...", defaultIngressCert[:30]))
358+
return defaultIngressCert, nil
359+
}
360+
361+
func getKubernetesAPIServerCertificates(ctx context.Context, oc *exutil.CLI) (string, error) {
362+
kubeNamespace := "openshift-kube-apiserver"
363+
secrets, err := oc.AdminKubeClient().CoreV1().Secrets(kubeNamespace).List(ctx, metav1.ListOptions{})
364+
if err != nil {
365+
return "", err
366+
}
367+
368+
certs := make([]string, 0, len(secrets.Items))
369+
for _, secret := range secrets.Items {
370+
if secret.Type != corev1.SecretTypeTLS {
371+
continue
372+
}
373+
certs = append(certs, string(secret.Data["tls.crt"]))
374+
}
375+
376+
kubeCerts := strings.Join(certs, "")
377+
framework.Logf("default Kubernetes certificates from TLS secrets in the %s namespace: %q", kubeNamespace, fmt.Sprintf("%s...", kubeCerts[:30]))
378+
return kubeCerts, nil
379+
}

0 commit comments

Comments
 (0)