OCPBUGS-33787: Tolerate the absence of ingress capability on HyperShift clusters

- Implement alternative ingress fields from the console operator config API
- Skip component route customizations if ingress capability is disabled
- Use NodePort type for console and downloads services if ingress capability is disabled
- Add document to describe how to implement alternative ingress on ROSA

Author: alebedev87

README.md

@@ -178,7 +178,7 @@ docker build -t <registry>/<your-username>/console-operator:<version> .
 # following: docker.io/openshift/origin-console-operator:latest
 # for development, you are going to push to an alternate registry.
 # specifically it can look something like this:
-docker build -t quay.io/benjaminapetersen/console-operator:latest .
+docker build -f Dockerfile.rhel7 -t quay.io/benjaminapetersen/console-operator:latest .
 ```
 You can optionally build a specific version.
 

bindata/assets/services/console-nodeport-service.yaml

@@ -0,0 +1,20 @@
+# This 'console' service manifest is used when the ingress cluster capability is disabled.
+# Service will be exposed using a NodePort to enable the alternative ingress.
+apiVersion: v1
+kind: Service
+metadata:
+  name: console
+  namespace: openshift-console
+  labels:
+    app: console
+spec:
+  ports:
+    - name: https
+      protocol: TCP
+      port: 443
+      targetPort: 8443
+  selector:
+    app: console
+    component: ui
+  type: NodePort
+  sessionAffinity: None

bindata/assets/services/downloads-nodeport-service.yaml

@@ -0,0 +1,18 @@
+# This 'downloads' service manifest is used when the ingress cluster capability is disabled.
+# Service will be exposed using a NodePort to enable the alternative ingress.
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: openshift-console
+  name: downloads
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    app: console
+    component: downloads
+  type: NodePort
+  sessionAffinity: None

docs/alb-ingress-rosa-hcp.md

@@ -0,0 +1,115 @@
+# Use AWS ALB as alternative ingress on ROSA HCP
+
+This doc aims at showing the effort needed to expose the OpenShift console via AWS ALB on a ROSA HCP cluster.
+The use case in mind is [HyperShift hosted clusters where the Ingress capability is disabled](https://github.com/openshift/enhancements/pull/1415).
+
+## Requirements
+
+- ROSA HCP OpenShift cluster.
+- [AWS Load Balancer Operator installed and its controller created](https://docs.openshift.com/rosa/networking/aws-load-balancer-operator.html).
+- User logged as a cluster admin.
+
+## Procedure
+
+### Create certificate in AWS Certificate Manager
+
+In order to configure an HTTPS listener on AWS ALB you need to have a certificate created in AWS Certificate Manager.
+You can import an existing certificate or request a new one. Make sure the certificate is created in the same region as your cluster.
+Note the certificate ARN, you will need it later.
+
+### Create Ingress resources for the NodePort services
+
+To provision ALBs create the following resources:
+```bash
+cat <<EOF | oc apply -f -
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/target-type: instance
+    alb.ingress.kubernetes.io/backend-protocol: HTTPS
+    alb.ingress.kubernetes.io/certificate-arn: ${CERTIFICATE_ARN}
+  name: console
+  namespace: openshift-console
+spec:
+  ingressClassName: alb
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: console
+                port:
+                  number: 443
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/target-type: instance
+    alb.ingress.kubernetes.io/backend-protocol: HTTP
+    alb.ingress.kubernetes.io/certificate-arn: ${CERTIFICATE_ARN}
+  name: downloads
+  namespace: openshift-console
+spec:
+  ingressClassName: alb
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: downloads
+                port:
+                  number: 80
+EOF
+```
+
+### Update console config
+
+Once the console ALBs are ready you need to let the console operator know which urls to use.
+Update the console operator config providing the custom urls:
+```bash
+$ CONSOLE_ALB_HOST=$(oc -n openshift-console get ing console -o yaml | yq .status.loadBalancer.ingress[0].hostname)
+$ DOWNLOADS_ALB_HOST=$(oc -n openshift-console get ing downloads -o yaml | yq .status.loadBalancer.ingress[0].hostname)
+$ oc patch console.operator.openshift.io cluster --type=merge -p "{\"spec\":{\"ingress\":{\"consoleURL\":\"https://${CONSOLE_ALB_HOST}\",\"clientDownloadsURL\":\"https://${DOWNLOADS_ALB_HOST}\"}}}"
+```
+
+## Notes
+
+1. ROSA HCP does not have the authentication operator, the authentication server is managed centrally by the HyperShift layer:
+```bash
+$ oc -n openshift-authentication-operator get deploy,route
+No resources found
+
+$ oc -n openshift-authentication get pods,routes
+No resources found
+
+$ oc get oauthclient | grep -v console
+NAME                           SECRET                                        WWW-CHALLENGE   TOKEN-MAX-AGE   REDIRECT URIS
+openshift-browser-client                                                     false           default         https://oauth.mytestcluster.5199.s3.devshift.org:443/oauth/token/display
+openshift-challenging-client                                                 true            default         https://oauth.mytestcluster.5199.s3.devshift.org:443/oauth/token/implicit
+
+$ oc -n openshift-console rsh deploy/console curl -k https://openshift.default.svc/.well-known/oauth-authorization-server
+{
+"issuer": "https://oauth.mytestcluster.5199.s3.devshift.org:443",
+"authorization_endpoint": "https://oauth.mytestcluster.5199.s3.devshift.org:443/oauth/authorize",
+"token_endpoint": "https://oauth.mytestcluster.5199.s3.devshift.org:443/oauth/token",
+```
+
+2. When the ingress capability is disabled, the console operator relies on the end user to provide the console and download URLs (using the operator API) for health checks and oauthclient.
+
+3. When the ingress capability is disabled, the console operator skips the implementation of the component route customization.
+
+4. To simulate the absence of ingress connectivity when the ingress capability is disabled, set the desired replicas to zero in the default ingress controller:
+```bash
+$ oc -n openshift-ingress-operator patch ingresscontroller default --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":0}]'
+```
+
+## Links
+- [Demo of ALB ingress for the console on ROSA HCP](https://drive.google.com/file/d/1uWZgFbSeZTlDzlFyPW7QcH-625JsbSbw/view)

pkg/console/controllers/clidownloads/controller.go

@@ -4,6 +4,7 @@ import (
 	// standard lib
 	"context"
 	"fmt"
+	"net/url"
 	"time"
 
 	// kube
@@ -114,25 +115,37 @@ func (c *CLIDownloadsSyncController) Sync(ctx context.Context, controllerContext
 	}
 
 	statusHandler := status.NewStatusHandler(c.operatorClient)
-	ingressConfig, err := c.ingressConfigLister.Get(api.ConfigResourceName)
-	if err != nil {
-		return statusHandler.FlushAndReturn(err)
-	}
 
-	activeRouteName := api.OpenShiftConsoleDownloadsRouteName
-	routeConfig := routesub.NewRouteConfig(updatedOperatorConfig, ingressConfig, activeRouteName)
-	if routeConfig.IsCustomHostnameSet() {
-		activeRouteName = api.OpenshiftDownloadsCustomRouteName
-	}
+	var (
+		downloadsURI *url.URL
+		downloadsErr error
+	)
+	if len(operatorConfig.Spec.Ingress.ClientDownloadsURL) == 0 {
+		ingressConfig, err := c.ingressConfigLister.Get(api.ConfigResourceName)
+		if err != nil {
+			return statusHandler.FlushAndReturn(err)
+		}
 
-	downloadsRoute, downloadsRouteErr := c.routeLister.Routes(api.TargetNamespace).Get(activeRouteName)
-	if downloadsRouteErr != nil {
-		return downloadsRouteErr
-	}
+		activeRouteName := api.OpenShiftConsoleDownloadsRouteName
+		routeConfig := routesub.NewRouteConfig(updatedOperatorConfig, ingressConfig, activeRouteName)
+		if routeConfig.IsCustomHostnameSet() {
+			activeRouteName = api.OpenshiftDownloadsCustomRouteName
+		}
+
+		downloadsRoute, downloadsRouteErr := c.routeLister.Routes(api.TargetNamespace).Get(activeRouteName)
+		if downloadsRouteErr != nil {
+			return downloadsRouteErr
+		}
 
-	downloadsURI, _, downloadsRouteErr := routeapihelpers.IngressURI(downloadsRoute, downloadsRoute.Spec.Host)
-	if downloadsRouteErr != nil {
-		return downloadsRouteErr
+		downloadsURI, _, downloadsErr = routeapihelpers.IngressURI(downloadsRoute, downloadsRoute.Spec.Host)
+		if downloadsErr != nil {
+			return downloadsErr
+		}
+	} else {
+		downloadsURI, downloadsErr = url.Parse(operatorConfig.Spec.Ingress.ClientDownloadsURL)
+		if downloadsErr != nil {
+			return fmt.Errorf("failed to parse downloads url: %w", downloadsErr)
+		}
 	}
 
 	ocConsoleCLIDownloads := PlatformBasedOCConsoleCLIDownloads(downloadsURI.String(), api.OCCLIDownloadsCustomResourceName)

pkg/console/controllers/healthcheck/controller.go

@@ -6,6 +6,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	// k8s
@@ -78,7 +79,6 @@ func NewHealthCheckController(
 			util.IncludeNamesFilter(api.ConfigResourceName),
 			operatorConfigInformer.Informer(),
 			configV1Informers.Ingresses().Informer(),
-			configV1Informers.Infrastructures().Informer(),
 		).WithFilteredEventsInformers( // service
 		util.IncludeNamesFilter(api.TrustedCAConfigMapName, api.OAuthServingCertConfigMapName),
 		configMapInformer.Informer(),
@@ -153,10 +153,22 @@ func (c *HealthCheckController) CheckRouteHealth(ctx context.Context, operatorCo
 		retry.DefaultRetry,
 		func(err error) bool { return err != nil },
 		func() error {
-			url, _, err := routeapihelpers.IngressURI(route, route.Spec.Host)
-			if err != nil {
-				reason = "RouteNotAdmitted"
-				return fmt.Errorf("console route is not admitted")
+			var (
+				url *url.URL
+				err error
+			)
+			if len(operatorConfig.Spec.Ingress.ConsoleURL) == 0 {
+				url, _, err = routeapihelpers.IngressURI(route, route.Spec.Host)
+				if err != nil {
+					reason = "RouteNotAdmitted"
+					return fmt.Errorf("console route is not admitted")
+				}
+			} else {
+				url, err = url.Parse(operatorConfig.Spec.Ingress.ConsoleURL)
+				if err != nil {
+					reason = "FailedParseConsoleURL"
+					return fmt.Errorf("failed to parse console url: %w", err)
+				}
 			}
 
 			caPool, err := c.getCA(ctx, route.Spec.TLS)

pkg/console/controllers/oauthclients/oauthclients.go

@@ -3,6 +3,7 @@ package oauthclients
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -131,15 +132,26 @@ func (c *oauthClientsController) sync(ctx context.Context, controllerContext fac
 		return err
 	}
 
-	routeName := api.OpenShiftConsoleRouteName
-	routeConfig := routesub.NewRouteConfig(operatorConfig, ingressConfig, routeName)
-	if routeConfig.IsCustomHostnameSet() {
-		routeName = api.OpenshiftConsoleCustomRouteName
-	}
-
-	_, consoleURL, _, routeErr := routesub.GetActiveRouteInfo(c.routesLister, routeName)
-	if routeErr != nil {
-		return routeErr
+	var consoleURL *url.URL
+
+	if len(operatorConfig.Spec.Ingress.ConsoleURL) == 0 {
+		routeName := api.OpenShiftConsoleRouteName
+		routeConfig := routesub.NewRouteConfig(operatorConfig, ingressConfig, routeName)
+		if routeConfig.IsCustomHostnameSet() {
+			routeName = api.OpenshiftConsoleCustomRouteName
+		}
+
+		_, url, _, routeErr := routesub.GetActiveRouteInfo(c.routesLister, routeName)
+		if routeErr != nil {
+			return routeErr
+		}
+		consoleURL = url
+	} else {
+		url, err := url.Parse(operatorConfig.Spec.Ingress.ConsoleURL)
+		if err != nil {
+			return fmt.Errorf("failed to parse console url: %w", err)
+		}
+		consoleURL = url
 	}
 
 	waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)

pkg/console/controllers/route/controller.go

@@ -40,11 +40,13 @@ type RouteSyncController struct {
 	routeName            string
 	isHealthCheckEnabled bool
 	// clients
-	operatorClient       v1helpers.OperatorClient
-	routeClient          routeclientv1.RoutesGetter
-	operatorConfigLister operatorv1listers.ConsoleLister
-	ingressConfigLister  configlistersv1.IngressLister
-	secretLister         corev1listers.SecretLister
+	operatorClient             v1helpers.OperatorClient
+	routeClient                routeclientv1.RoutesGetter
+	operatorConfigLister       operatorv1listers.ConsoleLister
+	ingressConfigLister        configlistersv1.IngressLister
+	secretLister               corev1listers.SecretLister
+	infrastructureConfigLister configlistersv1.InfrastructureLister
+	clusterVersionLister       configlistersv1.ClusterVersionLister
 }
 
 func NewRouteSyncController(
@@ -63,13 +65,15 @@ func NewRouteSyncController(
 	recorder events.Recorder,
 ) factory.Controller {
 	ctrl := &RouteSyncController{
-		routeName:            routeName,
-		isHealthCheckEnabled: isHealthCheckEnabled,
-		operatorClient:       operatorClient,
-		operatorConfigLister: operatorConfigInformer.Lister(),
-		ingressConfigLister:  configInformer.Config().V1().Ingresses().Lister(),
-		routeClient:          routev1Client,
-		secretLister:         secretInformer.Lister(),
+		routeName:                  routeName,
+		isHealthCheckEnabled:       isHealthCheckEnabled,
+		operatorClient:             operatorClient,
+		operatorConfigLister:       operatorConfigInformer.Lister(),
+		ingressConfigLister:        configInformer.Config().V1().Ingresses().Lister(),
+		routeClient:                routev1Client,
+		secretLister:               secretInformer.Lister(),
+		infrastructureConfigLister: configInformer.Config().V1().Infrastructures().Lister(),
+		clusterVersionLister:       configInformer.Config().V1().ClusterVersions().Lister(),
 	}
 
 	configV1Informers := configInformer.Config().V1()
@@ -114,6 +118,35 @@ func (c *RouteSyncController) Sync(ctx context.Context, controllerContext factor
 
 	statusHandler := status.NewStatusHandler(c.operatorClient)
 
+	// Do not proceed to the route checks if alternative ingress is requested.
+	switch c.routeName {
+	case api.OpenShiftConsoleRouteName:
+		if len(operatorConfig.Spec.Ingress.ConsoleURL) != 0 {
+			return statusHandler.FlushAndReturn(nil)
+		}
+	case api.OpenShiftConsoleDownloadsRouteName:
+		if len(operatorConfig.Spec.Ingress.ClientDownloadsURL) != 0 {
+			return statusHandler.FlushAndReturn(nil)
+		}
+	}
+
+	infrastructureConfig, err := c.infrastructureConfigLister.Get(api.ConfigResourceName)
+	if err != nil {
+		return statusHandler.FlushAndReturn(err)
+	}
+
+	clusterVersionConfig, err := c.clusterVersionLister.Get("version")
+	if err != nil {
+		return statusHandler.FlushAndReturn(err)
+	}
+
+	// Disable the route check for external control plane topology (hypershift) if the ingress capability is disabled.
+	// The components will miss the required RBAC to implement the custom hostname or TLS.
+	// Link: https://github.com/openshift/enhancements/blob/f5290a98ea4f23f8e76621806b656a3849c74a17/enhancements/ingress/optional-ingress-hypershift.md#component-routes.
+	if util.IsExternalControlPlaneWithIngressDisabled(infrastructureConfig, clusterVersionConfig) {
+		return statusHandler.FlushAndReturn(nil)
+	}
+
 	ingressConfig, err := c.ingressConfigLister.Get(api.ConfigResourceName)
 	if err != nil {
 		return statusHandler.FlushAndReturn(err)