feat: Support oauth2 service in proxy mode (#1561)

Author: nilsgstrabo

.vscode/launch.json

@@ -153,8 +153,8 @@
 				"RADIX_CONTAINER_REGISTRY": "radixdev.azurecr.io",
 				"RADIX_APP_CONTAINER_REGISTRY": "radixdevapp.azurecr.io",
 				"RADIX_OAUTH_PROXY_DEFAULT_OIDC_ISSUER_URL": "https://login.microsoftonline.com/3aa4a235-b6e2-48d5-9195-7fcf05b459b0/v2.0",
-				"RADIX_OAUTH_PROXY_IMAGE": "quay.io/oauth2-proxy/oauth2-proxy:v7.9.0",
-				"RADIX_OAUTH_REDIS_IMAGE": "docker.io/library/redis:8.0",
+				"RADIX_OAUTH_PROXY_IMAGE": "quay.io/oauth2-proxy/oauth2-proxy:v7.12.0",
+				"RADIX_OAUTH_REDIS_IMAGE": "docker.io/library/redis:8.2.2-alpine3.22",
 				"RADIXOPERATOR_TENANT_ID": "3aa4a235-b6e2-48d5-9195-7fcf05b459b0",
 				"KUBERNETES_SERVICE_PORT": "443",
 				"REGISTRATION_CONTROLLER_THREADS": "10",
@@ -168,10 +168,10 @@
 				"RADIXOPERATOR_PODSECURITYSTANDARD_ENFORCE_LEVEL": "baseline",
 				"RADIXOPERATOR_PODSECURITYSTANDARD_APP_NAMESPACE_ENFORCE_LEVEL": "privileged",
 				"RADIXOPERATOR_PODSECURITYSTANDARD_ENFORCE_VERSION": "v1.23",
-				"RADIXOPERATOR_PODSECURITYSTANDARD_AUDIT_LEVEL": "restricted",
-				"RADIXOPERATOR_PODSECURITYSTANDARD_AUDIT_VERSION": "v1.23",
-				"RADIXOPERATOR_PODSECURITYSTANDARD_WARN_LEVEL": "restricted",
-				"RADIXOPERATOR_PODSECURITYSTANDARD_WARN_VERSION": "v1.23",
+				"RADIXOPERATOR_PODSECURITYSTANDARD_AUDIT_LEVEL": "",
+				"RADIXOPERATOR_PODSECURITYSTANDARD_AUDIT_VERSION": "",
+				"RADIXOPERATOR_PODSECURITYSTANDARD_WARN_LEVEL": "",
+				"RADIXOPERATOR_PODSECURITYSTANDARD_WARN_VERSION": "",
 				"RADIX_ZONE": "dev",
 				"RADIX_DEPLOYMENTS_PER_ENVIRONMENT_HISTORY_LIMIT": "10",
 				"RADIX_PIPELINE_JOBS_HISTORY_LIMIT": "5",
@@ -186,7 +186,7 @@
 				"RADIXOPERATOR_ORPHANED_ENVIRONMENTS_CLEANUP_CRON": "0 0 * * *",
 				"LOG_LEVEL": "info",
 				"LOG_PRETTY": "true",
-				"RADIXOPERATOR_PIPELINE_IMAGE": "ghcr.io/equinor/radix/pipeline:1.104.1",
+				"RADIXOPERATOR_PIPELINE_IMAGE": "ghcr.io/equinor/radix/pipeline:1.107.1",
 				"RADIX_PIPELINE_GIT_CLONE_GIT_IMAGE": "docker.io/alpine/git:2.45.2"
 			},
 			"args": ["--useOutClusterClient=false"]

operator/deployment/controller.go

@@ -10,6 +10,7 @@ import (
 	"github.com/equinor/radix-operator/pkg/apis/kube"
 	"github.com/equinor/radix-operator/pkg/apis/metrics"
 	v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
+	"github.com/equinor/radix-operator/pkg/apis/utils/annotations"
 	radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
 	informers "github.com/equinor/radix-operator/pkg/client/informers/externalversions"
 	"github.com/rs/zerolog/log"
@@ -130,7 +131,8 @@ func NewController(ctx context.Context,
 			if radixutils.ArrayEqualElements(newRr.Spec.AdGroups, oldRr.Spec.AdGroups) &&
 				radixutils.ArrayEqualElements(newRr.Spec.AdUsers, oldRr.Spec.AdUsers) &&
 				radixutils.ArrayEqualElements(newRr.Spec.ReaderAdGroups, oldRr.Spec.ReaderAdGroups) &&
-				radixutils.ArrayEqualElements(newRr.Spec.ReaderAdUsers, oldRr.Spec.ReaderAdUsers) {
+				radixutils.ArrayEqualElements(newRr.Spec.ReaderAdUsers, oldRr.Spec.ReaderAdUsers) &&
+				oldRr.Annotations[annotations.PreviewOAuth2ProxyModeAnnotation] == newRr.Annotations[annotations.PreviewOAuth2ProxyModeAnnotation] {
 				return
 			}
 

operator/deployment/controller_test.go

@@ -5,7 +5,9 @@ import (
 	"testing"
 
 	"github.com/equinor/radix-operator/operator/common"
+	"github.com/equinor/radix-operator/pkg/apis/kube"
 	radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
+	"github.com/equinor/radix-operator/pkg/apis/utils/annotations"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/suite"
 	corev1 "k8s.io/api/core/v1"
@@ -21,7 +23,7 @@ func TestControllerSuite(t *testing.T) {
 }
 
 func (s *controllerTestSuite) Test_Controller_Calls_Handler() {
-	rdName, namespace := "any-rd", "any-ns"
+	appName, rdName, namespace := "any-app", "any-rd", "any-ns"
 
 	sut := NewController(context.Background(), s.KubeClient, s.RadixClient, s.Handler, s.KubeInformerFactory, s.RadixInformerFactory)
 	s.RadixInformerFactory.Start(s.Ctx.Done())
@@ -30,18 +32,22 @@ func (s *controllerTestSuite) Test_Controller_Calls_Handler() {
 		err := sut.Run(s.Ctx, 5)
 		s.Require().NoError(err)
 	}()
+	rr, err := s.RadixClient.RadixV1().RadixRegistrations().Create(context.Background(), &radixv1.RadixRegistration{
+		ObjectMeta: metav1.ObjectMeta{Name: appName, Annotations: map[string]string{}}},
+		metav1.CreateOptions{})
+	s.Require().NoError(err)
 
 	// Create RD should sync
 	rd := &radixv1.RadixDeployment{
-		ObjectMeta: metav1.ObjectMeta{Name: rdName, Namespace: namespace},
+		ObjectMeta: metav1.ObjectMeta{Name: rdName, Namespace: namespace, Labels: map[string]string{kube.RadixAppLabel: appName}},
 		Spec: radixv1.RadixDeploymentSpec{
 			Components: []radixv1.RadixDeployComponent{
 				{Name: "any", PublicPort: "http"},
 			},
 		},
 	}
 	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
-	rd, err := s.RadixClient.RadixV1().RadixDeployments(rd.Namespace).Create(s.Ctx, rd, metav1.CreateOptions{})
+	rd, err = s.RadixClient.RadixV1().RadixDeployments(rd.Namespace).Create(s.Ctx, rd, metav1.CreateOptions{})
 	s.Require().NoError(err)
 	s.WaitForSynced("Sync should be called")
 
@@ -54,7 +60,7 @@ func (s *controllerTestSuite) Test_Controller_Calls_Handler() {
 
 	// Update RD labels should sync.
 	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
-	rd.Labels = map[string]string{"key": "value"}
+	rd.Labels["key"] = "val"
 	rd, err = s.RadixClient.RadixV1().RadixDeployments(rd.ObjectMeta.Namespace).Update(s.Ctx, rd, metav1.UpdateOptions{})
 	s.Require().NoError(err)
 	s.WaitForSynced("Sync should be called")
@@ -107,4 +113,38 @@ func (s *controllerTestSuite) Test_Controller_Calls_Handler() {
 	s.Require().NoError(err)
 	s.WaitForSynced("Sync should be called")
 
+	// Sync should trigger when annotation radix.equinor.com/preview-oauth2-proxy-mode changes on RR
+	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
+	rr.Annotations[annotations.PreviewOAuth2ProxyModeAnnotation] = "any"
+	rr, err = s.RadixClient.RadixV1().RadixRegistrations().Update(context.Background(), rr, metav1.UpdateOptions{})
+	s.Require().NoError(err)
+	s.WaitForSynced("Sync should be called")
+
+	// Sync should trigger when AdGroups changes on RR
+	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
+	rr.Spec.AdGroups = []string{"new-admin-group"}
+	rr, err = s.RadixClient.RadixV1().RadixRegistrations().Update(context.Background(), rr, metav1.UpdateOptions{})
+	s.Require().NoError(err)
+	s.WaitForSynced("Sync should be called")
+
+	// Sync should trigger when AdUsers changes on RR
+	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
+	rr.Spec.AdUsers = []string{"new-admin-user"}
+	rr, err = s.RadixClient.RadixV1().RadixRegistrations().Update(context.Background(), rr, metav1.UpdateOptions{})
+	s.Require().NoError(err)
+	s.WaitForSynced("Sync should be called")
+
+	// Sync should trigger when ReaderAdGroups changes on RR
+	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
+	rr.Spec.ReaderAdGroups = []string{"new-reader-group"}
+	rr, err = s.RadixClient.RadixV1().RadixRegistrations().Update(context.Background(), rr, metav1.UpdateOptions{})
+	s.Require().NoError(err)
+	s.WaitForSynced("Sync should be called")
+
+	// Sync should trigger when ReaderAdUsers changes on RR
+	s.Handler.EXPECT().Sync(gomock.Any(), namespace, rdName).Times(1).DoAndReturn(s.SyncedChannelCallback())
+	rr.Spec.ReaderAdUsers = []string{"new-reader-user"}
+	_, err = s.RadixClient.RadixV1().RadixRegistrations().Update(context.Background(), rr, metav1.UpdateOptions{})
+	s.Require().NoError(err)
+	s.WaitForSynced("Sync should be called")
 }

operator/deployment/handler.go

@@ -135,13 +135,12 @@ func (t *handler) Sync(ctx context.Context, namespace, name string) error {
 		return err
 	}
 
-	ingressAnnotations := ingress.GetAnnotationProvider(t.ingressConfiguration, rd.Namespace, t.oauth2DefaultConfig)
-
 	auxResourceManagers := []deployment.AuxiliaryResourceManager{
-		deployment.NewOAuthProxyResourceManager(rd, radixRegistration, t.kubeutil, t.oauth2DefaultConfig, ingress.GetAuxOAuthProxyAnnotationProviders(), t.oauth2ProxyDockerImage, t.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
+		deployment.NewOAuthProxyResourceManager(rd, radixRegistration, t.kubeutil, t.oauth2DefaultConfig, ingress.GetOAuthAnnotationProviders(), ingress.GetOAuthProxyModeAnnotationProviders(t.ingressConfiguration, rd.Namespace), t.oauth2ProxyDockerImage, t.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
 		deployment.NewOAuthRedisResourceManager(rd, radixRegistration, t.kubeutil, t.oauth2RedisDockerImage, t.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
 	}
 
+	ingressAnnotations := ingress.GetComponentAnnotationProvider(t.ingressConfiguration, rd.Namespace, t.oauth2DefaultConfig)
 	syncRD := rd.DeepCopy()
 	deployment := t.deploymentSyncerFactory.CreateDeploymentSyncer(t.kubeclient, t.kubeutil, t.radixclient, t.prometheusperatorclient, t.certClient, radixRegistration, syncRD, ingressAnnotations, auxResourceManagers, t.config)
 	err = deployment.OnSync(ctx)

operator/deployment/handler_test.go

@@ -160,13 +160,13 @@ func (s *handlerSuite) Test_Sync() {
 			ingress.NewForceSslRedirectAnnotationProvider(),
 			ingress.NewIngressConfigurationAnnotationProvider(ingressConfig),
 			ingress.NewClientCertificateAnnotationProvider(activeRd.Namespace),
-			ingress.NewOAuth2AnnotationProvider(oauthConfig),
+			ingress.NewOAuth2AnnotationProvider(oauthConfig, activeRd.Namespace),
 			ingress.NewIngressPublicAllowListAnnotationProvider(),
 			ingress.NewIngressPublicConfigAnnotationProvider(),
 			ingress.NewRedirectErrorPageAnnotationProvider(),
 		}
 		expectedAuxResources := []deployment.AuxiliaryResourceManager{
-			deployment.NewOAuthProxyResourceManager(activeRd, rr, s.kubeUtil, oauthConfig, ingress.GetAuxOAuthProxyAnnotationProviders(), oauthProxyImage, s.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
+			deployment.NewOAuthProxyResourceManager(activeRd, rr, s.kubeUtil, oauthConfig, ingress.GetOAuthAnnotationProviders(), ingress.GetOAuthProxyModeAnnotationProviders(ingressConfig, activeRd.Namespace), oauthProxyImage, s.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
 			deployment.NewOAuthRedisResourceManager(activeRd, rr, s.kubeUtil, oauthRedisImage, s.config.ContainerRegistryConfig.ExternalRegistryAuthSecret),
 		}
 		factory.

operator/dnsalias/controller.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"reflect"
 
-	radixutils "github.com/equinor/radix-common/utils"
 	"github.com/equinor/radix-operator/operator/common"
 	"github.com/equinor/radix-operator/pkg/apis/metrics"
 	radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
+	"github.com/equinor/radix-operator/pkg/apis/utils/annotations"
 	radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels"
 	radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
 	informers "github.com/equinor/radix-operator/pkg/client/informers/externalversions"
@@ -50,24 +50,21 @@ func NewController(ctx context.Context, kubeClient kubernetes.Interface,
 	addEventHandlersForRadixDNSAliases(radixDNSAliasInformer, controller, &logger)
 	addEventHandlersForRadixDeployments(radixInformerFactory, controller, radixClient, &logger)
 	addEventHandlersForIngresses(ctx, kubeInformerFactory, controller, &logger)
-	addEventHandlersForRadixRegistrations(radixInformerFactory, controller, radixClient, &logger)
+	addEventHandlersForRadixRegistrations(ctx, radixInformerFactory, controller, radixClient, &logger)
 	return controller
 }
 
-func addEventHandlersForRadixRegistrations(radixInformerFactory informers.SharedInformerFactory, controller *common.Controller, radixClient radixclient.Interface, logger *zerolog.Logger) {
+func addEventHandlersForRadixRegistrations(ctx context.Context, radixInformerFactory informers.SharedInformerFactory, controller *common.Controller, radixClient radixclient.Interface, logger *zerolog.Logger) {
 	radixRegistrationInformer := radixInformerFactory.Radix().V1().RadixRegistrations()
 	if _, err := radixRegistrationInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		UpdateFunc: func(oldObj, newObj interface{}) {
 			oldRR := oldObj.(*radixv1.RadixRegistration)
 			newRR := newObj.(*radixv1.RadixRegistration)
-			if oldRR.GetResourceVersion() == newRR.GetResourceVersion() &&
-				radixutils.ArrayEqualElements(oldRR.Spec.AdGroups, newRR.Spec.AdGroups) &&
-				radixutils.ArrayEqualElements(oldRR.Spec.AdUsers, newRR.Spec.AdUsers) &&
-				radixutils.ArrayEqualElements(oldRR.Spec.ReaderAdGroups, newRR.Spec.ReaderAdGroups) &&
-				radixutils.ArrayEqualElements(oldRR.Spec.ReaderAdUsers, newRR.Spec.ReaderAdUsers) {
+
+			if oldRR.Annotations[annotations.PreviewOAuth2ProxyModeAnnotation] == newRR.Annotations[annotations.PreviewOAuth2ProxyModeAnnotation] {
 				return // updating RadixRegistration has the same resource version. Do nothing.
 			}
-			enqueueRadixDNSAliasesForAppName(controller, radixClient, newRR.GetName(), logger)
+			enqueueRadixDNSAliasesForAppName(ctx, controller, radixClient, newRR.GetName(), logger)
 		},
 	}); err != nil {
 		panic(err)
@@ -175,23 +172,21 @@ func enqueueRadixDNSAliasesForRadixDeployment(controller *common.Controller, rad
 		return
 	}
 	for _, radixDNSAlias := range radixDNSAliases {
-		radixDNSAlias := radixDNSAlias
 		logger.Debug().Msgf("re-sync RadixDNSAlias %s", radixDNSAlias.GetName())
 		if err := controller.Enqueue(&radixDNSAlias); err != nil {
 			logger.Error().Err(err).Msgf("failed to enqueue RadixDNSAlias %s", radixDNSAlias.GetName())
 		}
 	}
 }
 
-func enqueueRadixDNSAliasesForAppName(controller *common.Controller, radixClient radixclient.Interface, appName string, logger *zerolog.Logger) {
+func enqueueRadixDNSAliasesForAppName(ctx context.Context, controller *common.Controller, radixClient radixclient.Interface, appName string, logger *zerolog.Logger) {
 	logger.Debug().Msgf("Added or updated an RadixRegistration %s. Enqueue relevant RadixDNSAliases", appName)
-	radixDNSAliases, err := getRadixDNSAliasForApp(radixClient, appName)
+	radixDNSAliases, err := getRadixDNSAliasForApp(ctx, radixClient, appName)
 	if err != nil {
 		logger.Error().Err(err).Msgf("failed to get list of RadixDNSAliases for the application %s", appName)
 		return
 	}
 	for _, radixDNSAlias := range radixDNSAliases {
-		radixDNSAlias := radixDNSAlias
 		logger.Debug().Msgf("Enqueue RadixDNSAlias %s", radixDNSAlias.GetName())
 		if err := controller.Enqueue(&radixDNSAlias); err != nil {
 			logger.Error().Err(err).Msgf("failed to enqueue RadixDNSAlias %s", radixDNSAlias.GetName())
@@ -210,8 +205,8 @@ func getRadixDNSAliasForAppAndEnvironment(radixClient radixclient.Interface, app
 	return radixDNSAliasList.Items, err
 }
 
-func getRadixDNSAliasForApp(radixClient radixclient.Interface, appName string) ([]radixv1.RadixDNSAlias, error) {
-	radixDNSAliasList, err := radixClient.RadixV1().RadixDNSAliases().List(context.Background(), metav1.ListOptions{
+func getRadixDNSAliasForApp(ctx context.Context, radixClient radixclient.Interface, appName string) ([]radixv1.RadixDNSAlias, error) {
+	radixDNSAliasList, err := radixClient.RadixV1().RadixDNSAliases().List(ctx, metav1.ListOptions{
 		LabelSelector: radixlabels.ForApplicationName(appName).String(),
 	})
 	if err != nil {
@@ -222,10 +217,7 @@ func getRadixDNSAliasForApp(radixClient radixclient.Interface, appName string) (
 
 func deepEqual(old, new *radixv1.RadixDNSAlias) bool {
 	return reflect.DeepEqual(new.Spec, old.Spec) &&
-		reflect.DeepEqual(new.ObjectMeta.Labels, old.ObjectMeta.Labels) &&
-		reflect.DeepEqual(new.ObjectMeta.Annotations, old.ObjectMeta.Annotations) &&
-		reflect.DeepEqual(new.ObjectMeta.Finalizers, old.ObjectMeta.Finalizers) &&
-		reflect.DeepEqual(new.ObjectMeta.DeletionTimestamp, old.ObjectMeta.DeletionTimestamp)
+		reflect.DeepEqual(new.ObjectMeta.Labels, old.ObjectMeta.Labels)
 }
 
 func getOwner(ctx context.Context, radixClient radixclient.Interface, _, name string) (interface{}, error) {