add a controller that reconciles SCCs' volumes

The storage team added "ephemeral" and "csi" volumes to all SCCs
somewhere around OCP version 4.13. However, since the past us set
SCCs as "create-only" (due to the legacy permission system),
meaning the CVO won't reconcile the SCCs, this change
never made it to upgraded clusters' SCCs. People that lack SCC update
privileges were installing odd operators they found on the internets
in hope to fix their SCC issues.

Introduce a controller that adds these volumes to all SCCs that the
CVO does not automatically reconcile.

Author: stlaz

bindata/assets/config/config-overrides.yaml

@@ -17,4 +17,3 @@ serviceAccountPublicKeyFiles:
   # The following path contains the public keys needed to verify bound sa
   # tokens. This is only supported post-bootstrap.
   - /etc/kubernetes/static-pod-resources/configmaps/bound-sa-token-signing-certs
-

pkg/operator/sccreconcilecontroller/sccreconcilecontroller.go

@@ -0,0 +1,77 @@
+package sccreconcilecontroller
+
+import (
+	"context"
+
+	securityv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1"
+	securityv1informers "github.com/openshift/client-go/security/informers/externalversions/security/v1"
+	securityv1listers "github.com/openshift/client-go/security/listers/security/v1"
+	"github.com/openshift/library-go/pkg/controller/factory"
+	"github.com/openshift/library-go/pkg/operator/events"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+type sccReconcileController struct {
+	sccClient securityv1client.SecurityV1Interface
+	sccLister securityv1listers.SecurityContextConstraintsLister
+}
+
+var createOnlySCCs = []string{
+	"anyuid",
+	"hostaccess",
+	"hostmount-anyuid",
+	"hostnetwork",
+	"nonroot",
+	"restricted",
+}
+
+// NewSCCReconcileController reconciles the "ephemeral" and "csi" volumes into
+// otherwise "create-only" reconciled SCCs to make these volumes available to
+// regular users in upgraded clusters.
+//
+// TODO:(consider): In the past we tried to reconcile the original (non-v2) SCCs with CVO
+// but that backfired badly as people still used the original SCC permission model
+// directly writing to SCC's "users" and "groups" fields as compared to
+// the newer RBAC approach.
+// Consider using this controller to reconcile all the fields BUT "users" and "groups"
+// for these older SCCs.
+func NewSCCReconcileController(
+	sccClient securityv1client.SecurityV1Interface,
+	sccInformer securityv1informers.SecurityContextConstraintsInformer,
+	recorder events.Recorder,
+) (factory.Controller, error) {
+	c := &sccReconcileController{
+		sccClient: sccClient,
+		sccLister: sccInformer.Lister(),
+	}
+
+	return factory.New().
+		WithSync(c.sync).
+		WithFilteredEventsInformersQueueKeyFunc(
+			factory.ObjectNameToKey,
+			factory.NamesFilter(createOnlySCCs...),
+			sccInformer.Informer(),
+		).
+		ToController("SCCReconcileController", recorder.WithComponentSuffix("scc-reconcile-controller")), nil
+}
+
+func (c *sccReconcileController) sync(ctx context.Context, controllerContext factory.SyncContext) error {
+	sccName := controllerContext.QueueKey()
+	scc, err := c.sccLister.Get(sccName)
+	if err != nil {
+		return err
+	}
+
+	volumeSet := sets.New(scc.Volumes...)
+	if volumeSet.Has("ephemeral") && volumeSet.Has("csi") {
+		return nil
+	}
+
+	volumeSet.Insert("ephemeral", "csi")
+	sccCopy := scc.DeepCopy()
+	sccCopy.Volumes = volumeSet.UnsortedList()
+
+	_, err = c.sccClient.SecurityContextConstraints().Update(ctx, sccCopy, v1.UpdateOptions{})
+	return err
+}

pkg/operator/starter.go

@@ -15,6 +15,8 @@ import (
 	operatorv1client "github.com/openshift/client-go/operator/clientset/versioned"
 	operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions"
 	operatorcontrolplaneclient "github.com/openshift/client-go/operatorcontrolplane/clientset/versioned"
+	securityclient "github.com/openshift/client-go/security/clientset/versioned"
+	securityvnformers "github.com/openshift/client-go/security/informers/externalversions"
 	"github.com/openshift/cluster-kube-apiserver-operator/bindata"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/boundsatokensignercontroller"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/certrotationcontroller"
@@ -28,6 +30,7 @@ import (
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/podsecurityreadinesscontroller"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/resourcesynccontroller"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/sccreconcilecontroller"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/serviceaccountissuercontroller"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/startupmonitorreadiness"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/targetconfigcontroller"
@@ -84,6 +87,10 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	if err != nil {
 		return err
 	}
+	securityClient, err := securityclient.NewForConfig(controllerContext.KubeConfig)
+	if err != nil {
+		return err
+	}
 	operatorcontrolplaneClient, err := operatorcontrolplaneclient.NewForConfig(controllerContext.KubeConfig)
 	if err != nil {
 		return err
@@ -113,6 +120,8 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 		return err
 	}
 
+	securityInformers := securityvnformers.NewSharedInformerFactory(securityClient, 10*time.Minute)
+
 	desiredVersion := status.VersionForOperatorFromEnv()
 	missingVersion := "0.0.1-snapshot"
 
@@ -427,6 +436,12 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 		controllerContext.EventRecorder,
 	)
 
+	sccReconcileController, err := sccreconcilecontroller.NewSCCReconcileController(
+		securityClient.SecurityV1(),
+		securityInformers.Security().V1().SecurityContextConstraints(),
+		controllerContext.EventRecorder,
+	)
+
 	podSecurityReadinessController, err := podsecurityreadinesscontroller.NewPodSecurityReadinessController(
 		controllerContext.ProtoKubeConfig,
 		operatorClient,
@@ -448,6 +463,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	migrationInformer.Start(ctx.Done())
 	apiextensionsInformers.Start(ctx.Done())
 	operatorInformers.Start(ctx.Done())
+	securityInformers.Start(ctx.Done())
 
 	go staticPodControllers.Start(ctx)
 	go resourceSyncController.Run(ctx, 1)
@@ -470,6 +486,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	go webhookSupportabilityController.Run(ctx, 1)
 	go serviceAccountIssuerController.Run(ctx, 1)
 	go podSecurityReadinessController.Run(ctx, 1)
+	go sccReconcileController.Run(ctx, 1)
 
 	<-ctx.Done()
 	return nil