Fix the default Coherence Pod security context to be compatible with OpenShift and to be configurable. (#770)

Author: thegridman

api/v1/coherenceresourcespec_types.go

@@ -833,21 +833,11 @@ func (in *CoherenceResourceSpec) GetImagePullSecrets() []corev1.LocalObjectRefer
 // GetSecurityContext returns the Pod security context to use.
 func (in *CoherenceResourceSpec) GetSecurityContext() *corev1.PodSecurityContext {
 	if in == nil || in.SecurityContext == nil {
-		return DefaultSecurityContext()
+		return operator.DefaultSecurityContext()
 	}
 	return in.SecurityContext
 }
 
-func DefaultSecurityContext() *corev1.PodSecurityContext {
-	return &corev1.PodSecurityContext{
-		RunAsNonRoot:        ptr.To(DefaultRunAsNonRoot),
-		RunAsUser:           ptr.To(DefaultRunAsUser),
-		RunAsGroup:          ptr.To(DefaultRunAsGroup),
-		FSGroup:             ptr.To(DefaultFsGroup),
-		FSGroupChangePolicy: ptr.To(DefaultFSGroupChangePolicy),
-	}
-}
-
 // GetServiceAccountName returns the service account name for the cluster.
 func (in *CoherenceResourceSpec) GetServiceAccountName() string {
 	if in != nil {

api/v1/common_test.go

@@ -503,7 +503,7 @@ func createMinimalExpectedPodSpec(deployment coh.CoherenceResource) corev1.PodTe
 					VolumeSource: emptyVolume,
 				},
 			},
-			SecurityContext:           coh.DefaultSecurityContext(),
+			SecurityContext:           operator.DefaultSecurityContext(),
 			TopologySpreadConstraints: spec.EnsureTopologySpreadConstraints(deployment),
 			Affinity:                  spec.CreateDefaultPodAffinity(deployment),
 			ServiceAccountName:        spec.GetServiceAccountName(),

config/manager/manager.yaml

@@ -87,6 +87,9 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        - mountPath: /coherence-operator/config
+          name: config
+          readOnly: true
         readinessProbe:
           httpGet:
             port: health
@@ -114,6 +117,10 @@ spec:
             drop:
               - "ALL"
       volumes:
+      - name: config
+        configMap:
+          name: coherence-operator
+          optional: true
       - name: cert
         secret:
           defaultMode: 420

docs/about/04_coherence_spec.adoc

@@ -1051,6 +1051,9 @@ m| ipFamilies | IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned t
 This field may hold a maximum of two entries (dual-stack families, in either order).  These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. m| []https://pkg.go.dev/k8s.io/api/core/v1#IPFamily | false
 m| ipFamilyPolicy | IPFamilyPolicy represents the dual-stack-ness requested or required by this Service, and is gated by the "IPv6DualStack" feature gate.  If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field.  This field will be wiped when updating a service to type ExternalName. m| *https://{k8s-doc-link}/#ipfamilypolicy-v1-core[corev1.IPFamilyPolicy] | false
 m| allocateLoadBalancerNodePorts | allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer.  Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. allocateLoadBalancerNodePorts may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is alpha-level and is only honored by servers that enable the ServiceLBNodePortControl feature. m| *bool | false
+m| loadBalancerClass | loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. m| *string | false
+m| internalTrafficPolicy | InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). m| *https://{k8s-doc-link}/#serviceinternaltrafficpolicy-v1-core[corev1.ServiceInternalTrafficPolicy] | false
+m| trafficDistribution | TrafficDistribution offers a way to express preferences for how traffic is distributed to Service endpoints. Implementations can use this field as a hint, but are not required to guarantee strict adherence. If the field is not set, the implementation will apply its default routing strategy. If set to "PreferClose", implementations should prioritize endpoints that are in the same zone. m| *string | false
 |===
 
 <<Table of Contents,Back to TOC>>

docs/installation/011_install_manifests.adoc

@@ -103,9 +103,6 @@ below in <<manual-crd,Manually Install the CRDs>>.
 [#manual-crd]
 == Manually Install the CRDs
 
-Although by default the Operator will install its CRDs, they can be manually installed into Kubernetes.
-This may be required where the Operator is running with restricted permissions as described above.
-
 The Operator release artifacts include small versions of the two CRDs which can be installed with the following commands:
 
 [source,bash]
@@ -133,3 +130,4 @@ Or on MacOS, where `sed` is slightly different:
 ----
 sed -i '' -e 's/replicas: 3/replicas: 1/g' coherence-operator.yaml
 ----
+

docs/other/045_security_context.adoc

@@ -16,18 +16,23 @@ Kubernetes allows you to configure a https://kubernetes.io/docs/tasks/configure-
 
 For more details see the Kubernetes https://kubernetes.io/docs/tasks/configure-pod-container/security-context/[Security Context] documentation.
 
+=== The Default Security Context
+
 The Coherence Operator configures a default security context for the Coherence Pods is none is specified in the `Coherence` resource yaml.
 The default security context looks like this:
 [source,yaml]
 ----
 securityContext:
   runAsNonRoot: true
-  runAsUser: 1000
-  runAsGroup: 2000
-  fsGroup: 2000
+  runAsUser: 1000650000
+  runAsGroup: 1000650000
+  fsGroup: 1000650000
   fsGroupChangePolicy: "OnRootMismatch"
 ----
 
+It is possible to change the values used for the default security context by specifying them when the Operator is installed.
+See the <<config,Configure The Default Security Context>> section below.
+
 It is possible to override this as described below.
 
 === Setting the Pod Security Context
@@ -72,3 +77,116 @@ spec:
       capabilities:
         add: ["NET_ADMIN", "SYS_TIME"]
 ----
+
+[#config]
+=== Configure The Default Security Context
+
+As already mentioned above the default security context created by the operator looks like this:
+
+[source,yaml]
+----
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 1000650000
+  runAsGroup: 1000650000
+  fsGroup: 1000650000
+  fsGroupChangePolicy: "OnRootMismatch"
+----
+
+The default values used for `runAsUser`, `runAsGroup` and `fsGroup` can be configured using the Operator's configuration file.
+
+When the Operator is installed using the default installation it will read an optional configuration file from
+an optional `ConfigMap`. The `ConfigMap` must be created in the same namespace as the operator is running and
+should be named `coherence-operator`. The config map should contain a yaml file named `coherence-operator.yaml`.
+
+[IMPORTANT]
+====
+The `coherence-operator` config map MUST be created before the Operator is installed, even if the yaml file
+that it contains is empty.
+
+The Operator will watch the config file for changes, so if the `ConfigMap` is updated after the Operator is started,
+the changes will take effect. If the `ConfigMap` does not exist when the Operator is started then the config file
+cannot be mounted for the Operator to watch.
+====
+
+
+==== Disable The Default Security Context
+
+To disable the creation of a default Pod security context for Coherence Pods, create a configuration file
+name `coherence-operator.yaml` with the following contents.
+
+[source]
+.coherence-operator.yaml
+----
+coherenceSecurityContext:
+  enabled: false
+----
+
+Create the `ConfigMap` using the configuration file in the same namespace that the operator will be installed into.
+For example, if the operator is to be installed into a namespace named `coherence` the `ConfigMap` can be created
+using the following command:
+
+[source,bash]
+----
+kubectl -n coherence create configmap coherence-operator \
+  --from-file=coherence-operator.yaml
+----
+
+With the `coherenceSecurityContext.enabled` field set to false, the Operator will not apply a default security context
+to the Coherence Pods. This may be useful in environments such as OpenShift which already apply a default security
+configuration to Pods.
+
+==== Change The Default Security Context
+
+In the configuration file, any field under the `coherenceSecurityContext` section will be applied to
+the default security context and override the operators default values.
+
+For example, a `ConfigMap` could be created with the following file:
+
+[source]
+.coherence-operator.yaml
+----
+coherenceSecurityContext:
+  runAsUser: 1000
+----
+
+This will override the `runAsUser` field to be set to `1000` resulting in a default security context as shown below:
+
+[source,yaml]
+----
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 1000
+  runAsGroup: 1000650000
+  fsGroup: 1000650000
+  fsGroupChangePolicy: "OnRootMismatch"
+----
+
+If the config file contains an empty value, this will result in the corresponding value being unset in the
+security context.
+This is useful for unsetting fields that the operator has default values for such as `runAsUser`, `runAsGroup`,
+`runAsNonRoot`, `fsGroup` and `fsGroupChangePolicy`.
+
+
+For example, the default `runAsUser` value is `1000650000`.
+The configuration file can be created with a `runAsUser` field with no value as shown below
+
+[source]
+.coherence-operator.yaml
+----
+coherenceSecurityContext:
+  runAsUser:
+----
+
+This will result in a security context with the `runAsUser` unset.
+
+[source,yaml]
+----
+securityContext:
+  runAsNonRoot: true
+  runAsUser:
+  runAsGroup: 1000650000
+  fsGroup: 1000650000
+  fsGroupChangePolicy: "OnRootMismatch"
+----
+

helm-charts/coherence-operator/templates/deployment.yaml

@@ -275,6 +275,9 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        - mountPath: /coherence-operator/config
+          name: config
+          readOnly: true
         readinessProbe:
           httpGet:
             port: health
@@ -378,3 +381,7 @@ spec:
         secret:
           defaultMode: 420
           secretName: {{ .Values.webhookCertSecret }}
+      - name: config
+        configMap:
+          name: coherence-operator
+          optional: true

helm-charts/coherence-operator/values.yaml

@@ -75,7 +75,7 @@ globalAnnotations:
 
 # ---------------------------------------------------------------------------
 # Operator Pod securityContext
-# This sets the securityContext configuration for the Pod, for example
+# This sets the securityContext configuration for the Operator Pod, for example,
 # to run as a non-root user:
 #
 #  podSecurityContext:
@@ -86,7 +86,7 @@ podSecurityContext:
 
 # ---------------------------------------------------------------------------
 # Operator container securityContext
-# This sets the securityContext configuration for the container, for example
+# This sets the securityContext configuration for the container, for example,
 # to run as a non-root user:
 #
 #  securityContext:

pkg/operator/operator.go

@@ -19,6 +19,7 @@ import (
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/utils/ptr"
 	"os"
 	"path/filepath"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -46,6 +47,25 @@ const (
 	CertTypeManual        = "manual"
 	CertManagerIssuerName = "coherence-webhook-server-issuer"
 
+	// DefaultRunAsNonRoot is the default value for the runAsNonRoot field in the Pod security context
+	DefaultRunAsNonRoot = true
+	// DefaultRunAsUser is the default value for the runAsUser field in the Pod security context
+	DefaultRunAsUser int64 = 1000650000
+	// DefaultRunAsGroup is the default value for the runAsGroup field in the Pod security context
+	DefaultRunAsGroup int64 = 1000650000
+	// DefaultFsGroup is the default value for the fsGroup field in the Pod security context
+	DefaultFsGroup int64 = DefaultRunAsGroup
+	// DefaultFSGroupChangePolicy is the default value for the fsGroup field in the Pod security context
+	DefaultFSGroupChangePolicy = corev1.FSGroupChangeOnRootMismatch
+
+	ConfigKeyCoherenceSecurityContext                    = "coherenceSecurityContext"
+	ConfigKeyCoherenceSecurityContextEnabled             = "enabled"
+	ConfigKeyCoherenceSecurityContextRunAsUser           = "runAsUser"
+	ConfigKeyCoherenceSecurityContextRunAsGroup          = "runAsGroup"
+	ConfigKeyCoherenceSecurityContextRunAsNonRoot        = "runAsNonRoot"
+	ConfigKeyCoherenceSecurityContextFsGroup             = "fsGroup"
+	ConfigKeyCoherenceSecurityContextFSGroupChangePolicy = "fSGroupChangePolicy"
+
 	DefaultMutatingWebhookName   = "coherence-operator-mutating-webhook-configuration"
 	DefaultValidatingWebhookName = "coherence-operator-validating-webhook-configuration"
 
@@ -60,6 +80,8 @@ const (
 	FlagDevMode               = "coherence-dev-mode"
 	FlagCipherDenyList        = "cipher-deny-list"
 	FlagCipherAllowList       = "cipher-allow-list"
+	FlagConfig                = "config"
+	FlagConfigType            = "config-type"
 	FlagDryRun                = "dry-run"
 	FlagEnableWebhook         = "enable-webhook"
 	FlagEnableHttp2           = "enable-http2"
@@ -619,3 +641,46 @@ func RemoveFromUInt16Array(arr []uint16, toRemove uint16) []uint16 {
 	}
 	return arr
 }
+
+// DefaultSecurityContext returns the default Pod security context that the Operator will apply
+// to Coherence pods. The values used can be overridden using command line args.
+func DefaultSecurityContext() *corev1.PodSecurityContext {
+	v := GetViper()
+
+	m := v.GetStringMap(ConfigKeyCoherenceSecurityContext)
+	if m == nil {
+		m = make(map[string]interface{})
+	}
+
+	enabled, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextEnabled)]
+	if found && enabled == false {
+		return nil
+	}
+
+	sc := &corev1.PodSecurityContext{}
+
+	if v.IsSet(ConfigKeyCoherenceSecurityContext) {
+		err := v.UnmarshalKey(ConfigKeyCoherenceSecurityContext, sc)
+		if err != nil {
+			setupLog.Error(err, "unable to unmarshal coherenceSecurityContext from Operator config file")
+		}
+	}
+
+	if _, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextRunAsUser)]; !found {
+		sc.RunAsUser = ptr.To(DefaultRunAsUser)
+	}
+	if _, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextRunAsGroup)]; !found {
+		sc.RunAsGroup = ptr.To(DefaultRunAsGroup)
+	}
+	if _, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextRunAsNonRoot)]; !found {
+		sc.RunAsNonRoot = ptr.To(DefaultRunAsNonRoot)
+	}
+	if _, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextFsGroup)]; !found {
+		sc.FSGroup = ptr.To(DefaultFsGroup)
+	}
+	if _, found := m[strings.ToLower(ConfigKeyCoherenceSecurityContextFSGroupChangePolicy)]; !found {
+		sc.FSGroupChangePolicy = ptr.To(DefaultFSGroupChangePolicy)
+	}
+
+	return sc
+}

pkg/runner/TestOperatorWithDefaultSecurityContextDisabled.yaml

@@ -0,0 +1,3 @@
+coherenceSecurityContext:
+  enabled: false
+