Run glance-api as privileged container when image cache is enabled

ImageCache currently uses two different cronJobs associated to each
glance-api instance (-cleaner and -pruner cronJobs). They mount
/var/lib/glance/image-cache, a RWO Pvc, and execute a glance utility
on the filestem owned by glance kolla user/group. Without glance-api
being privileged, after the cronJob execution the Glance Pod is not
able to access the image-cache path anymore, resulting in a Permission
denied error.
This patch defines a FSGroup that should be set at Pod level to make
sure we always have the right privileges on the container fs, and,
in addition, it runs glance-api as a privileged container when Cache
is enabled.

Jira: https://issues.redhat.com/browse/OSPRH-9842

Signed-off-by: Francesco Pantano <[email protected]>

Author: fmount

api/v1beta1/common_types.go

@@ -108,6 +108,7 @@ type GlanceAPITemplate struct {
 	APITimeout int `json:"apiTimeout,omitempty"`
 }
 
+// Storage -
 type Storage struct {
 	// +kubebuilder:validation:Optional
 	// StorageClass -

api/v1beta1/glance_types.go

@@ -35,7 +35,7 @@ const (
 	APIEdge = "edge"
 )
 
-// GlanceSpec defines the desired state of Glance
+// GlanceSpecCore defines the desired state of Glance
 type GlanceSpecCore struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default=glance

api/v1beta1/glance_webhook.go

@@ -346,6 +346,7 @@ func (r *GlanceSpec) ValidateUpdate(old GlanceSpec, basePath *field.Path) field.
 	return r.GlanceSpecCore.ValidateUpdate(old.GlanceSpecCore, basePath)
 }
 
+// ValidateUpdate -
 func (r *GlanceSpecCore) ValidateUpdate(old GlanceSpecCore, basePath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
 

controllers/glanceapi_controller.go

@@ -802,6 +802,8 @@ func (r *GlanceAPIReconciler) reconcileNormal(
 
 	// This is currently required because cleaner and pruner cronJobs
 	// mount the same pvc to clean data present in /var/lib/glance/image-cache
+	// TODO (fpantano) reference a Glance spec/proposal to move to a different
+	// approach
 	if len(instance.Spec.ImageCache.Size) > 0 {
 		privileged = true
 	}

pkg/glance/const.go

@@ -54,7 +54,7 @@ const (
 	GlanceInternalPort int32 = 9292
 	// GlanceUID - https://github.com/openstack/kolla/blob/master/kolla/common/users.py
 	GlanceUID int64 = 42415
-	// GlanceGid - https://github.com/openstack/kolla/blob/master/kolla/common/users.py
+	// GlanceGID - https://github.com/openstack/kolla/blob/master/kolla/common/users.py
 	GlanceGID int64 = 42415
 	// DefaultsConfigFileName -
 	DefaultsConfigFileName = "00-config.conf"

pkg/glance/funcs.go

@@ -2,6 +2,7 @@ package glance
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -13,19 +14,16 @@ func GetOwningGlanceName(instance client.Object) string {
 			return ownerRef.Name
 		}
 	}
-
 	return ""
 }
 
 // dbSyncSecurityContext - currently used to make sure we don't run db-sync as
 // root user
 func dbSyncSecurityContext() *corev1.SecurityContext {
-	runAsUser := int64(GlanceUID)
-	runAsGroup := int64(GlanceGID)
 
 	return &corev1.SecurityContext{
-		RunAsUser:  &runAsUser,
-		RunAsGroup: &runAsGroup,
+		RunAsUser:  ptr.To(GlanceUID),
+		RunAsGroup: ptr.To(GlanceGID),
 		Capabilities: &corev1.Capabilities{
 			Drop: []corev1.Capability{
 				"MKNOD",
@@ -40,16 +38,12 @@ func dbSyncSecurityContext() *corev1.SecurityContext {
 // BaseSecurityContext - currently used to make sure we don't run cronJob and Log
 // Pods as root user, and we drop privileges and Capabilities we don't need
 func BaseSecurityContext() *corev1.SecurityContext {
-	falseVal := false
-	trueVal := true
-	runAsUser := int64(GlanceUID)
-	runAsGroup := int64(GlanceGID)
 
 	return &corev1.SecurityContext{
-		RunAsUser:                &runAsUser,
-		RunAsGroup:               &runAsGroup,
-		RunAsNonRoot:             &trueVal,
-		AllowPrivilegeEscalation: &falseVal,
+		RunAsUser:                ptr.To(GlanceUID),
+		RunAsGroup:               ptr.To(GlanceGID),
+		RunAsNonRoot:             ptr.To(true),
+		AllowPrivilegeEscalation: ptr.To(false),
 		Capabilities: &corev1.Capabilities{
 			Drop: []corev1.Capability{
 				"ALL",
@@ -63,11 +57,10 @@ func BaseSecurityContext() *corev1.SecurityContext {
 
 // APISecurityContext -
 func APISecurityContext(userID int64, privileged bool) *corev1.SecurityContext {
-	runAsUser := int64(userID)
-	trueVal := true
+
 	return &corev1.SecurityContext{
-		AllowPrivilegeEscalation: &trueVal,
-		RunAsUser:                &runAsUser,
+		AllowPrivilegeEscalation: ptr.To(true),
+		RunAsUser:                ptr.To(userID),
 		Privileged:               &privileged,
 		SeccompProfile: &corev1.SeccompProfile{
 			Type: corev1.SeccompProfileTypeRuntimeDefault,
@@ -77,16 +70,15 @@ func APISecurityContext(userID int64, privileged bool) *corev1.SecurityContext {
 
 // HttpdSecurityContext -
 func HttpdSecurityContext() *corev1.SecurityContext {
-	runAsUser := int64(0)
-	falseVal := false
+
 	return &corev1.SecurityContext{
-		AllowPrivilegeEscalation: &falseVal,
 		Capabilities: &corev1.Capabilities{
 			Drop: []corev1.Capability{
-				"ALL",
+				"MKNOD",
 			},
 		},
-		RunAsUser: &runAsUser,
+		RunAsUser:  ptr.To(GlanceUID),
+		RunAsGroup: ptr.To(GlanceGID),
 		SeccompProfile: &corev1.SeccompProfile{
 			Type: corev1.SeccompProfileTypeRuntimeDefault,
 		},

pkg/glanceapi/cachejob.go

@@ -16,22 +16,20 @@ limitations under the License.
 package glanceapi
 
 import (
+	"fmt"
 	glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/glance-operator/pkg/glance"
-
-	"fmt"
-
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 )
 
 // ImageCacheJob -
 func ImageCacheJob(
 	instance *glancev1.GlanceAPI,
 	cronSpec glance.CronJobSpec,
 ) *batchv1.CronJob {
-	runAsUser := int64(0)
 	var config0644AccessMode int32 = 0644
 
 	cronCommand := fmt.Sprintf(
@@ -100,6 +98,9 @@ func ImageCacheJob(
 					Completions: &completions,
 					Template: corev1.PodTemplateSpec{
 						Spec: corev1.PodSpec{
+							SecurityContext: &corev1.PodSecurityContext{
+								FSGroup: ptr.To(glance.GlanceUID),
+							},
 							Affinity: GetGlanceAPIPodAffinity(instance),
 							Containers: []corev1.Container{
 								{
@@ -108,11 +109,9 @@ func ImageCacheJob(
 									Command: []string{
 										"/bin/bash",
 									},
-									Args:         args,
-									VolumeMounts: cronJobVolumeMounts,
-									SecurityContext: &corev1.SecurityContext{
-										RunAsUser: &runAsUser,
-									},
+									Args:            args,
+									VolumeMounts:    cronJobVolumeMounts,
+									SecurityContext: glance.BaseSecurityContext(),
 								},
 							},
 							Volumes:            cronJobVolume,

pkg/glanceapi/statefulset.go

@@ -179,6 +179,9 @@ func StatefulSet(
 					Labels:      labels,
 				},
 				Spec: corev1.PodSpec{
+					SecurityContext: &corev1.PodSecurityContext{
+						FSGroup: &userID,
+					},
 					ServiceAccountName: instance.Spec.ServiceAccount,
 					// When using Cinder we run as privileged, but also some
 					// commands need to be run on the host using nsenter (eg: