Allow customize http vhost config using HttpdCustomization.CustomConfigSecret

stuggi · stuggi · commit 0249ba14f654 · 2025-01-23T12:03:53.000+01:00
This change allows to customize the httpd vhost config using this parameter to specify a secret that contains service config data. The content of each provided snippet gets rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<endpoint>_<key> . At the end of the vhost config in the default httpd template these custom configs get included using `Include conf/httpd_custom_<endpoint>_*`. For information on how sections in httpd configuration get merged, check section "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging All possible parameters which can be use in a template can be looked up in the <service>-config-data secret of the service like: $ oc get secret -n openstack glance-config-data -o json | jq -r .data.TemplateParameters | base64 -d or in the running pod of the service in the file: $ cat /var/lib/config-data/default/TemplateParameters The content is a versioned dump of the parameters of the service operator, like: ~~~ DatabaseConnection: mysql+pymysql://user:pwd@openstack.openstack.svc/keystone?read_default_file=/etc/my.cnf KeystoneEndpointInternal: https://keystone-internal.openstack.svc:5000 KeystoneEndpointPublic: https://keystone-public-openstack.apps-crc.testing TransportURL: rabbit://user:pwd@rabbitmq.openstack.svc:5671/?ssl=1 VHosts: internal: Override: false SSLCertificateFile: /etc/pki/tls/certs/internal.crt SSLCertificateKeyFile: /etc/pki/tls/private/internal.key ServerName: glance-internal.openstack.svc TLS: true public: Override: false SSLCertificateFile: /etc/pki/tls/certs/public.crt SSLCertificateKeyFile: /etc/pki/tls/private/public.key ServerName: glance-public.openstack.svc TLS: true ... ~~~ Depends-On: openstack-k8s-operators/lib-common#591 Depends-On: openstack-k8s-operators/lib-common#593 Jira: https://issues.redhat.com/browse/OSPRH-13100 Signed-off-by: Martin Schuppert <mschuppert@redhat.com>
diff --git a/api/bases/glance.openstack.org_glanceapis.yaml b/api/bases/glance.openstack.org_glanceapis.yaml
@@ -546,6 +546,11 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              httpdCustomization:
+                properties:
+                  customConfigSecret:
+                    type: string
+                type: object
               imageCache:
                 properties:
                   cleanerScheduler:
diff --git a/api/bases/glance.openstack.org_glances.yaml b/api/bases/glance.openstack.org_glances.yaml
@@ -559,6 +559,11 @@ spec:
                       items:
                         type: string
                       type: array
+                    httpdCustomization:
+                      properties:
+                        customConfigSecret:
+                          type: string
+                      type: object
                     imageCache:
                       properties:
                         cleanerScheduler:
@@ -708,6 +713,11 @@ spec:
                   type: object
                 default: {}
                 type: object
+              httpdCustomization:
+                properties:
+                  customConfigSecret:
+                    type: string
+                type: object
               imageCache:
                 properties:
                   cleanerScheduler:
diff --git a/api/go.mod b/api/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/google/go-cmp v0.6.0
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e
 	k8s.io/api v0.29.12
 	k8s.io/apimachinery v0.29.12
diff --git a/api/go.sum b/api/go.sum
@@ -75,8 +75,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E=
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7 h1:vXHpH93PjbAgg5ZN6n5WmxkybVQOs0nhXvVw62o7aZs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e h1:Qz0JFEoRDUyjEWorNY3LggwxTsmpMtQkcpmZDQulGHQ=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:tfgBeLRqmlH/NQkLPe7396rj+t0whv2wPuMb8Ttvh8w=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go
@@ -106,6 +106,10 @@ type GlanceAPITemplate struct {
 	// +kubebuilder:validation:Minimum=1
 	// APITimeout for HAProxy and Apache defaults to GlanceSpecCore APITimeout
 	APITimeout int `json:"apiTimeout,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// HttpdCustomization - customize the httpd service
+	HttpdCustomization HttpdCustomization `json:"httpdCustomization,omitempty"`
 }
 
 // Storage -
@@ -144,6 +148,19 @@ type APIOverrideSpec struct {
 	Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"`
 }
 
+// HttpdCustomization - customize the httpd service
+type HttpdCustomization struct {
+	// +kubebuilder:validation:Optional
+	// CustomConfigSecret - customize the httpd vhost config using this parameter to specify
+	// a secret that contains service config data. The content of each provided snippet gets
+	// rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
+	// In the default httpd template at the end of the vhost those custom configs get
+	// included using `Include conf/httpd_custom_<endpoint>_*`.
+	// For information on how sections in httpd configuration get merged, check section
+	// "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
+	CustomConfigSecret *string `json:"customConfigSecret,omitempty"`
+}
+
 // SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks)
 func SetupDefaults() {
 	// Acquire environmental defaults and initialize Glance defaults with them
diff --git a/api/v1beta1/glance_types.go b/api/v1beta1/glance_types.go
@@ -125,6 +125,10 @@ type GlanceSpecCore struct {
 	// +kubebuilder:validation:Minimum=1
 	// Default APITimeout for HAProxy and Apache, defaults to 60 seconds
 	APITimeout int `json:"apiTimeout"`
+
+	// +kubebuilder:validation:Optional
+	// HttpdCustomization - customize the httpd service of all GlanceAPIs
+	HttpdCustomization HttpdCustomization `json:"httpdCustomization,omitempty"`
 }
 
 // GlanceSpec defines the desired state of Glance
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/glance.openstack.org_glanceapis.yaml b/config/crd/bases/glance.openstack.org_glanceapis.yaml
@@ -546,6 +546,11 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              httpdCustomization:
+                properties:
+                  customConfigSecret:
+                    type: string
+                type: object
               imageCache:
                 properties:
                   cleanerScheduler:
diff --git a/config/crd/bases/glance.openstack.org_glances.yaml b/config/crd/bases/glance.openstack.org_glances.yaml
@@ -559,6 +559,11 @@ spec:
                       items:
                         type: string
                       type: array
+                    httpdCustomization:
+                      properties:
+                        customConfigSecret:
+                          type: string
+                      type: object
                     imageCache:
                       properties:
                         cleanerScheduler:
@@ -708,6 +713,11 @@ spec:
                   type: object
                 default: {}
                 type: object
+              httpdCustomization:
+                properties:
+                  customConfigSecret:
+                    type: string
+                type: object
               imageCache:
                 properties:
                   cleanerScheduler:
diff --git a/controllers/glance_common.go b/controllers/glance_common.go
@@ -23,6 +23,7 @@ import (
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
+	"gopkg.in/yaml.v2"
 	"k8s.io/apimachinery/pkg/types"
 
 	glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1"
@@ -45,10 +46,11 @@ import (
 
 // fields to index to reconcile when change
 const (
-	passwordSecretField     = ".spec.secret"
-	caBundleSecretNameField = ".spec.tls.caBundleSecretName"
-	tlsAPIInternalField     = ".spec.tls.api.internal.secretName"
-	tlsAPIPublicField       = ".spec.tls.api.public.secretName"
+	passwordSecretField                 = ".spec.secret"
+	caBundleSecretNameField             = ".spec.tls.caBundleSecretName"
+	tlsAPIInternalField                 = ".spec.tls.api.internal.secretName"
+	tlsAPIPublicField                   = ".spec.tls.api.public.secretName"
+	httpdCustomServiceConfigSecretField = ".spec.httpdCustomization.customServiceConfigSecret"
 )
 
 var (
@@ -60,6 +62,7 @@ var (
 		caBundleSecretNameField,
 		tlsAPIInternalField,
 		tlsAPIPublicField,
+		httpdCustomServiceConfigSecretField,
 	}
 )
 
@@ -162,18 +165,27 @@ func GenerateConfigsGeneric(
 	customData map[string]string,
 	cmLabels map[string]string,
 	scripts bool,
+	customTemplates map[string]string,
 ) error {
 
+	// Marshal the templateParameters map to YAML
+	yamlData, err := yaml.Marshal(templateParameters)
+	if err != nil {
+		return fmt.Errorf("Error marshalling to YAML: %w", err)
+	}
+	customData[common.TemplateParameters] = string(yamlData)
+
 	cms := []util.Template{
 		// Templates where the GlanceAPI config is stored
 		{
-			Name:          fmt.Sprintf("%s-config-data", instance.GetName()),
-			Namespace:     instance.GetNamespace(),
-			Type:          util.TemplateTypeConfig,
-			InstanceType:  instance.GetObjectKind().GroupVersionKind().Kind,
-			ConfigOptions: templateParameters,
-			CustomData:    customData,
-			Labels:        cmLabels,
+			Name:           fmt.Sprintf("%s-config-data", instance.GetName()),
+			Namespace:      instance.GetNamespace(),
+			Type:           util.TemplateTypeConfig,
+			InstanceType:   instance.GetObjectKind().GroupVersionKind().Kind,
+			ConfigOptions:  templateParameters,
+			CustomData:     customData,
+			StringTemplate: customTemplates,
+			Labels:         cmLabels,
 		},
 	}
 	// TODO: Scripts have no reason to be secrets, should move to configmap
diff --git a/controllers/glance_controller.go b/controllers/glance_controller.go
@@ -838,6 +838,10 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(
 		apiSpec.GlanceAPITemplate.NodeSelector = instance.Spec.NodeSelector
 	}
 
+	if apiSpec.GlanceAPITemplate.HttpdCustomization.CustomConfigSecret == nil {
+		apiSpec.GlanceAPITemplate.HttpdCustomization.CustomConfigSecret = instance.Spec.HttpdCustomization.CustomConfigSecret
+	}
+
 	// Inherit the ImageCacheSize from the top level if not specified
 	if apiSpec.ImageCache.Size == "" {
 		apiSpec.ImageCache.Size = instance.Spec.ImageCache.Size
@@ -967,7 +971,7 @@ func (r *GlanceReconciler) generateServiceConfig(
 	}
 
 	// Generate both default 00-config.conf and -scripts
-	return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, true)
+	return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, true, map[string]string{})
 }
 
 // ensureRegisteredLimits - create registered limits in keystone that will be
diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go
@@ -241,6 +241,18 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 
+	// index httpdOverrideSecretField
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &glancev1.GlanceAPI{}, httpdCustomServiceConfigSecretField, func(rawObj client.Object) []string {
+		// Extract the secret name from the spec, if one is provided
+		cr := rawObj.(*glancev1.GlanceAPI)
+		if cr.Spec.HttpdCustomization.CustomConfigSecret == nil {
+			return nil
+		}
+		return []string{*cr.Spec.HttpdCustomization.CustomConfigSecret}
+	}); err != nil {
+		return err
+	}
+
 	// Watch for changes to any CustomServiceConfigSecrets. Global secrets
 	svcSecretFn := func(_ context.Context, o client.Object) []reconcile.Request {
 		var namespace string = o.GetNamespace()
@@ -1011,6 +1023,14 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
 	}
 	customData[glance.CustomServiceConfigSecretsFileName] = customSecrets
 
+	httpdOverrideSecret := &corev1.Secret{}
+	if instance.Spec.HttpdCustomization.CustomConfigSecret != nil && *instance.Spec.HttpdCustomization.CustomConfigSecret != "" {
+		httpdOverrideSecret, _, err = secret.GetSecret(ctx, h, *instance.Spec.HttpdCustomization.CustomConfigSecret, instance.Namespace)
+		if err != nil {
+			return err
+		}
+	}
+
 	keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{})
 	// KeystoneAPI not available we should not aggregate the error and continue
 	if err != nil {
@@ -1043,6 +1063,7 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
 	if instance.Spec.APIType != glancev1.APISingle {
 		endptName = fmt.Sprintf("%s-api", instance.Name)
 	}
+	customTemplates := map[string]string{}
 	httpdVhostConfig := map[string]interface{}{}
 	for endpt := range glanceEndpoints {
 		endptConfig := map[string]interface{}{}
@@ -1055,6 +1076,16 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
 			endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String())
 			endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String())
 		}
+
+		endptConfig["Override"] = false
+		if len(httpdOverrideSecret.Data) > 0 {
+			endptConfig["Override"] = true
+			for key, data := range httpdOverrideSecret.Data {
+				if len(data) > 0 {
+					customTemplates["httpd_custom_"+endpt.String()+"_"+key] = string(data)
+				}
+			}
+		}
 		httpdVhostConfig[endpt.String()] = endptConfig
 	}
 
@@ -1113,7 +1144,7 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
 	// 00-default.conf will be regenerated as we have a ln -s of the
 	// templates/glance/config directory
 	// Do not generate -scripts as they are inherited from the top-level CR
-	return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, false)
+	return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, false, customTemplates)
 }
 
 // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart
diff --git a/go.mod b/go.mod
@@ -13,12 +13,13 @@ require (
 	github.com/openstack-k8s-operators/glance-operator/api v0.0.0-00010101000000-000000000000
 	github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20250108092548-58707fa645ce
 	github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20250107165241-16c3ed8e549f
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7
 	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e
 	github.com/openstack-k8s-operators/mariadb-operator/api v0.5.1-0.20250108071621-aa59f25c5b1a
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.29.12
 	k8s.io/apimachinery v0.29.12
@@ -74,7 +75,6 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiextensions-apiserver v0.29.12 // indirect
 	k8s.io/component-base v0.29.12 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
diff --git a/go.sum b/go.sum
@@ -84,8 +84,8 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20250108092548-5
 github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20250108092548-58707fa645ce/go.mod h1:TDaE7BVQvJwJGFm33R6xcPTeF8LKAnMh+a1ho+YqJHs=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20250107165241-16c3ed8e549f h1:jlUo93FAwlDll1bJRxJO5B1Vi3t3wCoHQuy5HEO96ME=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20250107165241-16c3ed8e549f/go.mod h1:CyuEOM1TpXKNUR1n8cudNtRzTEwkzv90JFkpDPPId8E=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7 h1:vXHpH93PjbAgg5ZN6n5WmxkybVQOs0nhXvVw62o7aZs=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250116145727-01a8948d5dd7/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e h1:HFo4OqPY0x4ZQeaWI2YGonTXAGTQFt+rOEJlfZVhS7s=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:IASoGvp5QM/tBJUd/8i8uIjj4DBnI+64Ydh4r7pmnvA=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e h1:Qz0JFEoRDUyjEWorNY3LggwxTsmpMtQkcpmZDQulGHQ=
diff --git a/templates/common/config/10-glance-httpd.conf b/templates/common/config/10-glance-httpd.conf
@@ -33,5 +33,10 @@
   SSLCertificateFile      "{{ $vhost.SSLCertificateFile }}"
   SSLCertificateKeyFile   "{{ $vhost.SSLCertificateKeyFile }}"
 {{- end }}
+
+{{- if $vhost.Override }}
+  Include conf/httpd_custom_{{ $endpt }}_*
+{{- end }}
+
 </VirtualHost>
 {{ end }}
diff --git a/templates/common/config/glance-api-config.json b/templates/common/config/glance-api-config.json
@@ -61,6 +61,13 @@
         "dest": "/usr/local/bin/kolla_extend_start",
         "owner": "root:root",
         "perm": "0755"
+      },
+      {
+        "source": "/var/lib/config-data/default/httpd_custom_*",
+        "dest": "/etc/httpd/conf/",
+        "owner": "apache",
+        "perm": "0444",
+        "optional": true
       }
     ],
     "permissions": [
diff --git a/test/functional/glance_controller_test.go b/test/functional/glance_controller_test.go
diff --git a/test/functional/glance_test_data.go b/test/functional/glance_test_data.go
diff --git a/test/functional/glanceapi_controller_test.go b/test/functional/glanceapi_controller_test.go
