feat: add support for multiple SCP composition

Author: pkoutsovasilis

config/crds/v1/all-crds.yaml

@@ -10932,6 +10932,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

config/crds/v1/resources/stackconfigpolicy.k8s.elastic.co_stackconfigpolicies.yaml

@@ -288,6 +288,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -11002,6 +11002,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

docs/reference/api-reference/main.md

@@ -2068,6 +2068,7 @@ StackConfigPolicy represents a StackConfigPolicy resource in a Kubernetes cluste
 | Field | Description |
 | --- | --- |
 | *`resourceSelector`* __[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#labelselector-v1-meta)__ |  |
+| *`weight`* __integer__ | Weight determines the priority of this policy when multiple policies target the same resource.<br>Lower weight values take precedence. Defaults to 0. |
 | *`secureSettings`* __[SecretSource](#secretsource) array__ | Deprecated: SecureSettings only applies to Elasticsearch and is deprecated. It must be set per application instead. |
 | *`elasticsearch`* __[ElasticsearchConfigPolicySpec](#elasticsearchconfigpolicyspec)__ |  |
 | *`kibana`* __[KibanaConfigPolicySpec](#kibanaconfigpolicyspec)__ |  |

pkg/apis/stackconfigpolicy/v1alpha1/stackconfigpolicy_types.go

@@ -55,6 +55,10 @@ type StackConfigPolicyList struct {
 
 type StackConfigPolicySpec struct {
 	ResourceSelector metav1.LabelSelector `json:"resourceSelector,omitempty"`
+	// Weight determines the priority of this policy when multiple policies target the same resource.
+	// Lower weight values take precedence. Defaults to 0.
+	// +kubebuilder:default=0
+	Weight int32 `json:"weight,omitempty"`
 	// Deprecated: SecureSettings only applies to Elasticsearch and is deprecated. It must be set per application instead.
 	SecureSettings []commonv1.SecretSource       `json:"secureSettings,omitempty"`
 	Elasticsearch  ElasticsearchConfigPolicySpec `json:"elasticsearch,omitempty"`

pkg/controller/common/annotation/constants.go

@@ -25,4 +25,6 @@ const (
 
 	ElasticsearchConfigAndSecretMountsHashAnnotation = "policy.k8s.elastic.co/elasticsearch-config-mounts-hash" //nolint:gosec
 	SourceSecretAnnotationName                       = "policy.k8s.elastic.co/source-secret-name"               //nolint:gosec
+
+	SoftOwnerRefsAnnotation = "eck.k8s.elastic.co/owner-refs"
 )

pkg/controller/common/reconciler/secret.go

@@ -6,7 +6,9 @@ package reconciler
 
 import (
 	"context"
+	"encoding/json"
 	"reflect"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -17,6 +19,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	policyv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/stackconfigpolicy/v1alpha1"
+	commonannotation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/annotation"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s"
 	ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/utils/maps"
@@ -92,6 +95,49 @@ func SoftOwnerRefFromLabels(labels map[string]string) (SoftOwnerRef, bool) {
 	return SoftOwnerRef{Namespace: namespace, Name: name, Kind: kind}, true
 }
 
+// SoftOwnerRefs returns the soft owner references of the given object.
+func SoftOwnerRefs(obj metav1.Object) ([]SoftOwnerRef, error) {
+	// Check if this Secret has a soft-owner kind label set
+	ownerKind, exists := obj.GetLabels()[SoftOwnerKindLabel]
+	if !exists {
+		// Not a soft-owned secret
+		return nil, nil
+	}
+
+	// Check for multi-policy ownership (annotation-based)
+	if ownerRefsBytes, exists := obj.GetAnnotations()[commonannotation.SoftOwnerRefsAnnotation]; exists {
+		// Multi-policy soft owned secret - parse the JSON map of owners
+		var ownerRefs map[string]struct{}
+		if err := json.Unmarshal([]byte(ownerRefsBytes), &ownerRefs); err != nil {
+			return nil, err
+		}
+
+		// Convert the map keys (namespaced name strings) back to NamespacedName objects
+		var ownerRefsNsn []SoftOwnerRef
+		for nsnStr := range ownerRefs {
+			// Split the string format "namespace/name" into components
+			nsnComponents := strings.Split(nsnStr, string(types.Separator))
+			if len(nsnComponents) != 2 {
+				// Skip malformed entries
+				continue
+			}
+			ownerRefsNsn = append(ownerRefsNsn, SoftOwnerRef{Namespace: nsnComponents[0], Name: nsnComponents[1], Kind: ownerKind})
+		}
+
+		return ownerRefsNsn, nil
+	}
+
+	// Fall back to single-policy ownership (label-based)
+	currentOwner, referenced := SoftOwnerRefFromLabels(obj.GetLabels())
+	if !referenced {
+		// No soft owner found in labels
+		return nil, nil
+	}
+
+	// Return the single owner as a slice with one element
+	return []SoftOwnerRef{currentOwner}, nil
+}
+
 // ReconcileSecretNoOwnerRef should be called to reconcile a Secret for which we explicitly don't want
 // an owner reference to be set, and want existing ownerReferences from previous operator versions to be removed,
 // because of this k8s bug: https://github.com/kubernetes/kubernetes/issues/65200 (fixed in k8s 1.20).
@@ -200,43 +246,54 @@ func GarbageCollectAllSoftOwnedOrphanSecrets(ctx context.Context, c k8s.Client,
 	var secrets corev1.SecretList
 	if err := c.List(ctx,
 		&secrets,
-		client.HasLabels{SoftOwnerNamespaceLabel, SoftOwnerNameLabel, SoftOwnerKindLabel},
+		client.HasLabels{SoftOwnerKindLabel},
 	); err != nil {
 		return err
 	}
 	// remove any secret whose owner doesn't exist
 	for i := range secrets.Items {
 		secret := secrets.Items[i]
-		softOwner, referenced := SoftOwnerRefFromLabels(secret.Labels)
-		if !referenced {
-			continue
-		}
-		if restrictedToOwnerNamespace(softOwner.Kind) && softOwner.Namespace != secret.Namespace {
-			// Secret references an owner in a different namespace: this likely results
-			// from a "manual" copy of the secret in another namespace, not handled by the operator.
-			// We don't want to touch that secret.
-			continue
+		softOwners, err := SoftOwnerRefs(&secret)
+		if err != nil {
+			return err
 		}
-		owner, managed := ownerKinds[softOwner.Kind]
-		if !managed {
+		if len(softOwners) == 0 {
 			continue
 		}
-		owner = k8s.DeepCopyObject(owner)
-		err := c.Get(ctx, types.NamespacedName{Namespace: softOwner.Namespace, Name: softOwner.Name}, owner)
-		if err != nil {
-			if apierrors.IsNotFound(err) {
-				// owner doesn't exit anymore
-				ulog.FromContext(ctx).Info("Deleting secret as part of garbage collection",
-					"namespace", secret.Namespace, "secret_name", secret.Name,
-					"owner_kind", softOwner.Kind, "owner_namespace", softOwner.Namespace, "owner_name", softOwner.Name,
-				)
-				options := client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &secret.UID}}
-				if err := c.Delete(ctx, &secret, &options); err != nil && !apierrors.IsNotFound(err) {
-					return err
-				}
+
+		missingOwners := make(map[types.NamespacedName]client.Object)
+		for _, softOwner := range softOwners {
+			if restrictedToOwnerNamespace(softOwner.Kind) && softOwner.Namespace != secret.Namespace {
+				// Secret references an owner in a different namespace: this likely results
+				// from a "manual" copy of the secret in another namespace, not handled by the operator.
+				// We don't want to touch that secret.
 				continue
 			}
-			return err
+			owner, managed := ownerKinds[softOwner.Kind]
+			if !managed {
+				continue
+			}
+			owner = k8s.DeepCopyObject(owner)
+			err := c.Get(ctx, types.NamespacedName{Namespace: softOwner.Namespace, Name: softOwner.Name}, owner)
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					// owner doesn't exit anymore
+					ulog.FromContext(ctx).Info("Deleting secret as part of garbage collection",
+						"namespace", secret.Namespace, "secret_name", secret.Name,
+						"owner_kind", softOwner.Kind, "owner_namespace", softOwner.Namespace, "owner_name", softOwner.Name,
+					)
+					missingOwners[types.NamespacedName{Namespace: softOwner.Namespace, Name: softOwner.Name}] = owner
+					continue
+				}
+				return err
+			}
+		}
+
+		if len(missingOwners) == len(softOwners) {
+			options := client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &secret.UID}}
+			if err := c.Delete(ctx, &secret, &options); err != nil && !apierrors.IsNotFound(err) {
+				return err
+			}
 		}
 		// owner still exists, keep the secret
 	}

pkg/controller/elasticsearch/filesettings/reconciler.go

@@ -29,7 +29,7 @@ var (
 
 	// managedAnnotations are the annotations managed by the operator for the stack config policy related secrets, which means that the operator
 	// will always take precedence to update or remove these annotations.
-	managedAnnotations = []string{commonannotation.SecureSettingsSecretsAnnotationName, commonannotation.SettingsHashAnnotationName, commonannotation.ElasticsearchConfigAndSecretMountsHashAnnotation, commonannotation.KibanaConfigHashAnnotation}
+	managedAnnotations = []string{commonannotation.SecureSettingsSecretsAnnotationName, commonannotation.SettingsHashAnnotationName, commonannotation.ElasticsearchConfigAndSecretMountsHashAnnotation, commonannotation.KibanaConfigHashAnnotation, commonannotation.SoftOwnerRefsAnnotation}
 )
 
 // ReconcileEmptyFileSettingsSecret reconciles an empty File settings Secret for the given Elasticsearch only when there is no Secret.

pkg/controller/elasticsearch/filesettings/secret.go

@@ -85,9 +85,6 @@ func newSettingsSecret(version int64, es types.NamespacedName, currentSecret *co
 	}
 
 	if policy != nil {
-		// set this policy as soft owner of this Secret
-		SetSoftOwner(settingsSecret, *policy)
-
 		// add the Secure Settings Secret sources to the Settings Secret
 		if err := setSecureSettings(settingsSecret, *policy); err != nil {
 			return corev1.Secret{}, 0, err
@@ -160,19 +157,6 @@ func setSecureSettings(settingsSecret *corev1.Secret, policy policyv1alpha1.Stac
 	return nil
 }
 
-// CanBeOwnedBy return true if the Settings Secret can be owned by the given StackConfigPolicy, either because the Secret
-// belongs to no one or because it already belongs to the given policy.
-func CanBeOwnedBy(settingsSecret corev1.Secret, policy policyv1alpha1.StackConfigPolicy) (reconciler.SoftOwnerRef, bool) {
-	currentOwner, referenced := reconciler.SoftOwnerRefFromLabels(settingsSecret.Labels)
-	// either there is no soft owner
-	if !referenced {
-		return reconciler.SoftOwnerRef{}, true
-	}
-	// or the owner is already the given policy
-	canBeOwned := currentOwner.Kind == policyv1alpha1.Kind && currentOwner.Namespace == policy.Namespace && currentOwner.Name == policy.Name
-	return currentOwner, canBeOwned
-}
-
 // getSecureSettings returns the SecureSettings Secret sources stores in an annotation of the given file settings Secret.
 func getSecureSettings(settingsSecret corev1.Secret) ([]commonv1.NamespacedSecretSource, error) {
 	rawString, ok := settingsSecret.Annotations[commonannotation.SecureSettingsSecretsAnnotationName]

pkg/controller/elasticsearch/filesettings/secret_test.go

@@ -101,53 +101,6 @@ func Test_SettingsSecret_hasChanged(t *testing.T) {
 	assert.Equal(t, strconv.FormatInt(newVersion, 10), newSettings.Metadata.Version)
 }
 
-func Test_SettingsSecret_setSoftOwner_canBeOwnedBy(t *testing.T) {
-	es := types.NamespacedName{
-		Namespace: "esNs",
-		Name:      "esName",
-	}
-	policy := policyv1alpha1.StackConfigPolicy{
-		TypeMeta: metav1.TypeMeta{
-			Kind: policyv1alpha1.Kind,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "policyNs",
-			Name:      "policyName",
-		},
-	}
-	otherPolicy := policyv1alpha1.StackConfigPolicy{
-		TypeMeta: metav1.TypeMeta{
-			Kind: policyv1alpha1.Kind,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "otherPolicyNs",
-			Name:      "otherPolicyName",
-		},
-	}
-
-	// empty settings can be owned by any policy
-	secret, _, err := NewSettingsSecretWithVersion(es, nil, nil, metadata.Metadata{})
-	assert.NoError(t, err)
-	_, canBeOwned := CanBeOwnedBy(secret, policy)
-	assert.Equal(t, true, canBeOwned)
-	_, canBeOwned = CanBeOwnedBy(secret, otherPolicy)
-	assert.Equal(t, true, canBeOwned)
-
-	// set a policy soft owner
-	SetSoftOwner(&secret, policy)
-	_, canBeOwned = CanBeOwnedBy(secret, policy)
-	assert.Equal(t, true, canBeOwned)
-	_, canBeOwned = CanBeOwnedBy(secret, otherPolicy)
-	assert.Equal(t, false, canBeOwned)
-
-	// update the policy soft owner
-	SetSoftOwner(&secret, otherPolicy)
-	_, canBeOwned = CanBeOwnedBy(secret, policy)
-	assert.Equal(t, false, canBeOwned)
-	_, canBeOwned = CanBeOwnedBy(secret, otherPolicy)
-	assert.Equal(t, true, canBeOwned)
-}
-
 func Test_SettingsSecret_setSecureSettings_getSecureSettings(t *testing.T) {
 	es := types.NamespacedName{
 		Namespace: "esNs",