✨ global and per-spoke klusterlet annotations; bump to v0.0.6 (#32)

TylerGillson · web-flow · commit c861bc97d411 · 2025-07-22T15:24:09.000-06:00
* feat: global and per-spoke klusterlet annotations
* test: global klusterlet annotations, merge precedence
* fix: handle nil annotations
* fix: singleton -&gt; singletonControlPlane
* chore: bump to v0.0.6
* chore: make reviewable
* fix: move hubInitializedCond below handleHub
---------

Signed-off-by: Tyler Gillson &lt;tyler.gillson@gmail.com&gt;
diff --git a/fleetconfig-controller/api/v1alpha1/fleetconfig_types.go b/fleetconfig-controller/api/v1alpha1/fleetconfig_types.go
@@ -408,6 +408,11 @@ func (j *JoinedSpoke) conditionName() string {
 
 // Klusterlet is the configuration for a klusterlet.
 type Klusterlet struct {
+	// Annotations to apply to the spoke cluster. If not present, the 'agent.open-cluster-management.io/' prefix is added to each key.
+	// Each annotation is added to klusterlet.spec.registrationConfiguration.clusterAnnotations on the spoke and subsequently to the ManagedCluster on the hub.
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+
 	// A set of comma-separated pairs of the form 'key1=value1,key2=value2' that describe feature gates for alpha/experimental features.
 	// Options are:
 	//  - AddonManagement (ALPHA - default=true)
diff --git a/fleetconfig-controller/api/v1alpha1/zz_generated.deepcopy.go b/fleetconfig-controller/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/fleetconfig-controller/charts/fleetconfig-controller/README.md b/fleetconfig-controller/charts/fleetconfig-controller/README.md
diff --git a/fleetconfig-controller/charts/fleetconfig-controller/crds/fleetconfig.open-cluster-management.io-crds.yaml b/fleetconfig-controller/charts/fleetconfig-controller/crds/fleetconfig.open-cluster-management.io-crds.yaml
@@ -347,6 +347,13 @@ spec:
                       default: {}
                       description: Klusterlet configuration.
                       properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          description: |-
+                            Annotations to apply to the spoke cluster. If not present, the 'agent.open-cluster-management.io/' prefix is added to each key.
+                            Each annotation is added to klusterlet.spec.registrationConfiguration.clusterAnnotations on the spoke and subsequently to the ManagedCluster on the hub.
+                          type: object
                         featureGates:
                           default: AddonManagement=true,ClusterClaim=true
                           description: |-
diff --git a/fleetconfig-controller/charts/fleetconfig-controller/templates/fleetconfig.yaml b/fleetconfig-controller/charts/fleetconfig-controller/templates/fleetconfig.yaml
@@ -30,8 +30,8 @@ spec:
         bundleVersion: {{ .Values.fleetConfig.source.bundleVersion }}
         registry: {{ .Values.fleetConfig.source.registry }}
     {{- end }}
-    {{- if .Values.fleetConfig.hub.singleton }}
-    singleton: {{- toYaml .Values.fleetConfig.hub.singleton | nindent 6 }}
+    {{- if .Values.fleetConfig.hub.singletonControlPlane }}
+    singleton: {{- toYaml .Values.fleetConfig.hub.singletonControlPlane | nindent 6 }}
     {{- end }}
     createNamespace: {{ .Values.fleetConfig.hub.createNamespace }}
     force: {{ .Values.fleetConfig.hub.force }}
@@ -53,6 +53,13 @@ spec:
       clusterARN: {{ .clusterARN | quote }}
       {{- end }}
       klusterlet:
+        {{- if and .klusterlet.annotations $.Values.fleetConfig.spokeAnnotations }}
+        annotations: {{- toYaml (merge .klusterlet.annotations $.Values.fleetConfig.spokeAnnotations) | nindent 10 }}
+        {{- else if .klusterlet.annotations }}
+        annotations: {{- toYaml .klusterlet.annotations | nindent 10 }}
+        {{- else if $.Values.fleetConfig.spokeAnnotations }}
+        annotations: {{- toYaml $.Values.fleetConfig.spokeAnnotations | nindent 10 }}
+        {{- end }}
         mode: {{ .klusterlet.mode | quote }}
         purgeOperator: {{ .klusterlet.purgeOperator }}
         featureGates: {{ include "featureGates" (dict "dict" $spokeFeatureGates) | quote }}
diff --git a/fleetconfig-controller/charts/fleetconfig-controller/values.yaml b/fleetconfig-controller/charts/fleetconfig-controller/values.yaml
@@ -10,6 +10,8 @@
 fleetConfig:
   ## @param fleetConfig.enabled Whether to create a FleetConfig resource.
   enabled: true
+  ## @param fleetConfig.spokeAnnotations Global annotations to apply to all spoke clusters. If not present, the 'agent.open-cluster-management.io/' prefix is added to each key. Each annotation is added to klusterlet.spec.registrationConfiguration.clusterAnnotations on every spoke and subsequently to the ManagedClusters on the hub. Per-spoke annotations take precedence over the global annotations.
+  spokeAnnotations: {}
   ## @descriptionStart
   ## ### Spoke Feature Gates
   ## Uncomment and configure `fleetConfig.spokeFeatureGates` to enable feature gates for the Klusterlet on each Spoke.
@@ -157,6 +159,7 @@ fleetConfig:
   ## @param fleetConfig.spokes[0].ca Hub cluster CA certificate, optional.
   ## @param fleetConfig.spokes[0].proxyCa Proxy CA certificate, optional.
   ## @param fleetConfig.spokes[0].proxyUrl URL of a forward proxy server used by agents to connect to the Hub cluster, optional.
+  ## @param fleetConfig.spokes[0].klusterlet.annotations Annotations to apply to the spoke cluster. If not present, the 'agent.open-cluster-management.io/' prefix is added to each key. Each annotation is added to klusterlet.spec.registrationConfiguration.clusterAnnotations on the spoke and subsequently to the ManagedCluster on the hub. These annotations take precedence over the global spoke annotations.
   ## @param fleetConfig.spokes[0].klusterlet.mode Deployment mode for klusterlet. Options: Default (agents run on spoke cluster) | Hosted (agents run on hub cluster).
   ## @param fleetConfig.spokes[0].klusterlet.purgeOperator If set, the klusterlet operator will be purged and all open-cluster-management namespaces deleted when the klusterlet is unjoined from its Hub cluster.
   ## @param fleetConfig.spokes[0].klusterlet.forceInternalEndpointLookup If true, the klusterlet agent will start the cluster registration process by looking for the
@@ -187,6 +190,7 @@ fleetConfig:
     proxyUrl: ""
     ## Configuration for the Klusterlet on the Spoke cluster.
     klusterlet:
+      annotations: {}
       mode: "Default"
       purgeOperator: true
       # Reference the internal endpoint from the cluster-info ConfigMap in the Hub cluster instead of using the Hub's
@@ -269,7 +273,7 @@ imageRegistry: ""
 ## @param image.pullPolicy Image pull policy
 image:
   repository: quay.io/open-cluster-management/fleetconfig-controller
-  tag: v0.0.5
+  tag: v0.0.6
   pullPolicy: IfNotPresent
 
 ## @param imagePullSecrets Image pull secrets
diff --git a/fleetconfig-controller/config/crd/bases/fleetconfig.open-cluster-management.io_fleetconfigs.yaml b/fleetconfig-controller/config/crd/bases/fleetconfig.open-cluster-management.io_fleetconfigs.yaml
@@ -337,6 +337,13 @@ spec:
                       default: {}
                       description: Klusterlet configuration.
                       properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          description: |-
+                            Annotations to apply to the spoke cluster. If not present, the 'agent.open-cluster-management.io/' prefix is added to each key.
+                            Each annotation is added to klusterlet.spec.registrationConfiguration.clusterAnnotations on the spoke and subsequently to the ManagedCluster on the hub.
+                          type: object
                         featureGates:
                           default: AddonManagement=true,ClusterClaim=true
                           description: |-
diff --git a/fleetconfig-controller/internal/controller/fleetconfig_controller.go b/fleetconfig-controller/internal/controller/fleetconfig_controller.go
@@ -138,12 +138,12 @@ func (r *FleetConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	// Handle Hub cluster: initialization and/or upgrade
-	hubInitializedCond := fc.GetCondition(v1alpha1.FleetConfigHubInitialized)
 	if err := handleHub(ctx, r.Client, fc); err != nil {
 		logger.Error(err, "Failed to handle hub operations")
 		fc.Status.Phase = v1alpha1.FleetConfigUnhealthy
 	}
-	if hubInitializedCond.Status == metav1.ConditionFalse {
+	hubInitializedCond := fc.GetCondition(v1alpha1.FleetConfigHubInitialized)
+	if hubInitializedCond == nil || hubInitializedCond.Status == metav1.ConditionFalse {
 		return ret(ctx, ctrl.Result{Requeue: true}, nil)
 	}
 
diff --git a/fleetconfig-controller/internal/controller/spoke.go b/fleetconfig-controller/internal/controller/spoke.go
@@ -269,6 +269,10 @@ func joinSpoke(ctx context.Context, kClient client.Client, spec v1alpha1.FleetCo
 		"--image-registry", spoke.Klusterlet.Source.Registry,
 	}
 
+	for k, v := range spoke.Klusterlet.Annotations {
+		joinArgs = append(joinArgs, fmt.Sprintf("--klusterlet-annotation=%s=%s", k, v))
+	}
+
 	// resources args
 	joinArgs = append(joinArgs, common.PrepareResources(spoke.Klusterlet.Resources)...)
 
diff --git a/fleetconfig-controller/test/data/fleetconfig-values.yaml b/fleetconfig-controller/test/data/fleetconfig-values.yaml
@@ -1,4 +1,7 @@
 fleetConfig:
+  spokeAnnotations:
+    foo: "not-bar"
+    baz: "quux"
   spokes:
   - name: hub-as-spoke
     createNamespace: true
@@ -21,6 +24,8 @@ fleetConfig:
         namespace: "default"
         kubeconfigKey: "value"
     klusterlet:
+      annotations:
+        foo: "bar"
       mode: "Default"
       purgeOperator: true
       forceInternalEndpointLookup: true
diff --git a/fleetconfig-controller/test/e2e/fleetconfig.go b/fleetconfig-controller/test/e2e/fleetconfig.go
@@ -28,6 +28,7 @@ import (
 	kerrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ktypes "k8s.io/apimachinery/pkg/types"
+	operatorv1 "open-cluster-management.io/api/operator/v1"
 
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/api/v1alpha1"
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/pkg/common"
@@ -72,6 +73,22 @@ var _ = Describe("fleetconfig", Label("fleetconfig"), Ordered, func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
+		It("should verify spoke cluster annotations", func() {
+			EventuallyWithOffset(1, func() error {
+				klusterlet := &operatorv1.Klusterlet{}
+				if err := tc.kClientSpoke.Get(tc.ctx, klusterletNN, klusterlet); err != nil {
+					return err
+				}
+				if err := assertKlusterletAnnotation(klusterlet, "foo", "bar"); err != nil {
+					return err
+				}
+				if err := assertKlusterletAnnotation(klusterlet, "baz", "quux"); err != nil {
+					return err
+				}
+				return nil
+			}, 1*time.Minute, 1*time.Second).Should(Succeed())
+		})
+
 		It("should successfully create a namespace in the hub-as-spoke cluster", func() {
 
 			By("creating a ManifestWork in the hub-as-spoke cluster namespace")
diff --git a/fleetconfig-controller/test/e2e/helper.go b/fleetconfig-controller/test/e2e/helper.go
@@ -21,6 +21,7 @@ import (
 
 	clusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
 	clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta2"
+	operatorv1 "open-cluster-management.io/api/operator/v1"
 	workv1 "open-cluster-management.io/api/work/v1"
 
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/api/v1alpha1"
@@ -29,13 +30,14 @@ import (
 )
 
 const (
-	fcNamespace         = "fleetconfig-system"
-	spokeSecretName     = "test-fleetconfig-kubeconfig"
-	devspaceLocal       = "local"
-	devspaceCI          = "ci"
-	kubeconfigSecretKey = "value"
-	hubAsSpokeName      = v1alpha1.ManagedClusterTypeHubAsSpoke
-	spokeName           = v1alpha1.ManagedClusterTypeSpoke
+	fcNamespace                = "fleetconfig-system"
+	spokeSecretName            = "test-fleetconfig-kubeconfig"
+	klusterletAnnotationPrefix = "agent.open-cluster-management.io"
+	devspaceLocal              = "local"
+	devspaceCI                 = "ci"
+	kubeconfigSecretKey        = "value"
+	hubAsSpokeName             = v1alpha1.ManagedClusterTypeHubAsSpoke
+	spokeName                  = v1alpha1.ManagedClusterTypeSpoke
 )
 
 var (
@@ -45,6 +47,7 @@ var (
 
 	// global test variables
 	fleetConfigNN = ktypes.NamespacedName{Name: "fleetconfig", Namespace: fcNamespace}
+	klusterletNN  = ktypes.NamespacedName{Name: "klusterlet"}
 )
 
 // E2EContext holds all the test-specific state.
@@ -121,6 +124,7 @@ func setupTestEnvironment() *E2EContext {
 	Expect(v1alpha1.AddToScheme(scheme.Scheme)).To(Succeed())
 	Expect(clusterv1beta1.AddToScheme(scheme.Scheme)).To(Succeed())
 	Expect(clusterv1beta2.AddToScheme(scheme.Scheme)).To(Succeed())
+	Expect(operatorv1.AddToScheme(scheme.Scheme)).To(Succeed())
 	Expect(workv1.AddToScheme(scheme.Scheme)).To(Succeed())
 
 	By("creating a kubernetes client for the hub cluster")
@@ -329,3 +333,15 @@ func assertNamespace(ctx context.Context, cluster string, kClient client.Client)
 	utils.Info("Namespace %s is now created in cluster '%s'", namespaceName, cluster)
 	return nil
 }
+
+func assertKlusterletAnnotation(klusterlet *operatorv1.Klusterlet, key, expectedValue string) error {
+	expectedKey := fmt.Sprintf("%s/%s", klusterletAnnotationPrefix, key)
+	v, ok := klusterlet.Spec.RegistrationConfiguration.ClusterAnnotations[expectedKey]
+	if !ok {
+		return fmt.Errorf("expected annotation, %s, not found", expectedKey)
+	}
+	if v != expectedValue {
+		return fmt.Errorf("expected %s=%s, got %s", expectedKey, expectedValue, v)
+	}
+	return nil
+}

Original file line number	Diff line number	Diff line change
`@@ -138,12 +138,12 @@ func (r *FleetConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)`
`138`	`138`	`}`
`139`	`139`
`140`	`140`	`// Handle Hub cluster: initialization and/or upgrade`
`141`		`- hubInitializedCond := fc.GetCondition(v1alpha1.FleetConfigHubInitialized)`
`142`	`141`	`if err := handleHub(ctx, r.Client, fc); err != nil {`
`143`	`142`	`logger.Error(err, "Failed to handle hub operations")`
`144`	`143`	`fc.Status.Phase = v1alpha1.FleetConfigUnhealthy`
`145`	`144`	`}`
`146`		`- if hubInitializedCond.Status == metav1.ConditionFalse {`
	`145`	`+ hubInitializedCond := fc.GetCondition(v1alpha1.FleetConfigHubInitialized)`
	`146`	`+ if hubInitializedCond == nil \|\| hubInitializedCond.Status == metav1.ConditionFalse {`
`147`	`147`	`return ret(ctx, ctrl.Result{Requeue: true}, nil)`
`148`	`148`	`}`
`149`	`149`
Original file line number	Diff line number	Diff line change
`@@ -269,6 +269,10 @@ func joinSpoke(ctx context.Context, kClient client.Client, spec v1alpha1.FleetCo`
`269`	`269`	`"--image-registry", spoke.Klusterlet.Source.Registry,`
`270`	`270`	`}`
`271`	`271`
	`272`	`+ for k, v := range spoke.Klusterlet.Annotations {`
	`273`	`+ joinArgs = append(joinArgs, fmt.Sprintf("--klusterlet-annotation=%s=%s", k, v))`
	`274`	`+ }`
	`275`	`+`
`272`	`276`	`// resources args`
`273`	`277`	`joinArgs = append(joinArgs, common.PrepareResources(spoke.Klusterlet.Resources)...)`
`274`	`278`