fix new member cluster helm install

Signed-off-by: Arvind Thirumurugan <[email protected]>

Author: Arvind Thirumurugan

charts/member-agent/templates/crds/placement.kubernetes-fleet.io_metriccollectors.yaml

@@ -51,7 +51,6 @@ import (
 	clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1"
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	imcv1beta1 "github.com/kubefleet-dev/kubefleet/pkg/controllers/internalmembercluster/v1beta1"
-	"github.com/kubefleet-dev/kubefleet/pkg/controllers/metriccollector"
 	"github.com/kubefleet-dev/kubefleet/pkg/controllers/workapplier"
 	"github.com/kubefleet-dev/kubefleet/pkg/propertyprovider"
 	"github.com/kubefleet-dev/kubefleet/pkg/propertyprovider/azure"
@@ -465,15 +464,15 @@ func Start(ctx context.Context, hubCfg, memberConfig *rest.Config, hubOpts, memb
 			return fmt.Errorf("failed to set up InternalMemberCluster v1beta1 controller with the controller manager: %w", err)
 		}
 
-		// Set up the MetricCollector controller.
-		mcReconciler := &metriccollector.Reconciler{
-			MemberClient: memberMgr.GetClient(),
-			HubClient:    hubMgr.GetClient(),
-		}
-		if err := mcReconciler.SetupWithManager(memberMgr); err != nil {
-			klog.ErrorS(err, "Failed to set up MetricCollector controller with the controller manager")
-			return fmt.Errorf("failed to set up MetricCollector controller with the controller manager: %w", err)
-		}
+		// // Set up the MetricCollector controller.
+		// mcReconciler := &metriccollector.Reconciler{
+		// 	MemberClient: memberMgr.GetClient(),
+		// 	HubClient:    hubMgr.GetClient(),
+		// }
+		// if err := mcReconciler.SetupWithManager(memberMgr); err != nil {
+		// 	klog.ErrorS(err, "Failed to set up MetricCollector controller with the controller manager")
+		// 	return fmt.Errorf("failed to set up MetricCollector controller with the controller manager: %w", err)
+		// }
 	}
 
 	klog.InfoS("starting hub manager")

cmd/memberagent/main.go

@@ -109,6 +109,13 @@ func (r *Reconciler) reconcileApprovalRequestObj(ctx context.Context, approvalRe
 		return r.handleDelete(ctx, approvalReqObj, isClusterScoped)
 	}
 
+	// Check if the approval request is already approved or rejected - stop reconciliation if so
+	approvedCond := meta.FindStatusCondition(approvalReqObj.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApproved))
+	if approvedCond != nil && approvedCond.Status == metav1.ConditionTrue {
+		klog.V(2).InfoS("ApprovalRequest has been approved, stopping reconciliation", "approvalRequest", approvalReqRef)
+		return ctrl.Result{}, nil
+	}
+
 	// Add finalizer if not present
 	if !controllerutil.ContainsFinalizer(obj, metricCollectorFinalizer) {
 		controllerutil.AddFinalizer(obj, metricCollectorFinalizer)
@@ -119,13 +126,6 @@ func (r *Reconciler) reconcileApprovalRequestObj(ctx context.Context, approvalRe
 		klog.V(2).InfoS("Added finalizer to ApprovalRequest", "approvalRequest", approvalReqRef)
 	}
 
-	// Check if the approval request is approved
-	approvedCond := meta.FindStatusCondition(approvalReqObj.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApproved))
-	if approvedCond != nil && approvedCond.Status == metav1.ConditionTrue {
-		klog.V(2).InfoS("ApprovalRequest has been approved, skipping", "approvalRequest", approvalReqRef)
-		return ctrl.Result{}, nil
-	}
-
 	// Get the UpdateRun (ClusterStagedUpdateRun or StagedUpdateRun)
 	spec := approvalReqObj.GetApprovalRequestSpec()
 	updateRunName := spec.TargetUpdateRun
@@ -190,13 +190,13 @@ func (r *Reconciler) reconcileApprovalRequestObj(ctx context.Context, approvalRe
 	klog.V(2).InfoS("Successfully ensured MetricCollector resources", "approvalRequest", approvalReqRef, "clusters", clusterNames)
 
 	// Check workload health and approve if all workloads are healthy
-	result, err := r.checkWorkloadHealthAndApprove(ctx, approvalReqObj, clusterNames, updateRunName, stageName)
-	if err != nil {
+	if err := r.checkWorkloadHealthAndApprove(ctx, approvalReqObj, clusterNames, updateRunName, stageName); err != nil {
 		klog.ErrorS(err, "Failed to check workload health", "approvalRequest", approvalReqRef)
-		return ctrl.Result{RequeueAfter: 30 * time.Second}, err
+		return ctrl.Result{RequeueAfter: 15 * time.Second}, err
 	}
 
-	return result, nil
+	// Requeue after 15 seconds to check again (will stop if approved in next reconciliation)
+	return ctrl.Result{RequeueAfter: 15 * time.Second}, nil
 }
 
 // ensureMetricCollectorResources creates the Namespace, MetricCollector, CRP, and ResourceOverrides
@@ -360,7 +360,7 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 	approvalReqObj placementv1beta1.ApprovalRequestObj,
 	clusterNames []string,
 	updateRunName, stageName string,
-) (ctrl.Result, error) {
+) error {
 	obj := approvalReqObj.(client.Object)
 	approvalReqRef := klog.KObj(obj)
 
@@ -370,12 +370,12 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 	workloadTrackerList := &placementv1beta1.WorkloadTrackerList{}
 	if err := r.Client.List(ctx, workloadTrackerList); err != nil {
 		klog.ErrorS(err, "Failed to list WorkloadTracker", "approvalRequest", approvalReqRef)
-		return ctrl.Result{}, fmt.Errorf("failed to list WorkloadTracker: %w", err)
+		return fmt.Errorf("failed to list WorkloadTracker: %w", err)
 	}
 
 	if len(workloadTrackerList.Items) == 0 {
 		klog.V(2).InfoS("No WorkloadTracker found, skipping health check", "approvalRequest", approvalReqRef)
-		return ctrl.Result{}, nil
+		return nil
 	}
 
 	// Use the first WorkloadTracker (assuming there's only one)
@@ -384,7 +384,7 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 
 	if len(workloadTracker.Workloads) == 0 {
 		klog.V(2).InfoS("WorkloadTracker has no workloads defined, skipping health check", "approvalRequest", approvalReqRef)
-		return ctrl.Result{}, nil
+		return nil
 	}
 
 	// MetricCollectorReport name is same as MetricCollector name
@@ -426,7 +426,7 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 				"cluster", clusterName,
 				"report", metricCollectorName,
 				"namespace", reportNamespace)
-			return ctrl.Result{}, fmt.Errorf("failed to get MetricCollectorReport for cluster %s: %w", clusterName, err)
+			return fmt.Errorf("failed to get MetricCollectorReport for cluster %s: %w", clusterName, err)
 		}
 
 		klog.V(2).InfoS("Found MetricCollectorReport",
@@ -500,7 +500,7 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 			approvalReqObj.SetApprovalRequestStatus(*status)
 			if err := r.Client.Status().Update(ctx, obj); err != nil {
 				klog.ErrorS(err, "Failed to approve ApprovalRequest", "approvalRequest", approvalReqRef)
-				return ctrl.Result{}, fmt.Errorf("failed to approve ApprovalRequest: %w", err)
+				return fmt.Errorf("failed to approve ApprovalRequest: %w", err)
 			}
 
 			klog.InfoS("Successfully approved ApprovalRequest", "approvalRequest", approvalReqRef)
@@ -509,17 +509,16 @@ func (r *Reconciler) checkWorkloadHealthAndApprove(
 			klog.V(2).InfoS("ApprovalRequest already approved", "approvalRequest", approvalReqRef)
 		}
 
-		// Stop reconciliation since we're approved
-		return ctrl.Result{}, nil
+		// Approval successful or already approved
+		return nil
 	}
 
-	// Not all workloads are healthy yet, requeue
-	klog.V(2).InfoS("Not all workloads are healthy yet, will requeue",
+	// Not all workloads are healthy yet, log details and return nil (reconcile will requeue)
+	klog.V(2).InfoS("Not all workloads are healthy yet",
 		"approvalRequest", approvalReqRef,
 		"unhealthyDetails", unhealthyDetails)
 
-	// Requeue after 30 seconds to check again
-	return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
+	return nil
 }
 
 // handleDelete handles the deletion of an ApprovalRequest or ClusterApprovalRequest

pkg/controllers/approvalrequest/controller.go

@@ -9,7 +9,7 @@ rules:
   # MetricCollector CRD access on member cluster
   - apiGroups: ["placement.kubernetes-fleet.io"]
     resources: ["metriccollectors"]
-    verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["placement.kubernetes-fleet.io"]
     resources: ["metriccollectors/status"]
     verbs: ["update", "patch"]

standalone-metric-collector/charts/metric-collector/templates/rbac-member.yaml

@@ -4,7 +4,7 @@
 
 # Controller image configuration
 image:
-  repository: ghcr.io/kubefleet-dev/metric-collector
+  repository: metric-collector
   pullPolicy: IfNotPresent
   tag: "latest"
 

standalone-metric-collector/charts/metric-collector/values.yaml

@@ -85,11 +85,14 @@ func buildHubConfig() (*rest.Config, error) {
 	// Check for custom headers
 	customHeader := os.Getenv("HUB_KUBE_HEADER")
 
+	// Check TLS insecure flag
+	tlsInsecure := os.Getenv("TLS_INSECURE") == "true"
+
 	// Initialize hub config
 	hubConfig := &rest.Config{
 		Host: hubURL,
 		TLSClientConfig: rest.TLSClientConfig{
-			Insecure: false,
+			Insecure: tlsInsecure,
 		},
 		WrapTransport: func(rt http.RoundTripper) http.RoundTripper {
 			if customHeader != "" {

standalone-metric-collector/cmd/metriccollector/main.go

@@ -1,4 +1,4 @@
-FROM golang:1.21 AS builder
+FROM golang:1.24 AS builder
 WORKDIR /workspace
 
 # Copy go mod files

standalone-metric-collector/docker/Dockerfile

@@ -0,0 +1,8 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: MetricCollector
+metadata:
+  name: mc-example-run-staging
+spec:
+  prometheusURL: "http://prometheus.test-ns:9090"
+  promQLQuery: "workload_health"
+  reportNamespace: "fleet-member-cluster-1"

standalone-metric-collector/examples/metriccollector-example.yaml

@@ -0,0 +1,140 @@
+#!/bin/bash
+set -e
+
+# Configuration
+HUB_CONTEXT="kind-hub"
+MEMBER_CONTEXT="kind-cluster-1"
+MEMBER_CLUSTER_NAME="cluster-1"
+MEMBER_NAMESPACE="default"
+HUB_NAMESPACE="fleet-member-${MEMBER_CLUSTER_NAME}"
+PROMETHEUS_URL="http://prometheus.test-ns:9090"
+IMAGE_NAME="metric-collector"
+IMAGE_TAG="latest"
+
+# Get hub cluster API server URL dynamically using docker inspect (following kubefleet pattern)
+HUB_API_SERVER="https://$(docker inspect hub-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443"
+
+echo "=== Installing MetricCollector on member cluster ==="
+echo "Hub cluster: ${HUB_CONTEXT}"
+echo "Hub API server: ${HUB_API_SERVER}"
+echo "Member cluster: ${MEMBER_CONTEXT}"
+echo "Member cluster name: ${MEMBER_CLUSTER_NAME}"
+echo ""
+
+# Step 0: Build and load Docker image
+echo "Step 0: Building and loading Docker image..."
+docker buildx build \
+  --file docker/Dockerfile \
+  --output=type=docker \
+  --platform=linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
+  --tag ${IMAGE_NAME}:${IMAGE_TAG} \
+  --build-arg GOARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
+  --build-arg GOOS=linux \
+  .
+kind load docker-image ${IMAGE_NAME}:${IMAGE_TAG} --name cluster-1
+echo "✓ Docker image built and loaded into kind cluster"
+echo ""
+
+# Step 1: Setup RBAC on hub cluster
+echo "Step 1: Setting up RBAC on hub cluster..."
+kubectl --context=${HUB_CONTEXT} create namespace ${HUB_NAMESPACE} --dry-run=client -o yaml | kubectl --context=${HUB_CONTEXT} apply -f -
+kubectl --context=${HUB_CONTEXT} create serviceaccount metric-collector-sa -n ${HUB_NAMESPACE} --dry-run=client -o yaml | kubectl --context=${HUB_CONTEXT} apply -f -
+
+cat <<EOF | kubectl --context=${HUB_CONTEXT} apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metric-collector-hub-access
+rules:
+  - apiGroups: ["placement.kubernetes-fleet.io"]
+    resources: ["metriccollectorreports"]
+    verbs: ["get", "list", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metric-collector-${MEMBER_CLUSTER_NAME}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metric-collector-hub-access
+subjects:
+  - kind: ServiceAccount
+    name: metric-collector-sa
+    namespace: ${HUB_NAMESPACE}
+EOF
+
+echo "✓ RBAC configured on hub cluster"
+echo ""
+
+# Step 2: Create token secret on hub cluster
+echo "Step 2: Creating token secret on hub cluster..."
+cat <<EOF | kubectl --context=${HUB_CONTEXT} apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metric-collector-token
+  namespace: ${HUB_NAMESPACE}
+  annotations:
+    kubernetes.io/service-account.name: metric-collector-sa
+type: kubernetes.io/service-account-token
+EOF
+
+# Wait for token to be created
+echo "Waiting for token to be created..."
+sleep 3
+
+# Get token
+TOKEN=$(kubectl --context=${HUB_CONTEXT} get secret metric-collector-token -n ${HUB_NAMESPACE} -o jsonpath='{.data.token}' | base64 -d)
+if [ -z "$TOKEN" ]; then
+    echo "Error: Failed to get token from hub cluster"
+    exit 1
+fi
+
+echo "✓ Token created on hub cluster"
+echo ""
+
+# Step 3: Create namespace and secrets on member cluster
+echo "Step 3: Creating secrets on member cluster..."
+
+kubectl --context=${MEMBER_CONTEXT} create secret generic hub-token \
+  --from-literal=token="${TOKEN}" \
+  -n ${MEMBER_NAMESPACE} \
+  --dry-run=client -o yaml | kubectl --context=${MEMBER_CONTEXT} apply -f -
+
+echo "✓ Secrets created on member cluster"
+echo ""
+
+# Step 4: Install helm chart on member cluster (includes CRD)
+echo "Step 4: Installing helm chart on member cluster..."
+helm upgrade --install metric-collector ./charts/metric-collector \
+  --kube-context=${MEMBER_CONTEXT} \
+  --namespace ${MEMBER_NAMESPACE} \
+  --set memberCluster.name=${MEMBER_CLUSTER_NAME} \
+  --set hubCluster.url=${HUB_API_SERVER} \
+  --set hubCluster.tls.insecure=true \
+  --set prometheus.url=${PROMETHEUS_URL} \
+  --set image.repository=${IMAGE_NAME} \
+  --set image.tag=${IMAGE_TAG} \
+  --set image.pullPolicy=IfNotPresent
+
+echo "✓ Helm chart installed on member cluster"
+echo ""
+
+# Step 5: Verify installation
+echo "Step 5: Verifying installation..."
+echo "Checking pods on member cluster..."
+kubectl --context=${MEMBER_CONTEXT} get pods -n ${MEMBER_NAMESPACE}
+
+echo ""
+echo "=== Installation Complete ==="
+echo ""
+echo "To check logs:"
+echo "  kubectl --context=${MEMBER_CONTEXT} logs -n ${MEMBER_NAMESPACE} -l app.kubernetes.io/name=metric-collector -f"
+echo ""
+echo "To check MetricCollectorReports on hub:"
+echo "  kubectl --context=${HUB_CONTEXT} get metriccollectorreports -n fleet-${MEMBER_CLUSTER_NAME}"
+echo ""