Add support for service annotation propagation

Enable propagation of Service annotations from workload clusters
to influence final Service creation.
Only propagates annotations when enable-service-annotation-propagation
is explicitly set true.

Author: silvery1622

pkg/cloudprovider/vsphereparavirtual/cloud.go

@@ -62,6 +62,9 @@ var (
 
 	// podIPPoolType specifies if Pod IP addresses are public or private.
 	podIPPoolType string
+
+	// serviceAnnotationPropagationEnabled if set to true, will propagate the service annotation to resource in supervisor cluster.
+	serviceAnnotationPropagationEnabled bool
 )
 
 func init() {
@@ -89,6 +92,7 @@ func init() {
 	flag.BoolVar(&vmservice.IsLegacy, "is-legacy-paravirtual", false, "If true, machine label selector will start with capw.vmware.com. By default, it's false, machine label selector will start with capv.vmware.com.")
 	flag.BoolVar(&vpcModeEnabled, "enable-vpc-mode", false, "If true, routable pod controller will start with VPC mode. It is useful only when route controller is enabled in vsphereparavirtual mode")
 	flag.StringVar(&podIPPoolType, "pod-ip-pool-type", "", "Specify if Pod IP address is Public or Private routable in VPC network. Valid values are Public and Private")
+	flag.BoolVar(&serviceAnnotationPropagationEnabled, "enable-service-annotation-propagation", false, "If true, will propagate the service annotation to resource in supervisor cluster.")
 }
 
 // Creates new Controller node interface and returns
@@ -139,7 +143,7 @@ func (cp *VSphereParavirtual) Initialize(clientBuilder cloudprovider.ControllerC
 	}
 	cp.routes = routes
 
-	lb, err := NewLoadBalancer(clusterNS, kcfg, cp.ownerReference)
+	lb, err := NewLoadBalancer(clusterNS, kcfg, cp.ownerReference, serviceAnnotationPropagationEnabled)
 	if err != nil {
 		klog.Errorf("Failed to init LoadBalancer: %v", err)
 	}

pkg/cloudprovider/vsphereparavirtual/loadbalancer.go

@@ -39,15 +39,15 @@ type loadBalancer struct {
 }
 
 // NewLoadBalancer returns an implementation of cloudprovider.LoadBalancer
-func NewLoadBalancer(clusterNS string, kcfg *rest.Config, ownerRef *metav1.OwnerReference) (cloudprovider.LoadBalancer, error) {
+func NewLoadBalancer(clusterNS string, kcfg *rest.Config, ownerRef *metav1.OwnerReference, serviceAnnotationPropagationEnabled bool) (cloudprovider.LoadBalancer, error) {
 	klog.V(1).Info("Create load balancer for vsphere paravirtual cloud provider")
 
 	client, err := vmservice.GetVmopClient(kcfg)
 	if err != nil {
 		klog.Errorf("failed to create load balancer: %v", err)
 		return nil, err
 	}
-	vmService := vmservice.NewVMService(client, clusterNS, ownerRef)
+	vmService := vmservice.NewVMService(client, clusterNS, ownerRef, serviceAnnotationPropagationEnabled)
 	return &loadBalancer{
 		vmService: vmService,
 	}, nil

pkg/cloudprovider/vsphereparavirtual/loadbalancer_test.go

@@ -60,26 +60,34 @@ func newTestLoadBalancer() (cloudprovider.LoadBalancer, *dynamicfake.FakeDynamic
 	fc := dynamicfake.NewSimpleDynamicClient(scheme)
 	fcw := vmopclient.NewFakeClientSet(fc)
 
-	vms := vmservice.NewVMService(fcw, testClusterNameSpace, &testOwnerReference)
+	vms := vmservice.NewVMService(fcw, testClusterNameSpace, &testOwnerReference, false)
 	return &loadBalancer{vmService: vms}, fc
 }
 
 func TestNewLoadBalancer(t *testing.T) {
 	testCases := []struct {
-		name   string
-		config *rest.Config
-		err    error
+		name                                string
+		config                              *rest.Config
+		serviceAnnotationPropagationEnabled bool
+		err                                 error
 	}{
 		{
-			name:   "NewLoadBalancer: when everything is ok",
-			config: &rest.Config{},
-			err:    nil,
+			name:                                "NewLoadBalancer: when serviceAnnotationPropagationEnabled is false",
+			config:                              &rest.Config{},
+			serviceAnnotationPropagationEnabled: false,
+			err:                                 nil,
+		},
+		{
+			name:                                "NewLoadBalancer: when serviceAnnotationPropagationEnabled is true",
+			config:                              &rest.Config{},
+			serviceAnnotationPropagationEnabled: true,
+			err:                                 nil,
 		},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			_, err := NewLoadBalancer(testClusterNameSpace, testCase.config, &testOwnerReference)
+			_, err := NewLoadBalancer(testClusterNameSpace, testCase.config, &testOwnerReference, testCase.serviceAnnotationPropagationEnabled)
 			assert.Equal(t, testCase.err, err)
 		})
 	}

pkg/cloudprovider/vsphereparavirtual/vmservice/types.go

@@ -41,7 +41,8 @@ type VMService interface {
 
 // vmService takes care of mapping of LB type of service to VM service in supervisor cluster
 type vmService struct {
-	vmClient       vmop.Interface
-	namespace      string
-	ownerReference *metav1.OwnerReference
+	vmClient                            vmop.Interface
+	namespace                           string
+	ownerReference                      *metav1.OwnerReference
+	serviceAnnotationPropagationEnabled bool
 }

pkg/cloudprovider/vsphereparavirtual/vmservice/vmservice.go

@@ -22,6 +22,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"reflect"
+	"slices"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -65,11 +66,21 @@ const (
 	// configuration to the supervisor cluster.
 	AnnotationServiceHealthCheckNodePortKey = "virtualmachineservice.vmoperator.vmware.com/service.healthCheckNodePort"
 
+	// AnnotationLastAppliedConfiguration is used by kubectl as a legacy mechanism to track changes.
+	// That mechanism has been superseded by Server-side apply.
+	AnnotationLastAppliedConfiguration = "kubectl.kubernetes.io/last-applied-configuration"
+
 	// MaxCheckSumLen is the maximum length of vmservice suffix: vsphere paravirtual name length cannot exceed 41 bytes in total, so we need to make sure vmservice suffix is 21 bytes (63 - 41 -1 = 21)
 	// https://gitlab.eng.vmware.com/core-build/guest-cluster-controller/blob/master/webhooks/validation/tanzukubernetescluster_validator.go#L56
 	MaxCheckSumLen = 21
 )
 
+var excludedAnnotations = []string{
+	AnnotationLastAppliedConfiguration,
+	AnnotationServiceExternalTrafficPolicyKey,
+	AnnotationServiceHealthCheckNodePortKey,
+}
+
 // A list of possible error messages
 var (
 	ErrCreateVMService     = errors.New("failed to create VirtualMachineService")
@@ -93,11 +104,12 @@ func GetVmopClient(config *rest.Config) (vmop.Interface, error) {
 }
 
 // NewVMService creates a vmService object
-func NewVMService(vmClient vmop.Interface, ns string, ownerRef *metav1.OwnerReference) VMService {
+func NewVMService(vmClient vmop.Interface, ns string, ownerRef *metav1.OwnerReference, serviceAnnotationPropagationEnabled bool) VMService {
 	return &vmService{
-		vmClient:       vmClient,
-		namespace:      ns,
-		ownerReference: ownerRef,
+		vmClient:                            vmClient,
+		namespace:                           ns,
+		ownerReference:                      ownerRef,
+		serviceAnnotationPropagationEnabled: serviceAnnotationPropagationEnabled,
 	}
 }
 
@@ -226,7 +238,7 @@ func (s *vmService) Update(ctx context.Context, service *v1.Service, clusterName
 		service.Spec.LoadBalancerSourceRanges = []string{}
 	}
 
-	annotations := getVMServiceAnnotations(vmService, service)
+	annotations := getVMServiceAnnotations(vmService, service, s.serviceAnnotationPropagationEnabled)
 
 	// VMService only has a few fields to be kept in sync so we will simply
 	// iterate over them
@@ -342,14 +354,14 @@ func (s *vmService) lbServiceToVMService(service *v1.Service, clusterName string
 		Spec: vmServiceSpec,
 	}
 
-	if annotations := getVMServiceAnnotations(vmService, service); len(annotations) != 0 {
+	if annotations := getVMServiceAnnotations(vmService, service, s.serviceAnnotationPropagationEnabled); len(annotations) != 0 {
 		vmService.Annotations = annotations
 	}
 
 	return vmService, nil
 }
 
-func getVMServiceAnnotations(vmService *vmopv1.VirtualMachineService, service *v1.Service) map[string]string {
+func getVMServiceAnnotations(vmService *vmopv1.VirtualMachineService, service *v1.Service, serviceAnnotationPropagationEnabled bool) map[string]string {
 	var annotations map[string]string
 	// When ExternalTrafficPolicy is set to Local in the Service, add its
 	// value and the healthCheckNodePort to VirtualMachineService
@@ -362,6 +374,21 @@ func getVMServiceAnnotations(vmService *vmopv1.VirtualMachineService, service *v
 		annotations[AnnotationServiceExternalTrafficPolicyKey] = string(service.Spec.ExternalTrafficPolicy)
 		annotations[AnnotationServiceHealthCheckNodePortKey] = strconv.Itoa(int(service.Spec.HealthCheckNodePort))
 	}
+
+	// Annotation propagation logic
+	if serviceAnnotationPropagationEnabled {
+		// Initialize annotations map if empty
+		if annotations == nil {
+			annotations = make(map[string]string)
+		}
+		// Merge service annotations
+		for k, v := range service.Annotations {
+			if !slices.Contains(excludedAnnotations, k) {
+				annotations[k] = v
+			}
+		}
+	}
+
 	return annotations
 }