diff --git a/cloudstack.go b/cloudstack.go index 8d78a861..f8f38ebb 100644 --- a/cloudstack.go +++ b/cloudstack.go @@ -25,8 +25,10 @@ import ( "fmt" "io" "os" + "strings" "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver/v4" "gopkg.in/gcfg.v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" @@ -53,6 +55,7 @@ type CSCloud struct { client *cloudstack.CloudStackClient projectID string // If non-"", all resources will be created within this project zone string + version semver.Version } func init() { @@ -85,6 +88,7 @@ func newCSCloud(cfg *CSConfig) (*CSCloud, error) { cs := &CSCloud{ projectID: cfg.Global.ProjectID, zone: cfg.Global.Zone, + version: semver.Version{}, } if cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != "" { @@ -95,9 +99,32 @@ func newCSCloud(cfg *CSConfig) (*CSCloud, error) { return nil, errors.New("no cloud provider config given") } + version, err := cs.getManagementServerVersion() + if err != nil { + return nil, err + } + cs.version = version + return cs, nil } +func (cs *CSCloud) getManagementServerVersion() (semver.Version, error) { + msServersResp, err := cs.client.Management.ListManagementServersMetrics(cs.client.Management.NewListManagementServersMetricsParams()) + if err != nil { + return semver.Version{}, err + } + if msServersResp.Count == 0 { + return semver.Version{}, errors.New("no management servers found") + } + version := msServersResp.ManagementServersMetrics[0].Version + v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) + if err != nil { + klog.Errorf("failed to parse management server version: %v", err) + return semver.Version{}, err + } + return v, nil +} + // Initialize passes a Kubernetes clientBuilder interface to the cloud provider func (cs *CSCloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { } diff --git a/cloudstack_loadbalancer.go b/cloudstack_loadbalancer.go index 00f536f0..3420e350 100644 --- a/cloudstack_loadbalancer.go +++ b/cloudstack_loadbalancer.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver/v4" "k8s.io/klog/v2" corev1 "k8s.io/api/core/v1" @@ -44,7 +45,12 @@ const ( // CloudStack >= 4.6 is required for it to work. ServiceAnnotationLoadBalancerProxyProtocol = "service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol" ServiceAnnotationLoadBalancerLoadbalancerHostname = "service.beta.kubernetes.io/cloudstack-load-balancer-hostname" - ServiceAnnotationLoadBalancerSourceCidrs = "service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs" + + // ServiceAnnotationLoadBalancerSourceCidrs is the annotation used on the + // service to specify the source CIDR list for a CloudStack load balancer. + // The CIDR list is a comma-separated list of CIDR ranges (e.g., "10.0.0.0/8,192.168.1.0/24"). + // If not specified, the default is to allow all sources ("0.0.0.0/0"). + ServiceAnnotationLoadBalancerSourceCidrs = "service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs" ) type loadBalancer struct { @@ -143,7 +149,7 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s lbRuleName := fmt.Sprintf("%s-%s-%d", lb.name, protocol, port.Port) // If the load balancer rule exists and is up-to-date, we move on to the next rule. - lbRule, needsUpdate, err := lb.checkLoadBalancerRule(lbRuleName, port, protocol) + lbRule, needsUpdate, err := lb.checkLoadBalancerRule(lbRuleName, port, protocol, service, cs.version) if err != nil { return nil, err } @@ -151,7 +157,7 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s if lbRule != nil { if needsUpdate { klog.V(4).Infof("Updating load balancer rule: %v", lbRuleName) - if err := lb.updateLoadBalancerRule(lbRuleName, protocol); err != nil { + if err := lb.updateLoadBalancerRule(lbRuleName, protocol, service, cs.version); err != nil { return nil, err } // Delete the rule from the map, to prevent it being deleted. @@ -561,37 +567,110 @@ func (lb *loadBalancer) releaseLoadBalancerIP() error { return nil } +func (lb *loadBalancer) getCIDRList(service *corev1.Service) ([]string, error) { + sourceCIDRs := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSourceCidrs, defaultAllowedCIDR) + var cidrList []string + if sourceCIDRs != "" { + cidrList = strings.Split(sourceCIDRs, ",") + for i, cidr := range cidrList { + cidr = strings.TrimSpace(cidr) + if _, _, err := net.ParseCIDR(cidr); err != nil { + return nil, fmt.Errorf("invalid CIDR %s in annotation %s: %w", cidr, ServiceAnnotationLoadBalancerSourceCidrs, err) + } + cidrList[i] = cidr + } + } + return cidrList, nil +} + // checkLoadBalancerRule checks if the rule already exists and if it does, if it can be updated. If // it does exist but cannot be updated, it will delete the existing rule so it can be created again. -func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol) (*cloudstack.LoadBalancerRule, bool, error) { +func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol, service *corev1.Service, version semver.Version) (*cloudstack.LoadBalancerRule, bool, error) { lbRule, ok := lb.rules[lbRuleName] if !ok { return nil, false, nil } - // Check if any of the values we cannot update (those that require a new load balancer rule) are changed. - if lbRule.Publicip == lb.ipAddr && lbRule.Privateport == strconv.Itoa(int(port.NodePort)) && lbRule.Publicport == strconv.Itoa(int(port.Port)) { - updateAlgo := lbRule.Algorithm != lb.algorithm - updateProto := lbRule.Protocol != protocol.CSProtocol() - return lbRule, updateAlgo || updateProto, nil + cidrList, err := lb.getCIDRList(service) + if err != nil { + return nil, false, err } - // Delete the load balancer rule so we can create a new one using the new values. - if err := lb.deleteLoadBalancerRule(lbRule); err != nil { - return nil, false, err + var lbRuleCidrList []string + if lbRule.Cidrlist != "" { + lbRuleCidrList = strings.Split(lbRule.Cidrlist, " ") + for i, cidr := range lbRuleCidrList { + cidr = strings.TrimSpace(cidr) + lbRuleCidrList[i] = cidr + } + } + + // Check if basic properties match (IP and ports). If not, we need to recreate the rule. + basicPropsMatch := lbRule.Publicip == lb.ipAddr && + lbRule.Privateport == strconv.Itoa(int(port.NodePort)) && + lbRule.Publicport == strconv.Itoa(int(port.Port)) + + cidrListChanged := len(cidrList) != len(lbRuleCidrList) || !setsEqual(cidrList, lbRuleCidrList) + + // Check if CIDR list also changed and version < 4.22, then we must recreate the rule. + if !basicPropsMatch || (cidrListChanged && version.LT(semver.Version{Major: 4, Minor: 22, Patch: 0})) { + // Delete the load balancer rule so we can create a new one using the new values. + if err := lb.deleteLoadBalancerRule(lbRule); err != nil { + return nil, false, err + } + return nil, false, nil + } + + // Rule can be updated. Check what needs updating. + updateAlgo := lbRule.Algorithm != lb.algorithm + updateProto := lbRule.Protocol != protocol.CSProtocol() + + return lbRule, updateAlgo || updateProto || cidrListChanged, nil +} + +// setsEqual checks if two slices contain the exact same unique elements, regardless of order. +func setsEqual(listA, listB []string) bool { + createSet := func(list []string) map[string]bool { + set := make(map[string]bool) + for _, item := range list { + set[item] = true + } + return set + } + + setA := createSet(listA) + setB := createSet(listB) + + if len(setA) != len(setB) { + return false } - return nil, false, nil + for item := range setA { + if _, found := setB[item]; !found { + return false + } + } + + return true } // updateLoadBalancerRule updates a load balancer rule. -func (lb *loadBalancer) updateLoadBalancerRule(lbRuleName string, protocol LoadBalancerProtocol) error { +func (lb *loadBalancer) updateLoadBalancerRule(lbRuleName string, protocol LoadBalancerProtocol, service *corev1.Service, version semver.Version) error { lbRule := lb.rules[lbRuleName] p := lb.LoadBalancer.NewUpdateLoadBalancerRuleParams(lbRule.Id) p.SetAlgorithm(lb.algorithm) p.SetProtocol(protocol.CSProtocol()) + // If version >= 4.22, we can update the CIDR list. + if version.GTE(semver.Version{Major: 4, Minor: 22, Patch: 0}) { + cidrList, err := lb.getCIDRList(service) + if err != nil { + return err + } + p.SetCidrlist(cidrList) + } + _, err := lb.LoadBalancer.UpdateLoadBalancerRule(p) return err } @@ -613,19 +692,9 @@ func (lb *loadBalancer) createLoadBalancerRule(lbRuleName string, port corev1.Se p.SetOpenfirewall(false) // Read the source CIDR annotation - sourceCIDRs, ok := service.Annotations[ServiceAnnotationLoadBalancerSourceCidrs] - var cidrList []string - if ok && sourceCIDRs != "" { - cidrList = strings.Split(sourceCIDRs, ",") - for i, cidr := range cidrList { - cidr = strings.TrimSpace(cidr) - if _, _, err := net.ParseCIDR(cidr); err != nil { - return nil, fmt.Errorf("invalid CIDR in annotation %s: %s", ServiceAnnotationLoadBalancerSourceCidrs, cidr) - } - cidrList[i] = cidr - } - } else { - cidrList = []string{defaultAllowedCIDR} + cidrList, err := lb.getCIDRList(service) + if err != nil { + return nil, err } // Set the CIDR list in the parameters diff --git a/go.mod b/go.mod index a6fe59e7..c4c5e764 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/apache/cloudstack-kubernetes-provider go 1.23.0 require ( - github.com/apache/cloudstack-go/v2 v2.17.1 + github.com/apache/cloudstack-go/v2 v2.19.0 + github.com/blang/semver/v4 v4.0.0 github.com/spf13/pflag v1.0.5 gopkg.in/gcfg.v1 v1.2.3 k8s.io/api v0.24.17 @@ -17,7 +18,6 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect diff --git a/go.sum b/go.sum index 8b618a15..85fd3e0f 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/cloudstack-go/v2 v2.17.1 h1:XD0bGDOv+MCavXJfc/qxILgJh+cHJbudpqQ1FzA2sDI= -github.com/apache/cloudstack-go/v2 v2.17.1/go.mod h1:p/YBUwIEkQN6CQxFhw8Ff0wzf1MY0qRRRuGYNbcb1F8= +github.com/apache/cloudstack-go/v2 v2.19.0 h1:YHLw770MmgiqXx6NRFYw2Nr7DpnylLhLG2KYNCftgnc= +github.com/apache/cloudstack-go/v2 v2.19.0/go.mod h1:p/YBUwIEkQN6CQxFhw8Ff0wzf1MY0qRRRuGYNbcb1F8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=