[feat] Add IP Reservation support for Services backed by NodeBalancers (#437)

* add annotation for Service reserved IP

* Add creation/update handling for Service LB reserved IP annotation

* Add testcases for reserved IP support to CCM

* Test fails when curl command is ran against the service ip. Added sleep delay for the service to come up

* Added firewall in lb-created-with-reserved-ip-linode-range and lb-created-with-reserved-ip-nb-range testcases. Removed the liveness check in lb-created-with-specified-nb-id-reserved and lb-created-with-reserved-ip-and-nb-id-annotations.

* add unit tests and mocks

* add reserve IP fake API endpoint

* add unit test for reserved IP LB creation/update

* add annotation for Service reserved IP

* Add creation/update handling for Service LB reserved IP annotation

* Add testcases for reserved IP support to CCM

* Added firewall in lb-created-with-reserved-ip-linode-range and lb-created-with-reserved-ip-nb-range testcases. Removed the liveness check in lb-created-with-specified-nb-id-reserved and lb-created-with-reserved-ip-and-nb-id-annotations.

* add initial unit tests and mocks

* add reserve IP fake API endpoint

* add unit test for reserved IP LB creation/update

* update lb docs
---------

Co-authored-by: Henry Wagner <[email protected]>

Author: hcwagner

cloud/annotations/annotations.go

@@ -25,6 +25,10 @@ const (
 	// same client IP. Options are a number between 1-20, or 0 to disable. Defaults to 20.
 	AnnLinodeThrottle = "service.beta.kubernetes.io/linode-loadbalancer-throttle"
 
+	// AnnLinodeLoadBalancerIPv4 is the annotation used to specify a reserved IPv4 address
+	// for the NodeBalancer. If not specified, Linode will automatically assign an IPv4 address.
+	AnnLinodeLoadBalancerReservedIPv4 = "service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4"
+
 	AnnLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve"
 	AnnLinodeNodeBalancerID       = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id"
 	AnnLinodeNodeBalancerType     = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-type"

cloud/linode/client/client.go

@@ -66,6 +66,9 @@ type Client interface {
 	GetFirewall(context.Context, int) (*linodego.Firewall, error)
 	UpdateFirewallRules(context.Context, int, linodego.FirewallRuleSet) (*linodego.FirewallRuleSet, error)
 
+	ReserveIPAddress(ctx context.Context, opts linodego.ReserveIPOptions) (*linodego.InstanceIP, error)
+	DeleteReservedIPAddress(ctx context.Context, ipAddress string) error
+
 	GetProfile(ctx context.Context) (*linodego.Profile, error)
 }
 

cloud/linode/client/client_with_metrics.go

@@ -305,6 +305,34 @@ func (f *fakeAPI) setupRoutes() {
 		_, _ = w.Write(rr)
 	})
 
+	f.mux.HandleFunc("POST /v4/networking/reserved/ips", func(w http.ResponseWriter, r *http.Request) {
+		rico := linodego.ReserveIPOptions{}
+		if err := json.NewDecoder(r.Body).Decode(&rico); err != nil {
+			f.t.Fatal(err)
+		}
+
+		ip := net.IPv4(byte(rand.Intn(100)), byte(rand.Intn(100)), byte(rand.Intn(100)), byte(rand.Intn(100))).String()
+
+		rip := linodego.InstanceIP{
+			Address:    ip,
+			SubnetMask: "32",
+			Prefix:     0,
+			Type:       linodego.IPTypeIPv4,
+			Public:     true,
+			RDNS:       "",
+			LinodeID:   0,
+			Region:     rico.Region,
+			VPCNAT1To1: nil,
+			Reserved:   true,
+		}
+
+		resp, err := json.Marshal(rip)
+		if err != nil {
+			f.t.Fatal(err)
+		}
+		_, _ = w.Write(resp)
+	})
+
 	f.mux.HandleFunc("POST /v4/nodebalancers", func(w http.ResponseWriter, r *http.Request) {
 		nbco := linodego.NodeBalancerCreateOptions{}
 		if err := json.NewDecoder(r.Body).Decode(&nbco); err != nil {

cloud/linode/client/mocks/mock_client.go

@@ -34,8 +34,9 @@ import (
 )
 
 var (
-	errNoNodesAvailable          = errors.New("no nodes available for nodebalancer")
-	maxConnThrottleStringLen int = 20
+	errNoNodesAvailable             = errors.New("no nodes available for nodebalancer")
+	maxConnThrottleStringLen    int = 20
+	eventIPChangeIgnoredWarning     = "nodebalancer-ipv4-change-ignored"
 
 	// validProtocols is a map of valid protocols
 	validProtocols = map[string]bool{
@@ -362,6 +363,30 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 	return lbStatus, nil
 }
 
+func (l *loadbalancers) createIPChangeWarningEvent(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer, newIP string) {
+	_, err := l.kubeClient.CoreV1().Events(service.Namespace).Create(ctx, &v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%d", eventIPChangeIgnoredWarning, time.Now().Unix()),
+			Namespace: service.Namespace,
+		},
+		InvolvedObject: v1.ObjectReference{
+			Kind:      "Service",
+			Namespace: service.Namespace,
+			Name:      service.Name,
+			UID:       service.UID,
+		},
+		Type:    "Warning",
+		Reason:  "NodeBalancerIPChangeIgnored",
+		Message: fmt.Sprintf("IPv4 annotation changed to %s, but NodeBalancer (%d) IP cannot be updated after creation. It will remain %s", newIP, nb.ID, *nb.IPv4),
+		Source: v1.EventSource{
+			Component: "linode-cloud-controller-manager",
+		},
+	}, metav1.CreateOptions{})
+	if err != nil {
+		klog.Errorf("failed to create NodeBalancerIPChangeIgnored event for service %s: %s", getServiceNn(service), err)
+	}
+}
+
 func (l *loadbalancers) updateNodeBalancer(
 	ctx context.Context,
 	clusterName string,
@@ -373,6 +398,16 @@ func (l *loadbalancers) updateNodeBalancer(
 		return fmt.Errorf("%w: service %s", errNoNodesAvailable, getServiceNn(service))
 	}
 
+	// Check for IPv4 annotation change
+	if ipv4, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerReservedIPv4]; ok && ipv4 != *nb.IPv4 {
+		// Log the error in the CCM's logfile
+		klog.Warningf("IPv4 annotation has changed for service (%s) from %s to %s, but NodeBalancer (%d) IP cannot be updated after creation",
+			getServiceNn(service), *nb.IPv4, ipv4, nb.ID)
+
+		// Issue a k8s cluster event warning
+		l.createIPChangeWarningEvent(ctx, service, nb, ipv4)
+	}
+
 	connThrottle := getConnectionThrottle(service)
 	if connThrottle != nb.ClientConnThrottle {
 		update := nb.GetUpdateOptions()
@@ -839,6 +874,11 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri
 		}
 	}
 
+	// Check for static IPv4 address annotation
+	if ipv4, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerReservedIPv4]; ok {
+		createOpts.IPv4 = &ipv4
+	}
+
 	fwid, ok := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallID]
 	if ok {
 		firewallID, err := strconv.Atoi(fwid)

cloud/linode/fake_linode_test.go

@@ -195,6 +195,10 @@ func TestCCMLoadBalancers(t *testing.T) {
 			name: "Create Load Balancer With Global Tags set",
 			f:    testCreateNodeBalancerWithGlobalTags,
 		},
+		{
+			name: "Create Load Balancer With Reserved IP",
+			f:    testCreateNodeBalancerWithReservedIP,
+		},
 		{
 			name: "Update Load Balancer - Add Node",
 			f:    testUpdateLoadBalancerAddNode,
@@ -255,6 +259,10 @@ func TestCCMLoadBalancers(t *testing.T) {
 			name: "Update Load Balancer - Add a new Firewall ACL",
 			f:    testUpdateLoadBalancerAddNewFirewallACL,
 		},
+		{
+			name: "Update Load Balancer - Add Reserved IP",
+			f:    testUpdateLoadBalancerAddReservedIP,
+		},
 		{
 			name: "Build Load Balancer Request",
 			f:    testBuildLoadBalancerRequest,
@@ -447,6 +455,18 @@ func testCreateNodeBalancer(t *testing.T, client *linodego.Client, _ *fakeAPI, a
 	return nil
 }
 
+func testCreateNodeBalancerWithReservedIP(t *testing.T, client *linodego.Client, f *fakeAPI) {
+	t.Helper()
+
+	annMap := map[string]string{
+		annotations.AnnLinodeLoadBalancerReservedIPv4: "156.1.1.101",
+	}
+	err := testCreateNodeBalancer(t, client, f, annMap, nil)
+	if err != nil {
+		t.Fatalf("expected a nil error, got %v", err)
+	}
+}
+
 func testCreateNodeBalancerWithOutFirewall(t *testing.T, client *linodego.Client, f *fakeAPI) {
 	t.Helper()
 
@@ -2987,6 +3007,93 @@ func testUpdateLoadBalancerDeleteFirewallRemoveID(t *testing.T, client *linodego
 	}
 }
 
+func testUpdateLoadBalancerAddReservedIP(t *testing.T, client *linodego.Client, fakeAPI *fakeAPI) {
+	t.Helper()
+	clusterName := "linodelb"
+	region := "us-west"
+
+	svc := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        randString(),
+			UID:         "foobar123",
+			Annotations: map[string]string{},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Name:     randString(),
+					Protocol: "http",
+					Port:     int32(80),
+					NodePort: int32(8080),
+				},
+			},
+		},
+	}
+
+	nodes := []*v1.Node{
+		{
+			Status: v1.NodeStatus{
+				Addresses: []v1.NodeAddress{
+					{
+						Type:    v1.NodeInternalIP,
+						Address: "127.0.0.1",
+					},
+				},
+			},
+		},
+	}
+
+	lb, assertion := newLoadbalancers(client, region).(*loadbalancers)
+	if !assertion {
+		t.Error("type assertion failed")
+	}
+	defer func() {
+		_ = lb.EnsureLoadBalancerDeleted(t.Context(), clusterName, svc)
+	}()
+
+	fakeClientset := fake.NewSimpleClientset()
+	lb.kubeClient = fakeClientset
+
+	nodeBalancer, err := client.CreateNodeBalancer(t.Context(), linodego.NodeBalancerCreateOptions{
+		Region: lb.zone,
+	})
+	if err != nil {
+		t.Fatalf("failed to create NodeBalancer: %s", err)
+	}
+
+	initialIP := *nodeBalancer.IPv4
+	svc.Status.LoadBalancer = *makeLoadBalancerStatus(svc, nodeBalancer)
+
+	ipaddr, err := client.ReserveIPAddress(t.Context(), linodego.ReserveIPOptions{
+		Region: lb.zone,
+	})
+	if err != nil {
+		t.Fatalf("failed to reserve IP address: %s", err)
+	}
+
+	stubService(fakeClientset, svc)
+	svc.SetAnnotations(map[string]string{
+		annotations.AnnLinodeLoadBalancerReservedIPv4: ipaddr.Address,
+	})
+
+	err = lb.UpdateLoadBalancer(t.Context(), "", svc, nodes)
+	if err != nil {
+		t.Errorf("UpdateLoadBalancer returned an error while updated annotations: %s", err)
+	}
+
+	status, _, err := lb.GetLoadBalancer(t.Context(), clusterName, svc)
+	if status.Ingress[0].IP != initialIP {
+		t.Fatalf("IP should not have changed in service status: %s", err)
+	}
+
+	event, _ := fakeClientset.CoreV1().Events("").Get(t.Context(),
+		eventIPChangeIgnoredWarning,
+		metav1.GetOptions{})
+	if event == nil {
+		t.Fatalf("failed to generate %s event: %s", eventIPChangeIgnoredWarning, err)
+	}
+}
+
 func testUpdateLoadBalancerAddNodeBalancerID(t *testing.T, client *linodego.Client, fakeAPI *fakeAPI) {
 	t.Helper()
 

cloud/linode/loadbalancers.go

@@ -73,9 +73,9 @@ type cloudAllocator struct {
 }
 
 const (
-	providerIDPrefix = "linode://"
-	ipv6BitLen       = 128
-	ipv6PodCIDRMaskSize  = 112
+	providerIDPrefix    = "linode://"
+	ipv6BitLen          = 128
+	ipv6PodCIDRMaskSize = 112
 )
 
 var _ CIDRAllocator = &cloudAllocator{}

cloud/linode/loadbalancers_test.go

@@ -43,6 +43,7 @@ The keys and the values in [annotations must be strings](https://kubernetes.io/d
 | `backend-ipv4-range` | string | | The IPv4 range from VPC subnet to be applied to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |
 | `backend-vpc-name` | string | | VPC which is connected to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |
 | `backend-subnet-name` | string | | Subnet within VPC which is connected to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |
+| `reserved-ipv4` | string | | An existing Reserved IPv4 address that wil be used to initialize the NodeBalancer instance. See [LoadBalancer Configuration](loadbalancer.md#reserved-ipv4-addresses)) |
 
 ### Port Specific Configuration
 

cloud/nodeipam/ipam/cloud_allocator.go

@@ -211,6 +211,17 @@ metadata:
     service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id: "12345"
 ```
 
+### Reserved IPv4 addresses
+
+Create an new NodeBalancer with an existing Reserved IPv4 Address:
+```
+metadata:
+  annotations:
+    service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4: "100.100.100.100"
+```
+The annotation must be present when the Service is created in order to take effect.
+
+
 ### NodeBalancer Preservation
 
 Prevent NodeBalancer deletion when service is deleted: