[feat] Add IP Reservation support for Services backed by NodeBalancers

cloud/annotations/annotations.go

@@ -25,6 +25,10 @@ const (
 	// same client IP. Options are a number between 1-20, or 0 to disable. Defaults to 20.
 	AnnLinodeThrottle = "service.beta.kubernetes.io/linode-loadbalancer-throttle"
 
+	// AnnLinodeLoadBalancerIPv4 is the annotation used to specify a reserved IPv4 address
+	// for the NodeBalancer. If not specified, Linode will automatically assign an IPv4 address.
+	AnnLinodeLoadBalancerIPv4     = "service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4"
+
 	AnnLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve"
 	AnnLinodeNodeBalancerID       = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id"
 	AnnLinodeNodeBalancerType     = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-type"

cloud/linode/loadbalancers.go

@@ -361,6 +361,35 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 	return lbStatus, nil
 }
 
+func (l *loadbalancers) createIPChangeWarningEvent(ctx context.Context, service *v1.Service, nb *linodego.NodeBalancer, newIP string) {
+	if l.kubeClient == nil {
+		err := l.retrieveKubeClient()
+        if err != nil {
+			fmt.Errorf("%w: Error retrieving kube client", err)
+            return
+        }
+	}
+
+	l.kubeClient.CoreV1().Events(service.Namespace).Create(ctx, &v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("nodebalancer-ipv4-change-ignored-%d", time.Now().Unix()),
+			Namespace: service.Namespace,
+		},
+		InvolvedObject: v1.ObjectReference{
+			Kind:      "Service",
+			Namespace: service.Namespace,
+			Name:      service.Name,
+			UID:       service.UID,
+		},
+		Type:    "Warning",
+		Reason:  "NodeBalancerIPChangeIgnored",
+		Message: fmt.Sprintf("IPv4 annotation changed to %s, but NodeBalancer (%d) IP cannot be updated after creation. It will remain %s", newIP, nb.ID, *nb.IPv4),
+		Source: v1.EventSource{
+			Component: "linode-cloud-controller-manager",
+		},
+	}, metav1.CreateOptions{})
+}
+
 func (l *loadbalancers) updateNodeBalancer(
 	ctx context.Context,
 	clusterName string,
@@ -372,6 +401,16 @@ func (l *loadbalancers) updateNodeBalancer(
 		return fmt.Errorf("%w: service %s", errNoNodesAvailable, getServiceNn(service))
 	}
 
+	// Check for IPv4 annotation change
+	if ipv4, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerIPv4]; ok && ipv4 != *nb.IPv4 {
+		// Log the error in the CCM's logfile
+		klog.Warningf("IPv4 annotation has changed for service (%s) from %s to %s, but NodeBalancer (%d) IP cannot be updated after creation",
+			getServiceNn(service), *nb.IPv4, ipv4, nb.ID)
+
+		// Issue a k8s cluster event warning
+		l.createIPChangeWarningEvent(ctx, service, nb, ipv4)
+	}
+
 	connThrottle := getConnectionThrottle(service)
 	if connThrottle != nb.ClientConnThrottle {
 		update := nb.GetUpdateOptions()
@@ -838,6 +877,14 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri
 		}
 	}
 
+	// Check for static IPv4 address annotation
+	if ipv4, ok := service.GetAnnotations()[annotations.AnnLinodeLoadBalancerIPv4]; ok {
+		if err := isValidPublicIPv4(ipv4); err != nil {
+			return nil, fmt.Errorf("invalid IPv4 address in annotation %s: %v", annotations.AnnLinodeLoadBalancerIPv4, err)
+		}
+		createOpts.IPv4 = &ipv4
+	}
+
 	fwid, ok := service.GetAnnotations()[annotations.AnnLinodeCloudFirewallID]
 	if ok {
 		firewallID, err := strconv.Atoi(fwid)
@@ -866,6 +913,31 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri
 	return l.client.CreateNodeBalancer(ctx, createOpts)
 }
 
+// isValidPublicIPv4 checks if the given string is a valid public IPv4 address
+func isValidPublicIPv4(ipStr string) error {
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return fmt.Errorf("invalid IP address format: %s", ipStr)
+	}
+
+	ipv4 := ip.To4()
+	if ipv4 == nil {
+		return fmt.Errorf("not an IPv4 address: %s", ipStr)
+	}
+
+	// Check if it's a public IP (not private, not multicast, not experimental)
+	if ipv4[0] == 10 || // 10.0.0.0/8
+		(ipv4[0] == 172 && ipv4[1] >= 16 && ipv4[1] <= 31) || // 172.16.0.0/12
+		(ipv4[0] == 192 && ipv4[1] == 168) || // 192.168.0.0/16
+		ipv4[0] >= 224 || // Class D & E
+		ipv4[0] == 0 || // 0.0.0.0/8
+		ipv4[0] == 127 { // 127.0.0.0/8
+		return fmt.Errorf("not a public IPv4 address: %s", ipStr)
+	}
+
+	return nil
+}
+
 func (l *loadbalancers) buildNodeBalancerConfig(ctx context.Context, service *v1.Service, port v1.ServicePort) (linodego.NodeBalancerConfig, error) {
 	portConfigResult, err := getPortConfig(service, port)
 	if err != nil {

e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml

@@ -0,0 +1,94 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: lb-created-with-invalid-ip
+  labels:
+    all:
+    lke:
+spec:
+  namespace: "lb-created-with-invalid-ip"
+  steps:
+    - name: create reserved ip and nodebalancer resources
+      try:
+        - script:
+            content: |
+              set -e
+
+              invalid_ip="100.1000.1000.1000"
+
+              create_cm=$(kubectl -n $NAMESPACE create configmap invalid-ip-config --from-literal=InvalidIP=$invalid_ip -o yaml --dry-run=client | kubectl apply -f -)
+              if [[ "$create_cm" != "configmap/invalid-ip-config created" ]]; then
+                echo "Unable to create configmap. Error: $create_cm"
+              fi
+
+              echo "{\"invalid_ip\": \"$invalid_ip\"}"
+            check:
+              ($error == null): true
+              (contains($stdout, 'Unable to create configmap')): false
+            outputs:
+              - name: ip
+                value: (json_parse($stdout))
+        - apply:
+            file: create-pods-services.yaml
+      catch:
+        - describe:
+            apiVersion: v1
+            kind: Pod
+        - describe:
+            apiVersion: v1
+            kind: Service
+    - name: Check that loadbalancer ip is not assigned
+      try:
+      - assert:
+          resource:
+            apiVersion: v1
+            kind: Service
+            metadata:
+              name: svc-test
+            status:
+              (loadBalancer.ingress[0].ip == null): true
+    - name: get service ip and compare with invalid ip
+      try:
+        - script: 
+            content: |
+              set -e
+
+              invalid_ip=$(kubectl get configmap invalid-ip-config -o=jsonpath='{.data.InvalidIP}' -n $NAMESPACE)
+              if [[ -z "$invalid_ip" ]]; then
+                  echo "Error: No invalid ip found in configmap"
+              fi
+
+              annotation="service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4"
+              events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
+              message=$(echo $events | jq .items[0].message)
+
+              if [[ "$message" =~ ^\"Error\ syncing\ load\ balancer:\ failed\ to\ ensure\ load\ balancer:\ invalid\ IPv4\ address\ in\ annotation\ $annotation:\ invalid\ IP\ address\ format:\ $invalid_ip\"$ ]]; then
+                echo "Warning event found"
+              else
+                echo "Warning event not found"
+              fi
+
+              service_ip=$(kubectl get svc svc-test -n $NAMESPACE -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+              if [[ "$service_ip" != "" ]]; then
+                  echo "Error: service ip found for service svc-test"
+              fi
+              echo "{\"service_ip\": $service_ip}"
+              echo "{\"invalid_ip\": $invalid_ip}"
+              if [[ "$service_ip" == "$invalid_ip" ]]; then
+                echo "Expected service ip to be null got the invalid ip: $invalid_ip"
+              fi
+
+              delete_cm=$(kubectl delete configmap invalid-ip-config -n $NAMESPACE)
+              if [[ "$delete_cm" == "configmap \"invalid-ip-config\" deleted" ]]; then
+                echo "Configmap deleted successfully"
+              else
+                echo "Unable to delete the configmap: $delete_cm. Error: $delete_cm"
+              fi
+
+            check:
+              ($error == null): true
+              (contains($stdout, 'Warning event not found')): false
+              (contains($stdout, 'No invalid ip found in configmap')): false
+              (contains($stdout, 'Unable to delete the configmap')): false
+              (contains($stdout, 'Expected service ip to be null got the invalid ip')): false

e2e/test/lb-created-with-invalid-ip/create-pods-services.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: created-with-invalid-ip
+  name: test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: created-with-invalid-ip
+  template:
+    metadata:
+      labels:
+        app: created-with-invalid-ip
+    spec:
+      containers:
+      - image: appscode/test-server:2.3
+        name: test
+        ports:
+        - name: http-1
+          containerPort: 8080
+          protocol: TCP
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: svc-test
+  annotations:
+    service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4: ($ip.invalid_ip)
+  labels:
+    app: created-with-invalid-ip
+spec:
+  type: LoadBalancer
+  selector:
+    app: created-with-invalid-ip
+  ports:
+    - name: http-1
+      protocol: TCP
+      port: 80
+      targetPort: 8080
+  sessionAffinity: None