Merge pull request #8570 from tloesch/feature/cluster-autoscaler-hetzner-ip-range-config

feat(hetzner): add IP range configuration for private network

Author: k8s-ci-robot

cluster-autoscaler/cloudprovider/hetzner/README.md

@@ -22,6 +22,7 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes.
         "arm64": "",
         "amd64": ""
     },
+    "defaultSubnetIPRange": "10.0.0.0/16", // Optional, if not set the hetzner cloud default will be used - make sure this subnet exists within you private network and to use the cidr notation
     "nodeConfigs": {
         "pool1": { // This equals the pool name. Required for each pool that you have
             "cloudInit": "", // HCLOUD_CLOUD_INIT make sure it isn't base64 encoded twice ;]
@@ -35,7 +36,8 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes.
                     "value": "autoscaler-node",
                     "effect": "NoExecute"
                 }
-            ]
+            ],
+            "subnetIPRange": "10.0.0.0/24" // Optional, if not set the defaultSubnetIPRange will be used - make sure this subnet exists within you private network and to use the cidr notation
         }
     }
 }
@@ -55,6 +57,15 @@ The image selection logic works as follows:
 1. If a nodepool doesn't have `imagesForArch` configured, the global `imagesForArch` configuration will be used as a fallback
 1. If neither is configured, the legacy `HCLOUD_IMAGE` environment variable will be used
 
+
+The `defaultSubnetIPRange` and `subnetIPRange` configuration can be used to place nodes within a specific IP range.
+This only applies to private networks. Make sure that the subnet exists within your private network.
+If you do not set this value, the default setting from Hetzner Cloud will be used.
+
+The global `defaultSubnetIPRange` can be overridden on a per-nodepool basis by adding a `subnetIPRange` field to individual nodepool configurations.
+
+
+
 `HCLOUD_NETWORK` Default empty , The id or name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks
 
 `HCLOUD_FIREWALL` Default empty , The id or name of the firewall that is used in the cluster , @see https://docs.hetzner.cloud/#firewalls

cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -198,6 +200,17 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 		klog.Fatalf("No cluster config present provider: %v", err)
 	}
 
+	var defaultSubnetIPRange *net.IPNet
+	if manager.clusterConfig.IsUsingNewFormat && manager.network != nil && manager.clusterConfig.DefaultSubnetIPRange != "" {
+		_, defaultSubnetIPRange, err = net.ParseCIDR(manager.clusterConfig.DefaultSubnetIPRange)
+		if err != nil {
+			klog.Fatalf("failed to parse default subnet ip range %s: %s", manager.clusterConfig.DefaultSubnetIPRange, err)
+		}
+		if !isIpRangeInNetwork(defaultSubnetIPRange, manager.network) {
+			klog.Fatalf("default subnet ip range %s is not part of network %s", manager.clusterConfig.DefaultSubnetIPRange, manager.network.Name)
+		}
+	}
+
 	validNodePoolName := regexp.MustCompile(`^[a-z0-9A-Z]+[a-z0-9A-Z\-\.\_]*[a-z0-9A-Z]+$|^[a-z0-9A-Z]{1}$`)
 	clusterUpdateLock := sync.Mutex{}
 	placementGroupTotals := make(map[string]int)
@@ -214,6 +227,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 		}
 
 		var placementGroup *hcloud.PlacementGroup
+		var subnetIPRange *net.IPNet
 		if manager.clusterConfig.IsUsingNewFormat {
 			_, ok := manager.clusterConfig.NodeConfigs[spec.name]
 			if !ok {
@@ -232,6 +246,20 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 				}
 				placementGroupTotals[placementGroup.Name] += spec.maxSize
 			}
+			if manager.network != nil {
+				if manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange != "" {
+					_, subnetIPRange, err = net.ParseCIDR(manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange)
+					if err != nil {
+						klog.Fatalf("failed to parse subnet ip range %s for node group %s: %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, err)
+					}
+					if !isIpRangeInNetwork(subnetIPRange, manager.network) {
+						klog.Fatalf("subnet ip range %s for node group %s is not part of network %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, manager.network.Name)
+					}
+				} else {
+					subnetIPRange = defaultSubnetIPRange
+				}
+			}
+
 		}
 
 		manager.nodeGroups[spec.name] = &hetznerNodeGroup{
@@ -244,6 +272,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 			targetSize:         len(servers),
 			clusterUpdateMutex: &clusterUpdateLock,
 			placementGroup:     placementGroup,
+			subnetIPRange:      subnetIPRange,
 		}
 	}
 
@@ -262,6 +291,15 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 	return provider
 }
 
+func isIpRangeInNetwork(ipRange *net.IPNet, network *hcloud.Network) bool {
+	return slices.ContainsFunc(network.Subnets, func(subnet hcloud.NetworkSubnet) bool {
+		if ipRange == nil || subnet.IPRange == nil {
+			return false
+		}
+		return subnet.IPRange.IP.Equal(ipRange.IP) && len(subnet.IPRange.Mask) == len(ipRange.Mask)
+	})
+}
+
 func getPlacementGroup(manager *hetznerManager, placementGroupRef string) (*hcloud.PlacementGroup, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()

cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go

@@ -59,10 +59,11 @@ type hetznerManager struct {
 
 // ClusterConfig holds the configuration for all the nodepools
 type ClusterConfig struct {
-	ImagesForArch    ImageList
-	NodeConfigs      map[string]*NodeConfig
-	IsUsingNewFormat bool
-	LegacyConfig     LegacyConfig
+	ImagesForArch        ImageList
+	NodeConfigs          map[string]*NodeConfig
+	IsUsingNewFormat     bool
+	LegacyConfig         LegacyConfig
+	DefaultSubnetIPRange string
 }
 
 // ImageList holds the image id/names for the different architectures
@@ -78,6 +79,7 @@ type NodeConfig struct {
 	Taints         []apiv1.Taint
 	Labels         map[string]string
 	ImagesForArch  *ImageList
+	SubnetIPRange  string
 }
 
 // LegacyConfig holds the configuration in the legacy format

cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"maps"
 	"math/rand"
+	"net"
 	"strings"
 	"sync"
 
@@ -49,6 +50,7 @@ type hetznerNodeGroup struct {
 
 	clusterUpdateMutex *sync.Mutex
 	placementGroup     *hcloud.PlacementGroup
+	subnetIPRange      *net.IPNet
 }
 
 type hetznerNodeGroupSpec struct {
@@ -472,7 +474,8 @@ func createServer(n *hetznerNodeGroup) error {
 		cloudInit = n.manager.clusterConfig.NodeConfigs[n.id].CloudInit
 	}
 
-	StartAfterCreate := true
+	// dont start the server if we need to attach the server to a private subnet network
+	StartAfterCreate := n.manager.network != nil && n.subnetIPRange == nil
 	opts := hcloud.ServerCreateOpts{
 		Name:             newNodeName(n),
 		UserData:         cloudInit,
@@ -492,7 +495,7 @@ func createServer(n *hetznerNodeGroup) error {
 	if n.manager.sshKey != nil {
 		opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey}
 	}
-	if n.manager.network != nil {
+	if n.manager.network != nil && n.subnetIPRange == nil {
 		opts.Networks = []*hcloud.Network{n.manager.network}
 	}
 	if n.manager.firewall != nil {
@@ -516,6 +519,34 @@ func createServer(n *hetznerNodeGroup) error {
 		return fmt.Errorf("failed to start server %s error: %v", server.Name, err)
 	}
 
+	if n.manager.network != nil && n.subnetIPRange != nil {
+		// Attach server to private network with subnetIPRange
+		attachAction, _, err := n.manager.client.Server.AttachToNetwork(ctx, server, hcloud.ServerAttachToNetworkOpts{
+			Network: n.manager.network,
+			IPRange: n.subnetIPRange,
+		})
+		if err != nil {
+			_ = n.manager.deleteServer(server)
+			return fmt.Errorf("failed to attach server %s to network %s with IP range %s error: %v", server.Name, n.manager.network.Name, n.subnetIPRange.String(), err)
+		}
+		if err = n.manager.client.Action.WaitFor(ctx, attachAction); err != nil {
+			_ = n.manager.deleteServer(server)
+			return fmt.Errorf("failed waiting for network action for server %s error: %v", server.Name, err)
+		}
+	}
+
+	if !StartAfterCreate {
+		powerOnAction, _, err := n.manager.client.Server.Poweron(ctx, server)
+		if err != nil {
+			_ = n.manager.deleteServer(server)
+			return fmt.Errorf("failed to power on server %s error: %v", server.Name, err)
+		}
+		if err = n.manager.client.Action.WaitFor(ctx, powerOnAction); err != nil {
+			_ = n.manager.deleteServer(server)
+			return fmt.Errorf("failed waiting for power on action for server %s error: %v", server.Name, err)
+		}
+	}
+
 	return nil
 }