From 58386d93cb25400765055a691050e019b15cefa8 Mon Sep 17 00:00:00 2001 From: Timo Loesch Date: Thu, 25 Sep 2025 10:13:58 +0200 Subject: [PATCH 1/3] feat(hetzner): add IP range configuration for private network --- .../cloudprovider/hetzner/README.md | 7 +- .../cloudprovider/hetzner/hetzner_manager.go | 2 + .../hetzner/hetzner_node_group.go | 75 ++++++++++++++++++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index a5b18f080f46..d55fbd06cffe 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -22,6 +22,7 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. "arm64": "", "amd64": "" }, + "ipRange": "10.0.0.0/16", // Optional, make sure to match your private network configuration and to use the cidr notation - if not set the hetzner cloud default will be used "nodeConfigs": { "pool1": { // This equals the pool name. Required for each pool that you have "cloudInit": "", // HCLOUD_CLOUD_INIT make sure it isn't base64 encoded twice ;] @@ -47,7 +48,11 @@ Can be useful when you have many different node pools and run into issues of the **NOTE**: In contrast to `HCLOUD_CLUSTER_CONFIG`, this file is not base64 encoded. -The global `imagesForArch` configuration can be overridden on a per-nodepool basis by adding an `imagesForArch` field to individual nodepool configurations. +The `ipRange` configuration can be used to place nodes within a specific IP range. This only applies to private networks. Make sure that the IP range is within the configured private network IP Range. If you do not set this value, the default setting from Hetzner Cloud will be used. + +Following global configuration options can be overriden on a per-nodepool basis by adding them to the individual nodepool configurations: +- `imagesForArch` +- `ipRange` The image selection logic works as follows: diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go index 1d72ab17bdf2..989c2a27b53e 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go @@ -63,6 +63,7 @@ type ClusterConfig struct { NodeConfigs map[string]*NodeConfig IsUsingNewFormat bool LegacyConfig LegacyConfig + IPRange string } // ImageList holds the image id/names for the different architectures @@ -78,6 +79,7 @@ type NodeConfig struct { Taints []apiv1.Taint Labels map[string]string ImagesForArch *ImageList + IPRange string } // LegacyConfig holds the configuration in the legacy format diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go index 103138994a3d..8f0ef96219e8 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go @@ -22,6 +22,7 @@ import ( "fmt" "maps" "math/rand" + "net" "strings" "sync" @@ -452,6 +453,39 @@ func instanceTypeArch(manager *hetznerManager, instanceType string) (string, err } } +func findIpRange(nodeId string, clusterConfig *ClusterConfig) (*net.IPNet, error) { + if !clusterConfig.IsUsingNewFormat { + return nil, nil + } + if nodeConfig, exists := clusterConfig.NodeConfigs[nodeId]; exists && nodeConfig.IPRange != "" { + _, ipNet, err := net.ParseCIDR(nodeConfig.IPRange) + if err != nil { + return nil, fmt.Errorf("failed to parse IP range %s for node %s: %v", nodeConfig.IPRange, nodeId, err) + } + return ipNet, nil + } + if clusterConfig.IPRange != "" { + _, ipNet, err := net.ParseCIDR(clusterConfig.IPRange) + if err != nil { + return nil, fmt.Errorf("failed to parse global IP range %s: %v", clusterConfig.IPRange, err) + } + return ipNet, nil + } + return nil, nil +} + +func isIpRangeInNetwork(ipRange *net.IPNet, network *hcloud.Network) bool { + found := false + for _, nSubnet := range network.Subnets { + _, nSubnetRange, _ := net.ParseCIDR(nSubnet.IPRange.String()) + if nSubnetRange.String() == ipRange.String() { + found = true + break + } + } + return found +} + func createServer(n *hetznerNodeGroup) error { ctx, cancel := context.WithTimeout(n.manager.apiCallContext, n.manager.createTimeout) defer cancel() @@ -466,13 +500,22 @@ func createServer(n *hetznerNodeGroup) error { return err } + ipRange, err := findIpRange(n.Id(), n.manager.clusterConfig) + if err != nil { + return err + } + if ipRange != nil && n.manager.network != nil && !isIpRangeInNetwork(ipRange, n.manager.network) { + return fmt.Errorf("the specified IP range %s for node %s is not part of the network %s", ipRange.String(), n.Id(), n.manager.network.Name) + } + cloudInit := n.manager.clusterConfig.LegacyConfig.CloudInit if n.manager.clusterConfig.IsUsingNewFormat { cloudInit = n.manager.clusterConfig.NodeConfigs[n.id].CloudInit } - StartAfterCreate := true + // dont start the server if we need to attach the server to a private subnet network + StartAfterCreate := ipRange == nil opts := hcloud.ServerCreateOpts{ Name: newNodeName(n), UserData: cloudInit, @@ -492,7 +535,7 @@ func createServer(n *hetznerNodeGroup) error { if n.manager.sshKey != nil { opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey} } - if n.manager.network != nil { + if n.manager.network != nil && ipRange == nil { opts.Networks = []*hcloud.Network{n.manager.network} } if n.manager.firewall != nil { @@ -516,6 +559,34 @@ func createServer(n *hetznerNodeGroup) error { return fmt.Errorf("failed to start server %s error: %v", server.Name, err) } + if n.manager.network != nil && ipRange != nil { + // Attach server to private network with ipRange + attachAction, _, err := n.manager.client.Server.AttachToNetwork(ctx, server, hcloud.ServerAttachToNetworkOpts{ + Network: n.manager.network, + IPRange: ipRange, + }) + if err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed to attach server %s to network %s with IP range %s error: %v", server.Name, n.manager.network.Name, ipRange.String(), err) + } + if err = n.manager.client.Action.WaitFor(ctx, attachAction); err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed waiting for network action for server %s error: %v", server.Name, err) + } + } + + if !StartAfterCreate { + powerOnAction, _, err := n.manager.client.Server.Poweron(ctx, server) + if err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed to power on server %s error: %v", server.Name, err) + } + if err = n.manager.client.Action.WaitFor(ctx, powerOnAction); err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed waiting for power on action for server %s error: %v", server.Name, err) + } + } + return nil } From 3ab4087e41213984ab91d2d8284d21ef2a5ab230 Mon Sep 17 00:00:00 2001 From: Timo Loesch Date: Thu, 25 Sep 2025 10:55:52 +0200 Subject: [PATCH 2/3] fix typo --- cluster-autoscaler/cloudprovider/hetzner/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index d55fbd06cffe..67ab5c344f4c 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -50,7 +50,7 @@ Can be useful when you have many different node pools and run into issues of the The `ipRange` configuration can be used to place nodes within a specific IP range. This only applies to private networks. Make sure that the IP range is within the configured private network IP Range. If you do not set this value, the default setting from Hetzner Cloud will be used. -Following global configuration options can be overriden on a per-nodepool basis by adding them to the individual nodepool configurations: +Following global configuration options can be overridden on a per-nodepool basis by adding them to the individual nodepool configurations: - `imagesForArch` - `ipRange` From 6a0f0afdcfada973a1e30b0de6583d4b74e7dfe1 Mon Sep 17 00:00:00 2001 From: Timo Loesch Date: Mon, 6 Oct 2025 16:52:56 +0200 Subject: [PATCH 3/3] change implementation to parse the iprange clusterconfig early refactor some code update README.md --- .../cloudprovider/hetzner/README.md | 20 ++++--- .../hetzner/hetzner_cloud_provider.go | 42 +++++++++++++++ .../cloudprovider/hetzner/hetzner_manager.go | 12 ++--- .../hetzner/hetzner_node_group.go | 54 +++---------------- 4 files changed, 68 insertions(+), 60 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index 67ab5c344f4c..0c7c530035d3 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -22,7 +22,7 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. "arm64": "", "amd64": "" }, - "ipRange": "10.0.0.0/16", // Optional, make sure to match your private network configuration and to use the cidr notation - if not set the hetzner cloud default will be used + "defaultSubnetIPRange": "10.0.0.0/16", // Optional, if not set the hetzner cloud default will be used - make sure this subnet exists within you private network and to use the cidr notation "nodeConfigs": { "pool1": { // This equals the pool name. Required for each pool that you have "cloudInit": "", // HCLOUD_CLOUD_INIT make sure it isn't base64 encoded twice ;] @@ -36,7 +36,8 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. "value": "autoscaler-node", "effect": "NoExecute" } - ] + ], + "subnetIPRange": "10.0.0.0/24", // Optional, if not set the defaultSubnetIPRange will be used - make sure this subnet exists within you private network and to use the cidr notation } } } @@ -48,11 +49,7 @@ Can be useful when you have many different node pools and run into issues of the **NOTE**: In contrast to `HCLOUD_CLUSTER_CONFIG`, this file is not base64 encoded. -The `ipRange` configuration can be used to place nodes within a specific IP range. This only applies to private networks. Make sure that the IP range is within the configured private network IP Range. If you do not set this value, the default setting from Hetzner Cloud will be used. - -Following global configuration options can be overridden on a per-nodepool basis by adding them to the individual nodepool configurations: -- `imagesForArch` -- `ipRange` +The global `imagesForArch` configuration can be overridden on a per-nodepool basis by adding an `imagesForArch` field to individual nodepool configurations. The image selection logic works as follows: @@ -60,6 +57,15 @@ The image selection logic works as follows: 1. If a nodepool doesn't have `imagesForArch` configured, the global `imagesForArch` configuration will be used as a fallback 1. If neither is configured, the legacy `HCLOUD_IMAGE` environment variable will be used + +The `defaultSubnetIPRange` and `subnetIPRange` configuration can be used to place nodes within a specific IP range. +This only applies to private networks. Make sure that the subnet exists within your private network. +If you do not set this value, the default setting from Hetzner Cloud will be used. + +The global `defaultSubnetIPRange` can be overridden on a per-nodepool basis by adding a `subnetIPRange` field to individual nodepool configurations. + + + `HCLOUD_NETWORK` Default empty , The id or name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks `HCLOUD_FIREWALL` Default empty , The id or name of the firewall that is used in the cluster , @see https://docs.hetzner.cloud/#firewalls diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go index 07e73cfd3c6e..faff857fc124 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go @@ -20,7 +20,9 @@ import ( "context" "errors" "fmt" + "net" "regexp" + "slices" "strconv" "strings" "sync" @@ -198,6 +200,17 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove klog.Fatalf("No cluster config present provider: %v", err) } + var defaultSubnetIPRange *net.IPNet + if manager.clusterConfig.IsUsingNewFormat && manager.network != nil && manager.clusterConfig.DefaultSubnetIPRange != "" { + _, defaultSubnetIPRange, err = net.ParseCIDR(manager.clusterConfig.DefaultSubnetIPRange) + if err != nil { + klog.Fatalf("failed to parse default subnet ip range %s: %s", manager.clusterConfig.DefaultSubnetIPRange, err) + } + if !isIpRangeInNetwork(defaultSubnetIPRange, manager.network) { + klog.Fatalf("default subnet ip range %s is not part of network %s", manager.clusterConfig.DefaultSubnetIPRange, manager.network.Name) + } + } + validNodePoolName := regexp.MustCompile(`^[a-z0-9A-Z]+[a-z0-9A-Z\-\.\_]*[a-z0-9A-Z]+$|^[a-z0-9A-Z]{1}$`) clusterUpdateLock := sync.Mutex{} placementGroupTotals := make(map[string]int) @@ -214,6 +227,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove } var placementGroup *hcloud.PlacementGroup + var subnetIPRange *net.IPNet if manager.clusterConfig.IsUsingNewFormat { _, ok := manager.clusterConfig.NodeConfigs[spec.name] if !ok { @@ -232,6 +246,24 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove } placementGroupTotals[placementGroup.Name] += spec.maxSize } + if manager.network != nil { + if manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange != "" { + _, subnetIPRange, err = net.ParseCIDR(manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange) + if err != nil { + klog.Fatalf("failed to parse subnet ip range %s for node group %s: %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, err) + } + if !isIpRangeInNetwork(subnetIPRange, manager.network) { + klog.Fatalf("subnet ip range %s for node group %s is not part of network %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, manager.network.Name) + } + } else { + subnetIPRange = defaultSubnetIPRange + } + klog.Infof("[DEBUG]Using subnet ip range %s for node group %s", subnetIPRange, spec.name) + klog.Infof("[DEBUG]Default SubnetIPRange is %s", defaultSubnetIPRange) + klog.Infof("[DEBUG]Default and SubnetIPRange in Config is: %s + %s", manager.clusterConfig.DefaultSubnetIPRange, manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange) + + } + } manager.nodeGroups[spec.name] = &hetznerNodeGroup{ @@ -244,6 +276,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove targetSize: len(servers), clusterUpdateMutex: &clusterUpdateLock, placementGroup: placementGroup, + subnetIPRange: subnetIPRange, } } @@ -262,6 +295,15 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove return provider } +func isIpRangeInNetwork(ipRange *net.IPNet, network *hcloud.Network) bool { + return slices.ContainsFunc(network.Subnets, func(subnet hcloud.NetworkSubnet) bool { + if ipRange == nil || subnet.IPRange == nil { + return false + } + return subnet.IPRange.IP.Equal(ipRange.IP) && len(subnet.IPRange.Mask) == len(ipRange.Mask) + }) +} + func getPlacementGroup(manager *hetznerManager, placementGroupRef string) (*hcloud.PlacementGroup, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go index 989c2a27b53e..0a4a609a1da1 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go @@ -59,11 +59,11 @@ type hetznerManager struct { // ClusterConfig holds the configuration for all the nodepools type ClusterConfig struct { - ImagesForArch ImageList - NodeConfigs map[string]*NodeConfig - IsUsingNewFormat bool - LegacyConfig LegacyConfig - IPRange string + ImagesForArch ImageList + NodeConfigs map[string]*NodeConfig + IsUsingNewFormat bool + LegacyConfig LegacyConfig + DefaultSubnetIPRange string } // ImageList holds the image id/names for the different architectures @@ -79,7 +79,7 @@ type NodeConfig struct { Taints []apiv1.Taint Labels map[string]string ImagesForArch *ImageList - IPRange string + SubnetIPRange string } // LegacyConfig holds the configuration in the legacy format diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go index 8f0ef96219e8..4376f6fe0ffc 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go @@ -50,6 +50,7 @@ type hetznerNodeGroup struct { clusterUpdateMutex *sync.Mutex placementGroup *hcloud.PlacementGroup + subnetIPRange *net.IPNet } type hetznerNodeGroupSpec struct { @@ -453,39 +454,6 @@ func instanceTypeArch(manager *hetznerManager, instanceType string) (string, err } } -func findIpRange(nodeId string, clusterConfig *ClusterConfig) (*net.IPNet, error) { - if !clusterConfig.IsUsingNewFormat { - return nil, nil - } - if nodeConfig, exists := clusterConfig.NodeConfigs[nodeId]; exists && nodeConfig.IPRange != "" { - _, ipNet, err := net.ParseCIDR(nodeConfig.IPRange) - if err != nil { - return nil, fmt.Errorf("failed to parse IP range %s for node %s: %v", nodeConfig.IPRange, nodeId, err) - } - return ipNet, nil - } - if clusterConfig.IPRange != "" { - _, ipNet, err := net.ParseCIDR(clusterConfig.IPRange) - if err != nil { - return nil, fmt.Errorf("failed to parse global IP range %s: %v", clusterConfig.IPRange, err) - } - return ipNet, nil - } - return nil, nil -} - -func isIpRangeInNetwork(ipRange *net.IPNet, network *hcloud.Network) bool { - found := false - for _, nSubnet := range network.Subnets { - _, nSubnetRange, _ := net.ParseCIDR(nSubnet.IPRange.String()) - if nSubnetRange.String() == ipRange.String() { - found = true - break - } - } - return found -} - func createServer(n *hetznerNodeGroup) error { ctx, cancel := context.WithTimeout(n.manager.apiCallContext, n.manager.createTimeout) defer cancel() @@ -500,14 +468,6 @@ func createServer(n *hetznerNodeGroup) error { return err } - ipRange, err := findIpRange(n.Id(), n.manager.clusterConfig) - if err != nil { - return err - } - if ipRange != nil && n.manager.network != nil && !isIpRangeInNetwork(ipRange, n.manager.network) { - return fmt.Errorf("the specified IP range %s for node %s is not part of the network %s", ipRange.String(), n.Id(), n.manager.network.Name) - } - cloudInit := n.manager.clusterConfig.LegacyConfig.CloudInit if n.manager.clusterConfig.IsUsingNewFormat { @@ -515,7 +475,7 @@ func createServer(n *hetznerNodeGroup) error { } // dont start the server if we need to attach the server to a private subnet network - StartAfterCreate := ipRange == nil + StartAfterCreate := n.manager.network != nil && n.subnetIPRange == nil opts := hcloud.ServerCreateOpts{ Name: newNodeName(n), UserData: cloudInit, @@ -535,7 +495,7 @@ func createServer(n *hetznerNodeGroup) error { if n.manager.sshKey != nil { opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey} } - if n.manager.network != nil && ipRange == nil { + if n.manager.network != nil && n.subnetIPRange == nil { opts.Networks = []*hcloud.Network{n.manager.network} } if n.manager.firewall != nil { @@ -559,15 +519,15 @@ func createServer(n *hetznerNodeGroup) error { return fmt.Errorf("failed to start server %s error: %v", server.Name, err) } - if n.manager.network != nil && ipRange != nil { - // Attach server to private network with ipRange + if n.manager.network != nil && n.subnetIPRange != nil { + // Attach server to private network with subnetIPRange attachAction, _, err := n.manager.client.Server.AttachToNetwork(ctx, server, hcloud.ServerAttachToNetworkOpts{ Network: n.manager.network, - IPRange: ipRange, + IPRange: n.subnetIPRange, }) if err != nil { _ = n.manager.deleteServer(server) - return fmt.Errorf("failed to attach server %s to network %s with IP range %s error: %v", server.Name, n.manager.network.Name, ipRange.String(), err) + return fmt.Errorf("failed to attach server %s to network %s with IP range %s error: %v", server.Name, n.manager.network.Name, n.subnetIPRange.String(), err) } if err = n.manager.client.Action.WaitFor(ctx, attachAction); err != nil { _ = n.manager.deleteServer(server)