diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index a5b18f080f46..0c7c530035d3 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -22,6 +22,7 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. "arm64": "", "amd64": "" }, + "defaultSubnetIPRange": "10.0.0.0/16", // Optional, if not set the hetzner cloud default will be used - make sure this subnet exists within you private network and to use the cidr notation "nodeConfigs": { "pool1": { // This equals the pool name. Required for each pool that you have "cloudInit": "", // HCLOUD_CLOUD_INIT make sure it isn't base64 encoded twice ;] @@ -35,7 +36,8 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. "value": "autoscaler-node", "effect": "NoExecute" } - ] + ], + "subnetIPRange": "10.0.0.0/24", // Optional, if not set the defaultSubnetIPRange will be used - make sure this subnet exists within you private network and to use the cidr notation } } } @@ -55,6 +57,15 @@ The image selection logic works as follows: 1. If a nodepool doesn't have `imagesForArch` configured, the global `imagesForArch` configuration will be used as a fallback 1. If neither is configured, the legacy `HCLOUD_IMAGE` environment variable will be used + +The `defaultSubnetIPRange` and `subnetIPRange` configuration can be used to place nodes within a specific IP range. +This only applies to private networks. Make sure that the subnet exists within your private network. +If you do not set this value, the default setting from Hetzner Cloud will be used. + +The global `defaultSubnetIPRange` can be overridden on a per-nodepool basis by adding a `subnetIPRange` field to individual nodepool configurations. + + + `HCLOUD_NETWORK` Default empty , The id or name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks `HCLOUD_FIREWALL` Default empty , The id or name of the firewall that is used in the cluster , @see https://docs.hetzner.cloud/#firewalls diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go index 07e73cfd3c6e..faff857fc124 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go @@ -20,7 +20,9 @@ import ( "context" "errors" "fmt" + "net" "regexp" + "slices" "strconv" "strings" "sync" @@ -198,6 +200,17 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove klog.Fatalf("No cluster config present provider: %v", err) } + var defaultSubnetIPRange *net.IPNet + if manager.clusterConfig.IsUsingNewFormat && manager.network != nil && manager.clusterConfig.DefaultSubnetIPRange != "" { + _, defaultSubnetIPRange, err = net.ParseCIDR(manager.clusterConfig.DefaultSubnetIPRange) + if err != nil { + klog.Fatalf("failed to parse default subnet ip range %s: %s", manager.clusterConfig.DefaultSubnetIPRange, err) + } + if !isIpRangeInNetwork(defaultSubnetIPRange, manager.network) { + klog.Fatalf("default subnet ip range %s is not part of network %s", manager.clusterConfig.DefaultSubnetIPRange, manager.network.Name) + } + } + validNodePoolName := regexp.MustCompile(`^[a-z0-9A-Z]+[a-z0-9A-Z\-\.\_]*[a-z0-9A-Z]+$|^[a-z0-9A-Z]{1}$`) clusterUpdateLock := sync.Mutex{} placementGroupTotals := make(map[string]int) @@ -214,6 +227,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove } var placementGroup *hcloud.PlacementGroup + var subnetIPRange *net.IPNet if manager.clusterConfig.IsUsingNewFormat { _, ok := manager.clusterConfig.NodeConfigs[spec.name] if !ok { @@ -232,6 +246,24 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove } placementGroupTotals[placementGroup.Name] += spec.maxSize } + if manager.network != nil { + if manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange != "" { + _, subnetIPRange, err = net.ParseCIDR(manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange) + if err != nil { + klog.Fatalf("failed to parse subnet ip range %s for node group %s: %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, err) + } + if !isIpRangeInNetwork(subnetIPRange, manager.network) { + klog.Fatalf("subnet ip range %s for node group %s is not part of network %s", manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange, spec.name, manager.network.Name) + } + } else { + subnetIPRange = defaultSubnetIPRange + } + klog.Infof("[DEBUG]Using subnet ip range %s for node group %s", subnetIPRange, spec.name) + klog.Infof("[DEBUG]Default SubnetIPRange is %s", defaultSubnetIPRange) + klog.Infof("[DEBUG]Default and SubnetIPRange in Config is: %s + %s", manager.clusterConfig.DefaultSubnetIPRange, manager.clusterConfig.NodeConfigs[spec.name].SubnetIPRange) + + } + } manager.nodeGroups[spec.name] = &hetznerNodeGroup{ @@ -244,6 +276,7 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove targetSize: len(servers), clusterUpdateMutex: &clusterUpdateLock, placementGroup: placementGroup, + subnetIPRange: subnetIPRange, } } @@ -262,6 +295,15 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove return provider } +func isIpRangeInNetwork(ipRange *net.IPNet, network *hcloud.Network) bool { + return slices.ContainsFunc(network.Subnets, func(subnet hcloud.NetworkSubnet) bool { + if ipRange == nil || subnet.IPRange == nil { + return false + } + return subnet.IPRange.IP.Equal(ipRange.IP) && len(subnet.IPRange.Mask) == len(ipRange.Mask) + }) +} + func getPlacementGroup(manager *hetznerManager, placementGroupRef string) (*hcloud.PlacementGroup, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go index 1d72ab17bdf2..0a4a609a1da1 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go @@ -59,10 +59,11 @@ type hetznerManager struct { // ClusterConfig holds the configuration for all the nodepools type ClusterConfig struct { - ImagesForArch ImageList - NodeConfigs map[string]*NodeConfig - IsUsingNewFormat bool - LegacyConfig LegacyConfig + ImagesForArch ImageList + NodeConfigs map[string]*NodeConfig + IsUsingNewFormat bool + LegacyConfig LegacyConfig + DefaultSubnetIPRange string } // ImageList holds the image id/names for the different architectures @@ -78,6 +79,7 @@ type NodeConfig struct { Taints []apiv1.Taint Labels map[string]string ImagesForArch *ImageList + SubnetIPRange string } // LegacyConfig holds the configuration in the legacy format diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go index 103138994a3d..4376f6fe0ffc 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go @@ -22,6 +22,7 @@ import ( "fmt" "maps" "math/rand" + "net" "strings" "sync" @@ -49,6 +50,7 @@ type hetznerNodeGroup struct { clusterUpdateMutex *sync.Mutex placementGroup *hcloud.PlacementGroup + subnetIPRange *net.IPNet } type hetznerNodeGroupSpec struct { @@ -472,7 +474,8 @@ func createServer(n *hetznerNodeGroup) error { cloudInit = n.manager.clusterConfig.NodeConfigs[n.id].CloudInit } - StartAfterCreate := true + // dont start the server if we need to attach the server to a private subnet network + StartAfterCreate := n.manager.network != nil && n.subnetIPRange == nil opts := hcloud.ServerCreateOpts{ Name: newNodeName(n), UserData: cloudInit, @@ -492,7 +495,7 @@ func createServer(n *hetznerNodeGroup) error { if n.manager.sshKey != nil { opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey} } - if n.manager.network != nil { + if n.manager.network != nil && n.subnetIPRange == nil { opts.Networks = []*hcloud.Network{n.manager.network} } if n.manager.firewall != nil { @@ -516,6 +519,34 @@ func createServer(n *hetznerNodeGroup) error { return fmt.Errorf("failed to start server %s error: %v", server.Name, err) } + if n.manager.network != nil && n.subnetIPRange != nil { + // Attach server to private network with subnetIPRange + attachAction, _, err := n.manager.client.Server.AttachToNetwork(ctx, server, hcloud.ServerAttachToNetworkOpts{ + Network: n.manager.network, + IPRange: n.subnetIPRange, + }) + if err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed to attach server %s to network %s with IP range %s error: %v", server.Name, n.manager.network.Name, n.subnetIPRange.String(), err) + } + if err = n.manager.client.Action.WaitFor(ctx, attachAction); err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed waiting for network action for server %s error: %v", server.Name, err) + } + } + + if !StartAfterCreate { + powerOnAction, _, err := n.manager.client.Server.Poweron(ctx, server) + if err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed to power on server %s error: %v", server.Name, err) + } + if err = n.manager.client.Action.WaitFor(ctx, powerOnAction); err != nil { + _ = n.manager.deleteServer(server) + return fmt.Errorf("failed waiting for power on action for server %s error: %v", server.Name, err) + } + } + return nil }