Skip to content

Commit effc737

Browse files
ederstederst
authored
Support specifying a sort order for node addresses (kubernetes#1946)
This introduces a configuration key which influences the way the provider reports the node addresses to the Kubernetes node resource. The default order depends on the hard-coded order the provider queries the addresses and what the cloud returns, which does not guarantee a specific order. To override this behavior it is possible to specify a comma separated list of CIDRs. Essentially, this will sort and group all addresses matching a CIDR in a prioritized manner, where the first item having a higher priority than the last. All non-matching addresses will remain in the same order they are already in. For example, this option can be useful when having multiple or dual-stack interfaces attached to a node and needing a user-controlled, deterministic way of sorting the addresses.
1 parent 47fcb98 commit effc737

File tree

5 files changed

+357
-0
lines changed

5 files changed

+357
-0
lines changed

‎docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,13 @@ The options in `Global` section are used for openstack-cloud-controller-manager
166166
The name of Neutron external network. openstack-cloud-controller-manager uses this option when getting the external IP of the Kubernetes node. Can be specified multiple times. Specified network names will be ORed. Default: ""
167167
* `internal-network-name`
168168
The name of Neutron internal network. openstack-cloud-controller-manager uses this option when getting the internal IP of the Kubernetes node, this is useful if the node has multiple interfaces. Can be specified multiple times. Specified network names will be ORed. Default: ""
169+
* `address-sort-order`
170+
This configuration key influences the way the provider reports the node addresses to the Kubernetes node resource. The default order depends on the hard-coded order the provider queries the addresses and what the cloud returns, which does not guarantee a specific order.
171+
172+
To override this behavior it is possible to specify a comma separated list of CIDRs. Essentially, this will sort and group all addresses matching a CIDR in a prioritized manner, where the first item having a higher priority than the last. All non-matching addresses will remain in the same order they are already in.
173+
174+
For example, this option can be useful when having multiple or dual-stack interfaces attached to a node and needing a user-controlled, deterministic way of sorting the addresses.
175+
Default: ""
169176
170177
### Load Balancer
171178

‎pkg/openstack/instances.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package openstack
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"fmt"
2223
"net"
@@ -56,10 +57,79 @@ type Instances struct {
5657
const (
5758
instanceShutoff = "SHUTOFF"
5859
RegionalProviderIDEnv = "OS_CCM_REGIONAL"
60+
noSortPriority = 0
5961
)
6062

6163
var _ cloudprovider.Instances = &Instances{}
6264

65+
// buildAddressSortOrderList builds a list containing only valid CIDRs based on the content of addressSortOrder.
66+
//
67+
// It will ignore and warn about invalid sort order items.
68+
func buildAddressSortOrderList(addressSortOrder string) []*net.IPNet {
69+
var list []*net.IPNet
70+
for _, item := range strings.Split(addressSortOrder, ",") {
71+
item = strings.TrimSpace(item)
72+
73+
_, cidr, err := net.ParseCIDR(item)
74+
if err != nil {
75+
klog.Warningf("Ignoring invalid sort order item '%s': %v.", item, err)
76+
continue
77+
}
78+
79+
list = append(list, cidr)
80+
}
81+
82+
return list
83+
}
84+
85+
// getSortPriority returns the priority as int of an address.
86+
//
87+
// The priority depends on the index of the CIDR in the list the address is matching,
88+
// where the first item of the list has higher priority than the last.
89+
//
90+
// If the address does not match any CIDR or is not an IP address the function returns noSortPriority.
91+
func getSortPriority(list []*net.IPNet, address string) int {
92+
parsedAddress := net.ParseIP(address)
93+
if parsedAddress == nil {
94+
return noSortPriority
95+
}
96+
97+
for i, cidr := range list {
98+
if cidr.Contains(parsedAddress) {
99+
fmt.Println(i, cidr, len(list)-i)
100+
return len(list) - i
101+
}
102+
}
103+
104+
return noSortPriority
105+
}
106+
107+
// sortNodeAddresses sorts node addresses based on comma separated list of CIDRs represented by addressSortOrder.
108+
//
109+
// The function only sorts addresses which match the CIDR and leaves the other addresses in the same order they are in.
110+
// Essentially, it will also group the addresses matching a CIDR together and sort them ascending in this group,
111+
// whereas the inter-group sorting depends on the priority.
112+
//
113+
// The priority depends on the order of the item in addressSortOrder, where the first item has higher priority than the last.
114+
func sortNodeAddresses(addresses []v1.NodeAddress, addressSortOrder string) {
115+
list := buildAddressSortOrderList(addressSortOrder)
116+
117+
sort.SliceStable(addresses, func(i int, j int) bool {
118+
addressLeft := addresses[i]
119+
addressRight := addresses[j]
120+
121+
priorityLeft := getSortPriority(list, addressLeft.Address)
122+
priorityRight := getSortPriority(list, addressRight.Address)
123+
124+
// ignore priorities of value 0 since this means the address has noSortPriority and we need to sort by priority
125+
if priorityLeft > noSortPriority && priorityLeft == priorityRight {
126+
return bytes.Compare(net.ParseIP(addressLeft.Address), net.ParseIP(addressRight.Address)) < 0
127+
}
128+
129+
return priorityLeft > priorityRight
130+
})
131+
}
132+
63133
// Instances returns an implementation of Instances for OpenStack.
64134
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
65135
return os.instances()
@@ -608,6 +678,10 @@ func nodeAddresses(srv *servers.Server, interfaces []attachinterfaces.Interface,
608678
}
609679
}
610680

681+
if networkingOpts.AddressSortOrder != "" {
682+
sortNodeAddresses(addrs, networkingOpts.AddressSortOrder)
683+
}
684+
611685
return addrs, nil
612686
}
613687

‎pkg/openstack/instances_test.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package openstack
18+
19+
import (
20+
"fmt"
21+
"net"
22+
"reflect"
23+
"testing"
24+
25+
v1 "k8s.io/api/core/v1"
26+
)
27+
28+
func TestBuildAddressSortOrderList(t *testing.T) {
29+
var emptyList []*net.IPNet
30+
31+
_, cidrIPv4, _ := net.ParseCIDR("192.168.0.0/16")
32+
_, cidrIPv6, _ := net.ParseCIDR("2001:4800:790e::/64")
33+
34+
emptyOption := ""
35+
multipleInvalidOptions := "InvalidOption, AnotherInvalidOption"
36+
multipleOptionsWithInvalidOption := fmt.Sprintf("%s, %s, %s", cidrIPv4, multipleInvalidOptions, cidrIPv6)
37+
38+
tests := map[string][]*net.IPNet{
39+
emptyOption: emptyList,
40+
multipleInvalidOptions: emptyList,
41+
multipleOptionsWithInvalidOption: {cidrIPv4, cidrIPv6},
42+
}
43+
44+
for option, want := range tests {
45+
actual := buildAddressSortOrderList(option)
46+
if !reflect.DeepEqual(want, actual) {
47+
t.Errorf("assignSortOrderPriorities returned incorrect value for '%v', want %+v but got %+v", option, want, actual)
48+
}
49+
}
50+
}
51+
52+
func TestGetSortPriority(t *testing.T) {
53+
_, cidrIPv4, _ := net.ParseCIDR("192.168.100.0/24")
54+
_, cidrIPv6, _ := net.ParseCIDR("2001:4800:790e::/64")
55+
56+
list := []*net.IPNet{cidrIPv4, cidrIPv6}
57+
t.Log(list)
58+
tests := map[string]int{
59+
"": noSortPriority,
60+
"some-host.exam.ple": noSortPriority,
61+
"2001:4800:790e::82a8": 1,
62+
"2001:cafe:babe::82a8": noSortPriority,
63+
"192.168.100.200": 2,
64+
"192.168.101.123": noSortPriority,
65+
}
66+
67+
for option, want := range tests {
68+
actual := getSortPriority(list, option)
69+
if !reflect.DeepEqual(want, actual) {
70+
t.Errorf("assignSortOrderPriorities returned incorrect value for '%v', want %+v but got %+v", option, want, actual)
71+
}
72+
}
73+
}
74+
75+
func executeSortNodeAddressesTest(t *testing.T, addressSortOrder string, want []v1.NodeAddress) {
76+
addresses := []v1.NodeAddress{
77+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
78+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
79+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
80+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
81+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
82+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
83+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
84+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
85+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
86+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
87+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
88+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
89+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
90+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
91+
}
92+
93+
sortNodeAddresses(addresses, addressSortOrder)
94+
95+
t.Logf("addresses are %v", addresses)
96+
if !reflect.DeepEqual(want, addresses) {
97+
t.Fatalf("sortNodeAddresses returned incorrect value, want %v", want)
98+
}
99+
}
100+
101+
func TestSortNodeAddressesWithAnInvalidCIDR(t *testing.T) {
102+
addressSortOrder := "10.0.0.0/244"
103+
104+
want := []v1.NodeAddress{
105+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
106+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
107+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
108+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
109+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
110+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
111+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
112+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
113+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
114+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
115+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
116+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
117+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
118+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
119+
}
120+
121+
executeSortNodeAddressesTest(t, addressSortOrder, want)
122+
}
123+
124+
func TestSortNodeAddressesWithOneIPv4CIDR(t *testing.T) {
125+
addressSortOrder := "10.0.0.0/8"
126+
127+
want := []v1.NodeAddress{
128+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
129+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
130+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
131+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
132+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
133+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
134+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
135+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
136+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
137+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
138+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
139+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
140+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
141+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
142+
}
143+
144+
executeSortNodeAddressesTest(t, addressSortOrder, want)
145+
}
146+
147+
func TestSortNodeAddressesWithOneIPv6CIDR(t *testing.T) {
148+
addressSortOrder := "fd08:1374:fcee:916b::/64"
149+
150+
want := []v1.NodeAddress{
151+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
152+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
153+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
154+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
155+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
156+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
157+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
158+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
159+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
160+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
161+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
162+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
163+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
164+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
165+
}
166+
167+
executeSortNodeAddressesTest(t, addressSortOrder, want)
168+
}
169+
170+
func TestSortNodeAddressesWithMultipleCIDRs(t *testing.T) {
171+
addressSortOrder := "10.0.0.0/8, 172.16.0.0/16, 192.168.0.0/24, fd08:1374:fcee:916b::/64, 50.56.176.0/24, 2001:cafe:babe::/64"
172+
173+
want := []v1.NodeAddress{
174+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
175+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
176+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
177+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
178+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
179+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
180+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
181+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
182+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
183+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
184+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
185+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
186+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
187+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
188+
}
189+
190+
executeSortNodeAddressesTest(t, addressSortOrder, want)
191+
}

‎pkg/openstack/openstack.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ type NetworkingOpts struct {
124124
IPv6SupportDisabled bool `gcfg:"ipv6-support-disabled"`
125125
PublicNetworkName []string `gcfg:"public-network-name"`
126126
InternalNetworkName []string `gcfg:"internal-network-name"`
127+
AddressSortOrder string `gcfg:"address-sort-order"`
127128
}
128129

129130
// RouterOpts is used for Neutron routes

0 commit comments

Comments
 (0)