Skip to content

Commit ac3a18d

Browse files
ederstederst
authored
Support specifying a sort order for node addresses (kubernetes#2030)
This introduces a configuration key which influences the way the provider reports the node addresses to the Kubernetes node resource. The default order depends on the hard-coded order the provider queries the addresses and what the cloud returns, which does not guarantee a specific order. To override this behavior it is possible to specify a comma separated list of CIDRs. Essentially, this will sort and group all addresses matching a CIDR in a prioritized manner, where the first item having a higher priority than the last. All non-matching addresses will remain in the same order they are already in. For example, this option can be useful when having multiple or dual-stack interfaces attached to a node and needing a user-controlled, deterministic way of sorting the addresses.
1 parent f09c5d3 commit ac3a18d

File tree

5 files changed

+357
-0
lines changed

5 files changed

+357
-0
lines changed

‎docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,13 @@ The options in `Global` section are used for openstack-cloud-controller-manager
166166
The name of Neutron external network. openstack-cloud-controller-manager uses this option when getting the external IP of the Kubernetes node. Can be specified multiple times. Specified network names will be ORed. Default: ""
167167
* `internal-network-name`
168168
The name of Neutron internal network. openstack-cloud-controller-manager uses this option when getting the internal IP of the Kubernetes node, this is useful if the node has multiple interfaces. Can be specified multiple times. Specified network names will be ORed. Default: ""
169+
* `address-sort-order`
170+
This configuration key influences the way the provider reports the node addresses to the Kubernetes node resource. The default order depends on the hard-coded order the provider queries the addresses and what the cloud returns, which does not guarantee a specific order.
171+
172+
To override this behavior it is possible to specify a comma separated list of CIDRs. Essentially, this will sort and group all addresses matching a CIDR in a prioritized manner, where the first item having a higher priority than the last. All non-matching addresses will remain in the same order they are already in.
173+
174+
For example, this option can be useful when having multiple or dual-stack interfaces attached to a node and needing a user-controlled, deterministic way of sorting the addresses.
175+
Default: ""
169176
170177
### Load Balancer
171178

‎pkg/openstack/instances.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package openstack
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"fmt"
2223
"io/ioutil"
@@ -53,10 +54,79 @@ type Instances struct {
5354

5455
const (
5556
instanceShutoff = "SHUTOFF"
57+
noSortPriority = 0
5658
)
5759

5860
var _ cloudprovider.Instances = &Instances{}
5961

62+
// buildAddressSortOrderList builds a list containing only valid CIDRs based on the content of addressSortOrder.
63+
//
64+
// It will ignore and warn about invalid sort order items.
65+
func buildAddressSortOrderList(addressSortOrder string) []*net.IPNet {
66+
var list []*net.IPNet
67+
for _, item := range strings.Split(addressSortOrder, ",") {
68+
item = strings.TrimSpace(item)
69+
70+
_, cidr, err := net.ParseCIDR(item)
71+
if err != nil {
72+
klog.Warningf("Ignoring invalid sort order item '%s': %v.", item, err)
73+
continue
74+
}
75+
76+
list = append(list, cidr)
77+
}
78+
79+
return list
80+
}
81+
82+
// getSortPriority returns the priority as int of an address.
83+
//
84+
// The priority depends on the index of the CIDR in the list the address is matching,
85+
// where the first item of the list has higher priority than the last.
86+
//
87+
// If the address does not match any CIDR or is not an IP address the function returns noSortPriority.
88+
func getSortPriority(list []*net.IPNet, address string) int {
89+
parsedAddress := net.ParseIP(address)
90+
if parsedAddress == nil {
91+
return noSortPriority
92+
}
93+
94+
for i, cidr := range list {
95+
if cidr.Contains(parsedAddress) {
96+
fmt.Println(i, cidr, len(list)-i)
97+
return len(list) - i
98+
}
99+
}
100+
101+
return noSortPriority
102+
}
103+
104+
// sortNodeAddresses sorts node addresses based on comma separated list of CIDRs represented by addressSortOrder.
105+
//
106+
// The function only sorts addresses which match the CIDR and leaves the other addresses in the same order they are in.
107+
// Essentially, it will also group the addresses matching a CIDR together and sort them ascending in this group,
108+
// whereas the inter-group sorting depends on the priority.
109+
//
110+
// The priority depends on the order of the item in addressSortOrder, where the first item has higher priority than the last.
111+
func sortNodeAddresses(addresses []v1.NodeAddress, addressSortOrder string) {
112+
list := buildAddressSortOrderList(addressSortOrder)
113+
114+
sort.SliceStable(addresses, func(i int, j int) bool {
115+
addressLeft := addresses[i]
116+
addressRight := addresses[j]
117+
118+
priorityLeft := getSortPriority(list, addressLeft.Address)
119+
priorityRight := getSortPriority(list, addressRight.Address)
120+
121+
// ignore priorities of value 0 since this means the address has noSortPriority and we need to sort by priority
122+
if priorityLeft > noSortPriority && priorityLeft == priorityRight {
123+
return bytes.Compare(net.ParseIP(addressLeft.Address), net.ParseIP(addressRight.Address)) < 0
124+
}
125+
126+
return priorityLeft > priorityRight
127+
})
128+
}
129+
60130
// Instances returns an implementation of Instances for OpenStack.
61131
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
62132
return os.instances()
@@ -570,6 +640,10 @@ func nodeAddresses(srv *servers.Server, interfaces []attachinterfaces.Interface,
570640
}
571641
}
572642

643+
if networkingOpts.AddressSortOrder != "" {
644+
sortNodeAddresses(addrs, networkingOpts.AddressSortOrder)
645+
}
646+
573647
return addrs, nil
574648
}
575649

‎pkg/openstack/instances_test.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package openstack
18+
19+
import (
20+
"fmt"
21+
"net"
22+
"reflect"
23+
"testing"
24+
25+
v1 "k8s.io/api/core/v1"
26+
)
27+
28+
func TestBuildAddressSortOrderList(t *testing.T) {
29+
var emptyList []*net.IPNet
30+
31+
_, cidrIPv4, _ := net.ParseCIDR("192.168.0.0/16")
32+
_, cidrIPv6, _ := net.ParseCIDR("2001:4800:790e::/64")
33+
34+
emptyOption := ""
35+
multipleInvalidOptions := "InvalidOption, AnotherInvalidOption"
36+
multipleOptionsWithInvalidOption := fmt.Sprintf("%s, %s, %s", cidrIPv4, multipleInvalidOptions, cidrIPv6)
37+
38+
tests := map[string][]*net.IPNet{
39+
emptyOption: emptyList,
40+
multipleInvalidOptions: emptyList,
41+
multipleOptionsWithInvalidOption: {cidrIPv4, cidrIPv6},
42+
}
43+
44+
for option, want := range tests {
45+
actual := buildAddressSortOrderList(option)
46+
if !reflect.DeepEqual(want, actual) {
47+
t.Errorf("assignSortOrderPriorities returned incorrect value for '%v', want %+v but got %+v", option, want, actual)
48+
}
49+
}
50+
}
51+
52+
func TestGetSortPriority(t *testing.T) {
53+
_, cidrIPv4, _ := net.ParseCIDR("192.168.100.0/24")
54+
_, cidrIPv6, _ := net.ParseCIDR("2001:4800:790e::/64")
55+
56+
list := []*net.IPNet{cidrIPv4, cidrIPv6}
57+
t.Log(list)
58+
tests := map[string]int{
59+
"": noSortPriority,
60+
"some-host.exam.ple": noSortPriority,
61+
"2001:4800:790e::82a8": 1,
62+
"2001:cafe:babe::82a8": noSortPriority,
63+
"192.168.100.200": 2,
64+
"192.168.101.123": noSortPriority,
65+
}
66+
67+
for option, want := range tests {
68+
actual := getSortPriority(list, option)
69+
if !reflect.DeepEqual(want, actual) {
70+
t.Errorf("assignSortOrderPriorities returned incorrect value for '%v', want %+v but got %+v", option, want, actual)
71+
}
72+
}
73+
}
74+
75+
func executeSortNodeAddressesTest(t *testing.T, addressSortOrder string, want []v1.NodeAddress) {
76+
addresses := []v1.NodeAddress{
77+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
78+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
79+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
80+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
81+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
82+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
83+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
84+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
85+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
86+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
87+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
88+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
89+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
90+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
91+
}
92+
93+
sortNodeAddresses(addresses, addressSortOrder)
94+
95+
t.Logf("addresses are %v", addresses)
96+
if !reflect.DeepEqual(want, addresses) {
97+
t.Fatalf("sortNodeAddresses returned incorrect value, want %v", want)
98+
}
99+
}
100+
101+
func TestSortNodeAddressesWithAnInvalidCIDR(t *testing.T) {
102+
addressSortOrder := "10.0.0.0/244"
103+
104+
want := []v1.NodeAddress{
105+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
106+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
107+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
108+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
109+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
110+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
111+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
112+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
113+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
114+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
115+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
116+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
117+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
118+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
119+
}
120+
121+
executeSortNodeAddressesTest(t, addressSortOrder, want)
122+
}
123+
124+
func TestSortNodeAddressesWithOneIPv4CIDR(t *testing.T) {
125+
addressSortOrder := "10.0.0.0/8"
126+
127+
want := []v1.NodeAddress{
128+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
129+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
130+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
131+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
132+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
133+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
134+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
135+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
136+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
137+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
138+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
139+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
140+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
141+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
142+
}
143+
144+
executeSortNodeAddressesTest(t, addressSortOrder, want)
145+
}
146+
147+
func TestSortNodeAddressesWithOneIPv6CIDR(t *testing.T) {
148+
addressSortOrder := "fd08:1374:fcee:916b::/64"
149+
150+
want := []v1.NodeAddress{
151+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
152+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
153+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
154+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
155+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
156+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
157+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
158+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
159+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
160+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
161+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
162+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
163+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
164+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
165+
}
166+
167+
executeSortNodeAddressesTest(t, addressSortOrder, want)
168+
}
169+
170+
func TestSortNodeAddressesWithMultipleCIDRs(t *testing.T) {
171+
addressSortOrder := "10.0.0.0/8, 172.16.0.0/16, 192.168.0.0/24, fd08:1374:fcee:916b::/64, 50.56.176.0/24, 2001:cafe:babe::/64"
172+
173+
want := []v1.NodeAddress{
174+
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
175+
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
176+
{Type: v1.NodeInternalIP, Address: "172.16.0.1"},
177+
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
178+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:82a8"},
179+
{Type: v1.NodeInternalIP, Address: "fd08:1374:fcee:916b:be76:4eff:fe04:84a8"},
180+
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
181+
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
182+
{Type: v1.NodeInternalIP, Address: "50.56.176.37"},
183+
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
184+
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
185+
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
186+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
187+
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.exam.ple"},
188+
}
189+
190+
executeSortNodeAddressesTest(t, addressSortOrder, want)
191+
}

‎pkg/openstack/openstack.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ type NetworkingOpts struct {
116116
IPv6SupportDisabled bool `gcfg:"ipv6-support-disabled"`
117117
PublicNetworkName []string `gcfg:"public-network-name"`
118118
InternalNetworkName []string `gcfg:"internal-network-name"`
119+
AddressSortOrder string `gcfg:"address-sort-order"`
119120
}
120121

121122
// RouterOpts is used for Neutron routes

0 commit comments

Comments
 (0)