Skip to content

Commit 45bf0b3

Browse files
qinqonjcaamano
qinqon
authored and
jcaamano
committed
Revert "e2e: Use ovnk allocator and reserve IPs"
This reverts commit 9fed90c. Signed-off-by: Enrique Llorente <[email protected]>
1 parent 9b56178 commit 45bf0b3

File tree

3 files changed

+238
-65
lines changed

3 files changed

+238
-65
lines changed

‎test/e2e/ipalloc/ipalloc.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package ipalloc
2+
3+
import (
4+
"fmt"
5+
"math/big"
6+
"net"
7+
)
8+
9+
type ipAllocator struct {
10+
net *net.IPNet
11+
// base is a cached version of the start IP in the CIDR range as a *big.Int
12+
base *big.Int
13+
// max is the maximum size of the usable addresses in the range
14+
max int
15+
count int
16+
}
17+
18+
func newIPAllocator(cidr *net.IPNet) *ipAllocator {
19+
return &ipAllocator{net: cidr, base: getBaseInt(cidr.IP), max: limit(cidr)}
20+
}
21+
22+
func (n *ipAllocator) AllocateNextIP() (net.IP, error) {
23+
if n.count >= n.max {
24+
return net.IP{}, fmt.Errorf("limit of %d reached", n.max)
25+
}
26+
n.base.Add(n.base, big.NewInt(1))
27+
n.count += 1
28+
b := n.base.Bytes()
29+
b = append(make([]byte, 16), b...)
30+
return b[len(b)-16:], nil
31+
}
32+
33+
func getBaseInt(ip net.IP) *big.Int {
34+
return big.NewInt(0).SetBytes(ip.To16())
35+
}
36+
37+
func limit(subnet *net.IPNet) int {
38+
ones, bits := subnet.Mask.Size()
39+
if bits == 32 && (bits-ones) >= 31 || bits == 128 && (bits-ones) >= 127 {
40+
return 0
41+
}
42+
// limit to 2^8 (256) IPs for e2es
43+
if bits == 128 && (bits-ones) >= 8 {
44+
return int(1) << uint(8)
45+
}
46+
return int(1) << uint(bits-ones)
47+
}

‎test/e2e/ipalloc/primaryipalloc.go‎

Lines changed: 133 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,19 @@ package ipalloc
33
import (
44
"context"
55
"fmt"
6-
"net"
7-
"sync"
8-
9-
ipallocator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/ip"
106
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
7+
corev1 "k8s.io/api/core/v1"
118
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
129
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
10+
"net"
11+
"sync"
1312
)
1413

1514
// primaryIPAllocator attempts to allocate an IP in the same subnet as a nodes primary network
1615
type primaryIPAllocator struct {
1716
mu *sync.Mutex
18-
v4 *ipallocator.Range
19-
v6 *ipallocator.Range
17+
v4 *ipAllocator
18+
v6 *ipAllocator
2019
nodeClient v1.NodeInterface
2120
}
2221

@@ -48,37 +47,91 @@ func newPrimaryIPAllocator(nodeClient v1.NodeInterface) (*primaryIPAllocator, er
4847
if len(nodes.Items) == 0 {
4948
return ipa, fmt.Errorf("expected at least one node but found zero")
5049
}
50+
// FIXME: the approach taken here to find the first node IP+mask and then to increment the second last octet wont work in
51+
// all scenarios (node with /24). We should generate an EgressIP compatible with a Node providers primary network and then take care its unique globally.
5152

52-
for _, node := range nodes.Items {
53+
// The approach here is to grab initial starting IP from first node found, increment the second last octet.
54+
// Approach taken here won't work for Nodes handed /24 subnets.
55+
nodePrimaryIPs, err := util.ParseNodePrimaryIfAddr(&nodes.Items[0])
56+
if err != nil {
57+
return ipa, fmt.Errorf("failed to parse node primary interface address from Node object: %v", err)
58+
}
59+
if nodePrimaryIPs.V4.IP != nil {
60+
// should be ok with /16 and /64 node primary provider subnets
61+
// TODO; fixme; what about /24 subnet Nodes like GCP
62+
nodePrimaryIPs.V4.IP[len(nodePrimaryIPs.V4.IP)-2]++
63+
ipa.v4 = newIPAllocator(&net.IPNet{IP: nodePrimaryIPs.V4.IP, Mask: nodePrimaryIPs.V4.Net.Mask})
64+
}
65+
if nodePrimaryIPs.V6.IP != nil {
66+
nodePrimaryIPs.V6.IP[len(nodePrimaryIPs.V6.IP)-2]++
67+
ipa.v6 = newIPAllocator(&net.IPNet{IP: nodePrimaryIPs.V6.IP, Mask: nodePrimaryIPs.V6.Net.Mask})
68+
}
69+
// verify the new starting base IP is within all Nodes subnets
70+
if nodePrimaryIPs.V4.IP != nil {
71+
ipNets, err := getNodePrimaryProviderIPs(nodes.Items, false)
72+
if err != nil {
73+
return ipa, err
74+
}
75+
nextIP, err := ipa.v4.AllocateNextIP()
76+
if err != nil {
77+
return ipa, err
78+
}
79+
if !isIPWithinAllSubnets(ipNets, nextIP) {
80+
return ipa, fmt.Errorf("IP %s is not within all Node subnets", nextIP)
81+
}
82+
}
83+
if nodePrimaryIPs.V6.IP != nil {
84+
ipNets, err := getNodePrimaryProviderIPs(nodes.Items, true)
85+
if err != nil {
86+
return ipa, err
87+
}
88+
nextIP, err := ipa.v6.AllocateNextIP()
89+
if err != nil {
90+
return ipa, err
91+
}
92+
if !isIPWithinAllSubnets(ipNets, nextIP) {
93+
return ipa, fmt.Errorf("IP %s is not within all Node subnets", nextIP)
94+
}
95+
}
96+
97+
return ipa, nil
98+
}
99+
100+
func getNodePrimaryProviderIPs(nodes []corev1.Node, isIPv6 bool) ([]*net.IPNet, error) {
101+
ipNets := make([]*net.IPNet, 0, len(nodes))
102+
for _, node := range nodes {
53103
nodePrimaryIPs, err := util.ParseNodePrimaryIfAddr(&node)
54104
if err != nil {
55-
return ipa, fmt.Errorf("failed to parse node primary interface address from Node %s object: %v", node.Name, err)
56-
}
57-
if nodePrimaryIPs.V4.IP != nil {
58-
if ipa.v4 == nil {
59-
ipa.v4, err = ipallocator.NewCIDRRange(nodePrimaryIPs.V4.Net)
60-
if err != nil {
61-
return ipa, fmt.Errorf("failed to create new CIDR range for IPv4: %v", err)
62-
}
63-
}
64-
if err := ipa.v4.Allocate(nodePrimaryIPs.V4.IP); err != nil {
65-
return ipa, fmt.Errorf("failed to allocate IPv4 %s: %v", nodePrimaryIPs.V4.IP, err)
66-
}
67-
}
68-
if nodePrimaryIPs.V6.IP != nil {
69-
if ipa.v6 == nil {
70-
ipa.v6, err = ipallocator.NewCIDRRange(nodePrimaryIPs.V6.Net)
71-
if err != nil {
72-
return ipa, fmt.Errorf("failed to create new CIDR range for IPv6: %v", err)
73-
}
74-
}
75-
if err := ipa.v6.Allocate(nodePrimaryIPs.V6.IP); err != nil {
76-
return ipa, fmt.Errorf("failed to allocate IPv6 %s: %v", nodePrimaryIPs.V6.IP, err)
77-
}
105+
return nil, fmt.Errorf("failed to parse node primary interface address from Node %s object: %v", node.Name, err)
78106
}
107+
var mask net.IPMask
108+
var ip net.IP
79109

110+
if isIPv6 {
111+
ip = nodePrimaryIPs.V6.IP
112+
mask = nodePrimaryIPs.V6.Net.Mask
113+
} else {
114+
ip = nodePrimaryIPs.V4.IP
115+
mask = nodePrimaryIPs.V4.Net.Mask
116+
}
117+
if len(ip) == 0 || len(mask) == 0 {
118+
return nil, fmt.Errorf("failed to find Node %s primary Node IP and/or mask", node.Name)
119+
}
120+
ipNets = append(ipNets, &net.IPNet{IP: ip, Mask: mask})
80121
}
81-
return ipa, nil
122+
return ipNets, nil
123+
}
124+
125+
func isIPWithinAllSubnets(ipNets []*net.IPNet, ip net.IP) bool {
126+
if len(ipNets) == 0 {
127+
return false
128+
}
129+
for _, ipNet := range ipNets {
130+
if !ipNet.Contains(ip) {
131+
return false
132+
}
133+
}
134+
return true
82135
}
83136

84137
func (pia *primaryIPAllocator) IncrementAndGetNextV4(times int) (net.IP, error) {
@@ -95,9 +148,12 @@ func (pia *primaryIPAllocator) AllocateNextV4() (net.IP, error) {
95148
if pia.v4 == nil {
96149
return nil, fmt.Errorf("IPv4 is not enable ")
97150
}
151+
if pia.v4.net == nil {
152+
return nil, fmt.Errorf("IPv4 is not enabled but Allocation request was called")
153+
}
98154
pia.mu.Lock()
99155
defer pia.mu.Unlock()
100-
return pia.v4.AllocateNext()
156+
return allocateIP(pia.nodeClient, pia.v4.AllocateNextIP)
101157
}
102158

103159
func (pia *primaryIPAllocator) IncrementAndGetNextV6(times int) (net.IP, error) {
@@ -114,7 +170,51 @@ func (pia primaryIPAllocator) AllocateNextV6() (net.IP, error) {
114170
if pia.v6 == nil {
115171
return nil, fmt.Errorf("IPv6 is not enabled but Allocation request was called")
116172
}
173+
if pia.v6.net == nil {
174+
return nil, fmt.Errorf("ipv6 network is not set")
175+
}
117176
pia.mu.Lock()
118177
defer pia.mu.Unlock()
119-
return pia.v6.AllocateNext()
178+
return allocateIP(pia.nodeClient, pia.v6.AllocateNextIP)
179+
}
180+
181+
type allocNextFn func() (net.IP, error)
182+
183+
func allocateIP(nodeClient v1.NodeInterface, allocateFn allocNextFn) (net.IP, error) {
184+
nodeList, err := nodeClient.List(context.TODO(), metav1.ListOptions{})
185+
if err != nil {
186+
return nil, fmt.Errorf("failed to list nodes: %v", err)
187+
}
188+
for {
189+
nextIP, err := allocateFn()
190+
if err != nil {
191+
return nil, fmt.Errorf("failed to allocated next IP address: %v", err)
192+
}
193+
firstOctet := nextIP[len(nextIP)-1]
194+
// skip 0 and 1
195+
if firstOctet == 0 || firstOctet == 1 {
196+
continue
197+
}
198+
isConflict, err := isConflictWithExistingHostIPs(nodeList.Items, nextIP)
199+
if err != nil {
200+
return nil, fmt.Errorf("failed to determine if IP conflicts with existing IPs: %v", err)
201+
}
202+
if !isConflict {
203+
return nextIP, nil
204+
}
205+
}
206+
}
207+
208+
func isConflictWithExistingHostIPs(nodes []corev1.Node, ip net.IP) (bool, error) {
209+
ipStr := ip.String()
210+
for _, node := range nodes {
211+
nodeIPsSet, err := util.ParseNodeHostCIDRsDropNetMask(&node)
212+
if err != nil {
213+
return false, fmt.Errorf("failed to parse node %s primary annotation info: %v", node.Name, err)
214+
}
215+
if nodeIPsSet.Has(ipStr) {
216+
return true, nil
217+
}
218+
}
219+
return false, nil
120220
}

‎test/e2e/ipalloc/primaryipalloc_test.go‎

Lines changed: 58 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,48 @@ import (
1515
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1616
"k8s.io/apimachinery/pkg/runtime"
1717
"k8s.io/client-go/kubernetes/fake"
18+
utilsnet "k8s.io/utils/net"
1819
)
1920

2021
func TestUtilSuite(t *testing.T) {
2122
gomega.RegisterFailHandler(ginkgo.Fail)
2223
ginkgo.RunSpecs(t, "node ip alloc suite")
2324
}
2425

26+
func TestAllocateNext(t *testing.T) {
27+
tests := []struct {
28+
desc string
29+
input *net.IPNet
30+
output []net.IP
31+
}{
32+
{
33+
desc: "increments IPv4 address",
34+
input: mustParseCIDRIncIP("192.168.1.5/16"), // mask /24 would fail
35+
output: []net.IP{net.ParseIP("192.168.1.6"), net.ParseIP("192.168.1.7"), net.ParseIP("192.168.1.8")},
36+
},
37+
{
38+
desc: "increments IPv6 address",
39+
input: mustParseCIDRIncIP("fc00:f853:ccd:e793::6/64"),
40+
output: []net.IP{net.ParseIP("fc00:f853:ccd:e793::7"), net.ParseIP("fc00:f853:ccd:e793::8"), net.ParseIP("fc00:f853:ccd:e793::9")},
41+
},
42+
}
43+
44+
for i, tc := range tests {
45+
t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) {
46+
nodeIPAlloc := newIPAllocator(tc.input)
47+
for _, expectedIP := range tc.output {
48+
allocatedIP, err := nodeIPAlloc.AllocateNextIP()
49+
if err != nil {
50+
t.Errorf("failed to allocated next IP: %v", err)
51+
}
52+
if !allocatedIP.Equal(expectedIP) {
53+
t.Errorf("Expected IP %q, but got %q", expectedIP.String(), allocatedIP.String())
54+
}
55+
}
56+
})
57+
}
58+
}
59+
2560
// mustParseCIDRIncIP parses the IP and CIDR. It adds the IP to the returned IPNet.
2661
func mustParseCIDRIncIP(cidr string) *net.IPNet {
2762
ip, ipNet, err := net.ParseCIDR(cidr)
@@ -43,19 +78,20 @@ type node struct {
4378
}
4479

4580
func TestIPAlloc(t *testing.T) {
46-
g := gomega.NewWithT(t)
47-
4881
tests := []struct {
49-
desc string
50-
existingPrimaryNodeIPs []node
82+
desc string
83+
existingPrimaryNodeIPs []node
84+
expectedFromAllocateNext []string
5185
}{
5286
{
53-
desc: "IPv4",
54-
existingPrimaryNodeIPs: []node{{v4: network{ip: "192.168.1.1", mask: "16"}}, {v4: network{ip: "192.168.1.2", mask: "16"}}},
87+
desc: "IPv4",
88+
existingPrimaryNodeIPs: []node{{v4: network{ip: "192.168.1.1", mask: "16"}}, {v4: network{ip: "192.168.1.2", mask: "16"}}},
89+
expectedFromAllocateNext: []string{"192.168.2.3", "192.168.2.4"},
5590
},
5691
{
57-
desc: "IPv6",
58-
existingPrimaryNodeIPs: []node{{v6: network{ip: "fc00:f853:ccd:e793::5", mask: "64"}}, {v6: network{ip: "fc00:f853:ccd:e793::6", mask: "64"}}},
92+
desc: "IPv6",
93+
existingPrimaryNodeIPs: []node{{v4: network{ip: "fc00:f853:ccd:e793::5", mask: "64"}}, {v4: network{ip: "fc00:f853:ccd:e793::6", mask: "64"}}},
94+
expectedFromAllocateNext: []string{"fc00:f853:ccd:e793::8", "fc00:f853:ccd:e793::9"},
5995
},
6096
}
6197

@@ -67,32 +103,22 @@ func TestIPAlloc(t *testing.T) {
67103
t.Errorf(err.Error())
68104
return
69105
}
70-
existingIPv4IPs := []string{}
71-
existingIPv6IPs := []string{}
72-
allocatedIPv4IPs := []string{}
73-
allocatedIPv6IPs := []string{}
74-
for _, existingPrimaryNodeIP := range tc.existingPrimaryNodeIPs {
75-
if existingPrimaryNodeIP.v4.ip != "" {
76-
existingIPv4IPs = append(existingIPv4IPs, existingPrimaryNodeIP.v4.ip)
77-
nextIPv4, err := pipa.AllocateNextV4()
78-
g.Expect(err).ToNot(gomega.HaveOccurred(), "should succeed in allocating the next IPv4 address")
79-
g.Expect(nextIPv4).ToNot(gomega.BeNil(), "should allocate next IPv4 address")
80-
allocatedIPv4IPs = append(allocatedIPv4IPs, nextIPv4.String())
106+
for _, expectedIPStr := range tc.expectedFromAllocateNext {
107+
expectedIP := net.ParseIP(expectedIPStr)
108+
var nextIP net.IP
109+
var err error
110+
if utilsnet.IsIPv6(expectedIP) {
111+
nextIP, err = pipa.AllocateNextV6()
112+
} else {
113+
nextIP, err = pipa.AllocateNextV4()
81114
}
82-
83-
if existingPrimaryNodeIP.v6.ip != "" {
84-
existingIPv6IPs = append(existingIPv6IPs, existingPrimaryNodeIP.v6.ip)
85-
nextIPv6, err := pipa.AllocateNextV6()
86-
g.Expect(err).ToNot(gomega.HaveOccurred(), "should succeed in allocating the next IPv6 address")
87-
g.Expect(nextIPv6).ToNot(gomega.BeNil(), "should allocate next IPv6 address")
88-
allocatedIPv6IPs = append(allocatedIPv6IPs, nextIPv6.String())
115+
if err != nil || nextIP == nil {
116+
t.Errorf("failed to allocated next IPv4 or IPv6 address. err %v", err)
117+
return
118+
}
119+
if !nextIP.Equal(expectedIP) {
120+
t.Errorf("expected IP %q, but found %q", expectedIP, nextIP)
89121
}
90-
}
91-
if len(existingIPv4IPs) > 0 {
92-
g.Expect(allocatedIPv4IPs).NotTo(gomega.ContainElements(existingIPv4IPs))
93-
}
94-
if len(existingIPv6IPs) > 0 {
95-
g.Expect(allocatedIPv6IPs).NotTo(gomega.ContainElements(existingIPv6IPs))
96122
}
97123
})
98124
}

0 commit comments

Comments
 (0)