Skip to content

Commit 180af42

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#90288 from aojea/cidrsets
Add metrics to the cidr_sets used by the nodeipam range allocator
2 parents e4ece47 + 069707f commit 180af42

File tree

4 files changed

+369
-53
lines changed

4 files changed

+369
-53
lines changed

‎pkg/controller/nodeipam/ipam/cidrset/BUILD‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,23 @@ go_test(
1010
name = "go_default_test",
1111
srcs = ["cidr_set_test.go"],
1212
embed = [":go_default_library"],
13-
deps = ["//vendor/k8s.io/klog/v2:go_default_library"],
13+
deps = [
14+
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
15+
"//vendor/k8s.io/klog/v2:go_default_library",
16+
],
1417
)
1518

1619
go_library(
1720
name = "go_default_library",
18-
srcs = ["cidr_set.go"],
21+
srcs = [
22+
"cidr_set.go",
23+
"metrics.go",
24+
],
1925
importpath = "k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset",
26+
deps = [
27+
"//staging/src/k8s.io/component-base/metrics:go_default_library",
28+
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
29+
],
2030
)
2131

2232
filegroup(

‎pkg/controller/nodeipam/ipam/cidrset/cidr_set.go‎

Lines changed: 73 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,26 @@ import (
3030
// be allocated from.
3131
type CidrSet struct {
3232
sync.Mutex
33-
clusterCIDR *net.IPNet
34-
clusterIP net.IP
33+
// clusterCIDR is the CIDR assigned to the cluster
34+
clusterCIDR *net.IPNet
35+
// clusterMaskSize is the mask size, in bits, assigned to the cluster
36+
// caches the mask size to avoid the penalty of calling clusterCIDR.Mask.Size()
3537
clusterMaskSize int
36-
maxCIDRs int
37-
nextCandidate int
38-
used big.Int
39-
subNetMaskSize int
38+
// nodeMask is the network mask assigned to the nodes
39+
nodeMask net.IPMask
40+
// nodeMaskSize is the mask size, in bits,assigned to the nodes
41+
// caches the mask size to avoid the penalty of calling nodeMask.Size()
42+
nodeMaskSize int
43+
// maxCIDRs is the maximum number of CIDRs that can be allocated
44+
maxCIDRs int
45+
// allocatedCIDRs counts the number of CIDRs allocated
46+
allocatedCIDRs int
47+
// nextCandidate points to the next CIDR that should be free
48+
nextCandidate int
49+
// used is a bitmap used to track the CIDRs allocated
50+
used big.Int
51+
// label is used to identify the metrics
52+
label string
4053
}
4154

4255
const (
@@ -64,72 +77,71 @@ var (
6477
// NewCIDRSet creates a new CidrSet.
6578
func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) (*CidrSet, error) {
6679
clusterMask := clusterCIDR.Mask
67-
clusterMaskSize, _ := clusterMask.Size()
80+
clusterMaskSize, bits := clusterMask.Size()
6881

6982
var maxCIDRs int
7083
if (clusterCIDR.IP.To4() == nil) && (subNetMaskSize-clusterMaskSize > clusterSubnetMaxDiff) {
7184
return nil, ErrCIDRSetSubNetTooBig
7285
}
86+
87+
// register CidrSet metrics
88+
registerCidrsetMetrics()
89+
7390
maxCIDRs = 1 << uint32(subNetMaskSize-clusterMaskSize)
7491
return &CidrSet{
7592
clusterCIDR: clusterCIDR,
76-
clusterIP: clusterCIDR.IP,
93+
nodeMask: net.CIDRMask(subNetMaskSize, bits),
7794
clusterMaskSize: clusterMaskSize,
7895
maxCIDRs: maxCIDRs,
79-
subNetMaskSize: subNetMaskSize,
96+
nodeMaskSize: subNetMaskSize,
97+
label: clusterCIDR.String(),
8098
}, nil
8199
}
82100

83101
func (s *CidrSet) indexToCIDRBlock(index int) *net.IPNet {
84102
var ip []byte
85-
var mask int
86103
switch /*v4 or v6*/ {
87-
case s.clusterIP.To4() != nil:
104+
case s.clusterCIDR.IP.To4() != nil:
88105
{
89-
j := uint32(index) << uint32(32-s.subNetMaskSize)
90-
ipInt := (binary.BigEndian.Uint32(s.clusterIP)) | j
91-
ip = make([]byte, 4)
106+
j := uint32(index) << uint32(32-s.nodeMaskSize)
107+
ipInt := (binary.BigEndian.Uint32(s.clusterCIDR.IP)) | j
108+
ip = make([]byte, net.IPv4len)
92109
binary.BigEndian.PutUint32(ip, ipInt)
93-
mask = 32
94-
95110
}
96-
case s.clusterIP.To16() != nil:
111+
case s.clusterCIDR.IP.To16() != nil:
97112
{
98113
// leftClusterIP | rightClusterIP
99114
// 2001:0DB8:1234:0000:0000:0000:0000:0000
100115
const v6NBits = 128
101116
const halfV6NBits = v6NBits / 2
102-
leftClusterIP := binary.BigEndian.Uint64(s.clusterIP[:halfIPv6Len])
103-
rightClusterIP := binary.BigEndian.Uint64(s.clusterIP[halfIPv6Len:])
117+
leftClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[:halfIPv6Len])
118+
rightClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[halfIPv6Len:])
104119

105-
leftIP, rightIP := make([]byte, halfIPv6Len), make([]byte, halfIPv6Len)
120+
ip = make([]byte, net.IPv6len)
106121

107-
if s.subNetMaskSize <= halfV6NBits {
122+
if s.nodeMaskSize <= halfV6NBits {
108123
// We only care about left side IP
109-
leftClusterIP |= uint64(index) << uint(halfV6NBits-s.subNetMaskSize)
124+
leftClusterIP |= uint64(index) << uint(halfV6NBits-s.nodeMaskSize)
110125
} else {
111126
if s.clusterMaskSize < halfV6NBits {
112127
// see how many bits are needed to reach the left side
113-
btl := uint(s.subNetMaskSize - halfV6NBits)
128+
btl := uint(s.nodeMaskSize - halfV6NBits)
114129
indexMaxBit := uint(64 - bits.LeadingZeros64(uint64(index)))
115130
if indexMaxBit > btl {
116131
leftClusterIP |= uint64(index) >> btl
117132
}
118133
}
119134
// the right side will be calculated the same way either the
120135
// subNetMaskSize affects both left and right sides
121-
rightClusterIP |= uint64(index) << uint(v6NBits-s.subNetMaskSize)
136+
rightClusterIP |= uint64(index) << uint(v6NBits-s.nodeMaskSize)
122137
}
123-
binary.BigEndian.PutUint64(leftIP, leftClusterIP)
124-
binary.BigEndian.PutUint64(rightIP, rightClusterIP)
125-
126-
ip = append(leftIP, rightIP...)
127-
mask = 128
138+
binary.BigEndian.PutUint64(ip[:halfIPv6Len], leftClusterIP)
139+
binary.BigEndian.PutUint64(ip[halfIPv6Len:], rightClusterIP)
128140
}
129141
}
130142
return &net.IPNet{
131143
IP: ip,
132-
Mask: net.CIDRMask(s.subNetMaskSize, mask),
144+
Mask: s.nodeMask,
133145
}
134146
}
135147

@@ -139,22 +151,27 @@ func (s *CidrSet) AllocateNext() (*net.IPNet, error) {
139151
s.Lock()
140152
defer s.Unlock()
141153

142-
nextUnused := -1
143-
for i := 0; i < s.maxCIDRs; i++ {
144-
candidate := (i + s.nextCandidate) % s.maxCIDRs
154+
if s.allocatedCIDRs == s.maxCIDRs {
155+
return nil, ErrCIDRRangeNoCIDRsRemaining
156+
}
157+
candidate := s.nextCandidate
158+
var i int
159+
for i = 0; i < s.maxCIDRs; i++ {
145160
if s.used.Bit(candidate) == 0 {
146-
nextUnused = candidate
147161
break
148162
}
163+
candidate = (candidate + 1) % s.maxCIDRs
149164
}
150-
if nextUnused == -1 {
151-
return nil, ErrCIDRRangeNoCIDRsRemaining
152-
}
153-
s.nextCandidate = (nextUnused + 1) % s.maxCIDRs
154165

155-
s.used.SetBit(&s.used, nextUnused, 1)
166+
s.nextCandidate = (candidate + 1) % s.maxCIDRs
167+
s.used.SetBit(&s.used, candidate, 1)
168+
s.allocatedCIDRs++
169+
// Update metrics
170+
cidrSetAllocations.WithLabelValues(s.label).Inc()
171+
cidrSetAllocationTriesPerRequest.WithLabelValues(s.label).Observe(float64(i))
172+
cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))
156173

157-
return s.indexToCIDRBlock(nextUnused), nil
174+
return s.indexToCIDRBlock(candidate), nil
158175
}
159176

160177
func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err error) {
@@ -176,10 +193,9 @@ func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err
176193
if cidr.IP.To4() == nil {
177194
ipSize = net.IPv6len
178195
}
179-
subNetMask := net.CIDRMask(s.subNetMaskSize, ipSize*8)
180196
begin, err = s.getIndexForCIDR(&net.IPNet{
181-
IP: cidr.IP.Mask(subNetMask),
182-
Mask: subNetMask,
197+
IP: cidr.IP.Mask(s.nodeMask),
198+
Mask: s.nodeMask,
183199
})
184200
if err != nil {
185201
return -1, -1, err
@@ -197,8 +213,8 @@ func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err
197213
binary.BigEndian.PutUint64(ip[net.IPv6len/2:], ipIntRight)
198214
}
199215
end, err = s.getIndexForCIDR(&net.IPNet{
200-
IP: net.IP(ip).Mask(subNetMask),
201-
Mask: subNetMask,
216+
IP: net.IP(ip).Mask(s.nodeMask),
217+
Mask: s.nodeMask,
202218
})
203219
if err != nil {
204220
return -1, -1, err
@@ -217,7 +233,11 @@ func (s *CidrSet) Release(cidr *net.IPNet) error {
217233
defer s.Unlock()
218234
for i := begin; i <= end; i++ {
219235
s.used.SetBit(&s.used, i, 0)
236+
s.allocatedCIDRs--
237+
cidrSetReleases.WithLabelValues(s.label).Inc()
220238
}
239+
240+
cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))
221241
return nil
222242
}
223243

@@ -228,13 +248,15 @@ func (s *CidrSet) Occupy(cidr *net.IPNet) (err error) {
228248
if err != nil {
229249
return err
230250
}
231-
232251
s.Lock()
233252
defer s.Unlock()
234253
for i := begin; i <= end; i++ {
235254
s.used.SetBit(&s.used, i, 1)
255+
s.allocatedCIDRs++
256+
cidrSetAllocations.WithLabelValues(s.label).Inc()
236257
}
237258

259+
cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs))
238260
return nil
239261
}
240262

@@ -244,19 +266,19 @@ func (s *CidrSet) getIndexForCIDR(cidr *net.IPNet) (int, error) {
244266

245267
func (s *CidrSet) getIndexForIP(ip net.IP) (int, error) {
246268
if ip.To4() != nil {
247-
cidrIndex := (binary.BigEndian.Uint32(s.clusterIP) ^ binary.BigEndian.Uint32(ip.To4())) >> uint32(32-s.subNetMaskSize)
269+
cidrIndex := (binary.BigEndian.Uint32(s.clusterCIDR.IP) ^ binary.BigEndian.Uint32(ip.To4())) >> uint32(32-s.nodeMaskSize)
248270
if cidrIndex >= uint32(s.maxCIDRs) {
249-
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.subNetMaskSize)
271+
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize)
250272
}
251273
return int(cidrIndex), nil
252274
}
253275
if ip.To16() != nil {
254-
bigIP := big.NewInt(0).SetBytes(s.clusterIP)
276+
bigIP := big.NewInt(0).SetBytes(s.clusterCIDR.IP)
255277
bigIP = bigIP.Xor(bigIP, big.NewInt(0).SetBytes(ip))
256-
cidrIndexBig := bigIP.Rsh(bigIP, uint(net.IPv6len*8-s.subNetMaskSize))
278+
cidrIndexBig := bigIP.Rsh(bigIP, uint(net.IPv6len*8-s.nodeMaskSize))
257279
cidrIndex := cidrIndexBig.Uint64()
258280
if cidrIndex >= uint64(s.maxCIDRs) {
259-
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.subNetMaskSize)
281+
return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize)
260282
}
261283
return int(cidrIndex), nil
262284
}

0 commit comments

Comments
 (0)