Skip to content

Commit 596054a

Browse files
Gackonickorlow
Gacko
and
nickorlow
authored
Generate correct output on NumCPU() when using cgroups2 (#11778)
Co-authored-by: Nicholas Orlowsky <[email protected]>
1 parent c31c9b0 commit 596054a

File tree

3 files changed

+201
-8
lines changed

3 files changed

+201
-8
lines changed

‎pkg/util/runtime/cpu_linux.go‎

Lines changed: 85 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,42 @@ import (
3636
//
3737
// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
3838
func NumCPU() int {
39+
return NumCPUWithCustomPath("")
40+
}
41+
42+
func NumCPUWithCustomPath(path string) int {
3943
cpus := runtime.NumCPU()
4044

41-
cgroupPath, err := libcontainercgroups.FindCgroupMountpoint("", "cpu")
42-
if err != nil {
43-
return cpus
45+
cgroupVersionCheckPath := path
46+
47+
if cgroupVersionCheckPath == "" {
48+
cgroupVersionCheckPath = "/sys/fs/cgroup/"
4449
}
4550

46-
cpuQuota := readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us")
47-
cpuPeriod := readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us")
51+
cgroupVersion := GetCgroupVersion(cgroupVersionCheckPath)
52+
cpuQuota := int64(-1)
53+
cpuPeriod := int64(-1)
54+
55+
if cgroupVersion == 1 {
56+
cgroupPath := ""
57+
if path == "" {
58+
cgroupPathRd, err := libcontainercgroups.FindCgroupMountpoint("", "cpu")
59+
if err != nil {
60+
return cpus
61+
}
62+
cgroupPath = cgroupPathRd
63+
} else {
64+
cgroupPath = path
65+
}
66+
cpuQuota = readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us")
67+
cpuPeriod = readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us")
68+
} else if cgroupVersion == 2 {
69+
cgroupPath := "/sys/fs/cgroup/"
70+
if path != "" {
71+
cgroupPath = path
72+
}
73+
cpuQuota, cpuPeriod = readCgroup2FileToInt64Tuple(cgroupPath, "cpu.max")
74+
}
4875

4976
if cpuQuota == -1 || cpuPeriod == -1 {
5077
return cpus
@@ -53,16 +80,66 @@ func NumCPU() int {
5380
return int(math.Ceil(float64(cpuQuota) / float64(cpuPeriod)))
5481
}
5582

56-
func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 {
83+
func GetCgroupVersion(cgroupPath string) int64 {
84+
// /sys/fs/cgroup/cgroup.controllers will not exist with cgroupsv1
85+
if _, err := os.Stat(filepath.Join(cgroupPath, "cgroup.controllers")); err == nil {
86+
return 2
87+
}
88+
89+
return 1
90+
}
91+
92+
func readCgroup2StringToInt64Tuple(cgroupString string) (quota, period int64) {
93+
// file contents looks like: $MAX $PERIOD
94+
// $MAX can have value "max" indicating no limit
95+
// it is possible for $PERIOD to be unset
96+
97+
values := strings.Fields(cgroupString)
98+
99+
if values[0] == "max" {
100+
return -1, -1
101+
}
102+
103+
cpuQuota, err := strconv.ParseInt(values[0], 10, 64)
104+
if err != nil {
105+
return -1, -1
106+
}
107+
108+
if len(values) == 1 {
109+
return cpuQuota, 100000
110+
}
111+
112+
cpuPeriod, err := strconv.ParseInt(values[1], 10, 64)
113+
if err != nil {
114+
return -1, -1
115+
}
116+
117+
return cpuQuota, cpuPeriod
118+
}
119+
120+
func readCgroup2FileToInt64Tuple(cgroupPath, cgroupFile string) (quota, period int64) {
57121
contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile))
58122
if err != nil {
59-
return -1
123+
return -1, -1
60124
}
61125

62-
strValue := strings.TrimSpace(string(contents))
126+
return readCgroup2StringToInt64Tuple(string(contents))
127+
}
128+
129+
func readCgroupStringToInt64(contents string) int64 {
130+
strValue := strings.TrimSpace(contents)
63131
if value, err := strconv.ParseInt(strValue, 10, 64); err == nil {
64132
return value
65133
}
66134

67135
return -1
68136
}
137+
138+
func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 {
139+
contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile))
140+
if err != nil {
141+
return -1
142+
}
143+
144+
return readCgroupStringToInt64(string(contents))
145+
}

‎test/e2e/cgroups/cgroups.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cgroups
18+
19+
import (
20+
"log"
21+
"os"
22+
"path/filepath"
23+
24+
"github.com/onsi/ginkgo/v2"
25+
"github.com/stretchr/testify/assert"
26+
27+
"k8s.io/ingress-nginx/test/e2e/framework"
28+
29+
"k8s.io/ingress-nginx/pkg/util/runtime"
30+
)
31+
32+
var _ = framework.IngressNginxDescribeSerial("[CGroups] cgroups", func() {
33+
f := framework.NewDefaultFramework("cgroups")
34+
35+
ginkgo.BeforeEach(func() {
36+
f.NewEchoDeployment()
37+
f.NewSlowEchoDeployment()
38+
})
39+
40+
ginkgo.It("detects cgroups version v1", func() {
41+
cgroupPath := "/testing/sys/fs/cgroup/"
42+
if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil {
43+
log.Fatal(err)
44+
}
45+
46+
quotaFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_quota_us"))
47+
if err != nil {
48+
log.Fatal(err)
49+
}
50+
51+
periodFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_period_us"))
52+
if err != nil {
53+
log.Fatal(err)
54+
}
55+
56+
_, err = quotaFile.WriteString("4")
57+
if err != nil {
58+
log.Fatal(err)
59+
}
60+
61+
err = quotaFile.Sync()
62+
if err != nil {
63+
log.Fatal(err)
64+
}
65+
66+
_, err = periodFile.WriteString("2")
67+
if err != nil {
68+
log.Fatal(err)
69+
}
70+
71+
err = periodFile.Sync()
72+
if err != nil {
73+
log.Fatal(err)
74+
}
75+
76+
assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(1))
77+
assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2)
78+
79+
os.Remove(filepath.Join(cgroupPath, "cpu.cfs_quota_us"))
80+
os.Remove(filepath.Join(cgroupPath, "cpu.cfs_period_us"))
81+
})
82+
83+
ginkgo.It("detect cgroups version v2", func() {
84+
cgroupPath := "/testing/sys/fs/cgroup/"
85+
if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil {
86+
log.Fatal(err)
87+
}
88+
89+
_, err := os.Create(filepath.Join(cgroupPath, "cgroup.controllers"))
90+
if err != nil {
91+
log.Fatal(err)
92+
}
93+
94+
file, err := os.Create(filepath.Join(cgroupPath, "cpu.max"))
95+
if err != nil {
96+
log.Fatal(err)
97+
}
98+
99+
_, err = file.WriteString("4 2")
100+
if err != nil {
101+
log.Fatal(err)
102+
}
103+
104+
err = file.Sync()
105+
if err != nil {
106+
log.Fatal(err)
107+
}
108+
109+
assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(2))
110+
assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2)
111+
112+
os.Remove(filepath.Join(cgroupPath, "cpu.max"))
113+
os.Remove(filepath.Join(cgroupPath, "cgroup.controllers"))
114+
})
115+
})

‎test/e2e/e2e.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
_ "k8s.io/ingress-nginx/test/e2e/admission"
3333
_ "k8s.io/ingress-nginx/test/e2e/annotations"
3434
_ "k8s.io/ingress-nginx/test/e2e/annotations/modsecurity"
35+
_ "k8s.io/ingress-nginx/test/e2e/cgroups"
3536
_ "k8s.io/ingress-nginx/test/e2e/dbg"
3637
_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
3738
_ "k8s.io/ingress-nginx/test/e2e/disableleaderelection"

0 commit comments

Comments
 (0)