Skip to content

Commit 8de881b

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #367 from austinvazquez/add-cgroupv2-cpu-quota-period-usec
Add cgroupV2 CPUQuotaPeriodUSec support
2 parents 834324e + 88f5938 commit 8de881b

File tree

5 files changed

+223
-29
lines changed

5 files changed

+223
-29
lines changed

‎cgroup2/cpu.go‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,26 @@ import (
2424

2525
type CPUMax string
2626

27+
const (
28+
// Default kernel value for cpu quota period is 100000 us (100 ms), same for v1 and v2.
29+
// v1: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html and
30+
// v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
31+
defaultCPUMax = "max"
32+
defaultCPUMaxPeriod = 100000
33+
defaultCPUMaxPeriodStr = "100000"
34+
)
35+
2736
func NewCPUMax(quota *int64, period *uint64) CPUMax {
28-
max := "max"
37+
max := defaultCPUMax
2938
if quota != nil {
3039
max = strconv.FormatInt(*quota, 10)
3140
}
32-
return CPUMax(strings.Join([]string{max, strconv.FormatUint(*period, 10)}, " "))
41+
42+
duration := defaultCPUMaxPeriodStr
43+
if period != nil {
44+
duration = strconv.FormatUint(*period, 10)
45+
}
46+
return CPUMax(strings.Join([]string{max, duration}, " "))
3347
}
3448

3549
type CPU struct {
@@ -39,19 +53,34 @@ type CPU struct {
3953
Mems string
4054
}
4155

42-
func (c CPUMax) extractQuotaAndPeriod() (int64, uint64) {
56+
func (c CPUMax) extractQuotaAndPeriod() (int64, uint64, error) {
4357
var (
44-
quota int64
45-
period uint64
58+
quota int64 = math.MaxInt64
59+
period uint64 = defaultCPUMaxPeriod
60+
err error
4661
)
62+
63+
// value: quota [period]
4764
values := strings.Split(string(c), " ")
48-
if values[0] == "max" {
49-
quota = math.MaxInt64
50-
} else {
51-
quota, _ = strconv.ParseInt(values[0], 10, 64)
65+
if len(values) < 1 || len(values) > 2 {
66+
return 0, 0, ErrInvalidFormat
5267
}
53-
period, _ = strconv.ParseUint(values[1], 10, 64)
54-
return quota, period
68+
69+
if strings.ToLower(values[0]) != defaultCPUMax {
70+
quota, err = strconv.ParseInt(values[0], 10, 64)
71+
if err != nil {
72+
return 0, 0, err
73+
}
74+
}
75+
76+
if len(values) == 2 {
77+
period, err = strconv.ParseUint(values[1], 10, 64)
78+
if err != nil {
79+
return 0, 0, err
80+
}
81+
}
82+
83+
return quota, period, nil
5584
}
5685

5786
func (r *CPU) Values() (o []Value) {

‎cgroup2/cpuv2_test.go‎

Lines changed: 66 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -83,22 +83,72 @@ func TestSystemdCgroupCpuController_NilWeight(t *testing.T) {
8383
}
8484

8585
func TestExtractQuotaAndPeriod(t *testing.T) {
86-
var (
87-
period uint64
88-
quota int64
86+
const (
87+
defaultQuota int64 = math.MaxInt64
88+
defaultPeriod uint64 = 100000
8989
)
90-
quota = 10000
91-
period = 8000
92-
cpuMax := NewCPUMax(&quota, &period)
93-
tquota, tPeriod := cpuMax.extractQuotaAndPeriod()
94-
95-
assert.Equal(t, quota, tquota)
96-
assert.Equal(t, period, tPeriod)
97-
98-
// case with nil quota which makes it "max" - max int val
99-
cpuMax2 := NewCPUMax(nil, &period)
100-
tquota2, tPeriod2 := cpuMax2.extractQuotaAndPeriod()
10190

102-
assert.Equal(t, int64(math.MaxInt64), tquota2)
103-
assert.Equal(t, period, tPeriod2)
91+
require.Equal(t, defaultCPUMaxPeriodStr, strconv.Itoa(defaultCPUMaxPeriod), "Constant for default period does not match its string type constant.")
92+
93+
// Default "max 100000"
94+
cpuMax := NewCPUMax(nil, nil)
95+
assert.Equal(t, CPUMax("max 100000"), cpuMax)
96+
quota, period, err := cpuMax.extractQuotaAndPeriod()
97+
assert.NoError(t, err)
98+
assert.Equal(t, defaultQuota, quota)
99+
assert.Equal(t, defaultPeriod, period)
100+
101+
// Only specifing limit is valid.
102+
cpuMax = CPUMax("max")
103+
quota, period, err = cpuMax.extractQuotaAndPeriod()
104+
assert.NoError(t, err)
105+
assert.Equal(t, defaultQuota, quota)
106+
assert.Equal(t, defaultPeriod, period)
107+
108+
tests := []struct {
109+
cpuMax string
110+
quota int64
111+
period uint64
112+
}{
113+
{
114+
cpuMax: "0 0",
115+
quota: 0,
116+
period: 0,
117+
},
118+
{
119+
cpuMax: "10000 8000",
120+
quota: 10000,
121+
period: 8000,
122+
},
123+
{
124+
cpuMax: "42000 4200",
125+
quota: 42000,
126+
period: 4200,
127+
},
128+
{
129+
cpuMax: "9223372036854775807 18446744073709551615",
130+
quota: 9223372036854775807,
131+
period: 18446744073709551615,
132+
},
133+
}
134+
135+
for _, test := range tests {
136+
t.Run(test.cpuMax, func(t *testing.T) {
137+
cpuMax := NewCPUMax(&test.quota, &test.period)
138+
assert.Equal(t, CPUMax(test.cpuMax), cpuMax)
139+
140+
tquota, tPeriod, err := cpuMax.extractQuotaAndPeriod()
141+
assert.NoError(t, err)
142+
assert.Equal(t, test.quota, tquota)
143+
assert.Equal(t, test.period, tPeriod)
144+
})
145+
}
146+
147+
// Negative test cases result in errors.
148+
for i, cpuMax := range []string{"", " ", "max 100000 100000"} {
149+
t.Run(fmt.Sprintf("negative-test-%d", i+1), func(t *testing.T) {
150+
_, _, err = CPUMax(cpuMax).extractQuotaAndPeriod()
151+
assert.Error(t, err)
152+
})
153+
}
104154
}

‎cgroup2/manager.go‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"path/filepath"
2828
"strconv"
2929
"strings"
30+
"sync"
3031
"time"
3132

3233
"github.com/containerd/cgroups/v3/cgroup2/stats"
@@ -45,9 +46,17 @@ const (
4546
typeFile = "cgroup.type"
4647
defaultCgroup2Path = "/sys/fs/cgroup"
4748
defaultSlice = "system.slice"
49+
50+
// systemd only supports CPUQuotaPeriodUSec since v2.42.0
51+
cpuQuotaPeriodUSecSupportedVersion = 242
4852
)
4953

50-
var canDelegate bool
54+
var (
55+
canDelegate bool
56+
57+
versionOnce sync.Once
58+
version int
59+
)
5160

5261
type Event struct {
5362
Low uint64
@@ -881,7 +890,19 @@ func NewSystemd(slice, group string, pid int, resources *Resources) (*Manager, e
881890
}
882891

883892
if resources.CPU != nil && resources.CPU.Max != "" {
884-
quota, period := resources.CPU.Max.extractQuotaAndPeriod()
893+
quota, period, err := resources.CPU.Max.extractQuotaAndPeriod()
894+
if err != nil {
895+
return &Manager{}, err
896+
}
897+
898+
if period != 0 {
899+
if sdVer := systemdVersion(conn); sdVer >= cpuQuotaPeriodUSecSupportedVersion {
900+
properties = append(properties, newSystemdProperty("CPUQuotaPeriodUSec", period))
901+
} else {
902+
log.G(context.TODO()).WithField("version", sdVer).Debug("Systemd version is too old to support CPUQuotaPeriodUSec")
903+
}
904+
}
905+
885906
// cpu.cfs_quota_us and cpu.cfs_period_us are controlled by systemd.
886907
// corresponds to USEC_INFINITY in systemd
887908
// if USEC_INFINITY is provided, CPUQuota is left unbound by systemd
@@ -921,6 +942,41 @@ func NewSystemd(slice, group string, pid int, resources *Resources) (*Manager, e
921942
}, nil
922943
}
923944

945+
// Adapted from https://github.com/opencontainers/cgroups/blob/9657f5a18b8d60a0f39fbb34d0cb7771e28e6278/systemd/common.go#L245-L281
946+
func systemdVersion(conn *systemdDbus.Conn) int {
947+
versionOnce.Do(func() {
948+
version = -1
949+
verStr, err := conn.GetManagerProperty("Version")
950+
if err == nil {
951+
version, err = systemdVersionAtoi(verStr)
952+
}
953+
954+
if err != nil {
955+
log.G(context.TODO()).WithError(err).Error("Unable to get systemd version")
956+
}
957+
})
958+
959+
return version
960+
}
961+
962+
func systemdVersionAtoi(str string) (int, error) {
963+
// Unconditionally remove the leading prefix ("v).
964+
str = strings.TrimLeft(str, `"v`)
965+
// Match on the first integer we can grab.
966+
for i := range len(str) {
967+
if str[i] < '0' || str[i] > '9' {
968+
// First non-digit: cut the tail.
969+
str = str[:i]
970+
break
971+
}
972+
}
973+
ver, err := strconv.Atoi(str)
974+
if err != nil {
975+
return -1, fmt.Errorf("can't parse version: %w", err)
976+
}
977+
return ver, nil
978+
}
979+
924980
func startUnit(conn *systemdDbus.Conn, group string, properties []systemdDbus.Property, ignoreExists bool) error {
925981
ctx := context.TODO()
926982

‎cgroup2/manager_test.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717
package cgroup2
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"os"
2223
"os/exec"
2324
"syscall"
2425
"testing"
2526
"time"
2627

28+
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
2729
"github.com/opencontainers/runtime-spec/specs-go"
2830
"github.com/stretchr/testify/assert"
2931
"github.com/stretchr/testify/require"
@@ -343,6 +345,51 @@ func TestSystemdCgroupPSIController(t *testing.T) {
343345
}
344346
}
345347

348+
func TestCPUQuotaPeriodUSec(t *testing.T) {
349+
checkCgroupMode(t)
350+
requireSystemdVersion(t, cpuQuotaPeriodUSecSupportedVersion)
351+
352+
pid := os.Getpid()
353+
group := fmt.Sprintf("testing-cpu-period-%d.scope", pid)
354+
355+
tests := []struct {
356+
name string
357+
cpuMax CPUMax
358+
expectedCPUMax string
359+
expectedPeriod uint64
360+
}{
361+
{
362+
name: "default cpu.max",
363+
cpuMax: "max 100000",
364+
expectedCPUMax: "max 100000",
365+
expectedPeriod: 100000,
366+
},
367+
}
368+
369+
for _, test := range tests {
370+
t.Run(test.name, func(t *testing.T) {
371+
c, err := NewSystemd("", group, pid, &Resources{
372+
CPU: &CPU{
373+
Max: test.cpuMax,
374+
},
375+
})
376+
require.NoError(t, err, "failed to init new cgroup systemd manager")
377+
378+
conn, err := systemdDbus.NewWithContext(context.TODO())
379+
require.NoError(t, err, "failed to connect to systemd")
380+
defer conn.Close()
381+
382+
unitName := systemdUnitFromPath(c.path)
383+
props, err := conn.GetAllPropertiesContext(context.TODO(), unitName)
384+
require.NoError(t, err, "failed to get unit properties")
385+
386+
periodUSec, ok := props["CPUQuotaPeriodUSec"]
387+
require.True(t, ok, "CPUQuotaPeriodUSec property not found")
388+
require.Equal(t, test.expectedPeriod, periodUSec.(uint64), "CPUQuotaPeriodUSec value doesn't match expected period")
389+
})
390+
}
391+
}
392+
346393
func BenchmarkStat(b *testing.B) {
347394
checkCgroupMode(b)
348395
group := "/stat-test-cg"

‎cgroup2/testutils_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@
1717
package cgroup2
1818

1919
import (
20+
"context"
2021
"os"
2122
"path/filepath"
2223
"strings"
2324
"testing"
2425

26+
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
2527
"github.com/stretchr/testify/assert"
2628
"github.com/stretchr/testify/require"
2729
"golang.org/x/sys/unix"
@@ -38,6 +40,16 @@ func checkCgroupMode(tb testing.TB) {
3840
}
3941
}
4042

43+
func requireSystemdVersion(tb testing.TB, requiredMinVersion int) {
44+
conn, err := systemdDbus.NewWithContext(context.TODO())
45+
require.NoError(tb, err, "failed to connect to systemd")
46+
defer conn.Close()
47+
48+
if sdVer := systemdVersion(conn); sdVer < requiredMinVersion {
49+
tb.Skipf("Skipping test; systemd version %d < required version %d", sdVer, requiredMinVersion)
50+
}
51+
}
52+
4153
func checkCgroupControllerSupported(t *testing.T, controller string) {
4254
b, err := os.ReadFile(filepath.Join(defaultCgroup2Path, controllersFile))
4355
if err != nil || !strings.Contains(string(b), controller) {

0 commit comments

Comments
 (0)