Merge pull request #169 from coroot/cgroup_hybrid_mode

add support for hybrid cgroup mode with both V1 and V2 enabled

Author: def

cgroup/cgroup.go

@@ -14,7 +14,8 @@ import (
 )
 
 var (
-	cgRoot = *flags.CgroupRoot
+	cgRoot  = *flags.CgroupRoot
+	cg2Root = *flags.CgroupRoot
 
 	baseCgroupPath = ""
 
@@ -26,13 +27,6 @@ var (
 	talosIdRegexp       = regexp.MustCompile(`/(system|podruntime)/([^/]+)`)
 )
 
-type Version uint8
-
-const (
-	V1 Version = iota
-	V2
-)
-
 type ContainerType uint8
 
 const (
@@ -68,17 +62,23 @@ func (t ContainerType) String() string {
 
 type Cgroup struct {
 	Id            string
-	Version       Version
 	ContainerType ContainerType
 	ContainerId   string
 
 	subsystems map[string]string
 }
 
 func (cg *Cgroup) CreatedAt() time.Time {
-	p := path.Join(cgRoot, cg.subsystems[""]) //v2
-	if cg.Version == V1 {
-		p = path.Join(cgRoot, "cpu", cg.subsystems["cpu"])
+	var p string
+	if sp := cg.subsystems[""]; sp != "" { //v2
+		p = path.Join(cg2Root, sp)
+	} else if sp = cg.subsystems["cpu"]; sp != "" {
+		p = path.Join(cgRoot, "cpu", sp)
+	} else if sp = cg.subsystems["memory"]; sp != "" {
+		p = path.Join(cgRoot, "memory", sp)
+	}
+	if p == "" {
+		return time.Time{}
 	}
 	fi, err := os.Stat(p)
 	if err != nil {
@@ -104,15 +104,22 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 			continue
 		}
 		for _, cgType := range strings.Split(parts[1], ",") {
-			cg.subsystems[cgType] = path.Join(baseCgroupPath, parts[2])
+			p := path.Join(baseCgroupPath, parts[2])
+			switch p {
+			case "/", "/init.scope":
+				continue
+			}
+			cg.subsystems[cgType] = p
 		}
 	}
-	if cg.Id = cg.subsystems[""]; cg.Id != "" {
-		cg.Version = V2
-	} else if cg.Id = cg.subsystems["cpu"]; cg.Id != "" {
-		cg.Version = V1
+	cg.Id = cg.subsystems[""]
+	if cg.Id == "" {
+		cg.Id = cg.subsystems["cpu"]
+	}
+	if cg.Id == "" {
+		cg.Id = cg.subsystems["memory"]
 	}
-	if (cg.Id == "" || cg.Id == "/") && cg.subsystems["name=systemd"] != "/" {
+	if cg.Id == "" && cg.subsystems["name=systemd"] != "" {
 		cg.Id = cg.subsystems["name=systemd"]
 	}
 	if cg.ContainerType, cg.ContainerId, err = containerByCgroup(cg.Id); err != nil {

cgroup/cgroup_linux.go

@@ -1,10 +1,13 @@
 package cgroup
 
 import (
+	"os"
+	"path"
 	"runtime"
 
 	"github.com/vishvananda/netns"
 	"golang.org/x/sys/unix"
+	"k8s.io/klog/v2"
 )
 
 func Init() error {
@@ -18,25 +21,25 @@ func Init() error {
 		return err
 	}
 	defer hostNs.Close()
-	if selfNs.Equal(hostNs) {
-		return nil
-	}
+	if !selfNs.Equal(hostNs) {
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+		if err := unix.Setns(int(hostNs), unix.CLONE_NEWCGROUP); err != nil {
+			return err
+		}
+		cg, err := NewFromProcessCgroupFile("/proc/self/cgroup")
+		if err != nil {
+			return err
+		}
+		baseCgroupPath = cg.Id
 
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
-	if err := unix.Setns(int(hostNs), unix.CLONE_NEWCGROUP); err != nil {
-		return err
+		if err := unix.Setns(int(selfNs), unix.CLONE_NEWCGROUP); err != nil {
+			return err
+		}
 	}
-
-	cg, err := NewFromProcessCgroupFile("/proc/self/cgroup")
-	if err != nil {
-		return err
+	if _, err := os.Stat(path.Join(cgRoot, "unified")); err == nil {
+		cg2Root = path.Join(cgRoot, "unified")
 	}
-	baseCgroupPath = cg.Id
-
-	if err := unix.Setns(int(selfNs), unix.CLONE_NEWCGROUP); err != nil {
-		return err
-	}
-
+	klog.Infoln("cgroup v2 root is", cg2Root)
 	return nil
 }

cgroup/cgroup_test.go

@@ -11,7 +11,6 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	cg, err := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
 	assert.Nil(t, err)
 	assert.Equal(t, "/system.slice/docker.service", cg.Id)
-	assert.Equal(t, V1, cg.Version)
 	assert.Equal(t, "/system.slice/docker.service", cg.ContainerId)
 	assert.Equal(t, ContainerTypeSystemdService, cg.ContainerType)
 
@@ -20,65 +19,52 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 			"blkio":        "/system.slice/docker.service",
 			"cpu":          "/system.slice/docker.service",
 			"cpuacct":      "/system.slice/docker.service",
-			"cpuset":       "/",
 			"devices":      "/system.slice/docker.service",
-			"freezer":      "/",
-			"hugetlb":      "/",
 			"memory":       "/system.slice/docker.service",
 			"name=systemd": "/system.slice/docker.service",
-			"net_cls":      "/",
-			"net_prio":     "/",
-			"perf_event":   "/",
 			"pids":         "/system.slice/docker.service",
 		},
 		cg.subsystems,
 	)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V1, cg.Version)
 	assert.Equal(t, "/docker/b43d92bf1e5c6f78bb9b7bc6f40721280299855ba692092716e3a1b6c0b86f3f", cg.Id)
 	assert.Equal(t, "b43d92bf1e5c6f78bb9b7bc6f40721280299855ba692092716e3a1b6c0b86f3f", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/300/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V1, cg.Version)
 	assert.Equal(t, "/kubepods/burstable/pod6a4ce4a0-ba47-11ea-b2a7-0cc47ac5979e/17db96a24ae1e9dd57143e62b1cb0d2d35e693c65c774c7470e87b0572e07c1a", cg.Id)
 	assert.Equal(t, "17db96a24ae1e9dd57143e62b1cb0d2d35e693c65c774c7470e87b0572e07c1a", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V2, cg.Version)
 	assert.Equal(t, "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope", cg.Id)
 	assert.Equal(t, "73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/600/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V1, cg.Version)
 	assert.Equal(t, "/system.slice/springboot.service", cg.Id)
 	assert.Equal(t, "/system.slice/springboot.service", cg.ContainerId)
 	assert.Equal(t, ContainerTypeSystemdService, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/700/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V2, cg.Version)
 	assert.Equal(t, "/podruntime/runtime", cg.Id)
 	assert.Equal(t, "/talos/runtime", cg.ContainerId)
 	assert.Equal(t, ContainerTypeTalosRuntime, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/800/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V2, cg.Version)
 	assert.Equal(t, "/system.slice/docker-cf87ba651579c9231db817909e7865e5747bd7abcac0c57ce23cf4abbaee046b.scope", cg.Id)
 	assert.Equal(t, "cf87ba651579c9231db817909e7865e5747bd7abcac0c57ce23cf4abbaee046b", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/900/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V1, cg.Version)
 	assert.Equal(t, "/system.slice/python-app.service", cg.Id)
 	assert.Equal(t, "/system.slice/python-app.service", cg.ContainerId)
 	assert.Equal(t, ContainerTypeSystemdService, cg.ContainerType)
@@ -89,7 +75,6 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	}()
 	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/500/cgroup"))
 	assert.Nil(t, err)
-	assert.Equal(t, V2, cg.Version)
 	assert.Equal(t, "/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope", cg.Id)
 	assert.Equal(t, "ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)

cgroup/cpu.go

@@ -17,16 +17,17 @@ type CPUStat struct {
 }
 
 func (cg Cgroup) CpuStat() *CPUStat {
-	if cg.Version == V1 {
-		st, _ := cg.cpuStatV1()
+	cpu, cpuacct := cg.subsystems["cpu"], cg.subsystems["cpuacct"]
+	if cpu == "" || cpuacct == "" {
+		st, _ := cg.cpuStatV2()
 		return st
 	}
-	st, _ := cg.cpuStatV2()
+	st, _ := cg.cpuStatV1()
 	return st
 }
 
 func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
-	if cg.subsystems["cpu"] == "/" || cg.subsystems["cpuacct"] == "/" {
+	if cg.subsystems["cpu"] == "" || cg.subsystems["cpuacct"] == "" {
 		return nil, nil
 	}
 	throttling, err := readVariablesFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.stat"))
@@ -56,15 +57,18 @@ func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
 }
 
 func (cg Cgroup) cpuStatV2() (*CPUStat, error) {
-	vars, err := readVariablesFromFile(path.Join(cgRoot, cg.subsystems[""], "cpu.stat"))
+	if cg.subsystems[""] == "" {
+		return nil, nil
+	}
+	vars, err := readVariablesFromFile(path.Join(cg2Root, cg.subsystems[""], "cpu.stat"))
 	if err != nil {
 		return nil, err
 	}
 	res := &CPUStat{
 		UsageSeconds:         float64(vars["usage_usec"]) / 1e6,
 		ThrottledTimeSeconds: float64(vars["throttled_usec"]) / 1e6,
 	}
-	if payload, err := os.ReadFile(path.Join(cgRoot, cg.subsystems[""], "cpu.max")); err == nil {
+	if payload, err := os.ReadFile(path.Join(cg2Root, cg.subsystems[""], "cpu.max")); err == nil {
 		data := strings.TrimSpace(string(payload))
 		parts := strings.Fields(data)
 		if len(parts) != 2 {

cgroup/cpu_test.go

@@ -9,6 +9,7 @@ import (
 
 func TestCgroup_CpuStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
+	cg2Root = "fixtures/cgroup"
 
 	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
 	s := cg.CpuStat()
@@ -36,4 +37,11 @@ func TestCgroup_CpuStat(t *testing.T) {
 	s, err := cg.cpuStatV1()
 	assert.NoError(t, err)
 	assert.Nil(t, s)
+
+	cg2Root = "fixtures/cgroup/unified"
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/550/cgroup"))
+	s = cg.CpuStat()
+	assert.Equal(t, 0., s.LimitCores)
+	assert.Equal(t, 0., s.ThrottledTimeSeconds)
+	assert.Equal(t, 151.439967, s.UsageSeconds)
 }

cgroup/fixtures/cgroup/memory/system.slice/ssh.service/memory.limit_in_bytes

@@ -0,0 +1 @@
+9223372036854771712

cgroup/fixtures/cgroup/memory/system.slice/ssh.service/memory.stat

@@ -0,0 +1,33 @@
+cache 7299072
+rss 3637248
+rss_huge 0
+shmem 0
+mapped_file 2703360
+dirty 0
+writeback 0
+pgpgin 1584132
+pgpgout 1581454
+pgfault 3320328
+pgmajfault 0
+inactive_anon 0
+active_anon 3649536
+inactive_file 3514368
+active_file 3514368
+unevictable 0
+hierarchical_memory_limit 9223372036854771712
+total_cache 7299072
+total_rss 3637248
+total_rss_huge 0
+total_shmem 0
+total_mapped_file 2703360
+total_dirty 0
+total_writeback 0
+total_pgpgin 1584132
+total_pgpgout 1581454
+total_pgfault 3320328
+total_pgmajfault 0
+total_inactive_anon 0
+total_active_anon 3649536
+total_inactive_file 3514368
+total_active_file 3514368
+total_unevictable 0

cgroup/fixtures/cgroup/unified/system.slice/ssh.service/cpu.stat

@@ -0,0 +1,3 @@
+usage_usec 151439967
+user_usec 99140470
+system_usec 52299496

cgroup/fixtures/proc/550/cgroup

@@ -0,0 +1,13 @@
+12:perf_event:/
+11:devices:/system.slice/ssh.service
+10:freezer:/
+9:hugetlb:/
+8:cpuset:/
+7:blkio:/
+6:memory:/system.slice/ssh.service
+5:pids:/system.slice/ssh.service
+4:cpu,cpuacct:/
+3:net_cls,net_prio:/
+2:rdma:/
+1:name=systemd:/system.slice/ssh.service
+0::/system.slice/ssh.service

cgroup/io.go

@@ -1,7 +1,7 @@
 package cgroup
 
 import (
-	"io/ioutil"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -17,16 +17,17 @@ type IOStat struct {
 }
 
 func (cg *Cgroup) IOStat() map[string]IOStat {
-	if cg.Version == V1 {
-		st, _ := cg.ioStatV1()
+	blkio := cg.subsystems["blkio"]
+	if blkio == "" {
+		st, _ := cg.ioStatV2()
 		return st
 	}
-	st, _ := cg.ioStatV2()
+	st, _ := cg.ioStatV1()
 	return st
 }
 
 func (cg *Cgroup) ioStatV1() (map[string]IOStat, error) {
-	if cg.subsystems["blkio"] == "/" {
+	if cg.subsystems["blkio"] == "" {
 		return nil, nil
 	}
 	ops, err := readBlkioStatFile(path.Join(cgRoot, "blkio", cg.subsystems["blkio"], "blkio.throttle.io_serviced"))
@@ -62,7 +63,10 @@ func (cg *Cgroup) ioStatV1() (map[string]IOStat, error) {
 }
 
 func (cg *Cgroup) ioStatV2() (map[string]IOStat, error) {
-	payload, err := ioutil.ReadFile(path.Join(cgRoot, cg.subsystems[""], "io.stat"))
+	if cg.subsystems[""] == "" {
+		return nil, nil
+	}
+	payload, err := os.ReadFile(path.Join(cg2Root, cg.subsystems[""], "io.stat"))
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +107,7 @@ type blkioVariable struct {
 }
 
 func readBlkioStatFile(filePath string) ([]blkioVariable, error) {
-	data, err := ioutil.ReadFile(filePath)
+	data, err := os.ReadFile(filePath)
 	if err != nil {
 		return nil, err
 	}