libcontainer: refactor resource manager interface

Currently, cgroups manager is the single resource manager in libcontainer.
Linux kernel 4.10 will introduce Intel RDT/CAT feature, the kernel interface
is exposed via "resource control" filesystem, which is a cgroup-like
interface. In order to support Intel RDT/CAT in libcontainer, we need a new
resource manager outside cgroups.

This patch adds a new "ResourceManager" structure as the base interface for
all resource managers, such as cgroups manager and incoming IntelRdt manager.

All registered resource managers are consolidated in linuxContainer structure.
We can apply to unified operations (e.g., Apply(), Set(), Destroy()) using
all of the registered resource managers.

Signed-off-by: Xiaochen Shen <[email protected]>

Author: xiaochenshen

libcontainer/cgroups/cgroups.go

@@ -5,40 +5,11 @@ package cgroups
 import (
 	"fmt"
 
-	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/resourcemanager"
 )
 
 type Manager interface {
-	// Applies cgroup configuration to the process with the specified pid
-	Apply(pid int) error
-
-	// Returns the PIDs inside the cgroup set
-	GetPids() ([]int, error)
-
-	// Returns the PIDs inside the cgroup set & all sub-cgroups
-	GetAllPids() ([]int, error)
-
-	// Returns statistics for the cgroup set
-	GetStats() (*Stats, error)
-
-	// Toggles the freezer cgroup according with specified state
-	Freeze(state configs.FreezerState) error
-
-	// Destroys the cgroup set
-	Destroy() error
-
-	// The option func SystemdCgroups() and Cgroupfs() require following attributes:
-	// 	Paths   map[string]string
-	// 	Cgroups *configs.Cgroup
-	// Paths maps cgroup subsystem to path at which it is mounted.
-	// Cgroups specifies specific cgroup settings for the various subsystems
-
-	// Returns cgroup paths to save in a state file and to be able to
-	// restore the object later.
-	GetPaths() map[string]string
-
-	// Sets the cgroup as configured.
-	Set(container *configs.Config) error
+	resourcemanager.ResourceManager
 }
 
 type NotFoundError struct {

libcontainer/cgroups/fs/apply_raw.go

@@ -171,7 +171,7 @@ func (m *Manager) GetPaths() map[string]string {
 	return paths
 }
 
-func (m *Manager) GetStats() (*cgroups.Stats, error) {
+func (m *Manager) GetStats() (interface{}, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	stats := cgroups.NewStats()

libcontainer/cgroups/systemd/apply_nosystemd.go

@@ -38,7 +38,7 @@ func (m *Manager) GetPaths() map[string]string {
 	return nil
 }
 
-func (m *Manager) GetStats() (*cgroups.Stats, error) {
+func (m *Manager) GetStats() (interface{}, error) {
 	return nil, fmt.Errorf("Systemd not supported")
 }
 

libcontainer/cgroups/systemd/apply_systemd.go

@@ -484,7 +484,7 @@ func (m *Manager) GetAllPids() ([]int, error) {
 	return cgroups.GetAllPids(path)
 }
 
-func (m *Manager) GetStats() (*cgroups.Stats, error) {
+func (m *Manager) GetStats() (interface{}, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	stats := cgroups.NewStats()

libcontainer/container_linux.go

@@ -22,6 +22,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/criurpc"
+	"github.com/opencontainers/runc/libcontainer/resourcemanager"
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/runc/libcontainer/utils"
 	"github.com/syndtr/gocapability/capability"
@@ -34,7 +35,7 @@ type linuxContainer struct {
 	id                   string
 	root                 string
 	config               *configs.Config
-	cgroupManager        cgroups.Manager
+	resourceManagers     map[string]resourcemanager.ResourceManager
 	initArgs             []string
 	initProcess          parentProcess
 	initProcessStartTime string
@@ -142,7 +143,7 @@ func (c *linuxContainer) State() (*State, error) {
 }
 
 func (c *linuxContainer) Processes() ([]int, error) {
-	pids, err := c.cgroupManager.GetAllPids()
+	pids, err := c.resourceManagers["cgroups"].GetAllPids()
 	if err != nil {
 		return nil, newSystemErrorWithCause(err, "getting all container pids from cgroups")
 	}
@@ -154,7 +155,9 @@ func (c *linuxContainer) Stats() (*Stats, error) {
 		err   error
 		stats = &Stats{}
 	)
-	if stats.CgroupStats, err = c.cgroupManager.GetStats(); err != nil {
+	cgroupStats, err := c.resourceManagers["cgroups"].GetStats()
+	stats.CgroupStats = cgroupStats.(*cgroups.Stats)
+	if err != nil {
 		return stats, newSystemErrorWithCause(err, "getting container stats from cgroups")
 	}
 	for _, iface := range c.config.Networks {
@@ -181,7 +184,12 @@ func (c *linuxContainer) Set(config configs.Config) error {
 		return newGenericError(fmt.Errorf("container not running"), ContainerNotRunning)
 	}
 	c.config = &config
-	return c.cgroupManager.Set(c.config)
+	for _, resourceManager := range c.resourceManagers {
+		if err := resourceManager.Set(c.config); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (c *linuxContainer) Start(process *Process) error {
@@ -283,7 +291,7 @@ func (c *linuxContainer) start(process *Process, isInit bool) error {
 
 func (c *linuxContainer) Signal(s os.Signal, all bool) error {
 	if all {
-		return signalAllProcesses(c.cgroupManager, s)
+		return signalAllProcesses(c.resourceManagers["cgroups"], s)
 	}
 	if err := c.initProcess.signal(s); err != nil {
 		return newSystemErrorWithCause(err, "signaling init process")
@@ -350,7 +358,7 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c
 		cmd:           cmd,
 		childPipe:     childPipe,
 		parentPipe:    parentPipe,
-		manager:       c.cgroupManager,
+		managers:      c.resourceManagers,
 		config:        c.newInitConfig(p),
 		container:     c,
 		process:       p,
@@ -376,7 +384,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe,
 	p.consoleChan = make(chan *os.File, 1)
 	return &setnsProcess{
 		cmd:           cmd,
-		cgroupPaths:   c.cgroupManager.GetPaths(),
+		cgroupPaths:   c.resourceManagers["cgroups"].GetPaths(),
 		childPipe:     childPipe,
 		parentPipe:    parentPipe,
 		config:        c.newInitConfig(p),
@@ -452,7 +460,7 @@ func (c *linuxContainer) Pause() error {
 	}
 	switch status {
 	case Running, Created:
-		if err := c.cgroupManager.Freeze(configs.Frozen); err != nil {
+		if err := c.resourceManagers["cgroups"].Freeze(configs.Frozen); err != nil {
 			return err
 		}
 		return c.state.transition(&pausedState{
@@ -472,7 +480,7 @@ func (c *linuxContainer) Resume() error {
 	if status != Paused {
 		return newGenericError(fmt.Errorf("container not paused"), ContainerNotPaused)
 	}
-	if err := c.cgroupManager.Freeze(configs.Thawed); err != nil {
+	if err := c.resourceManagers["cgroups"].Freeze(configs.Thawed); err != nil {
 		return err
 	}
 	return c.state.transition(&runningState{
@@ -481,11 +489,11 @@ func (c *linuxContainer) Resume() error {
 }
 
 func (c *linuxContainer) NotifyOOM() (<-chan struct{}, error) {
-	return notifyOnOOM(c.cgroupManager.GetPaths())
+	return notifyOnOOM(c.resourceManagers["cgroups"].GetPaths())
 }
 
 func (c *linuxContainer) NotifyMemoryPressure(level PressureLevel) (<-chan struct{}, error) {
-	return notifyMemoryPressure(c.cgroupManager.GetPaths(), level)
+	return notifyMemoryPressure(c.resourceManagers["cgroups"].GetPaths(), level)
 }
 
 // checkCriuVersion checks Criu version greater than or equal to minVersion
@@ -870,7 +878,7 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 }
 
 func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error {
-	if err := c.cgroupManager.Apply(pid); err != nil {
+	if err := c.resourceManagers["cgroups"].Apply(pid); err != nil {
 		return err
 	}
 
@@ -1224,7 +1232,7 @@ func (c *linuxContainer) runType() (Status, error) {
 }
 
 func (c *linuxContainer) isPaused() (bool, error) {
-	data, err := ioutil.ReadFile(filepath.Join(c.cgroupManager.GetPaths()["freezer"], "freezer.state"))
+	data, err := ioutil.ReadFile(filepath.Join(c.resourceManagers["cgroups"].GetPaths()["freezer"], "freezer.state"))
 	if err != nil {
 		// If freezer cgroup is not mounted, the container would just be not paused.
 		if os.IsNotExist(err) {
@@ -1254,7 +1262,7 @@ func (c *linuxContainer) currentState() (*State, error) {
 			InitProcessStartTime: startTime,
 			Created:              c.created,
 		},
-		CgroupPaths:         c.cgroupManager.GetPaths(),
+		CgroupPaths:         c.resourceManagers["cgroups"].GetPaths(),
 		NamespacePaths:      make(map[configs.NamespaceType]string),
 		ExternalDescriptors: externalDescriptors,
 	}

libcontainer/container_linux_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/resourcemanager"
 )
 
 type mockCgroupManager struct {
@@ -26,7 +27,7 @@ func (m *mockCgroupManager) GetAllPids() ([]int, error) {
 	return m.allPids, nil
 }
 
-func (m *mockCgroupManager) GetStats() (*cgroups.Stats, error) {
+func (m *mockCgroupManager) GetStats() (interface{}, error) {
 	return m.stats, nil
 }
 
@@ -88,10 +89,11 @@ func (m *mockProcess) setExternalDescriptors(newFds []string) {
 
 func TestGetContainerPids(t *testing.T) {
 	container := &linuxContainer{
-		id:            "myid",
-		config:        &configs.Config{},
-		cgroupManager: &mockCgroupManager{allPids: []int{1, 2, 3}},
+		id:     "myid",
+		config: &configs.Config{},
 	}
+	container.resourceManagers = make(map[string]resourcemanager.ResourceManager)
+	container.resourceManagers["cgroups"] = &mockCgroupManager{allPids: []int{1, 2, 3}}
 	pids, err := container.Processes()
 	if err != nil {
 		t.Fatal(err)
@@ -107,13 +109,14 @@ func TestGetContainerStats(t *testing.T) {
 	container := &linuxContainer{
 		id:     "myid",
 		config: &configs.Config{},
-		cgroupManager: &mockCgroupManager{
-			pids: []int{1, 2, 3},
-			stats: &cgroups.Stats{
-				MemoryStats: cgroups.MemoryStats{
-					Usage: cgroups.MemoryData{
-						Usage: 1024,
-					},
+	}
+	container.resourceManagers = make(map[string]resourcemanager.ResourceManager)
+	container.resourceManagers["cgroups"] = &mockCgroupManager{
+		pids: []int{1, 2, 3},
+		stats: &cgroups.Stats{
+			MemoryStats: cgroups.MemoryStats{
+				Usage: cgroups.MemoryData{
+					Usage: 1024,
 				},
 			},
 		},
@@ -152,18 +155,19 @@ func TestGetContainerState(t *testing.T) {
 			_pid:    pid,
 			started: "010",
 		},
-		cgroupManager: &mockCgroupManager{
-			pids: []int{1, 2, 3},
-			stats: &cgroups.Stats{
-				MemoryStats: cgroups.MemoryStats{
-					Usage: cgroups.MemoryData{
-						Usage: 1024,
-					},
+	}
+	container.resourceManagers = make(map[string]resourcemanager.ResourceManager)
+	container.resourceManagers["cgroups"] = &mockCgroupManager{
+		pids: []int{1, 2, 3},
+		stats: &cgroups.Stats{
+			MemoryStats: cgroups.MemoryStats{
+				Usage: cgroups.MemoryData{
+					Usage: 1024,
 				},
 			},
-			paths: map[string]string{
-				"memory": expectedMemoryPath,
-			},
+		},
+		paths: map[string]string{
+			"memory": expectedMemoryPath,
 		},
 	}
 	container.state = &createdState{c: container}

libcontainer/factory_linux.go

@@ -18,6 +18,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/configs/validate"
+	"github.com/opencontainers/runc/libcontainer/resourcemanager"
 	"github.com/opencontainers/runc/libcontainer/utils"
 )
 
@@ -172,13 +173,15 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err
 		return nil, newGenericError(err, SystemError)
 	}
 	c := &linuxContainer{
-		id:            id,
-		root:          containerRoot,
-		config:        config,
-		initArgs:      l.InitArgs,
-		criuPath:      l.CriuPath,
-		cgroupManager: l.NewCgroupsManager(config.Cgroups, nil),
+		id:       id,
+		root:     containerRoot,
+		config:   config,
+		initArgs: l.InitArgs,
+		criuPath: l.CriuPath,
 	}
+	resourceManagers := make(map[string]resourcemanager.ResourceManager)
+	resourceManagers["cgroups"] = l.NewCgroupsManager(config.Cgroups, nil)
+	c.resourceManagers = resourceManagers
 	c.state = &stoppedState{c: c}
 	return c, nil
 }
@@ -204,10 +207,12 @@ func (l *LinuxFactory) Load(id string) (Container, error) {
 		config:               &state.Config,
 		initArgs:             l.InitArgs,
 		criuPath:             l.CriuPath,
-		cgroupManager:        l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths),
 		root:                 containerRoot,
 		created:              state.Created,
 	}
+	resourceManagers := make(map[string]resourcemanager.ResourceManager)
+	resourceManagers["cgroups"] = l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths)
+	c.resourceManagers = resourceManagers
 	c.state = &loadedState{c: c}
 	if err := c.refreshState(); err != nil {
 		return nil, err

libcontainer/process_linux.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/resourcemanager"
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/runc/libcontainer/utils"
 )
@@ -209,7 +210,7 @@ type initProcess struct {
 	parentPipe    *os.File
 	childPipe     *os.File
 	config        *initConfig
-	manager       cgroups.Manager
+	managers      map[string]resourcemanager.ResourceManager
 	container     *linuxContainer
 	fds           []string
 	process       *Process
@@ -280,14 +281,18 @@ func (p *initProcess) start() error {
 	}
 	p.setExternalDescriptors(fds)
 	// Do this before syncing with child so that no children
-	// can escape the cgroup
-	if err := p.manager.Apply(p.pid()); err != nil {
-		return newSystemErrorWithCause(err, "applying cgroup configuration for process")
+	// can escape
+	for name, manager := range p.managers {
+		if err := manager.Apply(p.pid()); err != nil {
+			return newSystemErrorWithCause(err, name+": applying configuration for process")
+		}
 	}
 	defer func() {
 		if err != nil {
 			// TODO: should not be the responsibility to call here
-			p.manager.Destroy()
+			for _, manager := range p.managers {
+				manager.Destroy()
+			}
 		}
 	}()
 	if err := p.createNetworkInterfaces(); err != nil {
@@ -323,8 +328,10 @@ func (p *initProcess) start() error {
 				return newSystemErrorWithCause(err, "writing syncT 'ack fd'")
 			}
 		case procReady:
-			if err := p.manager.Set(p.config.Config); err != nil {
-				return newSystemErrorWithCause(err, "setting cgroup config for ready process")
+			for name, manager := range p.managers {
+				if err := manager.Set(p.config.Config); err != nil {
+					return newSystemErrorWithCause(err, name+": setting config for ready process")
+				}
 			}
 			// set oom_score_adj
 			if err := setOomScoreAdj(p.config.Config.OomScoreAdj, p.pid()); err != nil {
@@ -407,7 +414,7 @@ func (p *initProcess) wait() (*os.ProcessState, error) {
 	}
 	// we should kill all processes in cgroup when init is died if we use host PID namespace
 	if p.sharePidns {
-		signalAllProcesses(p.manager, syscall.SIGKILL)
+		signalAllProcesses(p.managers["cgroups"], syscall.SIGKILL)
 	}
 	return p.cmd.ProcessState, nil
 }