Merge pull request opencontainers#1279 from xiaochenshen/rdt-cat-resource-manager-v1

libcontainer: add support for Intel RDT/CAT in runc

Author: Mrunal Patel

events.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/intelrdt"
 
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -25,11 +26,12 @@ type event struct {
 
 // stats is the runc specific stats structure for stability when encoding and decoding stats.
 type stats struct {
-	CPU     cpu                `json:"cpu"`
-	Memory  memory             `json:"memory"`
-	Pids    pids               `json:"pids"`
-	Blkio   blkio              `json:"blkio"`
-	Hugetlb map[string]hugetlb `json:"hugetlb"`
+	CPU      cpu                `json:"cpu"`
+	Memory   memory             `json:"memory"`
+	Pids     pids               `json:"pids"`
+	Blkio    blkio              `json:"blkio"`
+	Hugetlb  map[string]hugetlb `json:"hugetlb"`
+	IntelRdt intelRdt           `json:"intel_rdt"`
 }
 
 type hugetlb struct {
@@ -96,6 +98,23 @@ type memory struct {
 	Raw       map[string]uint64 `json:"raw,omitempty"`
 }
 
+type l3CacheInfo struct {
+	CbmMask    string `json:"cbm_mask,omitempty"`
+	MinCbmBits uint64 `json:"min_cbm_bits,omitempty"`
+	NumClosids uint64 `json:"num_closids,omitempty"`
+}
+
+type intelRdt struct {
+	// The read-only L3 cache information
+	L3CacheInfo *l3CacheInfo `json:"l3_cache_info,omitempty"`
+
+	// The read-only L3 cache schema in root
+	L3CacheSchemaRoot string `json:"l3_cache_schema_root,omitempty"`
+
+	// The L3 cache schema in 'container_id' group
+	L3CacheSchema string `json:"l3_cache_schema,omitempty"`
+}
+
 var eventsCommand = cli.Command{
 	Name:  "events",
 	Usage: "display container events such as OOM notifications, cpu, memory, and IO usage statistics",
@@ -227,6 +246,13 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *stats {
 	for k, v := range cg.HugetlbStats {
 		s.Hugetlb[k] = convertHugtlb(v)
 	}
+
+	if is := ls.IntelRdtStats; is != nil {
+		s.IntelRdt.L3CacheInfo = convertL3CacheInfo(is.L3CacheInfo)
+		s.IntelRdt.L3CacheSchemaRoot = is.L3CacheSchemaRoot
+		s.IntelRdt.L3CacheSchema = is.L3CacheSchema
+	}
+
 	return &s
 }
 
@@ -259,3 +285,11 @@ func convertBlkioEntry(c []cgroups.BlkioStatEntry) []blkioEntry {
 	}
 	return out
 }
+
+func convertL3CacheInfo(i *intelrdt.L3CacheInfo) *l3CacheInfo {
+	return &l3CacheInfo{
+		CbmMask:    i.CbmMask,
+		MinCbmBits: i.MinCbmBits,
+		NumClosids: i.NumClosids,
+	}
+}

libcontainer/SPEC.md

@@ -154,6 +154,90 @@ that no processes or threads escape the cgroups.  This sync is
 done via a pipe ( specified in the runtime section below ) that the container's
 init process will block waiting for the parent to finish setup.
 
+### IntelRdt
+
+Intel platforms with new Xeon CPU support Intel Resource Director Technology
+(RDT). Cache Allocation Technology (CAT) is a sub-feature of RDT, which
+currently supports L3 cache resource allocation.
+
+This feature provides a way for the software to restrict cache allocation to a
+defined 'subset' of L3 cache which may be overlapping with other 'subsets'.
+The different subsets are identified by class of service (CLOS) and each CLOS
+has a capacity bitmask (CBM).
+
+It can be used to handle L3 cache resource allocation for containers if
+hardware and kernel support Intel RDT/CAT.
+
+In Linux 4.10 kernel or newer, the interface is defined and exposed via
+"resource control" filesystem, which is a "cgroup-like" interface.
+
+Comparing with cgroups, it has similar process management lifecycle and
+interfaces in a container. But unlike cgroups' hierarchy, it has single level
+filesystem layout.
+
+Intel RDT "resource control" filesystem hierarchy:
+```
+mount -t resctrl resctrl /sys/fs/resctrl
+tree /sys/fs/resctrl
+/sys/fs/resctrl/
+|-- info
+|   |-- L3
+|       |-- cbm_mask
+|       |-- min_cbm_bits
+|       |-- num_closids
+|-- cpus
+|-- schemata
+|-- tasks
+|-- <container_id>
+    |-- cpus
+    |-- schemata
+    |-- tasks
+
+```
+
+For runc, we can make use of `tasks` and `schemata` configuration for L3 cache
+resource constraints.
+
+The file `tasks` has a list of tasks that belongs to this group (e.g.,
+<container_id>" group). Tasks can be added to a group by writing the task ID
+to the "tasks" file  (which will automatically remove them from the previous
+group to which they belonged). New tasks created by fork(2) and clone(2) are
+added to the same group as their parent. If a pid is not in any sub group, it
+is in root group.
+
+The file `schemata` has allocation masks/values for L3 cache on each socket,
+which contains L3 cache id and capacity bitmask (CBM).
+```
+	Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
+```
+For example, on a two-socket machine, L3's schema line could be `L3:0=ff;1=c0`
+Which means L3 cache id 0's CBM is 0xff, and L3 cache id 1's CBM is 0xc0.
+
+The valid L3 cache CBM is a *contiguous bits set* and number of bits that can
+be set is less than the max bit. The max bits in the CBM is varied among
+supported Intel Xeon platforms. In Intel RDT "resource control" filesystem
+layout, the CBM in a group should be a subset of the CBM in root. Kernel will
+check if it is valid when writing. e.g., 0xfffff in root indicates the max bits
+of CBM is 20 bits, which mapping to entire L3 cache capacity. Some valid CBM
+values to set in a group: 0xf, 0xf0, 0x3ff, 0x1f00 and etc.
+
+For more information about Intel RDT/CAT kernel interface:  
+https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
+
+An example for runc:
+```
+Consider a two-socket machine with two L3 caches where the default CBM is
+0xfffff and the max CBM length is 20 bits. With this configuration, tasks
+inside the container only have access to the "upper" 80% of L3 cache id 0 and
+the "lower" 50% L3 cache id 1:
+
+"linux": {
+	"intelRdt": {
+		"l3CacheSchema": "L3:0=ffff0;1=3ff"
+	}
+}
+```
+
 ### Security 
 
 The standard set of Linux capabilities that are set in a container

libcontainer/configs/config.go

@@ -187,6 +187,10 @@ type Config struct {
 
 	// Rootless specifies whether the container is a rootless container.
 	Rootless bool `json:"rootless"`
+
+	// IntelRdt specifies settings for Intel RDT/CAT group that the container is placed into
+	// to limit the resources (e.g., L3 cache) the container has available
+	IntelRdt *IntelRdt `json:"intel_rdt,omitempty"`
 }
 
 type Hooks struct {

libcontainer/configs/intelrdt.go

@@ -0,0 +1,7 @@
+package configs
+
+type IntelRdt struct {
+	// The schema for L3 cache id and capacity bitmask (CBM)
+	// Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
+	L3CacheSchema string `json:"l3_cache_schema,omitempty"`
+}

libcontainer/configs/validate/validator.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/intelrdt"
 	selinux "github.com/opencontainers/selinux/go-selinux"
 )
 
@@ -40,6 +41,9 @@ func (v *ConfigValidator) Validate(config *configs.Config) error {
 	if err := v.sysctl(config); err != nil {
 		return err
 	}
+	if err := v.intelrdt(config); err != nil {
+		return err
+	}
 	if config.Rootless {
 		if err := v.rootless(config); err != nil {
 			return err
@@ -153,6 +157,19 @@ func (v *ConfigValidator) sysctl(config *configs.Config) error {
 	return nil
 }
 
+func (v *ConfigValidator) intelrdt(config *configs.Config) error {
+	if config.IntelRdt != nil {
+		if !intelrdt.IsIntelRdtEnabled() {
+			return fmt.Errorf("intelRdt is specified in config, but Intel RDT feature is not supported or enabled")
+		}
+		if config.IntelRdt.L3CacheSchema == "" {
+			return fmt.Errorf("intelRdt is specified in config, but intelRdt.l3CacheSchema is empty")
+		}
+	}
+
+	return nil
+}
+
 func isSymbolicLink(path string) (bool, error) {
 	fi, err := os.Lstat(path)
 	if err != nil {

libcontainer/container_linux.go

@@ -21,6 +21,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/criurpc"
+	"github.com/opencontainers/runc/libcontainer/intelrdt"
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/runc/libcontainer/utils"
 
@@ -38,6 +39,7 @@ type linuxContainer struct {
 	root                 string
 	config               *configs.Config
 	cgroupManager        cgroups.Manager
+	intelRdtManager      intelrdt.Manager
 	initArgs             []string
 	initProcess          parentProcess
 	initProcessStartTime uint64
@@ -67,6 +69,9 @@ type State struct {
 
 	// Container's standard descriptors (std{in,out,err}), needed for checkpoint and restore
 	ExternalDescriptors []string `json:"external_descriptors,omitempty"`
+
+	// Intel RDT "resource control" filesystem path
+	IntelRdtPath string `json:"intel_rdt_path"`
 }
 
 // Container is a libcontainer container object.
@@ -163,6 +168,11 @@ func (c *linuxContainer) Stats() (*Stats, error) {
 	if stats.CgroupStats, err = c.cgroupManager.GetStats(); err != nil {
 		return stats, newSystemErrorWithCause(err, "getting container stats from cgroups")
 	}
+	if c.intelRdtManager != nil {
+		if stats.IntelRdtStats, err = c.intelRdtManager.GetStats(); err != nil {
+			return stats, newSystemErrorWithCause(err, "getting container's Intel RDT stats")
+		}
+	}
 	for _, iface := range c.config.Networks {
 		switch iface.Type {
 		case "veth":
@@ -193,6 +203,15 @@ func (c *linuxContainer) Set(config configs.Config) error {
 		}
 		return err
 	}
+	if c.intelRdtManager != nil {
+		if err := c.intelRdtManager.Set(&config); err != nil {
+			// Set configs back
+			if err2 := c.intelRdtManager.Set(c.config); err2 != nil {
+				logrus.Warnf("Setting back intelrdt configs failed due to error: %v, your state.json and actual configs might be inconsistent.", err2)
+			}
+			return err
+		}
+	}
 	// After config setting succeed, update config and states
 	c.config = &config
 	_, err = c.updateState(nil)
@@ -434,15 +453,16 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c
 		return nil, err
 	}
 	return &initProcess{
-		cmd:           cmd,
-		childPipe:     childPipe,
-		parentPipe:    parentPipe,
-		manager:       c.cgroupManager,
-		config:        c.newInitConfig(p),
-		container:     c,
-		process:       p,
-		bootstrapData: data,
-		sharePidns:    sharePidns,
+		cmd:             cmd,
+		childPipe:       childPipe,
+		parentPipe:      parentPipe,
+		manager:         c.cgroupManager,
+		intelRdtManager: c.intelRdtManager,
+		config:          c.newInitConfig(p),
+		container:       c,
+		process:         p,
+		bootstrapData:   data,
+		sharePidns:      sharePidns,
 	}, nil
 }
 
@@ -461,6 +481,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe,
 	return &setnsProcess{
 		cmd:           cmd,
 		cgroupPaths:   c.cgroupManager.GetPaths(),
+		intelRdtPath:  state.IntelRdtPath,
 		childPipe:     childPipe,
 		parentPipe:    parentPipe,
 		config:        c.newInitConfig(p),
@@ -1519,6 +1540,10 @@ func (c *linuxContainer) currentState() (*State, error) {
 		startTime, _ = c.initProcess.startTime()
 		externalDescriptors = c.initProcess.externalDescriptors()
 	}
+	intelRdtPath, err := intelrdt.GetIntelRdtPath(c.ID())
+	if err != nil {
+		intelRdtPath = ""
+	}
 	state := &State{
 		BaseState: BaseState{
 			ID:                   c.ID(),
@@ -1529,6 +1554,7 @@ func (c *linuxContainer) currentState() (*State, error) {
 		},
 		Rootless:            c.config.Rootless,
 		CgroupPaths:         c.cgroupManager.GetPaths(),
+		IntelRdtPath:        intelRdtPath,
 		NamespacePaths:      make(map[configs.NamespaceType]string),
 		ExternalDescriptors: externalDescriptors,
 	}