Merge pull request opencontainers#2920 from marquiz/devel/rdt

libcontainer/intelrdt: support ClosID parameter

Author: kolyshkin

libcontainer/configs/intelrdt.go

@@ -1,6 +1,9 @@
 package configs
 
 type IntelRdt struct {
+	// The identity for RDT Class of Service
+	ClosID string `json:"closID,omitempty"`
+
 	// The schema for L3 cache id and capacity bitmask (CBM)
 	// Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
 	L3CacheSchema string `json:"l3_cache_schema,omitempty"`

libcontainer/configs/validate/validator.go

@@ -205,19 +205,16 @@ func (v *ConfigValidator) intelrdt(config *configs.Config) error {
 			return errors.New("intelRdt is specified in config, but Intel RDT is not supported or enabled")
 		}
 
+		if config.IntelRdt.ClosID == "." || config.IntelRdt.ClosID == ".." || strings.Contains(config.IntelRdt.ClosID, "/") {
+			return fmt.Errorf("invalid intelRdt.ClosID %q", config.IntelRdt.ClosID)
+		}
+
 		if !intelrdt.IsCATEnabled() && config.IntelRdt.L3CacheSchema != "" {
 			return errors.New("intelRdt.l3CacheSchema is specified in config, but Intel RDT/CAT is not enabled")
 		}
 		if !intelrdt.IsMBAEnabled() && config.IntelRdt.MemBwSchema != "" {
 			return errors.New("intelRdt.memBwSchema is specified in config, but Intel RDT/MBA is not enabled")
 		}
-
-		if intelrdt.IsCATEnabled() && config.IntelRdt.L3CacheSchema == "" {
-			return errors.New("Intel RDT/CAT is enabled and intelRdt is specified in config, but intelRdt.l3CacheSchema is empty")
-		}
-		if intelrdt.IsMBAEnabled() && config.IntelRdt.MemBwSchema == "" {
-			return errors.New("Intel RDT/MBA is enabled and intelRdt is specified in config, but intelRdt.memBwSchema is empty")
-		}
 	}
 
 	return nil

libcontainer/container_linux.go

@@ -1902,9 +1902,10 @@ func (c *linuxContainer) currentState() (*State, error) {
 		startTime, _ = c.initProcess.startTime()
 		externalDescriptors = c.initProcess.externalDescriptors()
 	}
-	intelRdtPath, err := intelrdt.GetIntelRdtPath(c.ID())
-	if err != nil {
-		intelRdtPath = ""
+
+	intelRdtPath := ""
+	if c.intelRdtManager != nil {
+		intelRdtPath = c.intelRdtManager.GetPath()
 	}
 	state := &State{
 		BaseState: BaseState{

libcontainer/intelrdt/intelrdt.go

@@ -72,7 +72,7 @@ import (
  * |-- ...
  * |-- schemata
  * |-- tasks
- * |-- <container_id>
+ * |-- <clos>
  *     |-- ...
  *     |-- schemata
  *     |-- tasks
@@ -155,7 +155,7 @@ type Manager interface {
 	// Returns statistics for Intel RDT
 	GetStats() (*Stats, error)
 
-	// Destroys the Intel RDT 'container_id' group
+	// Destroys the Intel RDT container-specific 'container_id' group
 	Destroy() error
 
 	// Returns Intel RDT path to save in a state file and to be able to
@@ -205,9 +205,7 @@ var (
 )
 
 type intelRdtData struct {
-	root   string
 	config *configs.Config
-	pid    int
 }
 
 // Check if Intel RDT sub-features are enabled in featuresInit()
@@ -405,18 +403,6 @@ func writeFile(dir, file, data string) error {
 	return nil
 }
 
-func getIntelRdtData(c *configs.Config, pid int) (*intelRdtData, error) {
-	rootPath, err := getIntelRdtRoot()
-	if err != nil {
-		return nil, err
-	}
-	return &intelRdtData{
-		root:   rootPath,
-		config: c,
-		pid:    pid,
-	}, nil
-}
-
 // Get the read-only L3 cache information
 func getL3CacheInfo() (*L3CacheInfo, error) {
 	l3CacheInfo := &L3CacheInfo{}
@@ -531,15 +517,19 @@ func IsMBAScEnabled() bool {
 	return mbaScEnabled
 }
 
-// Get the 'container_id' path in Intel RDT "resource control" filesystem
-func GetIntelRdtPath(id string) (string, error) {
+// Get the path of the clos group in "resource control" filesystem that the container belongs to
+func (m *intelRdtManager) getIntelRdtPath() (string, error) {
 	rootPath, err := getIntelRdtRoot()
 	if err != nil {
 		return "", err
 	}
 
-	path := filepath.Join(rootPath, id)
-	return path, nil
+	clos := m.id
+	if m.config.IntelRdt != nil && m.config.IntelRdt.ClosID != "" {
+		clos = m.config.IntelRdt.ClosID
+	}
+
+	return filepath.Join(rootPath, clos), nil
 }
 
 // Applies Intel RDT configuration to the process with the specified pid
@@ -548,38 +538,56 @@ func (m *intelRdtManager) Apply(pid int) (err error) {
 	if m.config.IntelRdt == nil {
 		return nil
 	}
-	d, err := getIntelRdtData(m.config, pid)
+
+	path, err := m.getIntelRdtPath()
 	if err != nil {
 		return err
 	}
 
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	path, err := d.join(m.id)
-	if err != nil {
-		return err
+
+	if m.config.IntelRdt.ClosID != "" && m.config.IntelRdt.L3CacheSchema == "" && m.config.IntelRdt.MemBwSchema == "" {
+		// Check that the CLOS exists, i.e. it has been pre-configured to
+		// conform with the runtime spec
+		if _, err := os.Stat(path); err != nil {
+			return fmt.Errorf("clos dir not accessible (must be pre-created when l3CacheSchema and memBwSchema are empty): %w", err)
+		}
+	}
+
+	if err := os.MkdirAll(path, 0o755); err != nil {
+		return newLastCmdError(err)
+	}
+
+	if err := WriteIntelRdtTasks(path, pid); err != nil {
+		return newLastCmdError(err)
 	}
 
 	m.path = path
 	return nil
 }
 
-// Destroys the Intel RDT 'container_id' group
+// Destroys the Intel RDT container-specific 'container_id' group
 func (m *intelRdtManager) Destroy() error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if err := os.RemoveAll(m.GetPath()); err != nil {
-		return err
+	// Don't remove resctrl group if closid has been explicitly specified. The
+	// group is likely externally managed, i.e. by some other entity than us.
+	// There are probably other containers/tasks sharing the same group.
+	if m.config.IntelRdt == nil || m.config.IntelRdt.ClosID == "" {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		if err := os.RemoveAll(m.GetPath()); err != nil {
+			return err
+		}
+		m.path = ""
 	}
-	m.path = ""
 	return nil
 }
 
 // Returns Intel RDT path to save in a state file and to be able to
 // restore the object later
 func (m *intelRdtManager) GetPath() string {
 	if m.path == "" {
-		m.path, _ = GetIntelRdtPath(m.id)
+		m.path, _ = m.getIntelRdtPath()
 	}
 	return m.path
 }
@@ -606,7 +614,7 @@ func (m *intelRdtManager) GetStats() (*Stats, error) {
 	}
 	schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
-	// The L3 cache and memory bandwidth schemata in 'container_id' group
+	// The L3 cache and memory bandwidth schemata in container's clos group
 	containerPath := m.GetPath()
 	tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
 	if err != nil {
@@ -629,7 +637,7 @@ func (m *intelRdtManager) GetStats() (*Stats, error) {
 			}
 		}
 
-		// The L3 cache schema in 'container_id' group
+		// The L3 cache schema in container's clos group
 		for _, schema := range schemaStrings {
 			if strings.Contains(schema, "L3") {
 				stats.L3CacheSchema = strings.TrimSpace(schema)
@@ -652,7 +660,7 @@ func (m *intelRdtManager) GetStats() (*Stats, error) {
 			}
 		}
 
-		// The memory bandwidth schema in 'container_id' group
+		// The memory bandwidth schema in container's clos group
 		for _, schema := range schemaStrings {
 			if strings.Contains(schema, "MB") {
 				stats.MemBwSchema = strings.TrimSpace(schema)
@@ -722,6 +730,12 @@ func (m *intelRdtManager) Set(container *configs.Config) error {
 		l3CacheSchema := container.IntelRdt.L3CacheSchema
 		memBwSchema := container.IntelRdt.MemBwSchema
 
+		// TODO: verify that l3CacheSchema and/or memBwSchema match the
+		// existing schemata if ClosID has been specified. This is a more
+		// involved than reading the file and doing plain string comparison as
+		// the value written in does not necessarily match what gets read out
+		// (leading zeros, cache id ordering etc).
+
 		// Write a single joint schema string to schemata file
 		if l3CacheSchema != "" && memBwSchema != "" {
 			if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
@@ -747,18 +761,6 @@ func (m *intelRdtManager) Set(container *configs.Config) error {
 	return nil
 }
 
-func (raw *intelRdtData) join(id string) (string, error) {
-	path := filepath.Join(raw.root, id)
-	if err := os.MkdirAll(path, 0o755); err != nil {
-		return "", newLastCmdError(err)
-	}
-
-	if err := WriteIntelRdtTasks(path, raw.pid); err != nil {
-		return "", err
-	}
-	return path, nil
-}
-
 func newLastCmdError(err error) error {
 	status, err1 := getLastCmdStatus()
 	if err1 == nil {

libcontainer/intelrdt/intelrdt_test.go

@@ -5,15 +5,13 @@ package intelrdt
 import (
 	"errors"
 	"io"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 )
 
 func TestIntelRdtSetL3CacheSchema(t *testing.T) {
-	if !IsCATEnabled() {
-		return
-	}
-
 	helper := NewIntelRdtTestUtil(t)
 
 	const (
@@ -44,10 +42,6 @@ func TestIntelRdtSetL3CacheSchema(t *testing.T) {
 }
 
 func TestIntelRdtSetMemBwSchema(t *testing.T) {
-	if !IsMBAEnabled() {
-		return
-	}
-
 	helper := NewIntelRdtTestUtil(t)
 
 	const (
@@ -78,10 +72,6 @@ func TestIntelRdtSetMemBwSchema(t *testing.T) {
 }
 
 func TestIntelRdtSetMemBwScSchema(t *testing.T) {
-	if !IsMBAScEnabled() {
-		return
-	}
-
 	helper := NewIntelRdtTestUtil(t)
 
 	const (
@@ -111,6 +101,35 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
 	}
 }
 
+func TestApply(t *testing.T) {
+	helper := NewIntelRdtTestUtil(t)
+
+	const closID = "test-clos"
+
+	helper.IntelRdtData.config.IntelRdt.ClosID = closID
+	intelrdt := NewManager(helper.IntelRdtData.config, "", helper.IntelRdtPath)
+	if err := intelrdt.Apply(1234); err == nil {
+		t.Fatal("unexpected success when applying pid")
+	}
+	if _, err := os.Stat(filepath.Join(helper.IntelRdtPath, closID)); err == nil {
+		t.Fatal("closid dir should not exist")
+	}
+
+	// Dir should be created if some schema has been specified
+	intelrdt.(*intelRdtManager).config.IntelRdt.L3CacheSchema = "L3:0=f"
+	if err := intelrdt.Apply(1235); err != nil {
+		t.Fatalf("Apply() failed: %v", err)
+	}
+
+	pids, err := getIntelRdtParamString(intelrdt.GetPath(), "tasks")
+	if err != nil {
+		t.Fatalf("failed to read tasks file: %v", err)
+	}
+	if pids != "1235" {
+		t.Fatalf("unexpected tasks file, expected '1235', got %q", pids)
+	}
+}
+
 const (
 	mountinfoValid = `18 40 0:18 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw
 19 40 0:3 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw

libcontainer/intelrdt/util_test.go

@@ -30,9 +30,9 @@ func NewIntelRdtTestUtil(t *testing.T) *intelRdtTestUtil {
 		config: &configs.Config{
 			IntelRdt: &configs.IntelRdt{},
 		},
-		root: t.TempDir(),
 	}
-	testIntelRdtPath := filepath.Join(d.root, "resctrl")
+	intelRdtRoot = t.TempDir()
+	testIntelRdtPath := filepath.Join(intelRdtRoot, "resctrl")
 
 	// Ensure the full mock Intel RDT "resource control" filesystem path exists
 	if err := os.MkdirAll(testIntelRdtPath, 0o755); err != nil {

libcontainer/specconv/spec_linux.go

@@ -301,6 +301,7 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 		}
 		if spec.Linux.IntelRdt != nil {
 			config.IntelRdt = &configs.IntelRdt{
+				ClosID:        spec.Linux.IntelRdt.ClosID,
 				L3CacheSchema: spec.Linux.IntelRdt.L3CacheSchema,
 				MemBwSchema:   spec.Linux.IntelRdt.MemBwSchema,
 			}