Skip to content

Commit 2852ea3

Browse files
author
Paweł Szulik
committed
Add mon groups for resctrl.
"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON group that is their ancestor. More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt Signed-off-by: Paweł Szulik <[email protected]>
1 parent e5f2eae commit 2852ea3

File tree

3 files changed

+207
-95
lines changed

3 files changed

+207
-95
lines changed

‎libcontainer/intelrdt/intelrdt.go‎

Lines changed: 167 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
"strings"
1313
"sync"
1414

15+
"github.com/sirupsen/logrus"
16+
1517
"github.com/opencontainers/runc/libcontainer/configs"
1618
)
1719

@@ -167,10 +169,17 @@ type IntelRdtManager struct {
167169
Config *configs.Config
168170
Id string
169171
Path string
172+
Type string
170173
}
171174

172175
const (
173-
IntelRdtTasks = "tasks"
176+
IntelRdtTasks = "tasks"
177+
MonitoringGroupDirectoryName = "mon_groups"
178+
179+
// See "Resource alloc and monitor groups" part here:
180+
// https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
181+
MonitoringGroupType = "MON"
182+
ControlGroupType = "CTRL_MON"
174183
)
175184

176185
var (
@@ -548,26 +557,47 @@ func GetIntelRdtPath(id string) (string, error) {
548557
return path, nil
549558
}
550559

560+
// Get the 'container_id" path in Intel RDT "monitoring group" filesystem.
561+
func getMonGroupIntelRdtPath(id string) (string, error) {
562+
rootPath, err := getIntelRdtRoot()
563+
if err != nil {
564+
return "", err
565+
}
566+
567+
path := filepath.Join(rootPath, MonitoringGroupDirectoryName, id)
568+
return path, nil
569+
}
570+
551571
// Applies Intel RDT configuration to the process with the specified pid
552572
func (m *IntelRdtManager) Apply(pid int) (err error) {
553-
// If intelRdt is not specified in config, we do nothing
554-
if m.Config.IntelRdt == nil {
573+
switch m.Type {
574+
case ControlGroupType:
575+
// If intelRdt is not specified in config, we do nothing
576+
if m.Config.IntelRdt == nil {
577+
return nil
578+
}
579+
rdtData, err := getIntelRdtData(m.Config, pid)
580+
if err != nil && !IsNotFound(err) {
581+
return err
582+
}
583+
584+
m.mu.Lock()
585+
defer m.mu.Unlock()
586+
path, err := rdtData.join(m.Id)
587+
if err != nil {
588+
return err
589+
}
590+
591+
m.Path = path
555592
return nil
556-
}
557-
d, err := getIntelRdtData(m.Config, pid)
558-
if err != nil && !IsNotFound(err) {
559-
return err
560-
}
561593

562-
m.mu.Lock()
563-
defer m.mu.Unlock()
564-
path, err := d.join(m.Id)
565-
if err != nil {
566-
return err
594+
case MonitoringGroupType:
595+
m.mu.Lock()
596+
defer m.mu.Unlock()
597+
return WriteIntelRdtTasks(m.Path, pid)
567598
}
568599

569-
m.Path = path
570-
return nil
600+
return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type)
571601
}
572602

573603
// Destroys the Intel RDT 'container_id' group
@@ -584,94 +614,124 @@ func (m *IntelRdtManager) Destroy() error {
584614
// Returns Intel RDT path to save in a state file and to be able to
585615
// restore the object later
586616
func (m *IntelRdtManager) GetPath() string {
617+
var err error
587618
if m.Path == "" {
588-
m.Path, _ = GetIntelRdtPath(m.Id)
619+
switch m.Type {
620+
case ControlGroupType:
621+
m.Path, err = GetIntelRdtPath(m.Id)
622+
if err != nil {
623+
logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err)
624+
}
625+
case MonitoringGroupType:
626+
flattedContainerID := strings.Replace(m.Id, "/", "-", -1)
627+
m.Path, err = getMonGroupIntelRdtPath(flattedContainerID)
628+
if err != nil {
629+
logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err)
630+
}
631+
}
589632
}
590633
return m.Path
591634
}
592635

593636
// Returns statistics for Intel RDT
594637
func (m *IntelRdtManager) GetStats() (*Stats, error) {
595-
// If intelRdt is not specified in config
596-
if m.Config.IntelRdt == nil {
597-
return nil, nil
598-
}
599-
600638
m.mu.Lock()
601639
defer m.mu.Unlock()
602640
stats := NewStats()
603641

604-
rootPath, err := getIntelRdtRoot()
605-
if err != nil {
606-
return nil, err
607-
}
608-
// The read-only L3 cache and memory bandwidth schemata in root
609-
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
610-
if err != nil {
611-
return nil, err
612-
}
613-
schemaRootStrings := strings.Split(tmpRootStrings, "\n")
614-
615-
// The L3 cache and memory bandwidth schemata in 'container_id' group
616-
containerPath := m.GetPath()
617-
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
618-
if err != nil {
619-
return nil, err
620-
}
621-
schemaStrings := strings.Split(tmpStrings, "\n")
642+
switch m.Type {
643+
case ControlGroupType:
644+
containerPath := m.GetPath()
622645

623-
if IsCatEnabled() {
624-
// The read-only L3 cache information
625-
l3CacheInfo, err := getL3CacheInfo()
646+
err := getMonitoringStats(containerPath, stats)
626647
if err != nil {
627648
return nil, err
628649
}
629-
stats.L3CacheInfo = l3CacheInfo
630650

631-
// The read-only L3 cache schema in root
632-
for _, schemaRoot := range schemaRootStrings {
633-
if strings.Contains(schemaRoot, "L3") {
634-
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
651+
// If intelRdt is not specified in config
652+
if m.Config != nil {
653+
if m.Config.IntelRdt == nil {
654+
return nil, nil
635655
}
636-
}
637656

638-
// The L3 cache schema in 'container_id' group
639-
for _, schema := range schemaStrings {
640-
if strings.Contains(schema, "L3") {
641-
stats.L3CacheSchema = strings.TrimSpace(schema)
657+
rootPath, err := getIntelRdtRoot()
658+
if err != nil {
659+
return nil, err
642660
}
643-
}
644-
}
661+
// The read-only L3 cache and memory bandwidth schemata in root
662+
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
663+
if err != nil {
664+
return nil, err
665+
}
666+
schemaRootStrings := strings.Split(tmpRootStrings, "\n")
645667

646-
if IsMbaEnabled() {
647-
// The read-only memory bandwidth information
648-
memBwInfo, err := getMemBwInfo()
649-
if err != nil {
650-
return nil, err
651-
}
652-
stats.MemBwInfo = memBwInfo
668+
// The L3 cache and memory bandwidth schemata in 'container_id' group
653669

654-
// The read-only memory bandwidth information
655-
for _, schemaRoot := range schemaRootStrings {
656-
if strings.Contains(schemaRoot, "MB") {
657-
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
670+
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
671+
if err != nil {
672+
return nil, err
658673
}
659-
}
674+
schemaStrings := strings.Split(tmpStrings, "\n")
675+
676+
if IsCatEnabled() {
677+
// The read-only L3 cache information
678+
l3CacheInfo, err := getL3CacheInfo()
679+
if err != nil {
680+
return nil, err
681+
}
682+
stats.L3CacheInfo = l3CacheInfo
660683

661-
// The memory bandwidth schema in 'container_id' group
662-
for _, schema := range schemaStrings {
663-
if strings.Contains(schema, "MB") {
664-
stats.MemBwSchema = strings.TrimSpace(schema)
684+
// The read-only L3 cache schema in root
685+
for _, schemaRoot := range schemaRootStrings {
686+
if strings.Contains(schemaRoot, "L3") {
687+
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
688+
}
689+
}
690+
691+
// The L3 cache schema in 'container_id' group
692+
for _, schema := range schemaStrings {
693+
if strings.Contains(schema, "L3") {
694+
stats.L3CacheSchema = strings.TrimSpace(schema)
695+
}
696+
}
697+
}
698+
699+
if IsMbaEnabled() {
700+
// The read-only memory bandwidth information
701+
memBwInfo, err := getMemBwInfo()
702+
if err != nil {
703+
return nil, err
704+
}
705+
stats.MemBwInfo = memBwInfo
706+
707+
// The read-only memory bandwidth information
708+
for _, schemaRoot := range schemaRootStrings {
709+
if strings.Contains(schemaRoot, "MB") {
710+
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
711+
}
712+
}
713+
714+
// The memory bandwidth schema in 'container_id' group
715+
for _, schema := range schemaStrings {
716+
if strings.Contains(schema, "MB") {
717+
stats.MemBwSchema = strings.TrimSpace(schema)
718+
}
719+
}
665720
}
666721
}
667-
}
668722

669-
err = getMonitoringStats(containerPath, stats)
670-
if err != nil {
671-
return nil, err
672-
}
723+
return stats, nil
724+
725+
case MonitoringGroupType:
726+
path := m.GetPath()
673727

674-
return stats, nil
728+
err := getMonitoringStats(path, stats)
729+
if err != nil {
730+
return nil, err
731+
}
732+
return stats, nil
733+
}
734+
return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type)
675735
}
676736

677737
// Set Intel RDT "resource control" filesystem as configured.
@@ -721,34 +781,46 @@ func (m *IntelRdtManager) Set(container *configs.Config) error {
721781
// For example, on a two-socket machine, the schema line could be
722782
// "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on
723783
// socket 0 and 7000 MBps memory bandwidth limit on socket 1.
724-
if container.IntelRdt != nil {
725-
path := m.GetPath()
726-
l3CacheSchema := container.IntelRdt.L3CacheSchema
727-
memBwSchema := container.IntelRdt.MemBwSchema
784+
switch m.Type {
785+
case ControlGroupType:
786+
if container.IntelRdt != nil {
787+
path := m.GetPath()
788+
l3CacheSchema := container.IntelRdt.L3CacheSchema
789+
memBwSchema := container.IntelRdt.MemBwSchema
790+
791+
// Write a single joint schema string to schemata file
792+
if l3CacheSchema != "" && memBwSchema != "" {
793+
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
794+
return NewLastCmdError(err)
795+
}
796+
}
728797

729-
// Write a single joint schema string to schemata file
730-
if l3CacheSchema != "" && memBwSchema != "" {
731-
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
732-
return NewLastCmdError(err)
798+
// Write only L3 cache schema string to schemata file
799+
if l3CacheSchema != "" && memBwSchema == "" {
800+
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
801+
return NewLastCmdError(err)
802+
}
733803
}
734-
}
735804

736-
// Write only L3 cache schema string to schemata file
737-
if l3CacheSchema != "" && memBwSchema == "" {
738-
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
739-
return NewLastCmdError(err)
805+
// Write only memory bandwidth schema string to schemata file
806+
if l3CacheSchema == "" && memBwSchema != "" {
807+
if err := writeFile(path, "schemata", memBwSchema); err != nil {
808+
return NewLastCmdError(err)
809+
}
740810
}
741811
}
742812

743-
// Write only memory bandwidth schema string to schemata file
744-
if l3CacheSchema == "" && memBwSchema != "" {
745-
if err := writeFile(path, "schemata", memBwSchema); err != nil {
746-
return NewLastCmdError(err)
747-
}
813+
case MonitoringGroupType:
814+
if _, err := os.Stat(m.GetPath()); err == nil {
815+
return nil
816+
}
817+
if err := os.Mkdir(m.GetPath(), 0755); err != nil {
818+
return err
748819
}
820+
return nil
749821
}
750822

751-
return nil
823+
return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type)
752824
}
753825

754826
func (raw *intelRdtData) join(id string) (string, error) {

‎libcontainer/intelrdt/intelrdt_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package intelrdt
44

55
import (
6+
"errors"
67
"strings"
78
"testing"
89
)
@@ -120,3 +121,41 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
120121
t.Fatal("Got the wrong value, set 'schemata' failed.")
121122
}
122123
}
124+
125+
func TestIntelRdtManager_GetStats_NotSupported_Type(t *testing.T) {
126+
manager := IntelRdtManager{
127+
Type: "not_supported",
128+
Id: "id",
129+
}
130+
131+
_, err := manager.GetStats()
132+
133+
expectedError := errors.New("couldn't obtain stats from: \"id\" resctrl manager of type: \"not_supported\"")
134+
135+
if err == nil {
136+
t.Fatalf("Expected error: %v, got nil.", expectedError)
137+
}
138+
139+
if err.Error() != expectedError.Error() {
140+
t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
141+
}
142+
}
143+
144+
func Test_IntelRdtManager_Set_NotSupported_Type(t *testing.T) {
145+
manager := IntelRdtManager{
146+
Type: "not_supported",
147+
Id: "id",
148+
}
149+
150+
err := manager.Set(nil)
151+
152+
expectedError := errors.New("couldn't set configuration for: \"id\" resctrl manager of type: \"not_supported\"")
153+
154+
if err == nil {
155+
t.Fatalf("Expected error: %v, got nil.", expectedError)
156+
}
157+
158+
if err.Error() != expectedError.Error() {
159+
t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
160+
}
161+
}

‎update.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ other options are ignored.
321321
Config: &config,
322322
Id: container.ID(),
323323
Path: state.IntelRdtPath,
324+
Type: intelrdt.ControlGroupType,
324325
}
325326
if err := intelRdtManager.Apply(state.InitProcessPid); err != nil {
326327
return err

0 commit comments

Comments
 (0)