libct/cg/sd/v2: support cpuset.* / Allowed*

 * cpuset.cpus -> AllowedCPUs
 * cpuset.mems -> AllowedMemoryNodes

No test for cgroup v2 resources.unified override, as this requires a
separate test case, and all the unified resources are handled uniformly
so there's little sense to test all parameters.

Signed-off-by: Kir Kolyshkin <[email protected]>

Author: kolyshkin

go.mod

@@ -22,5 +22,6 @@ require (
 	// NOTE: urfave/cli must be <= v1.22.1 due to a regression: https://github.com/urfave/cli/issues/1092
 	github.com/urfave/cli v1.22.1
 	github.com/vishvananda/netlink v1.1.0
+	github.com/willf/bitset v1.1.11
 	golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f
 )

go.sum

@@ -62,6 +62,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7Zo
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 h1:R43TdZy32XXSXjJn7M/HhALJ9imq6ztLnChfYJpVDnM=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
+github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

libcontainer/cgroups/systemd/cpuset.go

@@ -0,0 +1,67 @@
+package systemd
+
+import (
+	"encoding/binary"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/willf/bitset"
+)
+
+// rangeToBits converts a text representation of a CPU mask (as written to
+// or read from cgroups' cpuset.* files, e.g. "1,3-5") to a slice of bytes
+// with the corresponding bits set (as consumed by systemd over dbus as
+// AllowedCPUs/AllowedMemoryNodes unit property value).
+func rangeToBits(str string) ([]byte, error) {
+	bits := &bitset.BitSet{}
+
+	for _, r := range strings.Split(str, ",") {
+		// allow extra spaces around
+		r = strings.TrimSpace(r)
+		// allow empty elements (extra commas)
+		if r == "" {
+			continue
+		}
+		ranges := strings.SplitN(r, "-", 2)
+		if len(ranges) > 1 {
+			start, err := strconv.ParseUint(ranges[0], 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			end, err := strconv.ParseUint(ranges[1], 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			if start > end {
+				return nil, errors.New("invalid range: " + r)
+			}
+			for i := uint(start); i <= uint(end); i++ {
+				bits.Set(i)
+			}
+		} else {
+			val, err := strconv.ParseUint(ranges[0], 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			bits.Set(uint(val))
+		}
+	}
+
+	val := bits.Bytes()
+	if len(val) == 0 {
+		// do not allow empty values
+		return nil, errors.New("empty value")
+	}
+	ret := make([]byte, len(val)*8)
+	for i := range val {
+		// bitset uses BigEndian internally
+		binary.BigEndian.PutUint64(ret[i*8:], val[len(val)-1-i])
+	}
+	// remove upper all-zero bytes
+	for ret[0] == 0 {
+		ret = ret[1:]
+	}
+
+	return ret, nil
+}

libcontainer/cgroups/systemd/cpuset_test.go

@@ -0,0 +1,58 @@
+package systemd
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestRangeToBits(t *testing.T) {
+	testCases := []struct {
+		in    string
+		out   []byte
+		isErr bool
+	}{
+		{in: "", isErr: true},
+		{in: "0", out: []byte{1}},
+		{in: "1", out: []byte{2}},
+		{in: "0-1", out: []byte{3}},
+		{in: "0,1", out: []byte{3}},
+		{in: ",0,1,", out: []byte{3}},
+		{in: "0-3", out: []byte{0x0f}},
+		{in: "0,1,2-3", out: []byte{0x0f}},
+		{in: "4-7", out: []byte{0xf0}},
+		{in: "0-7", out: []byte{0xff}},
+		{in: "0-15", out: []byte{0xff, 0xff}},
+		{in: "16", out: []byte{1, 0, 0}},
+		{in: "0-3,32-33", out: []byte{3, 0, 0, 0, 0x0f}},
+		// extra spaces and tabs are ok
+		{in: "1, 2, 1-2", out: []byte{6}},
+		{in: "    , 1   , 3  ,  5-7,	", out: []byte{0xea}},
+		// somewhat large values
+		{in: "128-130,1", out: []byte{7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}},
+
+		{in: "-", isErr: true},
+		{in: "1-", isErr: true},
+		{in: "-3", isErr: true},
+		// bad range (start > end)
+		{in: "54-53", isErr: true},
+		// kernel does not allow extra spaces inside a range
+		{in: "1 - 2", isErr: true},
+	}
+
+	for _, tc := range testCases {
+		t.Logf("case: %q", tc.in)
+		out, err := rangeToBits(tc.in)
+		if err != nil {
+			t.Logf("   got error: %s", err)
+			if !tc.isErr {
+				t.Error("   ^^^ unexpected error")
+			}
+
+			continue
+		}
+		t.Logf("   expected %v, got %v", tc.out, out)
+		if !bytes.Equal(out, tc.out) {
+			t.Error("   ^^^ unexpected result")
+		}
+	}
+}

libcontainer/cgroups/systemd/v2.go

@@ -93,6 +93,18 @@ func unifiedResToSystemdProps(conn *systemdDbus.Conn, res map[string]string) (pr
 			props = append(props,
 				newProp("CPUWeight", num))
 
+		case "cpuset.cpus", "cpuset.mems":
+			bits, err := rangeToBits(v)
+			if err != nil {
+				return nil, fmt.Errorf("unified resource %q=%q conversion error: %w", k, v, err)
+			}
+			m := map[string]string{
+				"cpuset.cpus": "AllowedCPUs",
+				"cpuset.mems": "AllowedMemoryNodes",
+			}
+			props = append(props,
+				newProp(m[k], bits))
+
 		case "pids.max":
 			num := uint64(math.MaxUint64)
 			if v != "max" {

tests/integration/helpers.bash

@@ -357,6 +357,12 @@ function requires() {
 				skip_me=1
 			fi
 			;;
+		smp)
+			local cpu_count=$(grep -c '^processor' /proc/cpuinfo)
+			if [ "$cpu_count" -lt 2 ]; then
+				skip_me=1
+			fi
+			;;
 		systemd)
 			if [ -z "${RUNC_USE_SYSTEMD}" ]; then
 				skip_me=1

tests/integration/update.bats

@@ -392,6 +392,53 @@ EOF
 	check_systemd_value "TasksMax" 10
 }
 
+@test "update cpuset parameters via v2 unified map" {
+	[[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup
+	requires cgroups_v2 smp
+
+	update_config ' .linux.resources.unified |= {
+				"cpuset.cpus": "0",
+				"cpuset.mems": "0"
+			}' "${BUSYBOX_BUNDLE}"
+	runc run -d --console-socket "$CONSOLE_SOCKET" test_update
+	[ "$status" -eq 0 ]
+
+	# check that initial values were properly set
+	check_systemd_value "AllowedCPUs" 0
+	check_systemd_value "AllowedMemoryNodes" 0
+
+	runc update -r - test_update <<EOF
+{
+  "unified": {
+    "cpuset.cpus": "1"
+  }
+}
+EOF
+	[ "$status" -eq 0 ]
+
+	# check the updated systemd unit properties
+	check_systemd_value "AllowedCPUs" 1
+
+	# More than 1 numa memory node is required to test this
+	file="/sys/fs/cgroup/cpuset.mems.effective"
+	if ! test -r $file || grep -q '^0$' $file; then
+		# skip the rest of it
+		return 0
+	fi
+
+	runc update -r - test_update <<EOF
+{
+  "unified": {
+    "cpuset.mems": "1"
+  }
+}
+EOF
+	[ "$status" -eq 0 ]
+
+	# check the updated systemd unit properties
+	check_systemd_value "AllowedMemoryNodes" 1
+}
+
 @test "update rt period and runtime" {
 	[[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup
 	requires cgroups_v1 cgroups_rt no_systemd