Merge pull request #790 from mlaventure/runc-update-cgroup-kmem-limit

Runc update cgroup kmem limit

Author: hqhq

libcontainer/cgroups/fs/memory.go

@@ -5,13 +5,15 @@ package fs
 import (
 	"bufio"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/system"
 )
 
 type MemoryGroup struct {
@@ -33,7 +35,7 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
 			}
 		}
 		// We have to set kernel memory here, as we can't change it once
-		// processes have been attached.
+		// processes have been attached to the cgroup.
 		if err := s.SetKernelMemory(path, d.config); err != nil {
 			return err
 		}
@@ -55,9 +57,44 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
 }
 
 func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error {
-	// This has to be done separately because it has special constraints (it
-	// can't be done after there are processes attached to the cgroup).
-	if cgroup.Resources.KernelMemory > 0 {
+	// This has to be done separately because it has special
+	// constraints (it can only be initialized before setting up a
+	// hierarchy or adding a task to the cgroups. However, if
+	// sucessfully initialized, it can be updated anytime afterwards)
+	if cgroup.Resources.KernelMemory != 0 {
+		kmemInitialized := false
+		// Is kmem.limit_in_bytes already set?
+		kmemValue, err := getCgroupParamUint(path, "memory.kmem.limit_in_bytes")
+		if err != nil {
+			return err
+		}
+		switch system.GetLongBit() {
+		case 32:
+			kmemInitialized = uint32(kmemValue) != uint32(math.MaxUint32)
+		case 64:
+			kmemInitialized = kmemValue != uint64(math.MaxUint64)
+		}
+
+		if !kmemInitialized {
+			// If hierarchy is set, we can't change the limit
+			usesHierarchy, err := getCgroupParamUint(path, "memory.use_hierarchy")
+			if err != nil {
+				return err
+			}
+			if usesHierarchy != 0 {
+				return fmt.Errorf("cannot initialize kmem.limit_in_bytes if use_hierarchy is already set")
+			}
+
+			// If there's already tasks in the cgroup, we can't change the limit either
+			tasks, err := getCgroupParamString(path, "tasks")
+			if err != nil {
+				return err
+			}
+			if tasks != "" {
+				return fmt.Errorf("cannot initialize kmem.limit_in_bytes after task have joined this cgroup")
+			}
+		}
+
 		if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil {
 			return err
 		}
@@ -113,6 +150,10 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
 		return err
 	}
 
+	if err := s.SetKernelMemory(path, cgroup); err != nil {
+		return err
+	}
+
 	if cgroup.Resources.MemoryReservation != 0 {
 		if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
 			return err

libcontainer/cgroups/systemd/apply_systemd.go

@@ -214,11 +214,9 @@ func (m *Manager) Apply(pid int) error {
 			newProp("BlockIOWeight", uint64(c.Resources.BlkioWeight)))
 	}
 
-	// We need to set kernel memory before processes join cgroup because
-	// kmem.limit_in_bytes can only be set when the cgroup is empty.
-	// And swap memory limit needs to be set after memory limit, only
-	// memory limit is handled by systemd, so it's kind of ugly here.
-	if c.Resources.KernelMemory > 0 {
+	// We have to set kernel memory here, as we can't change it once
+	// processes have been attached to the cgroup.
+	if c.Resources.KernelMemory != 0 {
 		if err := setKernelMemory(c); err != nil {
 			return err
 		}
@@ -469,11 +467,5 @@ func setKernelMemory(c *configs.Cgroup) error {
 		return err
 	}
 
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return err
-	}
-
-	// This doesn't get called by manager.Set, so we need to do it here.
-	s := &fs.MemoryGroup{}
-	return s.SetKernelMemory(path, c)
+	return os.MkdirAll(path, 0755)
 }

libcontainer/system/sysconfig.go

@@ -10,3 +10,7 @@ import "C"
 func GetClockTicks() int {
 	return int(C.sysconf(C._SC_CLK_TCK))
 }
+
+func GetLongBit() int {
+	return int(C.sysconf(C._SC_LONG_BIT))
+}

main.go

@@ -18,7 +18,7 @@ const (
 	version    = "0.1.1"
 	specConfig = "config.json"
 	usage      = `Open Container Initiative runtime
-	
+
 runc is a command line client for running applications packaged according to
 the Open Container Initiative (OCI) format and is a compliant implementation of the
 Open Container Initiative specification.
@@ -30,7 +30,7 @@ direct child of the process supervisor.
 
 Containers are configured using bundles. A bundle for a container is a directory
 that includes a specification file named "` + specConfig + `" and a root filesystem.
-The root filesystem contains the contents of the container. 
+The root filesystem contains the contents of the container.
 
 To start a new instance of a container:
 
@@ -99,6 +99,7 @@ func main() {
 		specCommand,
 		startCommand,
 		stateCommand,
+		updateCommand,
 	}
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {

tests/integration/cgroups.bats

@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+
+load helpers
+
+UPDATE_TEST_RUNC_ROOT="$BATS_TMPDIR/runc-cgroups-integration-test"
+
+CGROUP_MEMORY=""
+
+TEST_CGROUP_NAME="runc-cgroups-integration-test"
+
+function init_cgroup_path() {
+   base_path=$(grep "rw,"  /proc/self/mountinfo | grep -i -m 1 'MEMORY$' | cut -d ' ' -f 5)
+   CGROUP_MEMORY="${base_path}/${TEST_CGROUP_NAME}"
+}
+
+function teardown() {
+    rm -f $BATS_TMPDIR/runc-update-integration-test.json
+    teardown_running_container_inroot test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
+    teardown_busybox
+}
+
+function setup() {
+    teardown
+    setup_busybox
+
+    init_cgroup_path
+}
+
+function check_cgroup_value() {
+    cgroup=$1
+    source=$2
+    expected=$3
+
+    current=$(cat $cgroup/$source)
+    echo  $cgroup/$source
+    echo "current" $current "!?" "$expected"
+    [ "$current" -eq "$expected" ]
+}
+
+@test "cgroups-kernel-memory-initialized" {
+    # Add cgroup path
+    sed -i 's/\("linux": {\)/\1\n    "cgroupsPath": "runc-cgroups-integration-test",/'  ${BUSYBOX_BUNDLE}/config.json
+
+    # Set some initial known values
+    DATA=$(cat <<-EOF
+    "memory": {
+        "kernel": 16777216
+    },
+EOF
+    )
+    DATA=$(echo ${DATA} | sed 's/\n/\\n/g')
+    sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json
+
+    # start a detached busybox to work with
+    "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem
+    [ "$status" -eq 0 ]
+    wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
+
+    # update kernel memory limit
+    "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648
+    [ "$status" -eq 0 ]
+    check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
+}
+
+@test "cgroups-kernel-memory-uninitialized" {
+    # Add cgroup path
+    sed -i 's/\("linux": {\)/\1\n    "cgroupsPath": "runc-cgroups-integration-test",/'  ${BUSYBOX_BUNDLE}/config.json
+
+    # start a detached busybox to work with
+    run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem
+    [ "$status" -eq 0 ]
+    wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
+
+    # update kernel memory limit
+    run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648
+    [ ! "$status" -eq 0 ]
+}