Merge pull request opencontainers#3030 from kolyshkin/openat2-improve

libct/cg/OpenFile: fix/improve openat2 handling

Author: thaJeztah

libcontainer/cgroups/file.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -13,7 +15,11 @@ import (
 )
 
 // OpenFile opens a cgroup file in a given dir with given flags.
-// It is supposed to be used for cgroup files only.
+// It is supposed to be used for cgroup files only, and returns
+// an error if the file is not a cgroup file.
+//
+// Arguments dir and file are joined together to form an absolute path
+// to a file being opened.
 func OpenFile(dir, file string, flags int) (*os.File, error) {
 	if dir == "" {
 		return nil, fmt.Errorf("no directory specified for %s", file)
@@ -109,43 +115,59 @@ func prepareOpenat2() error {
 	return prepErr
 }
 
-// OpenFile opens a cgroup file in a given dir with given flags.
-// It is supposed to be used for cgroup files only.
 func openFile(dir, file string, flags int) (*os.File, error) {
 	mode := os.FileMode(0)
 	if TestMode && flags&os.O_WRONLY != 0 {
 		// "emulate" cgroup fs for unit tests
 		flags |= os.O_TRUNC | os.O_CREATE
 		mode = 0o600
 	}
+	path := path.Join(dir, file)
 	if prepareOpenat2() != nil {
-		return openFallback(dir, file, flags, mode)
+		return openFallback(path, flags, mode)
 	}
-	reldir := strings.TrimPrefix(dir, cgroupfsPrefix)
-	if len(reldir) == len(dir) { // non-standard path, old system?
-		return openFallback(dir, file, flags, mode)
+	relPath := strings.TrimPrefix(path, cgroupfsPrefix)
+	if len(relPath) == len(path) { // non-standard path, old system?
+		return openFallback(path, flags, mode)
 	}
 
-	relname := reldir + "/" + file
-	fd, err := unix.Openat2(cgroupFd, relname,
+	fd, err := unix.Openat2(cgroupFd, relPath,
 		&unix.OpenHow{
 			Resolve: resolveFlags,
 			Flags:   uint64(flags) | unix.O_CLOEXEC,
 			Mode:    uint64(mode),
 		})
 	if err != nil {
-		return nil, &os.PathError{Op: "openat2", Path: dir + "/" + file, Err: err}
+		err = &os.PathError{Op: "openat2", Path: path, Err: err}
+		// Check if cgroupFd is still opened to cgroupfsDir
+		// (happens when this package is incorrectly used
+		// across the chroot/pivot_root/mntns boundary, or
+		// when /sys/fs/cgroup is remounted).
+		//
+		// TODO: if such usage will ever be common, amend this
+		// to reopen cgroupFd and retry openat2.
+		fdStr := strconv.Itoa(cgroupFd)
+		fdDest, _ := os.Readlink("/proc/self/fd/" + fdStr)
+		if fdDest != cgroupfsDir {
+			// Wrap the error so it is clear that cgroupFd
+			// is opened to an unexpected/wrong directory.
+			err = fmt.Errorf("cgroupFd %s unexpectedly opened to %s != %s: %w",
+				fdStr, fdDest, cgroupfsDir, err)
+		}
+		return nil, err
 	}
 
-	return os.NewFile(uintptr(fd), cgroupfsPrefix+relname), nil
+	return os.NewFile(uintptr(fd), path), nil
 }
 
 var errNotCgroupfs = errors.New("not a cgroup file")
 
-// openFallback is used when openat2(2) is not available. It checks the opened
+// Can be changed by unit tests.
+var openFallback = openAndCheck
+
+// openAndCheck is used when openat2(2) is not available. It checks the opened
 // file is on cgroupfs, returning an error otherwise.
-func openFallback(dir, file string, flags int, mode os.FileMode) (*os.File, error) {
-	path := dir + "/" + file
+func openAndCheck(path string, flags int, mode os.FileMode) (*os.File, error) {
 	fd, err := os.OpenFile(path, flags, mode)
 	if err != nil {
 		return nil, err

libcontainer/cgroups/file_test.go

@@ -1,6 +1,7 @@
 package cgroups
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -38,3 +39,35 @@ func TestWriteCgroupFileHandlesInterrupt(t *testing.T) {
 		}
 	}
 }
+
+func TestOpenat2(t *testing.T) {
+	if !IsCgroup2UnifiedMode() {
+		// The reason is many test cases below test opening files from
+		// the top-level directory, where cgroup v1 has no files.
+		t.Skip("test requires cgroup v2")
+	}
+
+	// Make sure we test openat2, not its fallback.
+	openFallback = func(_ string, _ int, _ os.FileMode) (*os.File, error) {
+		return nil, errors.New("fallback")
+	}
+	defer func() { openFallback = openAndCheck }()
+
+	for _, tc := range []struct{ dir, file string }{
+		{"/sys/fs/cgroup", "cgroup.controllers"},
+		{"/sys/fs/cgroup", "/cgroup.controllers"},
+		{"/sys/fs/cgroup/", "cgroup.controllers"},
+		{"/sys/fs/cgroup/", "/cgroup.controllers"},
+		{"/sys/fs/cgroup/user.slice", "cgroup.controllers"},
+		{"/sys/fs/cgroup/user.slice/", "/cgroup.controllers"},
+		{"/", "/sys/fs/cgroup/cgroup.controllers"},
+		{"/", "sys/fs/cgroup/cgroup.controllers"},
+		{"/sys/fs/cgroup/cgroup.controllers", ""},
+	} {
+		fd, err := OpenFile(tc.dir, tc.file, os.O_RDONLY)
+		if err != nil {
+			t.Errorf("case %+v: %v", tc, err)
+		}
+		fd.Close()
+	}
+}