Merge branch 'cherry-pick-for-v1.13.2' into 'release-1.13'

Cherry pick changes for 1.13.2

See merge request nvidia/container-toolkit/container-toolkit!407

Author: Evan Lezar

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
-	branch = main
+	branch = release-1.13
 [submodule "third_party/nvidia-container-runtime"]
 	path = third_party/nvidia-container-runtime
 	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git

CHANGELOG.md

@@ -1,5 +1,14 @@
 # NVIDIA Container Toolkit Changelog
 
+## v1.13.2
+
+* Add `nvidia-container-runtime-hook.path` config option to specify NVIDIA Container Runtime Hook path explicitly.
+* Fix bug in creation of `/dev/char` symlinks by failing operation if kernel modules are not loaded.
+* Add option to load kernel modules when creating device nodes
+* Add option to create device nodes when creating `/dev/char` symlinks
+* Treat failures to open debug log files as non-fatal.
+* Bump CUDA base image version to 12.1.1.
+
 ## v1.13.1
 
 * Update `update-ldcache` hook to only update ldcache if it exists.

cmd/nvidia-container-runtime/main_test.go

@@ -172,7 +172,7 @@ func TestDuplicateHook(t *testing.T) {
 // addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger())
+	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger(), nvidiaHook)
 	return m.Modify(spec)
 }
 

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -40,12 +40,23 @@ func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error
 	if err != nil {
 		return nil, fmt.Errorf("failed reading device majors: %v", err)
 	}
+
+	var requiredMajors []devices.Name
 	migCaps, err := nvcaps.NewMigCaps()
 	if err != nil {
 		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
 	}
 	if migCaps == nil {
 		migCaps = make(nvcaps.MigCaps)
+	} else {
+		requiredMajors = append(requiredMajors, devices.NVIDIACaps)
+	}
+
+	requiredMajors = append(requiredMajors, devices.NVIDIAGPU, devices.NVIDIAUVM)
+	for _, name := range requiredMajors {
+		if !deviceMajors.Exists(name) {
+			return nil, fmt.Errorf("missing required device major %s", name)
+		}
 	}
 
 	l := allPossible{

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -38,11 +39,13 @@ type command struct {
 }
 
 type config struct {
-	devCharPath string
-	driverRoot  string
-	dryRun      bool
-	watch       bool
-	createAll   bool
+	devCharPath       string
+	driverRoot        string
+	dryRun            bool
+	watch             bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
 }
 
 // NewCommand constructs a command sub-command with the specified logger
@@ -97,6 +100,18 @@ func (m command) build() *cli.Command {
 			Destination: &cfg.createAll,
 			EnvVars:     []string{"CREATE_ALL"},
 		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "Load the NVIDIA kernel modules before creating symlinks. This is only applicable when --create-all is set.",
+			Destination: &cfg.loadKernelModules,
+			EnvVars:     []string{"LOAD_KERNEL_MODULES"},
+		},
+		&cli.BoolFlag{
+			Name:        "create-device-nodes",
+			Usage:       "Create the NVIDIA control device nodes in the driver root if they do not exist. This is only applicable when --create-all is set",
+			Destination: &cfg.createDeviceNodes,
+			EnvVars:     []string{"CREATE_DEVICE_NODES"},
+		},
 		&cli.BoolFlag{
 			Name:        "dry-run",
 			Usage:       "If set, the command will not create any symlinks.",
@@ -114,6 +129,16 @@ func (m command) validateFlags(r *cli.Context, cfg *config) error {
 		return fmt.Errorf("create-all and watch are mutually exclusive")
 	}
 
+	if cfg.loadKernelModules && !cfg.createAll {
+		m.logger.Warn("load-kernel-modules is only applicable when create-all is set; ignoring")
+		cfg.loadKernelModules = false
+	}
+
+	if cfg.createDeviceNodes && !cfg.createAll {
+		m.logger.Warn("create-device-nodes is only applicable when create-all is set; ignoring")
+		cfg.createDeviceNodes = false
+	}
+
 	return nil
 }
 
@@ -137,6 +162,8 @@ func (m command) run(c *cli.Context, cfg *config) error {
 		WithDriverRoot(cfg.driverRoot),
 		WithDryRun(cfg.dryRun),
 		WithCreateAll(cfg.createAll),
+		WithLoadKernelModules(cfg.loadKernelModules),
+		WithCreateDeviceNodes(cfg.createDeviceNodes),
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create symlink creator: %v", err)
@@ -186,12 +213,14 @@ create:
 }
 
 type linkCreator struct {
-	logger      *logrus.Logger
-	lister      nodeLister
-	driverRoot  string
-	devCharPath string
-	dryRun      bool
-	createAll   bool
+	logger            *logrus.Logger
+	lister            nodeLister
+	driverRoot        string
+	devCharPath       string
+	dryRun            bool
+	createAll         bool
+	createDeviceNodes bool
+	loadKernelModules bool
 }
 
 // Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
@@ -218,6 +247,10 @@ func NewSymlinkCreator(opts ...Option) (Creator, error) {
 		c.devCharPath = defaultDevCharPath
 	}
 
+	if err := c.setup(); err != nil {
+		return nil, err
+	}
+
 	if c.createAll {
 		lister, err := newAllPossible(c.logger, c.driverRoot)
 		if err != nil {
@@ -230,6 +263,34 @@ func NewSymlinkCreator(opts ...Option) (Creator, error) {
 	return c, nil
 }
 
+func (m linkCreator) setup() error {
+	if !m.loadKernelModules && !m.createDeviceNodes {
+		return nil
+	}
+
+	s, err := system.New(
+		system.WithLogger(m.logger),
+		system.WithDryRun(m.dryRun),
+	)
+	if err != nil {
+		return err
+	}
+
+	if m.loadKernelModules {
+		if err := s.LoadNVIDIAKernelModules(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
+	}
+
+	if m.createDeviceNodes {
+		if err := s.CreateNVIDIAControlDeviceNodesAt(m.driverRoot); err != nil {
+			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
+		}
+	}
+
+	return nil
+}
+
 // WithDriverRoot sets the driver root path.
 func WithDriverRoot(root string) Option {
 	return func(c *linkCreator) {
@@ -265,6 +326,20 @@ func WithCreateAll(createAll bool) Option {
 	}
 }
 
+// WithLoadKernelModules sets the loadKernelModules flag for the linkCreator.
+func WithLoadKernelModules(loadKernelModules bool) Option {
+	return func(lc *linkCreator) {
+		lc.loadKernelModules = loadKernelModules
+	}
+}
+
+// WithCreateDeviceNodes sets the createDeviceNodes flag for the linkCreator.
+func WithCreateDeviceNodes(createDeviceNodes bool) Option {
+	return func(lc *linkCreator) {
+		lc.createDeviceNodes = createDeviceNodes
+	}
+}
+
 // CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
 func (m linkCreator) CreateLinks() error {
 	deviceNodes, err := m.lister.DeviceNodes()

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -34,6 +34,8 @@ type options struct {
 	dryRun bool
 
 	control bool
+
+	loadKernelModules bool
 }
 
 // NewCommand constructs a command sub-command with the specified logger
@@ -72,6 +74,11 @@ func (m command) build() *cli.Command {
 			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
 			Destination: &opts.control,
 		},
+		&cli.BoolFlag{
+			Name:        "load-kernel-modules",
+			Usage:       "load the NVIDIA Kernel Modules before creating devices nodes",
+			Destination: &opts.loadKernelModules,
+		},
 		&cli.BoolFlag{
 			Name:        "dry-run",
 			Usage:       "if set, the command will not create any symlinks.",
@@ -92,6 +99,7 @@ func (m command) run(c *cli.Context, opts *options) error {
 	s, err := system.New(
 		system.WithLogger(m.logger),
 		system.WithDryRun(opts.dryRun),
+		system.WithLoadKernelModules(opts.loadKernelModules),
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create library: %v", err)

go.mod

@@ -1,36 +1,36 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.18
+go 1.20
 
 require (
-	github.com/BurntSushi/toml v1.0.0
+	github.com/BurntSushi/toml v1.2.1
 	github.com/NVIDIA/go-nvml v0.12.0-0
 	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
 	github.com/fsnotify/fsnotify v1.5.4
-	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
+	github.com/opencontainers/runtime-spec v1.1.0-rc.2
 	github.com/pelletier/go-toml v1.9.4
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
 	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438
 	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
-	sigs.k8s.io/yaml v1.3.0
+	golang.org/x/sys v0.7.0
 )
 
 require (
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/runc v1.1.6 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
-	github.com/opencontainers/selinux v1.10.1 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )