DAOS-17113 control: Enable hotplug in config by default (#16264)

Deprecate enable_hotplug and add disable_hotplug (defaults to false).
Setting enable_hotplug and disable_hotplug will fail config validation.
enable_hotplug: true|false will still be accepted if disable_hotplug
is unset.

Features: control
Signed-off-by: Tom Nabarro <[email protected]>

Author: tanabarr

src/control/cmd/daos_server/auto_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2022-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -573,7 +574,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) {
 					}
 					return x.Equals(y)
 				}),
-				cmpopts.IgnoreUnexported(security.CertificateConfig{}),
+				cmpopts.IgnoreUnexported(security.CertificateConfig{}, config.Server{}),
 			}
 
 			if diff := cmp.Diff(tc.expCfg, gotCfg, cmpOpts...); diff != "" {

src/control/cmd/daos_server/start_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2019-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -261,6 +262,7 @@ func TestStartOptions(t *testing.T) {
 			cmpOpts := []cmp.Option{
 				cmpopts.IgnoreUnexported(
 					security.CertificateConfig{},
+					config.Server{},
 				),
 				cmpopts.SortSlices(func(a, b string) bool { return a < b }),
 			}

src/control/cmd/dmg/auto_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -520,7 +521,8 @@ func TestAuto_confGen(t *testing.T) {
 					}
 					return x.Equals(y)
 				}),
-				cmpopts.IgnoreUnexported(security.CertificateConfig{}),
+				cmpopts.IgnoreUnexported(security.CertificateConfig{},
+					config.Server{}),
 			}
 
 			if diff := cmp.Diff(tc.expCfg, gotCfg, cmpOpts...); diff != "" {
@@ -586,7 +588,7 @@ engines:
   pinned_numa_node: 1
 disable_vfio: false
 disable_vmd: false
-enable_hotplug: false
+disable_hotplug: false
 nr_hugepages: 0
 system_ram_reserved: 16
 disable_hugepages: false
@@ -608,6 +610,7 @@ hyperthreads: false
 		WithFabricProvider("ofi+verbs").
 		WithAccessPoints("hostX:10002").
 		WithDisableVMD(false).
+		WithDisableHotplug(false).
 		WithEngines(
 			engine.MockConfig().
 				WithTargetCount(defaultTargetCount).

src/control/fault/code/codes.go

@@ -197,6 +197,7 @@ const (
 	ServerConfigScmDiffClass
 	ServerConfigEngineBdevRolesMismatch
 	ServerConfigSysRsvdZero
+	ServerConfigEnableHotplugDeprecated
 )
 
 // SPDK library bindings codes

src/control/lib/control/auto_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -1772,7 +1773,8 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) {
 					}
 					return x.Equals(y)
 				}),
-				cmpopts.IgnoreUnexported(security.CertificateConfig{}),
+				cmpopts.IgnoreUnexported(security.CertificateConfig{},
+					config.Server{}),
 			}
 			cmpOpts = append(cmpOpts, defResCmpOpts()...)
 

src/control/lib/control/mocks.go

@@ -829,6 +829,7 @@ func MockServerCfg(provider string, ecs []*engine.Config) *config.Server {
 		WithControlLogFile(defaultControlLogFile).
 		WithFabricProvider(provider).
 		WithDisableVMD(false).
+		WithDisableHotplug(false).
 		WithEngines(ecs...)
 }
 

src/control/server/config/faults.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -111,6 +112,11 @@ var (
 		"`system_ram_reserved` is set to zero in server config",
 		"set `system_ram_reserved` to a positive integer value in config",
 	)
+	FaultConfigEnableHotplugDeprecated = serverConfigFault(
+		code.ServerConfigEnableHotplugDeprecated,
+		"'enable_hotplug' setting is deprecated and no longer supported",
+		"set 'disable_hotplug: true' in server config file to disable hotplug",
+	)
 )
 
 func FaultConfigDuplicateFabric(curIdx, seenIdx int) *fault.Fault {

src/control/server/config/server.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -36,6 +37,10 @@ const (
 	relConfExamplesPath = "../utils/config/examples/"
 )
 
+type deprecatedParams struct {
+	EnableHotplug *bool `yaml:"enable_hotplug,omitempty"` // deprecated in 2.6.5
+}
+
 // Server describes configuration options for DAOS control plane.
 // See utils/config/daos_server.yml for parameter descriptions.
 type Server struct {
@@ -46,7 +51,7 @@ type Server struct {
 	BdevExclude       []string                  `yaml:"bdev_exclude,omitempty"`
 	DisableVFIO       bool                      `yaml:"disable_vfio"`
 	DisableVMD        *bool                     `yaml:"disable_vmd"`
-	EnableHotplug     bool                      `yaml:"enable_hotplug"`
+	DisableHotplug    *bool                     `yaml:"disable_hotplug"`
 	NrHugepages       int                       `yaml:"nr_hugepages"`        // total for all engines
 	SystemRamReserved int                       `yaml:"system_ram_reserved"` // total for all engines
 	DisableHugepages  bool                      `yaml:"disable_hugepages"`
@@ -78,6 +83,8 @@ type Server struct {
 
 	// Behavior flags
 	AutoFormat bool `yaml:"-"`
+
+	deprecatedParams `yaml:",inline"`
 }
 
 // WithCoreDumpFilter sets the core dump filter written to /proc/self/coredump_filter.
@@ -182,7 +189,10 @@ func (cfg *Server) updateServerConfig(cfgPtr **engine.Config) {
 	engineCfg.SystemName = cfg.SystemName
 	engineCfg.SocketDir = cfg.SocketDir
 	engineCfg.Modules = cfg.Modules
-	engineCfg.Storage.EnableHotplug = cfg.EnableHotplug
+	engineCfg.Storage.EnableHotplug = true
+	if cfg.DisableHotplug != nil && *cfg.DisableHotplug {
+		engineCfg.Storage.EnableHotplug = false
+	}
 }
 
 // WithEngines sets the list of engine configurations.
@@ -245,9 +255,9 @@ func (cfg *Server) WithDisableVMD(disabled bool) *Server {
 	return cfg
 }
 
-// WithEnableHotplug can be used to enable hotplug
-func (cfg *Server) WithEnableHotplug(enabled bool) *Server {
-	cfg.EnableHotplug = enabled
+// WithDisableHotplug can be used to disable hotplug.
+func (cfg *Server) WithDisableHotplug(disabled bool) *Server {
+	cfg.DisableHotplug = &disabled
 	return cfg
 }
 
@@ -325,7 +335,6 @@ func DefaultServer() *Server {
 		SystemRamReserved: storage.DefaultSysMemRsvd / humanize.GiByte,
 		Path:              defaultConfigPath,
 		ControlLogMask:    common.ControlLogLevel(logging.LogLevelInfo),
-		EnableHotplug:     false, // disabled by default
 		// https://man7.org/linux/man-pages/man5/core.5.html
 		CoreDumpFilter: 0b00010011, // private, shared, ELF
 	}
@@ -639,13 +648,36 @@ func (cfg *Server) Validate(log logging.Logger) (err error) {
 		}
 	}()
 
+	if cfg.deprecatedParams.EnableHotplug != nil {
+		// Fail if conflicting EnableHotplug and DisableHotplug both set.
+		if cfg.DisableHotplug != nil {
+			return FaultConfigEnableHotplugDeprecated
+		}
+		log.Notice("enable_hotplug is deprecated; please use disable_hotplug instead " +
+			"(false by default)")
+
+		// Apply deprecated parameter updates.
+		eh := !*cfg.deprecatedParams.EnableHotplug
+		cfg.DisableHotplug = &eh
+		log.Debugf("deprecated param update: enable_hotplug: %v -> disable_hotplug: %v",
+			*cfg.deprecatedParams.EnableHotplug, *cfg.DisableHotplug)
+	}
+	// Set DisableHotplug reference if unset in config file.
+	if cfg.DisableHotplug == nil {
+		cfg.WithDisableHotplug(false)
+	}
+
 	// Set DisableVMD reference if unset in config file.
 	if cfg.DisableVMD == nil {
 		cfg.WithDisableVMD(false)
 	}
 
+	for i := range cfg.Engines {
+		cfg.updateServerConfig(&cfg.Engines[i])
+	}
+
 	log.Debugf("vfio=%v hotplug=%v vmd=%v requested in config", !cfg.DisableVFIO,
-		cfg.EnableHotplug, !(*cfg.DisableVMD))
+		!(*cfg.DisableHotplug), !(*cfg.DisableVMD))
 
 	// Update access point addresses with control port if port is not supplied.
 	newAPs := make([]string, 0, len(cfg.AccessPoints))

src/control/server/config/server_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -23,6 +24,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/pkg/errors"
 
+	"github.com/daos-stack/daos/src/control/build"
 	"github.com/daos-stack/daos/src/control/common"
 	"github.com/daos-stack/daos/src/control/common/test"
 	"github.com/daos-stack/daos/src/control/logging"
@@ -239,9 +241,9 @@ func TestServerConfig_Constructed(t *testing.T) {
 			DevicePath: "/dev/sdb1",
 		}).
 		WithBdevExclude("0000:81:00.1").
-		WithDisableVFIO(true).   // vfio enabled by default
-		WithDisableVMD(true).    // vmd enabled by default
-		WithEnableHotplug(true). // hotplug disabled by default
+		WithDisableVFIO(true).    // vfio enabled by default
+		WithDisableVMD(true).     // vmd enabled by default
+		WithDisableHotplug(true). // hotplug enabled by default
 		WithControlLogMask(common.ControlLogLevelError).
 		WithControlLogFile("/tmp/daos_server.log").
 		WithHelperLogFile("/tmp/daos_server_helper.log").
@@ -288,7 +290,7 @@ func TestServerConfig_Constructed(t *testing.T) {
 			WithEnvVars("CRT_TIMEOUT=30").
 			WithLogFile("/tmp/daos_engine.0.log").
 			WithLogMask("INFO").
-			WithStorageEnableHotplug(true).
+			WithStorageEnableHotplug(false).
 			WithStorageAutoFaultyCriteria(true, 100, 200),
 		engine.MockConfig().
 			WithSystemName("daos_server").
@@ -315,7 +317,7 @@ func TestServerConfig_Constructed(t *testing.T) {
 			WithEnvVars("CRT_TIMEOUT=100").
 			WithLogFile("/tmp/daos_engine.1.log").
 			WithLogMask("INFO").
-			WithStorageEnableHotplug(true).
+			WithStorageEnableHotplug(false).
 			WithStorageAutoFaultyCriteria(false, 0, 0),
 	}
 	constructed.Path = testFile // just to avoid failing the cmp
@@ -345,10 +347,9 @@ func TestServerConfig_updateServerConfig(t *testing.T) {
 		},
 		"basic": {
 			cfg: &Server{
-				SystemName:    "name",
-				SocketDir:     "socketdir",
-				Modules:       "modules",
-				EnableHotplug: true,
+				SystemName: "name",
+				SocketDir:  "socketdir",
+				Modules:    "modules",
 				Fabric: engine.FabricConfig{
 					Provider:              "provider",
 					Interface:             "iface",
@@ -381,6 +382,9 @@ func TestServerConfig_updateServerConfig(t *testing.T) {
 			},
 			expEngCfg: &engine.Config{
 				SystemName: "name",
+				Storage: storage.Config{
+					EnableHotplug: true,
+				},
 				Fabric: engine.FabricConfig{
 					Provider:              "p1 p2 p3",
 					NumSecondaryEndpoints: []int{2, 3, 4},
@@ -444,6 +448,7 @@ func TestServerConfig_MDonSSD_Constructed(t *testing.T) {
 					WithBdevDeviceList("0000:83:00.0").
 					WithBdevDeviceRoles(storage.BdevRoleData),
 			).
+			WithStorageEnableHotplug(true).
 			WithFabricInterface("ib0").
 			WithFabricInterfacePort(31316).
 			WithFabricProvider("ofi+tcp").
@@ -1566,6 +1571,31 @@ func TestServerConfig_Parsing(t *testing.T) {
 				return nil
 			},
 		},
+		"enable_hotplug and disable_hotplug both set": {
+			inTxt:          "disable_hugepages: false",
+			outTxt:         "enable_hotplug: true",
+			expValidateErr: FaultConfigEnableHotplugDeprecated,
+		},
+		"enable_hotplug false setting allowed": {
+			inTxt:  "disable_hotplug: true",
+			outTxt: "enable_hotplug: false",
+			expCheck: func(c *Server) error {
+				if c.Engines[0].Storage.EnableHotplug {
+					return errors.New("expecting hotplug to be disabled")
+				}
+				return nil
+			},
+		},
+		"enable_hotplug true setting allowed": {
+			inTxt:  "disable_hotplug: true",
+			outTxt: "enable_hotplug: true",
+			expCheck: func(c *Server) error {
+				if !c.Engines[0].Storage.EnableHotplug {
+					return errors.New("expecting hotplug to be enabled")
+				}
+				return nil
+			},
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			log, buf := logging.NewTestLogger(t.Name())
@@ -1671,7 +1701,8 @@ func TestServerConfig_WithEnginesInheritsMain(t *testing.T) {
 		WithFabricProvider(testFabric).
 		WithModules(testModules).
 		WithSocketDir(testSocketDir).
-		WithSystemName(testSystemName)
+		WithSystemName(testSystemName).
+		WithStorageEnableHotplug(true)
 
 	config := DefaultServer().
 		WithFabricProvider(testFabric).
@@ -1830,6 +1861,8 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) {
 
 			conf := DefaultServer().
 				WithFabricProvider("test").
+				WithAccessPoints(
+					fmt.Sprintf("localhost:%d", build.DefaultControlPort)).
 				WithEngines(tc.configA, tc.configB)
 
 			gotErr := conf.Validate(log)

utils/config/daos_server.yml

@@ -158,13 +158,14 @@
 #disable_vmd: true
 #
 #
-## Enable NVMe SSD Hotplug
+## Disable NVMe SSD Hotplug
 #
-## When hotplug is enabled, io engine will periodically check device hot
+## NVMe SSD hotplug is enabled by default but can be optionally disabled.
+## When enabled io engine will periodically check device hot
 ## plug/remove event, and setup/teardown the device automatically.
 #
 ## default: false
-#enable_hotplug: true
+#disable_hotplug: true
 #
 #
 ## Use Hyperthreads