Provide a mode to just disable kernel cache not the blobfuse cache (#1882)

Author: vibhansa-msft

cmd/mount.go

@@ -74,21 +74,22 @@ type mountOptions struct {
 	inputMountPath string
 	ConfigFile     string
 
-	Logging           LogOptions     `config:"logging"`
-	Components        []string       `config:"components"`
-	Foreground        bool           `config:"foreground"`
-	NonEmpty          bool           `config:"nonempty"`
-	DefaultWorkingDir string         `config:"default-working-dir"`
-	CPUProfile        string         `config:"cpu-profile"`
-	MemProfile        string         `config:"mem-profile"`
-	PassPhrase        string         `config:"passphrase"`
-	SecureConfig      bool           `config:"secure-config"`
-	DynamicProfiler   bool           `config:"dynamic-profile"`
-	ProfilerPort      int            `config:"profiler-port"`
-	ProfilerIP        string         `config:"profiler-ip"`
-	MonitorOpt        monitorOptions `config:"health_monitor"`
-	WaitForMount      time.Duration  `config:"wait-for-mount"`
-	LazyWrite         bool           `config:"lazy-write"`
+	Logging            LogOptions     `config:"logging"`
+	Components         []string       `config:"components"`
+	Foreground         bool           `config:"foreground"`
+	NonEmpty           bool           `config:"nonempty"`
+	DefaultWorkingDir  string         `config:"default-working-dir"`
+	CPUProfile         string         `config:"cpu-profile"`
+	MemProfile         string         `config:"mem-profile"`
+	PassPhrase         string         `config:"passphrase"`
+	SecureConfig       bool           `config:"secure-config"`
+	DynamicProfiler    bool           `config:"dynamic-profile"`
+	ProfilerPort       int            `config:"profiler-port"`
+	ProfilerIP         string         `config:"profiler-ip"`
+	MonitorOpt         monitorOptions `config:"health_monitor"`
+	WaitForMount       time.Duration  `config:"wait-for-mount"`
+	LazyWrite          bool           `config:"lazy-write"`
+	disableKernelCache bool           `config:"disable-kernel-cache"`
 
 	// v1 support
 	Streaming         bool     `config:"streaming"`
@@ -393,6 +394,11 @@ var mountCmd = &cobra.Command{
 			}
 		}
 
+		if config.IsSet("disable-kernel-cache") && directIO {
+			// Both flag shall not be enable together
+			return fmt.Errorf("direct-io and disable-kernel-cache cannot be enabled together")
+		}
+
 		if !config.IsSet("logging.file-path") {
 			options.Logging.LogFilePath = common.DefaultLogFilePath
 		}
@@ -870,6 +876,9 @@ func init() {
 
 	mountCmd.PersistentFlags().DurationVar(&options.WaitForMount, "wait-for-mount", 5*time.Second, "Let parent process wait for given timeout before exit")
 
+	mountCmd.PersistentFlags().Bool("disable-kernel-cache", false, "Disable kerneel cache, but keep blobfuse cache. Default value false.")
+	config.BindPFlag("disable-kernel-cache", mountCmd.PersistentFlags().Lookup("disable-kernel-cache"))
+
 	config.AttachToFlagSet(mountCmd.PersistentFlags())
 	config.AttachFlagCompletions(mountCmd)
 	config.AddConfigChangeEventListener(config.ConfigChangeEventHandlerFunc(OnConfigChange))

cmd/mount_test.go

@@ -360,6 +360,21 @@ func (suite *mountTestSuite) TestStreamAttrCacheOptionsV1() {
 	suite.assert.Contains(op, "failed to initialize new pipeline")
 }
 
+func (suite *mountTestSuite) TestDirectIODisableKernelCacheCombo() {
+	defer suite.cleanupTest()
+
+	mntDir, err := os.MkdirTemp("", "mntdir")
+	suite.assert.Nil(err)
+	defer os.RemoveAll(mntDir)
+
+	tempLogDir := "/tmp/templogs_" + randomString(6)
+	defer os.RemoveAll(tempLogDir)
+
+	op, err := executeCommandC(rootCmd, "mount", mntDir, "-o", "direct_io", "--disable-kernel-cache")
+	suite.assert.NotNil(err)
+	suite.assert.Contains(op, "direct-io and disable-kernel-cache cannot be enabled together")
+}
+
 // mount failure test where a libfuse option is incorrect
 func (suite *mountTestSuite) TestInvalidLibfuseOption() {
 	defer suite.cleanupTest()

component/libfuse/libfuse.go

@@ -75,6 +75,7 @@ type Libfuse struct {
 	maxFuseThreads        uint32
 	directIO              bool
 	umask                 uint32
+	disableKernelCache    bool
 }
 
 // To support pagination in readdir calls this structure holds a block of items for a given directory
@@ -195,6 +196,12 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error {
 	lf.ownerUID = opt.Uid
 	lf.umask = opt.Umask
 
+	if lf.disableKernelCache {
+		opt.DirectIO = true
+		lf.directIO = true
+		log.Crit("Libfuse::Validate : Kernel cache disabled, setting direct-io mode in fuse")
+	}
+
 	if opt.allowOther {
 		lf.dirPermission = uint(common.DefaultAllowOtherPermissionBits)
 		lf.filePermission = uint(common.DefaultAllowOtherPermissionBits)
@@ -326,6 +333,8 @@ func (lf *Libfuse) Configure(_ bool) error {
 		return err
 	}
 
+	_ = config.UnmarshalKey("disable-kernel-cache", &lf.disableKernelCache)
+
 	err = lf.Validate(&conf)
 	if err != nil {
 		log.Err("Libfuse::Configure : config error [invalid config settings]")
@@ -340,8 +349,8 @@ func (lf *Libfuse) Configure(_ bool) error {
 		}
 	}
 
-	log.Crit("Libfuse::Configure : read-only %t, allow-other %t, allow-root %t, default-perm %d, entry-timeout %d, attr-time %d, negative-timeout %d, ignore-open-flags %t, nonempty %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v",
-		lf.readOnly, lf.allowOther, lf.allowRoot, lf.filePermission, lf.entryExpiration, lf.attributeExpiration, lf.negativeTimeout, lf.ignoreOpenFlags, lf.nonEmptyMount, lf.directIO, lf.maxFuseThreads, lf.traceEnable, lf.extensionPath, lf.disableWritebackCache, lf.dirPermission, lf.mountPath, lf.umask)
+	log.Crit("Libfuse::Configure : read-only %t, allow-other %t, allow-root %t, default-perm %d, entry-timeout %d, attr-time %d, negative-timeout %d, ignore-open-flags %t, nonempty %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v, disableKernelCache %v",
+		lf.readOnly, lf.allowOther, lf.allowRoot, lf.filePermission, lf.entryExpiration, lf.attributeExpiration, lf.negativeTimeout, lf.ignoreOpenFlags, lf.nonEmptyMount, lf.directIO, lf.maxFuseThreads, lf.traceEnable, lf.extensionPath, lf.disableWritebackCache, lf.dirPermission, lf.mountPath, lf.umask, lf.disableKernelCache)
 
 	return nil
 }

component/libfuse/libfuse_handler_test.go

@@ -126,6 +126,20 @@ func (suite *libfuseTestSuite) TestConfigDefaultPermission() {
 	suite.assert.True(suite.libfuse.directIO)
 }
 
+func (suite *libfuseTestSuite) TestConfigDisableKernelCache() {
+	defer suite.cleanupTest()
+	suite.cleanupTest() // clean up the default libfuse generated
+	config := "read-only: true\ndisable-kernel-cache: true\n\n"
+	suite.setupTestHelper(config) // setup a new libfuse with a custom config (clean up will occur after the test as usual)
+
+	suite.assert.Equal(suite.libfuse.Name(), "libfuse")
+	suite.assert.Empty(suite.libfuse.mountPath)
+	suite.assert.Equal(suite.libfuse.entryExpiration, uint32(0))
+	suite.assert.Equal(suite.libfuse.attributeExpiration, uint32(0))
+	suite.assert.Equal(suite.libfuse.negativeTimeout, uint32(0))
+	suite.assert.True(suite.libfuse.directIO)
+}
+
 func (suite *libfuseTestSuite) TestConfigFuseTraceEnable() {
 	defer suite.cleanupTest()
 	suite.cleanupTest() // clean up the default libfuse generated

setup/advancedConfig.yaml

@@ -34,6 +34,7 @@ foreground: true|false <run blobfuse2 in foreground or background>
 # Common configurations
 allow-other: true|false <allow other users to access the mounted directory - used for FUSE and File Cache>
 nonempty: true|false <allow mounting on non-empty directory>
+disable-kernel-cache: true|false <disable kernel cache but keep blobfuse2 cache. Default - false>
 
 # Dynamic profiler related configuration. This helps to root-cause high memory/cpu usage related issues.
 dynamic-profile: true|false <allows to turn on dynamic profiler for cpu/memory usage monitoring. Only for debugging, shall not be used in production>
@@ -67,7 +68,7 @@ libfuse:
   negative-entry-expiration-sec: <time kernel can cache attributes of non existent paths (in sec). Default - 120 sec>
   fuse-trace: true|false <enable libfuse api trace logs for debugging>
   extension: <physical path to extension library>
-  direct-io: true|false <enable to bypass the kernel cache>
+  direct-io: true|false <enable to bypass the kernel cache. It also disables blobfuse2 data and metadata caching. Default - false>
 
 # Entry Cache configuration
 entry_cache: