Skip to content

Commit dbaa4ae

Browse files
motiejusmotiejus
committed
[btrfs] support dynamic_reclaim
Currently btrfs offers 3 methods to balance the filesystem: 1. beginning of time: `btrfs filesystem balance`. 2. kernel v5.19+: `/sys/fs/btrfs/UUID/allocation/{meta,}data/bg_reclaim_threshold`. 3. kernel v6.11+: `/sys/fs/btrfs/UUID/allocation/{meta,}data/dynamic_reclaim`. (3) is most robust out of the 3, and best documented in the kernel commit [f5ff64ccf7bb7274ed66b0d835b2f6ae10af5d7a](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=f5ff64ccf7bb7274ed66b0d835b2f6ae10af5d7a). Once GKE rolls up to the kernel v6.11, this will be the recommended option to set.
1 parent da59cfb commit dbaa4ae

File tree

4 files changed

+89
-47
lines changed

4 files changed

+89
-47
lines changed

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,10 @@ As noted in [GCP PD documentation](https://cloud.google.com/kubernetes-engine/do
100100

101101
`btrfs` filesystem accepts the following "special" mount options and the sysfs paths they target:
102102

103-
- `btrfs-data-bg_reclaim_threshold`: `/sys/fs/btrfs/FS-UUID/allocation/data/bg_reclaim_threshold`.
104-
- `btrfs-metadata-bg_reclaim_thresho: `/sys/fs/btrfs/FS-UUID/allocation/metadata/bg_reclaim_threshold`.
103+
- `btrfs-data-bg_reclaim_threshold`: `/sys/fs/btrfs/FS-UUID/allocation/data/bg_reclaim_threshold`. Supported on Linux v5.19+.
104+
- `btrfs-metadata-bg_reclaim_thresho: `/sys/fs/btrfs/FS-UUID/allocation/metadata/bg_reclaim_threshold`. Supported on Linux v5.19+.
105+
- `btrfs-data-dynamic_reclaim`: `/sys/fs/btrfs/FS-UUID/allocation/data/dynamic_reclaim`. Supported on Linux v6.11+.
106+
- `btrfs-metadata-bg_reclaim_thresho: `/sys/fs/btrfs/FS-UUID/allocation/metadata/bg_reclaim_threshold`. Supported on Linux v6.11+.
105107
- `btrfs-bdi-read_ahead_kb`: `/sys/fs/btrfs/FS-UUID/bdi/read_ahead_kb`.
106108

107109
See more in the [in btrfs docs](https://btrfs.readthedocs.io/en/latest/ch-sysfs.html#uuid-allocations-data-metadata-system).

‎pkg/gce-pd-csi-driver/node.go‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -121,19 +121,31 @@ const (
121121
fsTypeExt3 = "ext3"
122122
fsTypeBtrfs = "btrfs"
123123

124-
readAheadKBMountFlagRegexPattern = "^read_ahead_kb=(.+)$"
125-
btrfsReclaimDataRegexPattern = "^btrfs-allocation-data-bg_reclaim_threshold=(\\d{1,2})$" // 0-99 are valid, incl. 00
126-
btrfsReclaimMetadataRegexPattern = "^btrfs-allocation-metadata-bg_reclaim_threshold=(\\d{1,2})$" // ditto ^
127-
btrfsReadAheadKBRegexPattern = "^btrfs-bdi-read_ahead_kb=(\\d+)$"
124+
readAheadKBMountFlagRegexPattern = "^read_ahead_kb=(.+)$"
125+
btrfsReclaimDataRegexPattern = "^btrfs-allocation-data-bg_reclaim_threshold=(\\d{1,2})$" // 0-99 are valid, incl. 00
126+
btrfsReclaimMetadataRegexPattern = "^btrfs-allocation-metadata-bg_reclaim_threshold=(\\d{1,2})$" // ditto ^
127+
btrfsDynamicReclaimDataRegexPattern = "^btrfs-allocation-data-dynamic_reclaim=(0|1)$" // boolean in kernel, so accepting 0 or 1
128+
btrfsDynamicReclaimMetadataRegexPattern = "^btrfs-allocation-metadata-dynamic_reclaim=(0|1)$" // ditto ^
129+
btrfsReadAheadKBRegexPattern = "^btrfs-bdi-read_ahead_kb=(\\d+)$"
128130
)
129131

130132
var (
131-
readAheadKBMountFlagRegex = regexp.MustCompile(readAheadKBMountFlagRegexPattern)
132-
btrfsReclaimDataRegex = regexp.MustCompile(btrfsReclaimDataRegexPattern)
133-
btrfsReclaimMetadataRegex = regexp.MustCompile(btrfsReclaimMetadataRegexPattern)
134-
btrfsReadAheadKBRegex = regexp.MustCompile(btrfsReadAheadKBRegexPattern)
133+
readAheadKBMountFlagRegex = regexp.MustCompile(readAheadKBMountFlagRegexPattern)
134+
btrfsReclaimDataRegex = regexp.MustCompile(btrfsReclaimDataRegexPattern)
135+
btrfsReclaimMetadataRegex = regexp.MustCompile(btrfsReclaimMetadataRegexPattern)
136+
btrfsDynamicReclaimDataRegex = regexp.MustCompile(btrfsDynamicReclaimDataRegexPattern)
137+
btrfsDynamicReclaimMetadataRegex = regexp.MustCompile(btrfsDynamicReclaimMetadataRegexPattern)
138+
btrfsReadAheadKBRegex = regexp.MustCompile(btrfsReadAheadKBRegexPattern)
135139
)
136140

141+
type btrfsFlags struct {
142+
reclaimData,
143+
reclaimMetadata,
144+
dynamicReclaimData,
145+
dynamicReclaimMetadata,
146+
readAheadKb string
147+
}
148+
137149
func getDefaultFsType() string {
138150
if runtime.GOOS == "windows" {
139151
return defaultWindowsFsType
@@ -404,7 +416,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
404416
// Part 3: Mount device to stagingTargetPath
405417
fstype := getDefaultFsType()
406418

407-
var btrfsReclaimData, btrfsReclaimMetadata, btrfsReadAheadKb string
419+
var btrfsFlags btrfsFlags
408420
shouldUpdateReadAhead := false
409421
var readAheadKB int64
410422
options := []string{}
@@ -420,7 +432,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
420432
}
421433

422434
if mnt.FsType == fsTypeBtrfs {
423-
btrfsReclaimData, btrfsReclaimMetadata, btrfsReadAheadKb = extractBtrfsFlags(mnt.MountFlags)
435+
btrfsFlags = extractBtrfsFlags(mnt.MountFlags)
424436
}
425437
} else if blk := volumeCapability.GetBlock(); blk != nil {
426438
// Noop for Block NodeStageVolume
@@ -476,16 +488,22 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
476488

477489
btrfsSysfs := map[string]string{}
478490

479-
if btrfsReadAheadKb != "" {
480-
btrfsSysfs["bdi/read_ahead_kb"] = btrfsReadAheadKb
491+
if btrfsFlags.readAheadKb != "" {
492+
btrfsSysfs["bdi/read_ahead_kb"] = btrfsFlags.readAheadKb
481493
}
482494

483495
if !readonly {
484-
if btrfsReclaimData != "" {
485-
btrfsSysfs["allocation/data/bg_reclaim_threshold"] = btrfsReclaimData
496+
if btrfsFlags.reclaimData != "" {
497+
btrfsSysfs["allocation/data/bg_reclaim_threshold"] = btrfsFlags.reclaimData
498+
}
499+
if btrfsFlags.reclaimMetadata != "" {
500+
btrfsSysfs["allocation/metadata/bg_reclaim_threshold"] = btrfsFlags.reclaimMetadata
501+
}
502+
if btrfsFlags.dynamicReclaimData != "" {
503+
btrfsSysfs["allocation/data/dynamic_reclaim"] = btrfsFlags.dynamicReclaimData
486504
}
487-
if btrfsReclaimMetadata != "" {
488-
btrfsSysfs["allocation/metadata/bg_reclaim_threshold"] = btrfsReclaimMetadata
505+
if btrfsFlags.dynamicReclaimMetadata != "" {
506+
btrfsSysfs["allocation/metadata/dynamic_reclaim"] = btrfsFlags.dynamicReclaimMetadata
489507
}
490508
}
491509

@@ -552,18 +570,22 @@ func (ns *GCENodeServer) updateReadAhead(devicePath string, readAheadKB int64) e
552570
return nil
553571
}
554572

555-
func extractBtrfsFlags(mountFlags []string) (string, string, string) {
556-
var reclaimData, reclaimMetadata, readAheadKb string
573+
func extractBtrfsFlags(mountFlags []string) btrfsFlags {
574+
var flags btrfsFlags
557575
for _, mountFlag := range mountFlags {
558576
if got := btrfsReclaimDataRegex.FindStringSubmatch(mountFlag); len(got) == 2 {
559-
reclaimData = got[1]
577+
flags.reclaimData = got[1]
560578
} else if got := btrfsReclaimMetadataRegex.FindStringSubmatch(mountFlag); len(got) == 2 {
561-
reclaimMetadata = got[1]
579+
flags.reclaimMetadata = got[1]
562580
} else if got := btrfsReadAheadKBRegex.FindStringSubmatch(mountFlag); len(got) == 2 {
563-
readAheadKb = got[1]
581+
flags.readAheadKb = got[1]
582+
} else if got := btrfsDynamicReclaimDataRegex.FindStringSubmatch(mountFlag); len(got) == 2 {
583+
flags.dynamicReclaimData = got[1]
584+
} else if got := btrfsDynamicReclaimMetadataRegex.FindStringSubmatch(mountFlag); len(got) == 2 {
585+
flags.dynamicReclaimMetadata = got[1]
564586
}
565587
}
566-
return reclaimData, reclaimMetadata, readAheadKb
588+
return flags
567589
}
568590

569591
func extractReadAheadKBMountFlag(mountFlags []string) (int64, bool, error) {

‎pkg/gce-pd-csi-driver/node_test.go‎

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -631,7 +631,9 @@ func TestNodeStageVolume(t *testing.T) {
631631
btrfsPrefix = fmt.Sprintf("%s/sys/fs/btrfs/%s", tempDir, btrfsUUID)
632632
btrfsFixtures = map[string]string{
633633
"allocation/data/bg_reclaim_threshold": "0\n",
634+
"allocation/data/dynamic_reclaim": "0\n",
634635
"allocation/metadata/bg_reclaim_threshold": "0\n",
636+
"allocation/metadata/dynamic_reclaim": "0\n",
635637
"bdi/read_ahead_kb": "4096\n",
636638
}
637639
)
@@ -648,22 +650,24 @@ func TestNodeStageVolume(t *testing.T) {
648650
}
649651

650652
testCases := []struct {
651-
name string
652-
req *csi.NodeStageVolumeRequest
653-
deviceSize int
654-
blockExtSize int
655-
readonlyBit string
656-
expResize bool
657-
expReadAheadUpdate bool
658-
expReadAheadKB string
659-
expReadOnlyRemount bool
660-
expCommandList []fakeCmd
661-
readAheadSectors string
662-
btrfsReclaimData string
663-
btrfsReclaimMetadata string
664-
btrfsReadAheadKb string
665-
sectorSizeInBytes int
666-
expErrCode codes.Code
653+
name string
654+
req *csi.NodeStageVolumeRequest
655+
deviceSize int
656+
blockExtSize int
657+
readonlyBit string
658+
expResize bool
659+
expReadAheadUpdate bool
660+
expReadAheadKB string
661+
expReadOnlyRemount bool
662+
expCommandList []fakeCmd
663+
readAheadSectors string
664+
btrfsReclaimData string
665+
btrfsReclaimMetadata string
666+
btrfsDynamicReclaimData string
667+
btrfsDynamicReclaimMetadata string
668+
btrfsReadAheadKb string
669+
sectorSizeInBytes int
670+
expErrCode codes.Code
667671
}{
668672
{
669673
name: "Valid request, resize even though block and filesystem sizes match",
@@ -927,6 +931,8 @@ func TestNodeStageVolume(t *testing.T) {
927931
MountFlags: []string{
928932
"btrfs-allocation-data-bg_reclaim_threshold=90",
929933
"btrfs-allocation-metadata-bg_reclaim_threshold=91",
934+
"btrfs-allocation-data-dynamic_reclaim=1",
935+
"btrfs-allocation-metadata-dynamic_reclaim=1",
930936
"btrfs-bdi-read_ahead_kb=128",
931937
},
932938
},
@@ -936,12 +942,14 @@ func TestNodeStageVolume(t *testing.T) {
936942
},
937943
},
938944
},
939-
deviceSize: 1,
940-
blockExtSize: 1,
941-
readonlyBit: "0",
942-
btrfsReclaimData: "90",
943-
btrfsReclaimMetadata: "91",
944-
btrfsReadAheadKb: "128",
945+
deviceSize: 1,
946+
blockExtSize: 1,
947+
readonlyBit: "0",
948+
btrfsReclaimData: "90",
949+
btrfsReclaimMetadata: "91",
950+
btrfsDynamicReclaimData: "1",
951+
btrfsDynamicReclaimMetadata: "1",
952+
btrfsReadAheadKb: "128",
945953
expCommandList: []fakeCmd{
946954
{
947955
cmd: "blkid",
@@ -1267,13 +1275,17 @@ func TestNodeStageVolume(t *testing.T) {
12671275
if tc.expReadAheadUpdate == false && readAheadUpdateCalled == true {
12681276
t.Fatalf("Test updated read ahead, but it was not expected.")
12691277
}
1270-
if tc.btrfsReclaimData == "" && tc.btrfsReclaimMetadata == "" && tc.btrfsReadAheadKb == "" && blkidCalled {
1278+
if tc.btrfsReclaimData == "" && tc.btrfsReclaimMetadata == "" &&
1279+
tc.btrfsDynamicReclaimData == "" && tc.btrfsDynamicReclaimMetadata == "" &&
1280+
tc.btrfsReadAheadKb == "" && blkidCalled {
12711281
t.Fatalf("blkid was called, but was not expected.")
12721282
}
12731283

12741284
btrfsProps := map[string]string{
12751285
"/allocation/data/bg_reclaim_threshold": tc.btrfsReclaimData,
12761286
"/allocation/metadata/bg_reclaim_threshold": tc.btrfsReclaimMetadata,
1287+
"/allocation/data/dynamic_reclaim": tc.btrfsDynamicReclaimData,
1288+
"/allocation/metadata/dynamic_reclaim": tc.btrfsDynamicReclaimMetadata,
12771289
"/bdi/read_ahead_kb": tc.btrfsReadAheadKb,
12781290
}
12791291

‎pkg/gce-pd-csi-driver/utils.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,12 @@ func collectMountOptions(fsType string, mntFlags []string) []string {
317317
if btrfsReclaimMetadataRegex.FindString(opt) != "" {
318318
continue
319319
}
320+
if btrfsDynamicReclaimDataRegex.FindString(opt) != "" {
321+
continue
322+
}
323+
if btrfsDynamicReclaimMetadataRegex.FindString(opt) != "" {
324+
continue
325+
}
320326
if btrfsReadAheadKBRegex.FindString(opt) != "" {
321327
continue
322328
}

0 commit comments

Comments
 (0)