Skip to content

Commit 2769fa9

Browse files
sgalsalehajp-io
sgalsaleh
and
ajp-io
authored
Ensure inotify settings meet the minimum requirements (#1897)
* Ensure inotify settings meet the minimum requirements --------- Co-authored-by: Alex Parker <[email protected]>
1 parent 334c634 commit 2769fa9

File tree

3 files changed

+204
-6
lines changed

3 files changed

+204
-6
lines changed

‎pkg/configutils/runtime.go‎

Lines changed: 94 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os"
99
"os/exec"
1010
"path/filepath"
11+
"strconv"
1112
"strings"
1213

1314
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
@@ -19,17 +20,45 @@ import (
1920
// purposes.
2021
var sysctlConfigPath = "/etc/sysctl.d/99-embedded-cluster.conf"
2122

22-
var modulesLoadConfigPath = "/etc/modules-load.d/99-embedded-cluster.conf"
23+
// dynamicSysctlConfigPath is the path to the dynamic sysctl config file that is used to configure
24+
// the embedded cluster.
25+
const dynamicSysctlConfigPath = "/etc/sysctl.d/99-dynamic-embedded-cluster.conf"
26+
27+
// modulesLoadConfigPath is the path to the kernel modules config file that is used to configure
28+
// the embedded cluster.
29+
const modulesLoadConfigPath = "/etc/modules-load.d/99-embedded-cluster.conf"
2330

2431
//go:embed static/sysctl.d/99-embedded-cluster.conf
2532
var embeddedClusterSysctlConf []byte
2633

2734
//go:embed static/modules-load.d/99-embedded-cluster.conf
2835
var embeddedClusterModulesConf []byte
2936

30-
// ConfigureSysctl writes the sysctl config file for the embedded cluster and reloads the sysctl
31-
// configuration. This function has a distinct behavior: if the sysctl binary does not exist it
32-
// returns an error but if it fails to lay down the sysctl config on disk it simply returns nil.
37+
// dynamicSysctlConstraints are the constraints that are used to generate the dynamic sysctl
38+
// config file.
39+
var dynamicSysctlConstraints = []sysctlConstraint{
40+
// Increase inotify limits only if they are currently lower,
41+
// ensuring proper operation of applications that monitor filesystem events.
42+
{key: "fs.inotify.max_user_instances", value: 1024, operator: sysctlOperatorMin},
43+
{key: "fs.inotify.max_user_watches", value: 65536, operator: sysctlOperatorMin},
44+
}
45+
46+
type sysctlOperator string
47+
48+
const (
49+
sysctlOperatorMin sysctlOperator = "min"
50+
sysctlOperatorMax sysctlOperator = "max"
51+
)
52+
53+
type sysctlConstraint struct {
54+
key string
55+
value int64
56+
operator sysctlOperator
57+
}
58+
59+
type sysctlValueGetter func(key string) (int64, error)
60+
61+
// ConfigureSysctl writes the sysctl config files for the embedded cluster and reloads the sysctl configuration.
3362
// NOTE: do not run this after the cluster has already been installed as it may revert sysctl
3463
// settings set by k0s and its extensions.
3564
func ConfigureSysctl() error {
@@ -41,6 +70,10 @@ func ConfigureSysctl() error {
4170
return fmt.Errorf("materialize sysctl config: %w", err)
4271
}
4372

73+
if err := dynamicSysctlConfig(); err != nil {
74+
return fmt.Errorf("materialize dynamic sysctl config: %w", err)
75+
}
76+
4477
if _, err := helpers.RunCommand("sysctl", "--system"); err != nil {
4578
return fmt.Errorf("configure sysctl: %w", err)
4679
}
@@ -58,6 +91,63 @@ func sysctlConfig() error {
5891
return nil
5992
}
6093

94+
// dynamicSysctlConfig generates a dynamic sysctl config file based on current system values
95+
// and our constraints.
96+
func dynamicSysctlConfig() error {
97+
return generateDynamicSysctlConfig(getCurrentSysctlValue, dynamicSysctlConfigPath)
98+
}
99+
100+
// generateDynamicSysctlConfig is the testable version of dynamicSysctlConfig that accepts
101+
// a custom sysctl value getter and config path.
102+
func generateDynamicSysctlConfig(getter sysctlValueGetter, configPath string) error {
103+
if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
104+
return fmt.Errorf("create directory: %w", err)
105+
}
106+
107+
var config strings.Builder
108+
config.WriteString("# Dynamic sysctl configuration for embedded-cluster\n")
109+
config.WriteString("# This file is generated based on system values\n\n")
110+
111+
for _, constraint := range dynamicSysctlConstraints {
112+
currentValue, err := getter(constraint.key)
113+
if err != nil {
114+
return fmt.Errorf("check current value for %s: %w", constraint.key, err)
115+
}
116+
117+
needsUpdate := false
118+
switch constraint.operator {
119+
case sysctlOperatorMin:
120+
needsUpdate = currentValue < constraint.value
121+
case sysctlOperatorMax:
122+
needsUpdate = currentValue > constraint.value
123+
}
124+
125+
if needsUpdate {
126+
config.WriteString(fmt.Sprintf("%s = %d\n", constraint.key, constraint.value))
127+
}
128+
}
129+
130+
if err := os.WriteFile(configPath, []byte(config.String()), 0644); err != nil {
131+
return fmt.Errorf("write dynamic config file: %w", err)
132+
}
133+
return nil
134+
}
135+
136+
// getCurrentSysctlValue reads the current value of a sysctl parameter
137+
func getCurrentSysctlValue(key string) (int64, error) {
138+
out, err := helpers.RunCommand("sysctl", "-n", key)
139+
if err != nil {
140+
return 0, fmt.Errorf("get sysctl value: %w", err)
141+
}
142+
143+
value, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
144+
if err != nil {
145+
return 0, fmt.Errorf("parse sysctl value: %w", err)
146+
}
147+
148+
return value, nil
149+
}
150+
61151
// ConfigureKernelModules writes the kernel modules config file and ensures the kernel modules are
62152
// loaded that are listed in the file.
63153
func ConfigureKernelModules() error {

‎pkg/configutils/runtime_test.go‎

Lines changed: 94 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import (
99
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
1010
"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
1111
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
1213
)
1314

14-
func TestConfigureSysctl(t *testing.T) {
15+
func TestSysctlConfig(t *testing.T) {
1516
basedir, err := os.MkdirTemp("", "embedded-cluster-test-base-dir")
1617
assert.NoError(t, err)
1718
defer os.RemoveAll(basedir)
@@ -29,7 +30,7 @@ func TestConfigureSysctl(t *testing.T) {
2930
defer os.RemoveAll(dstdir)
3031

3132
sysctlConfigPath = filepath.Join(dstdir, "sysctl.conf")
32-
err = ConfigureSysctl()
33+
err = sysctlConfig()
3334
assert.NoError(t, err)
3435

3536
// check that the file exists.
@@ -42,6 +43,97 @@ func TestConfigureSysctl(t *testing.T) {
4243
assert.NoError(t, err)
4344
}
4445

46+
func TestDynamicSysctlConfig(t *testing.T) {
47+
// Create a temporary config file.
48+
configPath := filepath.Join(t.TempDir(), "99-dynamic-embedded-cluster.conf")
49+
50+
tests := []struct {
51+
name string
52+
mockValues map[string]int64
53+
expectedLines []string
54+
unexpectedLines []string
55+
}{
56+
{
57+
name: "inotify max_user values below minimum thresholds are updated",
58+
mockValues: map[string]int64{
59+
"fs.inotify.max_user_instances": 128, // Below min
60+
"fs.inotify.max_user_watches": 8192, // Below min
61+
},
62+
expectedLines: []string{
63+
"fs.inotify.max_user_instances = 1024",
64+
"fs.inotify.max_user_watches = 65536",
65+
},
66+
},
67+
{
68+
name: "only below minimum values of inotify max_user are updated",
69+
mockValues: map[string]int64{
70+
"fs.inotify.max_user_instances": 128, // Below min
71+
"fs.inotify.max_user_watches": 1048576, // Above min
72+
},
73+
expectedLines: []string{
74+
"fs.inotify.max_user_instances = 1024",
75+
},
76+
unexpectedLines: []string{
77+
"fs.inotify.max_user_watches",
78+
},
79+
},
80+
{
81+
name: "inotify max_user values above minimum thresholds are not updated",
82+
mockValues: map[string]int64{
83+
"fs.inotify.max_user_instances": 2048, // Above min
84+
"fs.inotify.max_user_watches": 1048576, // Above min
85+
},
86+
expectedLines: []string{}, // No updates needed
87+
unexpectedLines: []string{
88+
"fs.inotify.max_user_instances",
89+
"fs.inotify.max_user_watches",
90+
},
91+
},
92+
{
93+
name: "inotify max_user values equal to minimum thresholds are not updated",
94+
mockValues: map[string]int64{
95+
"fs.inotify.max_user_instances": 1024, // Equal to min
96+
"fs.inotify.max_user_watches": 65536, // Equal to min
97+
},
98+
expectedLines: []string{}, // No updates needed
99+
unexpectedLines: []string{
100+
"fs.inotify.max_user_instances",
101+
"fs.inotify.max_user_watches",
102+
},
103+
},
104+
}
105+
106+
for _, tt := range tests {
107+
t.Run(tt.name, func(t *testing.T) {
108+
// Create mock getter
109+
mockGetter := func(key string) (int64, error) {
110+
value, exists := tt.mockValues[key]
111+
if !exists {
112+
t.Fatalf("unexpected key requested: %s", key)
113+
}
114+
return value, nil
115+
}
116+
117+
err := generateDynamicSysctlConfig(mockGetter, configPath)
118+
require.NoError(t, err)
119+
120+
// Read generated file
121+
content, err := os.ReadFile(configPath)
122+
require.NoError(t, err)
123+
124+
// Check for expected lines
125+
for _, expectedLine := range tt.expectedLines {
126+
assert.Contains(t, string(content), expectedLine)
127+
}
128+
129+
// Check for unexpected lines
130+
for _, unexpectedLine := range tt.unexpectedLines {
131+
assert.NotContains(t, string(content), unexpectedLine)
132+
}
133+
})
134+
}
135+
}
136+
45137
func Test_ensureKernelModulesLoaded(t *testing.T) {
46138
// Create and set mock helper
47139
mock := &helpers.MockHelpers{

‎pkg/preflights/host-preflight.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,22 @@ spec:
10131013
- pass:
10141014
when: 'net.bridge.bridge-nf-call-iptables == 1'
10151015
message: "Bridge netfilter call iptables is enabled."
1016+
- sysctl:
1017+
checkName: "Maximum number of inotify instances per user"
1018+
outcomes:
1019+
- fail:
1020+
when: "fs.inotify.max_user_instances < 1024"
1021+
message: "The system limit for inotify instances per user must be at least 1024. To increase it, edit /etc/sysctl.conf, add or edit the line 'fs.inotify.max_user_instances=1024', and run 'sudo sysctl -p'."
1022+
- pass:
1023+
message: "The system allows at least 1024 inotify instances per user."
1024+
- sysctl:
1025+
checkName: "Maximum number of inotify watches per user"
1026+
outcomes:
1027+
- fail:
1028+
when: "fs.inotify.max_user_watches < 65536"
1029+
message: "The system limit for inotify watches per user must be at least 65536. To increase it, edit /etc/sysctl.conf, add or edit the line 'fs.inotify.max_user_watches=65536', and run 'sudo sysctl -p'."
1030+
- pass:
1031+
message: "The system allows at least 65536 inotify watches per user."
10161032
- kernelModules:
10171033
checkName: "Overlay kernel module"
10181034
outcomes:

0 commit comments

Comments
 (0)