diff --git a/config/file_config.go b/config/file_config.go index 4a5d921335..39b5719e2f 100644 --- a/config/file_config.go +++ b/config/file_config.go @@ -11,6 +11,8 @@ import ( "golang.org/x/exp/slices" "gopkg.in/yaml.v3" + + "github.com/honeycombio/refinery/internal/system" ) // In order to be able to unmarshal "15s" etc. into time.Duration, we need to @@ -544,6 +546,13 @@ func newFileConfig(opts *CmdEnv, cData, rulesData []configData, currentVersion . return nil, err } + // If AvailableMemory is not set, AND MaxAlloc is not set, try to detect it from the system + if mainconf.Collection.AvailableMemory == 0 && mainconf.Collection.MaxAlloc == 0 { + if mem, err := system.GetTotalMemory(); err == nil && mem > 0 { + allowed := uint64(float64(mem) * float64(mainconf.Collection.MaxMemoryPercentage) / 100.0) + mainconf.Collection.MaxAlloc = MemorySize(allowed) + } + } var rulesconf *V2SamplerConfig ruleshash, err := applyConfigInto(&rulesconf, rulesData, nil) if err != nil { diff --git a/config/metadata/configMeta.yaml b/config/metadata/configMeta.yaml index 754a7be812..0129f8799a 100644 --- a/config/metadata/configMeta.yaml +++ b/config/metadata/configMeta.yaml @@ -1320,8 +1320,9 @@ groups: description: > This value will typically be set through an environment variable controlled by the container or deploy script. If this value is zero or - not set, then `MaxMemoryPercentage` cannot be used to calculate the - maximum allocation and `MaxAlloc` will be used instead. If set, then + not set, Refinery will attempt to detect the total available memory from + the system (cgroups or /proc/meminfo). If detection fails, `MaxAlloc` + will be used. If set, then this must be a memory size. Sizes with standard unit suffixes (such as `MB` and `GiB`) and Kubernetes units (such as `M` and `Gi`) are supported. Fractional values with a suffix are supported. If @@ -1338,8 +1339,7 @@ groups: validations: - type: minimum arg: 10 - - type: requiredWith - arg: AvailableMemory + summary: is the maximum percentage of memory that should be allocated by the span collector. description: > If nonzero, then it must be an integer value between 1 and 100, diff --git a/internal/system/memory.go b/internal/system/memory.go new file mode 100644 index 0000000000..d8ce22fb64 --- /dev/null +++ b/internal/system/memory.go @@ -0,0 +1,82 @@ +package system + +import ( + "bufio" + "bytes" + "errors" + "math" + "os" + "strconv" + "strings" +) + +// GetTotalMemory returns the total available memory in bytes. +// It tries to read from cgroup limits first, then falls back to system memory. +func GetTotalMemory() (uint64, error) { + // Try cgroup v2 + if mem, err := getCgroupV2Memory(); err == nil { + return mem, nil + } + + // Try cgroup v1 + if mem, err := getCgroupV1Memory(); err == nil { + return mem, nil + } + + // Fall back to system memory + return getSystemMemory() +} + +func getCgroupV2Memory() (uint64, error) { + data, err := os.ReadFile("/sys/fs/cgroup/memory.max") + if err != nil { + return 0, err + } + s := bytes.TrimSpace(data) + if bytes.Equal(s, []byte("max")) { + return 0, errors.New("no limit set in cgroup v2") + } + return strconv.ParseUint(string(s), 10, 64) +} + +func getCgroupV1Memory() (uint64, error) { + data, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes") + if err != nil { + return 0, err + } + s := bytes.TrimSpace(data) + mem, err := strconv.ParseUint(string(s), 10, 64) + if err != nil { + return 0, err + } + // A very large number indicates no limit in cgroup v1 + if mem > math.MaxInt64 { + return 0, errors.New("no limit set in cgroup v1") + } + return mem, nil +} + +func getSystemMemory() (uint64, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "MemTotal:") { + parts := strings.Fields(line) + if len(parts) < 2 { + return 0, errors.New("invalid MemTotal format") + } + kb, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return 0, err + } + return kb * 1024, nil + } + } + return 0, errors.New("MemTotal not found in /proc/meminfo") +} diff --git a/refinery_config.md b/refinery_config.md index 7dc06d58f5..59d9e23638 100644 --- a/refinery_config.md +++ b/refinery_config.md @@ -884,7 +884,8 @@ Its minimum value should be at least three times the `CacheCapacity`. `AvailableMemory` is the amount of system memory available to the Refinery process. This value will typically be set through an environment variable controlled by the container or deploy script. -If this value is zero or not set, then `MaxMemoryPercentage` cannot be used to calculate the maximum allocation and `MaxAlloc` will be used instead. +If this value is zero or not set, Refinery will attempt to detect the total available memory from the system (cgroups or /proc/meminfo). +If detection fails, `MaxAlloc` will be used. If set, then this must be a memory size. Sizes with standard unit suffixes (such as `MB` and `GiB`) and Kubernetes units (such as `M` and `Gi`) are supported. Fractional values with a suffix are supported.