Skip to content

Commit 221e449

Browse files
Thejas-bhatabhinavdangeti
authored andcommitted
MB-60367: Accomodating stats from C heap in the utilization calculation
+ Applicable on linux and mac platforms only (for now). + nsstats and prometheusMetrics will now showcase a new stat num_bytes_used_ram_c (the C only report). Change-Id: Ia057c42ad1788e22638f6b50b15841f3a69e9109 Reviewed-on: https://review.couchbase.org/c/cbft/+/204392 Well-Formed: Restriction Checker Tested-by: Abhi Dangeti <[email protected]> Reviewed-by: Abhi Dangeti <[email protected]> Well-Formed: Build Bot <[email protected]>
1 parent c93bf24 commit 221e449

File tree

5 files changed

+174
-5
lines changed

5 files changed

+174
-5
lines changed

c_heap_mem_usage.h

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2024-Present Couchbase, Inc.
2+
//
3+
// Use of this software is governed by the Business Source License included
4+
// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified
5+
// in that file, in accordance with the Business Source License, use of this
6+
// software will be governed by the Apache License, Version 2.0, included in
7+
// the file licenses/APL2.txt.
8+
9+
#ifdef __cplusplus
10+
extern "C" {
11+
#endif
12+
// the first block is just to declare the heap stats function for all the
13+
// supported OS, whereas in the else block we have a no-op.
14+
#if (defined(__APPLE__) && defined(__MACH__)) || \
15+
(defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__))
16+
size_t get_total_heap_bytes();
17+
#else
18+
size_t get_total_heap_bytes() {
19+
return(size_t)0L;
20+
}
21+
#endif
22+
23+
#ifdef __cplusplus
24+
}
25+
#endif

c_heap_mem_usage_linux.cpp

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Copyright 2024-Present Couchbase, Inc.
2+
//
3+
// Use of this software is governed by the Business Source License included
4+
// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified
5+
// in that file, in accordance with the Business Source License, use of this
6+
// software will be governed by the Apache License, Version 2.0, included in
7+
// the file licenses/APL2.txt.
8+
9+
#if defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
10+
#include <cstdio>
11+
#include <cstdlib>
12+
#include <cstring>
13+
#include "c_heap_mem_usage.h"
14+
#include <malloc.h>
15+
16+
size_t get_attribute_value(const char *str, const char *substr) {
17+
const char *pos = strstr(str, substr);
18+
if (pos == NULL) {
19+
return 0;
20+
}
21+
pos += strlen(substr);
22+
23+
char* next_pos = NULL;
24+
size_t value = strtoull(pos, &next_pos, 10);
25+
if (next_pos == NULL || next_pos == pos) {
26+
return 0;
27+
}
28+
29+
return value;
30+
}
31+
32+
33+
size_t get_attribute_value(const char *str, const char *first_substr, const char* second_substr) {
34+
const char *pos = strstr(str, first_substr);
35+
if (pos == NULL) {
36+
return 0;
37+
}
38+
pos += strlen(first_substr);
39+
pos = strstr(pos, second_substr);
40+
if (pos == NULL) {
41+
return 0;
42+
}
43+
pos += strlen(second_substr);
44+
45+
char* next_pos = NULL;
46+
size_t value = strtoull(pos, &next_pos, 10);
47+
if (next_pos == NULL || next_pos == pos) {
48+
return 0;
49+
}
50+
51+
return value;
52+
}
53+
54+
55+
size_t get_total_heap_bytes(){
56+
// ref: https://gist.github.com/tadeu/95013963c64da4cd74a2c6f4fa4fd553
57+
// There are three functions in Linux libc API to retrieve heap
58+
// information: `malloc_stats`, `mallinfo` and `malloc_info`.
59+
// The first two are still broken for 64-bit systems and will
60+
// report wrong values if there are more than 4 Gb of allocated
61+
// memory. The latter works for more than 4 Gb, but it outputs
62+
// a XML to a file, so `open_memstream` is used to avoid writing
63+
// to the disk, and a very simple "parsing" is done here using
64+
// C-string search.
65+
//
66+
// More info:
67+
// https://stackoverflow.com/questions/40878169/64-bit-capable-alternative-to-mallinfo
68+
// https://stackoverflow.com/questions/3903807/how-does-malloc-info-work
69+
// https://stackoverflow.com/questions/34292457/gnu-malloc-info-get-really-allocated-memory
70+
71+
char* buf = NULL;
72+
size_t buf_size = 0;
73+
74+
FILE* f = open_memstream(&buf, &buf_size);
75+
// this doesn't include the golang or the other process's stats
76+
// as per local testing
77+
malloc_info(0, f);
78+
fclose(f);
79+
80+
// We are only interested in totals, so we skip everything until the
81+
// closing of the <heap>...</heap> block.
82+
const char* pos = strstr(buf, "</heap>");
83+
84+
// rest and fast are blocks that have been freed, we should subtract them
85+
size_t rest = get_attribute_value(pos, "<total type=\"rest\" count=\"", "\" size=\"");
86+
size_t fast = get_attribute_value(pos, "<total type=\"fast\" count=\"", "\" size=\"");
87+
88+
// mmap and current are totals (mmap is used for very large blocks)
89+
size_t mmap = get_attribute_value(pos, "<total type=\"mmap\" count=\"", "\" size=\"");
90+
size_t current = get_attribute_value(pos, "<system type=\"current\" size=\"");
91+
92+
size_t free_mem = rest + fast;
93+
size_t total_mem = mmap + current;
94+
95+
size_t allocated_mem = total_mem > free_mem ? total_mem - free_mem : 0;
96+
97+
free(buf);
98+
buf = NULL;
99+
return allocated_mem;
100+
}
101+
#endif

c_heap_mem_usage_mac.cpp

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2024-Present Couchbase, Inc.
2+
//
3+
// Use of this software is governed by the Business Source License included
4+
// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified
5+
// in that file, in accordance with the Business Source License, use of this
6+
// software will be governed by the Apache License, Version 2.0, included in
7+
// the file licenses/APL2.txt.
8+
9+
#if defined(__APPLE__) && defined(__MACH__)
10+
#include <malloc/malloc.h>
11+
#include <cstdio>
12+
#include <cstdlib>
13+
#include <cstring>
14+
#include "c_heap_mem_usage.h"
15+
16+
size_t get_total_heap_bytes() {
17+
// mstats on mac platform gives a copy of the struct which has information
18+
// like what's the bytes being used currently on the heap (the allocated bytes)
19+
// and also other information such as what's the total bytes that's been
20+
// allocated and the free bytes. this API call fetches the information by
21+
// talking to the memory manager which is responsible for tracking these
22+
// stats of malloc (which is what ultimately all dynamic datastructures call
23+
// underneath the hood for memory)
24+
// this doesn't include the golang or the other process's stats as per local testing
25+
//
26+
// https://opensource.apple.com/source/Libc/Libc-825.26/include/malloc/malloc.h.auto.html
27+
struct mstats ms = mstats();
28+
return ms.bytes_used;
29+
}
30+
#endif

ns_server.go

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88

99
package cbft
1010

11+
/*
12+
# include "c_heap_mem_usage.h"
13+
*/
14+
import "C"
15+
1116
import (
1217
"bytes"
1318
"encoding/json"
@@ -707,7 +712,8 @@ func obtainDestSeqsForIndex(indexDef *cbgt.IndexDef,
707712
return totDestSeq, 0, nil
708713
}
709714

710-
func getMemoryUtilization(memStats runtime.MemStats) uint64 {
715+
// Returns usage reported from go and c separately
716+
func getMemoryUtilization(memStats runtime.MemStats) (uint64, uint64) {
711717
// Memory utilization to account for:
712718
// - memory alloced for the process
713719
// - memory released by process back to the OS
@@ -720,7 +726,8 @@ func getMemoryUtilization(memStats runtime.MemStats) uint64 {
720726
// - HeapReleased is bytes of physical memory returned to the OS.
721727
// So our equation here should be ..
722728
// Sys - (HeapIdle - HeapReleased) - HeapReleased
723-
return memStats.Sys - memStats.HeapIdle
729+
cHeapBytes := C.get_total_heap_bytes()
730+
return memStats.Sys - memStats.HeapIdle, uint64(cHeapBytes)
724731
}
725732

726733
func gatherNodeUtilStats(mgr *cbgt.Manager,
@@ -742,8 +749,9 @@ func gatherNodeUtilStats(mgr *cbgt.Manager,
742749
rv["utilization:billableUnitsRate"] = DetermineNewAverage(
743750
"totalUnitsMetered", totalUnitsMetered)
744751

752+
goUtil, cUtil := getMemoryUtilization(rd.memStats)
745753
rv["utilization:memoryBytes"] = DetermineNewAverage(
746-
"memoryBytes", getMemoryUtilization(rd.memStats))
754+
"memoryBytes", goUtil + cUtil)
747755

748756
var size int64
749757
_ = filepath.Walk(mgr.DataDir(), func(_ string, info os.FileInfo, err error) error {
@@ -793,14 +801,17 @@ func gatherNodeUtilStats(mgr *cbgt.Manager,
793801
func gatherTopLevelStats(mgr *cbgt.Manager, rd *recentInfo) map[string]interface{} {
794802
topLevelStats := map[string]interface{}{}
795803

796-
memUtil := getMemoryUtilization(rd.memStats)
797804
var ftsMemoryQuota uint64
798805
if val := mgr.GetOption("ftsMemoryQuota"); len(val) > 0 {
799806
if valUint64, err := strconv.ParseUint(val, 10, 64); err == nil {
800807
ftsMemoryQuota = valUint64
801808
}
802809
}
810+
811+
goUtil, cUtil := getMemoryUtilization(rd.memStats)
812+
memUtil := goUtil + cUtil
803813
topLevelStats["num_bytes_used_ram"] = memUtil
814+
topLevelStats["num_bytes_used_ram_c"] = cUtil
804815
topLevelStats["num_bytes_ram_quota"] = ftsMemoryQuota
805816
topLevelStats["pct_used_ram"] =
806817
(float64(memUtil) / float64(ftsMemoryQuota)) * 100
@@ -1424,7 +1435,8 @@ func FetchCurMemoryUsed() uint64 {
14241435
}
14251436

14261437
func setCurMemoryUsedWith(memStats *runtime.MemStats) uint64 {
1427-
curMemoryUsed := getMemoryUtilization(*memStats)
1438+
goUtil, cUtil := getMemoryUtilization(*memStats)
1439+
curMemoryUsed := goUtil + cUtil
14281440
atomic.StoreUint64(&currentMemoryUsed, curMemoryUsed)
14291441
return curMemoryUsed
14301442
}

prometheus.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ var prometheusStats = map[string]string{
8686

8787
"pct_cpu_gc": "gauge",
8888
"num_bytes_used_ram": "gauge",
89+
"num_bytes_used_ram_c": "gauge",
8990
"num_bytes_ram_quota": "gauge",
9091
"pct_used_ram": "gauge",
9192
"avg_grpc_queries_latency": "gauge",

0 commit comments

Comments
 (0)