server: expose Go soft memory limit as a metric

If the Go memory usage gets close to the soft memory limit,
performance can plummet because the GC has to run constantly.

This commit adds a metric showing the soft memory limit, so it's easy
to see when the Go total and alloc graphs get close to that.

Epic: none
Release note: None

Author: RaduBerinde

docs/generated/eventlog.md

@@ -522,6 +522,7 @@ An event of type `runtime_stats` is recorded every 10 seconds as server health m
 | `GCRunCount` | The total number of GC runs. | no |
 | `NetHostRecvBytes` | The bytes received on all network interfaces since this process started. | no |
 | `NetHostSendBytes` | The bytes sent on all network interfaces since this process started. | no |
+| `GoLimitBytes` | The soft Go memory limit in bytes. | no |
 
 
 #### Common fields

docs/generated/metrics/metrics.yaml

@@ -10366,6 +10366,14 @@ layers:
       unit: BYTES
       aggregation: AVG
       derivative: NONE
+    - name: sys.go.limitbytes
+      exported_name: sys_go_limitbytes
+      description: Go soft memory limit
+      y_axis_label: Memory
+      type: GAUGE
+      unit: BYTES
+      aggregation: AVG
+      derivative: NONE
     - name: sys.go.pause.other.ns
       exported_name: sys_go_pause_other_ns
       description: Estimated non-GC-related total pause time

pkg/roachprod/opentelemetry/cockroachdb_metrics.go

@@ -2044,6 +2044,7 @@ var cockroachdbMetrics = map[string]string{
 	"sys_go_heap_heapfragmentbytes":                               "sys.go.heap.heapfragmentbytes",
 	"sys_go_heap_heapreleasedbytes":                               "sys.go.heap.heapreleasedbytes",
 	"sys_go_heap_heapreservedbytes":                               "sys.go.heap.heapreservedbytes",
+	"sys_go_limitbytes":                                           "sys.go.limitbytes",
 	"sys_go_pause_other_ns":                                       "sys.go.pause.other.ns",
 	"sys_go_stack_systembytes":                                    "sys.go.stack.systembytes",
 	"sys_go_stop_other_ns":                                        "sys.go.stop.other.ns",

pkg/server/status/runtime.go

@@ -67,6 +67,12 @@ var (
 		Measurement: "Memory",
 		Unit:        metric.Unit_BYTES,
 	}
+	metaGoLimitBytes = metric.Metadata{
+		Name:        "sys.go.limitbytes",
+		Help:        "Go soft memory limit",
+		Measurement: "Memory",
+		Unit:        metric.Unit_BYTES,
+	}
 	metaGoMemStackSysBytes = metric.Metadata{
 		Name:        "sys.go.stack.systembytes",
 		Help:        "Stack memory obtained from the OS.",
@@ -591,13 +597,17 @@ const runtimeMetricMemStackOSBytes = "/memory/classes/os-stacks:bytes"
 // metrics in /memory/classes.
 const runtimeMetricGoTotal = "/memory/classes/total:bytes"
 
+// Current soft memory limit (see debug.SetMemoryLimit).
+const runtimeMetricGoLimit = "/gc/gomemlimit:bytes"
+
 // Count of all completed GC cycles.
 const runtimeMetricGCCount = "/gc/cycles/total:gc-cycles"
 
 var runtimeMetrics = []string{
 	runtimeMetricGCAssist,
 	runtimeMetricGoTotal,
 	runtimeMetricHeapAlloc,
+	runtimeMetricGoLimit,
 	runtimeMetricHeapFragmentBytes,
 	runtimeMetricHeapReservedBytes,
 	runtimeMetricHeapReleasedBytes,
@@ -774,6 +784,7 @@ type RuntimeStatSampler struct {
 	RunnableGoroutinesPerCPU *metric.GaugeFloat64
 	GoAllocBytes             *metric.Gauge
 	GoTotalBytes             *metric.Gauge
+	GoLimitBytes             *metric.Gauge
 	GoMemStackSysBytes       *metric.Gauge
 	GoHeapFragmentBytes      *metric.Gauge
 	GoHeapReservedBytes      *metric.Gauge
@@ -875,6 +886,7 @@ func NewRuntimeStatSampler(ctx context.Context, clock hlc.WallClock) *RuntimeSta
 		RunnableGoroutinesPerCPU: metric.NewGaugeFloat64(metaRunnableGoroutinesPerCPU),
 		GoAllocBytes:             metric.NewGauge(metaGoAllocBytes),
 		GoTotalBytes:             metric.NewGauge(metaGoTotalBytes),
+		GoLimitBytes:             metric.NewGauge(metaGoLimitBytes),
 		GoMemStackSysBytes:       metric.NewGauge(metaGoMemStackSysBytes),
 		GoHeapFragmentBytes:      metric.NewGauge(metaGoHeapFragmentBytes),
 		GoHeapReservedBytes:      metric.NewGauge(metaGoHeapReservedBytes),
@@ -1136,6 +1148,10 @@ func (rsr *RuntimeStatSampler) SampleEnvironment(ctx context.Context, cs *CGoMem
 	goAlloc := rsr.goRuntimeSampler.uint64(runtimeMetricHeapAlloc)
 	goTotal := rsr.goRuntimeSampler.uint64(runtimeMetricGoTotal) -
 		rsr.goRuntimeSampler.uint64(runtimeMetricHeapReleasedBytes)
+	goLimit := rsr.goRuntimeSampler.uint64(runtimeMetricGoLimit)
+	if goLimit == math.MaxInt64 {
+		goLimit = 0
+	}
 	stackTotal := rsr.goRuntimeSampler.uint64(runtimeMetricMemStackHeapBytes) +
 		osStackBytes
 	heapFragmentBytes := rsr.goRuntimeSampler.uint64(runtimeMetricHeapFragmentBytes)
@@ -1147,6 +1163,7 @@ func (rsr *RuntimeStatSampler) SampleEnvironment(ctx context.Context, cs *CGoMem
 		MemStackSysBytes:  stackTotal,
 		GoAllocBytes:      goAlloc,
 		GoTotalBytes:      goTotal,
+		GoLimitBytes:      goLimit,
 		HeapFragmentBytes: heapFragmentBytes,
 		HeapReservedBytes: heapReservedBytes,
 		HeapReleasedBytes: heapReleasedBytes,
@@ -1168,6 +1185,7 @@ func (rsr *RuntimeStatSampler) SampleEnvironment(ctx context.Context, cs *CGoMem
 
 	rsr.GoAllocBytes.Update(int64(goAlloc))
 	rsr.GoTotalBytes.Update(int64(goTotal))
+	rsr.GoLimitBytes.Update(int64(goLimit))
 	rsr.GoMemStackSysBytes.Update(int64(osStackBytes))
 	rsr.GoHeapFragmentBytes.Update(int64(heapFragmentBytes))
 	rsr.GoHeapReservedBytes.Update(int64(heapReservedBytes))

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/runtime.tsx

@@ -51,6 +51,8 @@ export default function (props: GraphDashboardProps) {
             <dd>Memory allocated by the Go layer</dd>
             <dt>Go Total</dt>
             <dd>Total memory managed by the Go layer</dd>
+            <dt>Go Limit</dt>
+            <dd>Go soft memory limit</dd>
             <dt>C Allocated</dt>
             <dd>Memory allocated by the C layer</dd>
             <dt>C Total</dt>
@@ -64,6 +66,7 @@ export default function (props: GraphDashboardProps) {
         <Metric name="cr.node.sys.rss" title="Total memory (RSS)" />
         <Metric name="cr.node.sys.go.allocbytes" title="Go Allocated" />
         <Metric name="cr.node.sys.go.totalbytes" title="Go Total" />
+        <Metric name="cr.node.sys.go.limitbytes" title="Go Limit" />
         <Metric name="cr.node.sys.cgo.allocbytes" title="CGo Allocated" />
         <Metric name="cr.node.sys.cgo.totalbytes" title="CGo Total" />
       </Axis>

pkg/util/log/eventpb/health_events.proto

@@ -59,6 +59,8 @@ message RuntimeStats {
   uint64 net_host_recv_bytes = 18 [(gogoproto.jsontag) = ",omitempty"];
   // The bytes sent on all network interfaces since this process started.
   uint64 net_host_send_bytes = 19 [(gogoproto.jsontag) = ",omitempty"];
+  // The soft Go memory limit in bytes.
+  uint64 go_limit_bytes = 20 [(gogoproto.jsontag) = ",omitempty"];
 }
 
 // HotRangesStats