prometheusremotewriteexpoter: add WAL bytes read/write metrics. (open-telemetry#40586)

partly fixes open-telemetry#39556.

```zsh
# HELP otelcol_exporter_prometheusremotewrite_wal_bytes_read Total number of bytes read from the WAL
# TYPE otelcol_exporter_prometheusremotewrite_wal_bytes_read counter
otelcol_exporter_prometheusremotewrite_wal_bytes_read{service_instance_id="ee4cd5c5-5773-404b-a573-df85413a9adc",service_name="otelcontribcol",service_version="0.128.0-dev"} 224
# HELP otelcol_exporter_prometheusremotewrite_wal_bytes_written Total number of bytes written to the WAL
# TYPE otelcol_exporter_prometheusremotewrite_wal_bytes_written counter
otelcol_exporter_prometheusremotewrite_wal_bytes_written{service_instance_id="ee4cd5c5-5773-404b-a573-df85413a9adc",service_name="otelcontribcol",service_version="0.128.0-dev"} 112
```

Author: sujalshah-bit

.chloggen/sujal_shah_prometheusremotewriteexporter_feat-add-wal-bytes-metrics.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: prometheusremotewriteexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: |
+  Adds WAL bytes read/write metrics to the Prometheus Remote Write Exporter. The new metrics are:
+  - `otelcol_exporter_prometheusremotewrite_wal_bytes_written`: The total number of bytes written to the WAL.
+  - `otelcol_exporter_prometheusremotewrite_wal_bytes_read`: The total number of bytes reads from the WAL.
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [39556]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/prometheusremotewriteexporter/documentation.md

@@ -38,6 +38,22 @@ Number of Prometheus time series that were translated from OTel metrics
 | ---- | ----------- | ---------- | --------- |
 | 1 | Sum | Int | true |
 
+### otelcol_exporter_prometheusremotewrite_wal_bytes_read
+
+Total number of bytes read from the WAL
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| By | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_wal_bytes_written
+
+Total number of bytes written to the WAL
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| By | Sum | Int | true |
+
 ### otelcol_exporter_prometheusremotewrite_wal_read_latency
 
 Response latency in ms for the WAL reads.

exporter/prometheusremotewriteexporter/exporter.go

@@ -293,7 +293,7 @@ func (prwe *prwExporter) handleExport(ctx context.Context, tsMap map[string]*pro
 	// Otherwise the WAL is enabled, and just persist the requests to the WAL
 	prwe.wal.telemetry.recordWALWrites(ctx)
 	start := time.Now()
-	err = prwe.wal.persistToWAL(requests)
+	err = prwe.wal.persistToWAL(ctx, requests)
 	duration := time.Since(start)
 	prwe.wal.telemetry.recordWALWriteLatency(ctx, duration.Milliseconds())
 	if err != nil {

exporter/prometheusremotewriteexporter/internal/metadata/generated_telemetry.go

@@ -88,3 +88,18 @@ telemetry:
       histogram:
         value_type: int
         bucket_boundaries: [5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000]
+    exporter_prometheusremotewrite_wal_bytes_written:
+      enabled: true
+      description: Total number of bytes written to the WAL
+      unit: "By"
+      sum:
+        value_type: int
+        monotonic: true
+
+    exporter_prometheusremotewrite_wal_bytes_read:
+      enabled: true
+      description: Total number of bytes read from the WAL
+      unit: "By"
+      sum:
+        value_type: int
+        monotonic: true

exporter/prometheusremotewriteexporter/internal/metadatatest/generated_telemetrytest.go

@@ -31,6 +31,8 @@ type prwWalTelemetry interface {
 	recordWALReadLatency(ctx context.Context, durationMs int64)
 	recordWALReads(ctx context.Context)
 	recordWALReadsFailures(ctx context.Context)
+	recordWALBytesWritten(ctx context.Context, bytes int)
+	recordWALBytesRead(ctx context.Context, bytes int)
 }
 
 type prwWalTelemetryOTel struct {
@@ -62,6 +64,14 @@ func (p *prwWalTelemetryOTel) recordWALReadsFailures(ctx context.Context) {
 	p.telemetryBuilder.ExporterPrometheusremotewriteWalReadsFailures.Add(ctx, 1, metric.WithAttributes(p.otelAttrs...))
 }
 
+func (p *prwWalTelemetryOTel) recordWALBytesWritten(ctx context.Context, bytes int) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWalBytesWritten.Add(ctx, int64(bytes), metric.WithAttributes(p.otelAttrs...))
+}
+
+func (p *prwWalTelemetryOTel) recordWALBytesRead(ctx context.Context, bytes int) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWalBytesRead.Add(ctx, int64(bytes), metric.WithAttributes(p.otelAttrs...))
+}
+
 func newPRWWalTelemetry(set exporter.Settings) (prwWalTelemetry, error) {
 	telemetryBuilder, err := metadata.NewTelemetryBuilder(set.TelemetrySettings)
 	if err != nil {
@@ -365,7 +375,7 @@ func (prweWAL *prweWAL) exportThenFrontTruncateWAL(ctx context.Context, reqL []*
 // persistToWAL is the routine that'll be hooked into the exporter's receiving side and it'll
 // write them to the Write-Ahead-Log so that shutdowns won't lose data, and that the routine that
 // reads from the WAL can then process the previously serialized requests.
-func (prweWAL *prweWAL) persistToWAL(requests []*prompb.WriteRequest) error {
+func (prweWAL *prweWAL) persistToWAL(ctx context.Context, requests []*prompb.WriteRequest) error {
 	prweWAL.mu.Lock()
 	defer prweWAL.mu.Unlock()
 
@@ -376,6 +386,7 @@ func (prweWAL *prweWAL) persistToWAL(requests []*prompb.WriteRequest) error {
 		if err != nil {
 			return err
 		}
+		prweWAL.telemetry.recordWALBytesWritten(ctx, len(protoBlob))
 		wIndex := prweWAL.wWALIndex.Add(1)
 		batch.Write(wIndex, protoBlob)
 	}
@@ -414,6 +425,7 @@ func (prweWAL *prweWAL) readPrompbFromWAL(ctx context.Context, index uint64) (wr
 		protoBlob, err = prweWAL.wal.Read(index)
 		duration := time.Since(start)
 		prweWAL.telemetry.recordWALReadLatency(ctx, duration.Milliseconds())
+		prweWAL.telemetry.recordWALBytesRead(ctx, len(protoBlob))
 		if err == nil { // The read succeeded.
 			req := new(prompb.WriteRequest)
 			if err = proto.Unmarshal(protoBlob, req); err != nil {

exporter/prometheusremotewriteexporter/internal/metadatatest/generated_telemetrytest_test.go

@@ -151,7 +151,7 @@ func TestWAL_persist(t *testing.T) {
 		assert.NoError(t, pwal.stop())
 	})
 
-	require.NoError(t, pwal.persistToWAL(reqL))
+	require.NoError(t, pwal.persistToWAL(ctx, reqL))
 
 	// 2. Read all the entries from the WAL itself, guided by the indices available,
 	// and ensure that they are exactly in order as we'd expect them.
@@ -295,6 +295,9 @@ func TestWALWrite_Telemetry(t *testing.T) {
 
 	_, err = tel.GetMetric("otelcol_exporter_prometheusremotewrite_wal_write_latency")
 	require.NoError(t, err)
+
+	_, err = tel.GetMetric("otelcol_exporter_prometheusremotewrite_wal_bytes_written")
+	require.NoError(t, err)
 }
 
 func TestWALRead_Telemetry(t *testing.T) {
@@ -373,4 +376,7 @@ func TestWALRead_Telemetry(t *testing.T) {
 
 	_, err = tel.GetMetric("otelcol_exporter_prometheusremotewrite_wal_read_latency")
 	require.NoError(t, err)
+
+	_, err = tel.GetMetric("otelcol_exporter_prometheusremotewrite_wal_bytes_read")
+	require.NoError(t, err)
 }