Merge #155144 #155148

155144: roachtest: add benchmark for INSPECT under admission control r=spilchen a=spilchen

Add a new roachtest to measure the impact of admission control on INSPECT operations running concurrently with foreground workloads.

As part of this, introduce a cluster setting knob to control admission control behavior for INSPECT operations, allowing them to be run with either BulkLowQoS (enabled) or Normal QoS (disabled).

The test performs three INSPECT runs:
1. A calibration run (with no workload) to dynamically determine test duration.
2. An INSPECT run with admission control enabled (BulkLowQoS) during a foreground read workload.
3. An INSPECT run with admission control disabled (Normal QoS) during the same workload.

Each run measures and reports throughput (rows/s/CPU), enabling us to quantify:
- The performance impact of admission control on INSPECT operations.
- The effectiveness of admission control in protecting foreground workload latency.

The workload duration is dynamically calculated to ensure it's long enough to encompass both INSPECT runs.

Informs: #154457
Epic: CRDB-30356

Release note: none

155148: cli(tsdump): upsample counter metric of 30 min interval during Datadog upload r=aa-joshi a=aa-joshi

Previously, We were uploading counter metric types to Datadog without considering the metric interval. The Datadog honours the interval which is set in the metrics summary. Here, the  value is set as `10` seconds considering default scrape interval. This resulted into improper visulisation of the metrics which are generated at 30min interval. To address this, this patch upsample counter metric type of 30 min interval to 10 second interval. This means that we add 180 data points (30min/10sec=180) so that the counter metrics can be visualised properly irrespective of the metric interval.

Epic: none
Part of: CRDB-54228
Release note: None

Co-authored-by: Matt Spilchen <[email protected]>
Co-authored-by: Akshay Joshi <[email protected]>

Author: 3 people

pkg/cli/tsdump.go

@@ -76,9 +76,9 @@ var debugTimeSeriesDumpOpts = struct {
 var hostNameOverride string
 
 // datadogSeriesThreshold holds the threshold for the number of series
-// that will be uploaded to Datadog in a single request. We have capped it to 100
+// that will be uploaded to Datadog in a single request. We have capped it to 50
 // to avoid hitting the Datadog API limits.
-var datadogSeriesThreshold = 100
+var datadogSeriesThreshold = 50
 
 const uploadWorkerErrorMessage = "--upload-workers is set to an invalid value." +
 	" please select a value which between 1 and 100."

pkg/cli/tsdump_upload.go

@@ -86,6 +86,72 @@ type FailedRequestsFile struct {
 	Requests []FailedRequest `json:"requests"`
 }
 
+// GapFillProcessor interpolates 30-minute resolution counter metrics to 10-second resolution
+// by filling gaps with zero values while preserving the original data points.
+type GapFillProcessor struct{}
+
+func NewGapFillProcessor() *GapFillProcessor {
+	return &GapFillProcessor{}
+}
+
+// processCounterMetric interpolates 30-minute resolution counter metrics to 10-second resolution.
+// It checks if the metric is a counter type with 30-minute interval (1800 seconds) and converts
+// it to 10-second resolution by filling gaps with zero values between original data points.
+func (gfp *GapFillProcessor) processCounterMetric(series *datadogV2.MetricSeries) error {
+	// Only process counter metrics
+	if series.Type == nil || *series.Type != datadogV2.METRICINTAKETYPE_COUNT {
+		return nil
+	}
+
+	// Only process 30-minute resolution metrics (1800 seconds)
+	if series.Interval == nil || *series.Interval != 1800 {
+		return nil
+	}
+
+	// If no points or only one point, nothing to interpolate
+	if len(series.Points) <= 1 {
+		// Still update interval to 10 seconds for consistency
+		series.Interval = datadog.PtrInt64(10)
+		return nil
+	}
+
+	// Create new points array with interpolated values
+	var newPoints []datadogV2.MetricPoint
+
+	for i := 0; i < len(series.Points); i++ {
+		currentValue := *series.Points[i].Value
+		currentTimestamp := *series.Points[i].Timestamp
+
+		// Distribute the delta value across 180 points (1800s / 10s = 180)
+		distributedValue := currentValue / 180.0
+
+		newPoints = append(newPoints, datadogV2.MetricPoint{
+			Timestamp: datadog.PtrInt64(currentTimestamp),
+			Value:     datadog.PtrFloat64(distributedValue),
+		})
+
+		// Add 179 zero points (10-second intervals) between current and next
+		for j := 1; j < 180; j++ {
+			// We are adding delta of 0 so that same distributed value is getting published
+			// across all points. This would help us to perform roll ups with 10 seconds
+			// for metrics with 30 minute intervals.
+			// metric value = (metric value/180) * 180
+			interpolatedTimestamp := currentTimestamp + int64(j*10)
+			newPoints = append(newPoints, datadogV2.MetricPoint{
+				Timestamp: datadog.PtrInt64(interpolatedTimestamp),
+				Value:     datadog.PtrFloat64(0.0),
+			})
+		}
+
+	}
+
+	// Update the series with new points and 10-second interval
+	series.Points = newPoints
+	series.Interval = datadog.PtrInt64(10)
+
+	return nil
+}
+
 var newTsdumpUploadID = func(uploadTime time.Time) string {
 	clusterTagValue := "cluster-debug"
 	if debugTimeSeriesDumpOpts.clusterLabel != "" {
@@ -122,6 +188,8 @@ type datadogWriter struct {
 	hasFailedRequestsInUpload bool
 	// cumulativeToDeltaProcessor is used to convert cumulative counter metrics to delta metrics
 	cumulativeToDeltaProcessor *CumulativeToDeltaProcessor
+	// gapFillProcessor is used to interpolate 30-minute resolution counter metrics to 10-second resolution
+	gapFillProcessor *GapFillProcessor
 }
 
 func makeDatadogWriter(
@@ -185,6 +253,7 @@ func makeDatadogWriter(
 		noOfUploadWorkers:               noOfUploadWorkers,
 		isPartialUploadOfFailedRequests: isPartialUploadOfFailedRequests,
 		cumulativeToDeltaProcessor:      NewCumulativeToDeltaProcessor(),
+		gapFillProcessor:                NewGapFillProcessor(),
 	}, nil
 }
 
@@ -283,6 +352,11 @@ func (d *datadogWriter) dump(kv *roachpb.KeyValue) (*datadogV2.MetricSeries, err
 		}
 	}
 
+	// Process gap-filling for 30-minute resolution counter metrics
+	if err := d.gapFillProcessor.processCounterMetric(series); err != nil {
+		return nil, err
+	}
+
 	return series, nil
 }
 

pkg/cli/tsdump_upload_test.go

@@ -830,3 +830,68 @@ func TestDatadogInit(t *testing.T) {
 		require.Contains(t, receivedMetrics, expectedMetric, "expected metric %s should be present", expectedMetric)
 	}
 }
+
+// TestGapFillProcessor30MinTo10s tests the gap-filling functionality for 30-minute counter metrics.
+// It verifies that 30-minute resolution counter metrics are correctly interpolated to 10-second resolution
+// with zero-filled gaps between original data points.
+func TestGapFillProcessor30MinTo10s(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	processor := NewGapFillProcessor()
+
+	// Create a mock 30-minute resolution counter metric with 2 data points
+	series := &datadogV2.MetricSeries{
+		Metric:   "cr.node.test-counter-count",
+		Tags:     []string{"node_id:1"},
+		Type:     datadogV2.METRICINTAKETYPE_COUNT.Ptr(),
+		Interval: datadog.PtrInt64(1800), // 30 minutes in seconds
+		Points: []datadogV2.MetricPoint{
+			{
+				Timestamp: datadog.PtrInt64(1000), // t=1000
+				Value:     datadog.PtrFloat64(50), // first 30-min point
+			},
+			{
+				Timestamp: datadog.PtrInt64(2800), // t=2800 (1800 seconds later)
+				Value:     datadog.PtrFloat64(55), // second 30-min point
+			},
+		},
+	}
+
+	// Process the series
+	err := processor.processCounterMetric(series)
+	require.NoError(t, err)
+
+	// Verify the interval was updated to 10 seconds
+	require.NotNil(t, series.Interval)
+	require.Equal(t, int64(10), *series.Interval)
+
+	// Verify we have the correct number of points
+	// Original: 2 points
+	// These 2 points are uniformly distributed in 180 data points.
+	// So expected data points would be 2 * 180 = 360.
+	expectedPoints := 360
+	require.Len(t, series.Points, expectedPoints)
+
+	// Verify the first original point is preserved
+	require.Equal(t, int64(1000), *series.Points[0].Timestamp)
+	require.Equal(t, 50.0/180, *series.Points[0].Value)
+
+	// Verify the first & last interpolated points are zeros with correct timestamps
+	// for first data point
+	require.Equal(t, int64(1010), *series.Points[1].Timestamp)
+	require.Equal(t, 0.0, *series.Points[1].Value)
+
+	require.Equal(t, int64(2790), *series.Points[179].Timestamp)
+	require.Equal(t, 0.0, *series.Points[179].Value)
+
+	// Verify the first & last interpolated points are zeros with correct timestamps
+	index := 180
+	require.Equal(t, int64(2800), *series.Points[index].Timestamp)
+	require.Equal(t, 55.0/180, *series.Points[index].Value)
+
+	// Verify a point near the end has zero value (gap-filled)
+	lastIndex := len(series.Points) - 1
+	require.Equal(t, int64(4590), *series.Points[lastIndex].Timestamp)
+	require.Equal(t, 0.0, *series.Points[lastIndex].Value)
+}

pkg/cmd/roachtest/tests/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "admission_control_follower_overload.go",
         "admission_control_index_backfill.go",
         "admission_control_index_overload.go",
+        "admission_control_inspect.go",
         "admission_control_intent_resolution.go",
         "admission_control_multi_store_index_backfill.go",
         "admission_control_multi_store_overload.go",