Merge cockroachdb#153524

153524: cli(tsdump): remove tsdump file dependency from datadoginit r=arjunmahishi a=arjunmahishi

The `datadoginit` format previously required a tsdump file as input to extract metric names, then override their values to zero and timestamps to current time before uploading to Datadog. This created an unnecessary dependency and could miss metrics not present in the specific tsdump file.

This change makes `datadoginit` format work independently by leveraging the existing `generateMetricList()` mechanism that starts a test server and scrapes all metric registries. This ensures complete metric coverage and eliminates the tsdump file dependency.

The implementation uses a simple sequential approach without concurrency since there are only ~3153 metrics total, completing uploads in under 10 seconds.

Additionally, adds a hidden `--dd-metric-interval` flag to make the metric interval configurable for `datadoginit` uploads (defaults to 10 seconds). The flag is hidden to avoid confusing users as it only applies to `datadoginit`.

Release note: None
Part of: CRDB-52597
Epic: None

---

_Use of AI: Initially vibe coded the full end to end flow. The result was unnecessarily complex. Manually modified most of it_

Co-authored-by: Arjun Mahishi <[email protected]>

Author: craig[bot]

pkg/cli/debug.go

@@ -1620,6 +1620,8 @@ func init() {
 	f.IntVar(&debugTimeSeriesDumpOpts.noOfUploadWorkers, "upload-workers", 75, "number of workers to upload the time series data in parallel")
 	f.BoolVar(&debugTimeSeriesDumpOpts.retryFailedRequests, "retry-failed-requests", false, "retry previously failed requests from file")
 	f.BoolVar(&debugTimeSeriesDumpOpts.disableDeltaProcessing, "disable-delta-processing", false, "disable delta calculation for counter metrics (enabled by default)")
+	f.Int64Var(&debugTimeSeriesDumpOpts.ddMetricInterval, "dd-metric-interval", debugTimeSeriesDumpOpts.ddMetricInterval, "interval in seconds for datadoginit format only (default 10). Regular datadog format uses actual intervals from tsdump.")
+	f.Lookup("dd-metric-interval").Hidden = true // this is for internal use only
 
 	f = debugSendKVBatchCmd.Flags()
 	f.StringVar(&debugSendKVBatchContext.traceFormat, "trace", debugSendKVBatchContext.traceFormat,

pkg/cli/tsdump.go

@@ -57,6 +57,7 @@ var debugTimeSeriesDumpOpts = struct {
 	noOfUploadWorkers      int
 	retryFailedRequests    bool
 	disableDeltaProcessing bool
+	ddMetricInterval       int64 // interval for datadoginit format only
 }{
 	format:                 tsDumpText,
 	from:                   timestampValue{},
@@ -65,6 +66,10 @@ var debugTimeSeriesDumpOpts = struct {
 	yaml:                   "/tmp/tsdump.yaml",
 	retryFailedRequests:    false,
 	disableDeltaProcessing: false, // delta processing enabled by default
+
+	// default to 10 seconds interval for datadoginit.
+	// This is based on the scrape interval that is currently set accross all managed clusters
+	ddMetricInterval: 10,
 }
 
 // hostNameOverride is used to override the hostname for testing purpose.
@@ -125,7 +130,22 @@ will then convert it to the --format requested in the current invocation.
 				10_000_000, /* threshold */
 				doRequest,
 			)
-		case tsDumpDatadogInit, tsDumpDatadog:
+		case tsDumpDatadogInit:
+			datadogWriter, err := makeDatadogWriter(
+				debugTimeSeriesDumpOpts.ddSite,
+				true, /* init */
+				debugTimeSeriesDumpOpts.ddApiKey,
+				datadogSeriesThreshold,
+				hostNameOverride,
+				debugTimeSeriesDumpOpts.noOfUploadWorkers,
+				false, /* retryFailedRequests not applicable for init */
+			)
+			if err != nil {
+				return err
+			}
+
+			return datadogWriter.uploadInitMetrics()
+		case tsDumpDatadog:
 			if len(args) < 1 {
 				return errors.New("no input file provided")
 			}
@@ -136,7 +156,7 @@ will then convert it to the --format requested in the current invocation.
 
 			datadogWriter, err := makeDatadogWriter(
 				debugTimeSeriesDumpOpts.ddSite,
-				cmd == tsDumpDatadogInit,
+				false, /* init */
 				debugTimeSeriesDumpOpts.ddApiKey,
 				datadogSeriesThreshold,
 				hostNameOverride,

pkg/cli/tsdump_upload.go

@@ -933,3 +933,67 @@ func loadMetricTypesMap(ctx context.Context) (map[string]string, error) {
 
 	return metricTypeMap, nil
 }
+
+// uploadInitMetrics uploads all available metrics with zero values and current timestamp
+// This eliminates the need for a tsdump file in init mode
+func (d *datadogWriter) uploadInitMetrics() error {
+	if debugTimeSeriesDumpOpts.dryRun {
+		fmt.Println("Dry-run mode enabled. Not actually uploading data to Datadog.")
+	}
+
+	// get list of all metrics
+	metricLayers, err := generateMetricList(context.Background(), true /* skipFiltering */)
+	if err != nil {
+		return errors.Wrap(err, "failed to generate metric list for init upload")
+	}
+
+	currentTimestamp := getCurrentTime().Unix()
+	var successfulUploads, skippedMetrics int
+
+	// batch metrics based on threshold
+	currentBatch := make([]datadogV2.MetricSeries, 0, d.threshold)
+	for _, layer := range metricLayers {
+		for _, category := range layer.Categories {
+			for _, metric := range category.Metrics {
+				series := datadogV2.MetricSeries{
+					Metric: metric.Name,
+					Tags:   []string{},
+					Type:   d.resolveMetricType(metric.Name),
+					Points: []datadogV2.MetricPoint{{
+						Value:     datadog.PtrFloat64(0),
+						Timestamp: datadog.PtrInt64(currentTimestamp),
+					}},
+					Interval: datadog.PtrInt64(debugTimeSeriesDumpOpts.ddMetricInterval),
+				}
+
+				currentBatch = append(currentBatch, series)
+
+				// flush batch when threshold is reached
+				if len(currentBatch) >= d.threshold {
+					_, err := d.emitDataDogMetrics(currentBatch)
+					if err != nil {
+						fmt.Printf("Warning: Failed to upload batch of %d metrics: %v\n", len(currentBatch), err)
+						skippedMetrics += len(currentBatch)
+					} else {
+						successfulUploads += len(currentBatch)
+					}
+					currentBatch = make([]datadogV2.MetricSeries, 0, d.threshold)
+				}
+			}
+		}
+	}
+
+	// flush remaining metrics in the last batch
+	if len(currentBatch) > 0 {
+		_, err := d.emitDataDogMetrics(currentBatch)
+		if err != nil {
+			fmt.Printf("Warning: Failed to upload final batch of %d metrics: %v\n", len(currentBatch), err)
+			skippedMetrics += len(currentBatch)
+		} else {
+			successfulUploads += len(currentBatch)
+		}
+	}
+
+	fmt.Printf("Init upload completed: successfully uploaded %d metrics, skipped %d metrics\n", successfulUploads, skippedMetrics)
+	return nil
+}

pkg/cli/tsdump_upload_test.go

@@ -10,6 +10,7 @@ import (
 	"compress/gzip"
 	"encoding/csv"
 	"encoding/gob"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
@@ -751,3 +752,81 @@ func TestDeltaCalculationWithUnsortedTimestamps(t *testing.T) {
 	require.Equal(t, 50.0, *series.Points[1].Value)  // delta: 150 - 100
 	require.Equal(t, 150.0, *series.Points[2].Value) // delta: 300 - 150
 }
+
+func TestDatadogInit(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	now := time.Date(2025, 9, 23, 0, 0, 0, 0, time.UTC)
+	defer testutils.TestingHook(&getCurrentTime, func() time.Time {
+		return now
+	})()
+
+	var (
+		reqCount         int
+		expectedInterval = 30
+	)
+
+	// checkign "all" metrics will make this test flaky as new metrics are added.
+	// So, we can instead check for one representative metric from each namespace.
+	expectedMetrics := map[string]struct{}{
+		"cockroachdb.sql.txns.open":                               {},
+		"cockroachdb.storage.disk-slow":                           {},
+		"cockroachdb.storeliveness.callbacks.processing_duration": {},
+		"cockroachdb.sys.cgo.allocbytes":                          {},
+		"cockroachdb.timeseries.write.bytes":                      {},
+		"cockroachdb.totalbytes":                                  {},
+		"cockroachdb.txn.durations":                               {},
+		"cockroachdb.valbytes":                                    {},
+	}
+
+	receivedMetrics := make(map[string]struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "gzip", r.Header.Get("Content-Encoding"))
+
+		reader, err := gzip.NewReader(r.Body)
+		require.NoError(t, err)
+		defer reader.Close()
+
+		body, err := io.ReadAll(reader)
+		require.NoError(t, err)
+
+		var request struct {
+			Series []datadogV2.MetricSeries `json:"series"`
+		}
+
+		require.NoError(t, json.Unmarshal(body, &request))
+		if len(request.Series) > 0 {
+			for _, series := range request.Series {
+				receivedMetrics[series.Metric] = struct{}{}
+				require.NotNil(t, series.Interval, "interval should be set for datadoginit format")
+				require.EqualValues(t, expectedInterval, *series.Interval, "interval should match dd-metric-interval flag")
+
+				require.Contains(t, series.Tags, "cluster_label:\"test-cluster\"", "should include cluster_label tag")
+				require.Contains(t, series.Tags, "cluster_type:SELF_HOSTED", "should include cluster_type tag")
+
+				require.Len(t, series.Points, 1, "should have exactly one point for init")
+				require.EqualValues(t, now.Unix(), *series.Points[0].Timestamp, "timestamp should match mocked current time")
+				require.EqualValues(t, float64(0), *series.Points[0].Value, "value should be 0 for datadoginit")
+			}
+		}
+
+		w.WriteHeader(http.StatusOK)
+		reqCount++
+	}))
+	defer server.Close()
+	defer testutils.TestingHook(&hostNameOverride, server.Listener.Addr().String())()
+
+	cmd := `debug tsdump --format=datadoginit --dd-api-key="test-api-key" --cluster-label="test-cluster" --dd-metric-interval=30`
+	c := NewCLITest(TestCLIParams{})
+	defer c.Cleanup()
+
+	_, err := c.RunWithCapture(cmd)
+	require.NoError(t, err)
+	require.NotZero(t, reqCount, "should have made at least one request to the server")
+
+	// verify that our expected metrics are present in the received metrics
+	for expectedMetric := range expectedMetrics {
+		require.Contains(t, receivedMetrics, expectedMetric, "expected metric %s should be present", expectedMetric)
+	}
+}