cli: upload tsdump upload completion log to Datadog

Upload a log message to Datadog upon completion of a tsdump upload, with
structured information for the count of uploaded timeseries, the
duration, and the estimated cost of the operation.

Epic: none
Release note: None

Author: jordanlewis

pkg/cli/testdata/tsdump_upload_e2e

@@ -5,4 +5,9 @@ cr.node.sql.query.count,2021-01-01T00:00:00Z,1,100.5
 cr.node.sql.query.count,2021-01-01T00:00:10Z,1,102.3
 cr.store.rocksdb.block.cache.usage,2021-01-01T00:00:00Z,2,75.2
 ----
+----
 {"series":[{"interval":10,"metric":"crdb.tsdump.admission.admitted.elastic-cpu","points":[{"timestamp":1748248320,"value":1}],"tags":["node_id:1","cluster_type:SELF_HOSTED","cluster_label:\"test-cluster\"","cluster_id:test-cluster-id","zendesk_ticket:zd-test","org_name:test-org","user_name:test-user","upload_id:\"test-cluster\"-20241114000000","upload_timestamp:2024-11-14 00:00:00","upload_year:2024","upload_month:11","upload_day:14"],"type":0},{"interval":10,"metric":"crdb.tsdump.sql.query.count","points":[{"timestamp":1609459200,"value":100.5}],"tags":["node_id:1","cluster_type:SELF_HOSTED","cluster_label:\"test-cluster\"","cluster_id:test-cluster-id","zendesk_ticket:zd-test","org_name:test-org","user_name:test-user","upload_id:\"test-cluster\"-20241114000000","upload_timestamp:2024-11-14 00:00:00","upload_year:2024","upload_month:11","upload_day:14"],"type":0},{"interval":10,"metric":"crdb.tsdump.sql.query.count","points":[{"timestamp":1609459210,"value":102.3}],"tags":["node_id:1","cluster_type:SELF_HOSTED","cluster_label:\"test-cluster\"","cluster_id:test-cluster-id","zendesk_ticket:zd-test","org_name:test-org","user_name:test-user","upload_id:\"test-cluster\"-20241114000000","upload_timestamp:2024-11-14 00:00:00","upload_year:2024","upload_month:11","upload_day:14"],"type":0},{"interval":10,"metric":"crdb.tsdump.rocksdb.block.cache.usage","points":[{"timestamp":1609459200,"value":75.2}],"tags":["store:2","cluster_type:SELF_HOSTED","cluster_label:\"test-cluster\"","cluster_id:test-cluster-id","zendesk_ticket:zd-test","org_name:test-org","user_name:test-user","upload_id:\"test-cluster\"-20241114000000","upload_timestamp:2024-11-14 00:00:00","upload_year:2024","upload_month:11","upload_day:14"],"type":0}]}
+
+[{"ddsource":"tsdump_upload","ddtags":"cluster_type:SELF_HOSTED,cluster_label:\"test-cluster\",cluster_id:test-cluster-id,zendesk_ticket:zd-test,org_name:test-org,user_name:test-user,upload_id:\"test-cluster\"-20241114000000,upload_timestamp:2024-11-14 00:00:00,upload_year:2024,upload_month:11,upload_day:14,series_uploaded:4","hostname":"hostname","message":"{\"message\":\"tsdump upload completed: uploaded 4 series overall\",\"series_uploaded\":4,\"estimated_cost\":0.00024931506849315067,\"duration\":0,\"dry_run\":false,\"success\":true}","service":"tsdump_upload"}]
+----
+----

pkg/cli/tsdump_upload.go

@@ -156,6 +156,14 @@ var getCurrentTime = func() time.Time {
 	return timeutil.Now()
 }
 
+var getHostname = func() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return "unknown"
+	}
+	return hostname
+}
+
 // appendTag appends a formatted tag to the series tags.
 func appendTag(series *datadogV2.MetricSeries, tagKey, tagValue string) {
 	series.Tags = append(series.Tags, fmt.Sprintf("%s:%s", tagKey, tagValue))
@@ -472,12 +480,15 @@ func (d *datadogWriter) upload(fileName string) error {
 	estimatedCost := float64(seriesUploaded) * 4.55 / 100 / 730
 	fmt.Printf("Estimated cost of this upload: $%.2f\n", estimatedCost)
 
+	tags := getUploadTags(d)
+	success := metricsUploadState.isSingleUploadSucceeded
 	if metricsUploadState.isSingleUploadSucceeded {
 		var isDatadogUploadFailed = false
 		markDatadogUploadFailedOnce := sync.OnceFunc(func() {
 			isDatadogUploadFailed = true
 		})
 		if len(metricsUploadState.uploadFailedMetrics) != 0 {
+			success = false
 			fmt.Printf(partialFailureMessageFormat, len(metricsUploadState.uploadFailedMetrics), strings.Join(func() []string {
 				var failedMetricsList []string
 				index := 1
@@ -489,7 +500,7 @@ func (d *datadogWriter) upload(fileName string) error {
 				return failedMetricsList
 			}(), "\n"))
 
-			tags := strings.Join(getUploadTags(d), ",")
+			tags := strings.Join(tags, ",")
 			fmt.Println("\nPushing logs of metric upload failures to datadog...")
 			for metric := range metricsUploadState.uploadFailedMetrics {
 				wg.Add(1)
@@ -528,6 +539,40 @@ func (d *datadogWriter) upload(fileName string) error {
 		fmt.Println("All metric upload is failed. Please re-upload the Tsdump.")
 	}
 
+	eventTags := append(tags, makeDDTag("series_uploaded", strconv.Itoa(seriesUploaded)))
+
+	api := datadogV2.NewLogsApi(d.apiClient)
+	hostName := getHostname()
+	msgJson, _ := json.Marshal(struct {
+		Message        string  `json:"message"`
+		SeriesUploaded int     `json:"series_uploaded"`
+		EstimatedCost  float64 `json:"estimated_cost"`
+		Duration       int     `json:"duration"`
+		DryRun         bool    `json:"dry_run"`
+		Success        bool    `json:"success"`
+	}{
+		Message:        fmt.Sprintf("tsdump upload completed: uploaded %d series overall", seriesUploaded),
+		SeriesUploaded: seriesUploaded,
+		Duration:       int(getCurrentTime().Sub(d.uploadTime).Nanoseconds()),
+		DryRun:         debugTimeSeriesDumpOpts.dryRun,
+		EstimatedCost:  estimatedCost,
+		Success:        success,
+	})
+	_, _, err = api.SubmitLog(d.datadogContext, []datadogV2.HTTPLogItem{
+		{
+			Ddsource: datadog.PtrString("tsdump_upload"),
+			Ddtags:   datadog.PtrString(strings.Join(eventTags, ",")),
+			Message:  string(msgJson),
+			Service:  datadog.PtrString("tsdump_upload"),
+			Hostname: datadog.PtrString(hostName),
+		},
+	}, datadogV2.SubmitLogOptionalParameters{
+		ContentEncoding: datadogV2.CONTENTENCODING_GZIP.Ptr(),
+	})
+	if err != nil {
+		fmt.Printf("error submitting log to datadog: %v\n", err)
+	}
+
 	close(ch)
 	return nil
 }

pkg/cli/tsdump_upload_test.go

@@ -39,6 +39,9 @@ func TestTSDumpUploadE2E(t *testing.T) {
 	defer testutils.TestingHook(&getCurrentTime, func() time.Time {
 		return time.Date(2024, 11, 14, 0, 0, 0, 0, time.UTC)
 	})()
+	defer testutils.TestingHook(&getHostname, func() string {
+		return "hostname"
+	})()
 
 	datadriven.RunTest(t, "testdata/tsdump_upload_e2e", func(t *testing.T, d *datadriven.TestData) string {
 		var buf strings.Builder