cli(debug.zip): add fallback for Datadog multiline log upload request

We are relying on Datadog log API to upload logs and table data. We are
receiving cryptic `Decompression error` with 4xx error code. We are suspecting
due to request size >5MB post decompression on Datadog end. However, Datadog
log API documentation has mentioned that log lines greater than 5MB will get
truncated. To address this, we are introducing a fallback for Datadog multiline
uplaod where will make Datadog API request for each line from the respective
batch.

Epic: None
Part of: CRDB-51111
Release note: None

Author: aa-joshi

pkg/cli/zip_upload.go

@@ -720,7 +720,7 @@ func uploadZipTables(ctx context.Context, uploadID string, debugDirPath string)
 					if _, err := uploadLogsToDatadog(
 						chunk.payload, debugZipUploadOpts.ddAPIKey, debugZipUploadOpts.ddSite,
 					); err != nil {
-						fmt.Fprintf(os.Stderr, "failed to upload a part of %s: %s\n", chunk.tableName, err)
+						uploadIndividualLogToDatadog(chunk)
 					}
 				}()
 			}
@@ -748,6 +748,25 @@ func uploadZipTables(ctx context.Context, uploadID string, debugDirPath string)
 	return nil
 }
 
+// uploadIndividualLogToDatadog is a fallback function to upload the logs to datadog. We would receive cryptic "Decompression error"
+// errors from datadog. We are suspecting it is due to the logs being >5MB in size. So, we are uploading individual log
+// lines to datadog instead of the whole payload.
+func uploadIndividualLogToDatadog(chunk *tableDumpChunk) {
+	logs, _ := getLogLinesFromPayload(chunk.payload)
+	var stdErr error
+	for _, logMap := range logs {
+		logLine, _ := json.Marshal(logMap)
+		if _, err := uploadLogsToDatadog(
+			logLine, debugZipUploadOpts.ddAPIKey, debugZipUploadOpts.ddSite,
+		); err != nil && stdErr == nil {
+			stdErr = err
+		}
+	}
+	if stdErr != nil {
+		fmt.Fprintf(os.Stderr, "failed to upload a part of %s: %s\n", chunk.tableName, stdErr)
+	}
+}
+
 type ddArchivePayload struct {
 	Type       string              `json:"type"`
 	Attributes ddArchiveAttributes `json:"attributes"`
@@ -829,6 +848,7 @@ type logUploadSig struct {
 // number of lines and the size of the payload. But in case of CRDB logs, the
 // average size of 1000 lines is well within the limit (5MB). So, we are only
 // splitting based on the number of lines.
+// TODO(obs-india): consider log size in sig calculation
 func (s logUploadSig) split() []logUploadSig {
 	var (
 		noOfNewSignals = len(s.logLines)/datadogMaxLogLinesPerReq + 1
@@ -1232,6 +1252,15 @@ func makeDDMultiLineLogPayload(logLines [][]byte) []byte {
 	return buf.Bytes()
 }
 
+func getLogLinesFromPayload(payload []byte) ([]map[string]any, error) {
+	var logs []map[string]any
+	err := json.Unmarshal(payload, &logs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to log lines: %w", err)
+	}
+	return logs, nil
+}
+
 // humanReadableSize converts the given number of bytes to a human readable
 // format. Lowest unit is bytes and the highest unit is petabytes.
 func humanReadableSize(bytes int) string {