cli: debug zip captures execution traces from every node

dhartunian · dhartunian · commit 95006707f868 · 2025-07-03T14:36:47.000-04:00
debug zip will grab execution traces from each node if they're
available.

Release note: None
diff --git a/pkg/cli/testdata/zip/partial1 b/pkg/cli/testdata/zip/partial1
@@ -34,6 +34,9 @@ debug zip --concurrency=1 --cpu-profile-duration=0s --validate-zip-file=false /d
 [node 1] requesting cpu profile list... received response...
 [node 1] requesting cpu profile list: last request failed: rpc error: ...
 [node 1] requesting cpu profile list: creating error output: debug/nodes/1/cpuprof.err.txt... done
+[node 1] requesting execution trace list... received response...
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: creating error output: debug/nodes/1/executiontraces.err.txt... done
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
@@ -68,6 +71,9 @@ debug zip --concurrency=1 --cpu-profile-duration=0s --validate-zip-file=false /d
 [node 2] requesting cpu profile list... received response...
 [node 2] requesting cpu profile list: last request failed: rpc error: ...
 [node 2] requesting cpu profile list: creating error output: debug/nodes/2/cpuprof.err.txt... done
+[node 2] requesting execution trace list... received response...
+[node 2] requesting execution trace list: last request failed: rpc error: ...
+[node 2] requesting execution trace list: creating error output: debug/nodes/2/executiontraces.err.txt... done
 [node 2] requesting log files list... received response...
 [node 2] requesting log files list: last request failed: rpc error: ...
 [node 2] requesting log files list: creating error output: debug/nodes/2/logs.err.txt... done
@@ -92,6 +98,9 @@ debug zip --concurrency=1 --cpu-profile-duration=0s --validate-zip-file=false /d
 [node 3] requesting cpu profile list... received response...
 [node 3] requesting cpu profile list: last request failed: rpc error: ...
 [node 3] requesting cpu profile list: creating error output: debug/nodes/3/cpuprof.err.txt... done
+[node 3] requesting execution trace list... received response...
+[node 3] requesting execution trace list: last request failed: rpc error: ...
+[node 3] requesting execution trace list: creating error output: debug/nodes/3/executiontraces.err.txt... done
 [node 3] requesting log files list... received response... done
 [node ?] ? log files found
 [node 3] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/partial1_excluded b/pkg/cli/testdata/zip/partial1_excluded
@@ -34,6 +34,9 @@ debug zip /dev/null --concurrency=1 --exclude-nodes=2 --cpu-profile-duration=0 -
 [node 1] requesting cpu profile list... received response...
 [node 1] requesting cpu profile list: last request failed: rpc error: ...
 [node 1] requesting cpu profile list: creating error output: debug/nodes/1/cpuprof.err.txt... done
+[node 1] requesting execution trace list... received response...
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: creating error output: debug/nodes/1/executiontraces.err.txt... done
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
@@ -57,6 +60,9 @@ debug zip /dev/null --concurrency=1 --exclude-nodes=2 --cpu-profile-duration=0 -
 [node 3] requesting cpu profile list... received response...
 [node 3] requesting cpu profile list: last request failed: rpc error: ...
 [node 3] requesting cpu profile list: creating error output: debug/nodes/3/cpuprof.err.txt... done
+[node 3] requesting execution trace list... received response...
+[node 3] requesting execution trace list: last request failed: rpc error: ...
+[node 3] requesting execution trace list: creating error output: debug/nodes/3/executiontraces.err.txt... done
 [node 3] requesting log files list... received response... done
 [node ?] ? log files found
 [node 3] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/partial2 b/pkg/cli/testdata/zip/partial2
@@ -34,6 +34,9 @@ debug zip --concurrency=1 --cpu-profile-duration=0 --validate-zip-file=false /de
 [node 1] requesting cpu profile list... received response...
 [node 1] requesting cpu profile list: last request failed: rpc error: ...
 [node 1] requesting cpu profile list: creating error output: debug/nodes/1/cpuprof.err.txt... done
+[node 1] requesting execution trace list... received response...
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: creating error output: debug/nodes/1/executiontraces.err.txt... done
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
@@ -56,6 +59,9 @@ debug zip --concurrency=1 --cpu-profile-duration=0 --validate-zip-file=false /de
 [node 3] requesting cpu profile list... received response...
 [node 3] requesting cpu profile list: last request failed: rpc error: ...
 [node 3] requesting cpu profile list: creating error output: debug/nodes/3/cpuprof.err.txt... done
+[node 3] requesting execution trace list... received response...
+[node 3] requesting execution trace list: last request failed: rpc error: ...
+[node 3] requesting execution trace list: creating error output: debug/nodes/3/executiontraces.err.txt... done
 [node 3] requesting log files list... received response... done
 [node ?] ? log files found
 [node 3] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip b/pkg/cli/testdata/zip/testzip
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_concurrent b/pkg/cli/testdata/zip/testzip_concurrent
@@ -73,6 +73,11 @@ zip
 [node 1] requesting engine stats: done
 [node 1] requesting engine stats: received response...
 [node 1] requesting engine stats: writing binary output: debug/nodes/1/lsm.txt...
+[node 1] requesting execution trace list...
+[node 1] requesting execution trace list: creating error output: debug/nodes/1/executiontraces.err.txt...
+[node 1] requesting execution trace list: done
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: received response...
 [node 1] requesting goroutine dump list...
 [node 1] requesting goroutine dump list: creating error output: debug/nodes/1/goroutines.err.txt...
 [node 1] requesting goroutine dump list: done
@@ -125,6 +130,11 @@ zip
 [node 2] requesting engine stats: done
 [node 2] requesting engine stats: received response...
 [node 2] requesting engine stats: writing binary output: debug/nodes/2/lsm.txt...
+[node 2] requesting execution trace list...
+[node 2] requesting execution trace list: creating error output: debug/nodes/2/executiontraces.err.txt...
+[node 2] requesting execution trace list: done
+[node 2] requesting execution trace list: last request failed: rpc error: ...
+[node 2] requesting execution trace list: received response...
 [node 2] requesting goroutine dump list...
 [node 2] requesting goroutine dump list: creating error output: debug/nodes/2/goroutines.err.txt...
 [node 2] requesting goroutine dump list: done
@@ -177,6 +187,11 @@ zip
 [node 3] requesting engine stats: done
 [node 3] requesting engine stats: received response...
 [node 3] requesting engine stats: writing binary output: debug/nodes/3/lsm.txt...
+[node 3] requesting execution trace list...
+[node 3] requesting execution trace list: creating error output: debug/nodes/3/executiontraces.err.txt...
+[node 3] requesting execution trace list: done
+[node 3] requesting execution trace list: last request failed: rpc error: ...
+[node 3] requesting execution trace list: received response...
 [node 3] requesting goroutine dump list...
 [node 3] requesting goroutine dump list: creating error output: debug/nodes/3/goroutines.err.txt...
 [node 3] requesting goroutine dump list: done
diff --git a/pkg/cli/testdata/zip/testzip_exclude_goroutine_stacks b/pkg/cli/testdata/zip/testzip_exclude_goroutine_stacks
@@ -33,6 +33,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false --
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_exclude_range_info b/pkg/cli/testdata/zip/testzip_exclude_range_info
@@ -30,6 +30,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --include-range-info=false -
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [cluster] pprof summary script... writing binary output: debug/pprof-summary.sh... done
diff --git a/pkg/cli/testdata/zip/testzip_external_process_virtualization b/pkg/cli/testdata/zip/testzip_external_process_virtualization
@@ -50,6 +50,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_fallback b/pkg/cli/testdata/zip/testzip_fallback
@@ -80,6 +80,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_include_goroutine_stacks b/pkg/cli/testdata/zip/testzip_include_goroutine_stacks
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_include_range_info b/pkg/cli/testdata/zip/testzip_include_range_info
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --include-range-info --valid
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_redacted b/pkg/cli/testdata/zip/testzip_redacted
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --redact --validate-zip-file
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_shared_process_virtualization b/pkg/cli/testdata/zip/testzip_shared_process_virtualization
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
@@ -92,6 +94,9 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node 1] requesting cpu profile list... received response...
 [node 1] requesting cpu profile list: last request failed: rpc error: ...
 [node 1] requesting cpu profile list: creating error output: debug/cluster/test-tenant/nodes/1/cpuprof.err.txt... done
+[node 1] requesting execution trace list... received response...
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: creating error output: debug/cluster/test-tenant/nodes/1/executiontraces.err.txt... done
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/testdata/zip/testzip_shared_process_virtualization_with_default_tenant b/pkg/cli/testdata/zip/testzip_shared_process_virtualization_with_default_tenant
@@ -34,6 +34,8 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node ?] ? goroutine dumps found
 [node 1] requesting cpu profile list... received response... done
 [node ?] ? cpu profiles found
+[node 1] requesting execution trace list... received response... done
+[node ?] ? execution traces found
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
@@ -92,6 +94,9 @@ debug zip --concurrency=1 --cpu-profile-duration=1s --validate-zip-file=false /d
 [node 1] requesting cpu profile list... received response...
 [node 1] requesting cpu profile list: last request failed: rpc error: ...
 [node 1] requesting cpu profile list: creating error output: debug/cluster/test-tenant/nodes/1/cpuprof.err.txt... done
+[node 1] requesting execution trace list... received response...
+[node 1] requesting execution trace list: last request failed: rpc error: ...
+[node 1] requesting execution trace list: creating error output: debug/cluster/test-tenant/nodes/1/executiontraces.err.txt... done
 [node 1] requesting log files list... received response... done
 [node ?] ? log files found
 [node 1] requesting ranges... received response... done
diff --git a/pkg/cli/zip_per_node.go b/pkg/cli/zip_per_node.go
@@ -271,6 +271,9 @@ func (zc *debugZipContext) collectFileList(
 	case serverpb.FileType_CPU:
 		fileKind = "cpu profile"
 		prefix = prefix + "/cpuprof"
+	case serverpb.FileType_EXECUTIONTRACE:
+		fileKind = "execution trace"
+		prefix = prefix + "/executiontraces"
 	default:
 		return errors.AssertionFailedf("unknown file type: %v", fileType)
 	}
@@ -514,6 +517,12 @@ func (zc *debugZipContext) getProfiles(
 	if err := zc.collectFileList(ctx, nodePrinter, id, prefix, serverpb.FileType_CPU); err != nil {
 		return err
 	}
+
+	// Collect all relevant execution traces.
+	if err := zc.collectFileList(ctx, nodePrinter, id, prefix, serverpb.FileType_EXECUTIONTRACE); err != nil {
+		return err
+	}
+
 	return nil
 }
 
diff --git a/pkg/cli/zip_test.go b/pkg/cli/zip_test.go
@@ -687,6 +687,8 @@ func eraseNonDeterministicZipOutput(out string) string {
 	out = re.ReplaceAllString(out, ``)
 	re = regexp.MustCompile(`(?m)^\[node \d+\] retrieving goroutine_dump.*$` + "\n")
 	out = re.ReplaceAllString(out, ``)
+	re = regexp.MustCompile(`(?m)^\[node \d+\] \d+ execution traces found$`)
+	out = re.ReplaceAllString(out, `[node ?] ? execution traces found`)
 
 	return out
 }
