[8.19] Fix synthtrace OOM exception (#216686) (#219990)

# Backport

This will backport the following commits from `main` to `8.19`:
- [Fix synthtrace OOM exception
(#216686)](#216686)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Carlos
Crespo","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-04-03T09:10:19Z","message":"Fix
synthtrace OOM exception (#216686)\n\n## Summary\n\nSynthtrace was not
properly freeing memory when `generatorStream` ended.\nThis PR fixes
that.\n\nThe log entries below were printed at 5s
interval\n\n```before\n\n debg cpu time: (user: 718mss, sys: 43ms),
memory: 150mb/194mb\n debg cpu time: (user: 2326mss, sys: 550ms),
memory: 123mb/173mb\n debg cpu time: (user: 1424mss, sys: 114ms),
memory: 167mb/192mb\n debg cpu time: (user: 2829mss, sys: 603ms),
memory: 147mb/201mb\n debg cpu time: (user: 1999mss, sys: 155ms),
memory: 176mb/210mb\n debg cpu time: (user: 3350mss, sys: 651ms),
memory: 192mb/228mb\n debg cpu time: (user: 2517mss, sys: 197ms),
memory: 199mb/242mb\n debg cpu time: (user: 4006mss, sys: 697ms),
memory: 210mb/257mb\n debg cpu time: (user: 2748mss, sys: 216ms),
memory: 238mb/277mb\n debg cpu time: (user: 5557mss, sys: 878ms),
memory: 211mb/250mb\n debg cpu time: (user: 3060mss, sys: 251ms),
memory: 270mb/297mb\n```\nafter \n\n```bash\ndebg cpu time: (user:
725mss, sys: 41ms), memory: 136mb/189mb\ndebg cpu time: (user: 2232mss,
sys: 544ms), memory: 106mb/194mb\ndebg cpu time: (user: 1122mss, sys:
69ms), memory: 117mb/179mb\ndebg cpu time: (user: 2670mss, sys: 577ms),
memory: 117mb/185mb\ndebg cpu time: (user: 1438mss, sys: 82ms), memory:
166mb/193mb\ndebg cpu time: (user: 3148mss, sys: 606ms), memory:
149mb/187mb\ndebg cpu time: (user: 1875mss, sys: 109ms), memory:
129mb/190mb\ndebg cpu time: (user: 3669mss, sys: 629ms), memory:
133mb/178mb\ndebg cpu time: (user: 2252mss, sys: 150ms), memory:
114mb/182mb\ndebg cpu time: (user: 4158mss, sys: 663ms), memory:
104mb/187mb\ndebg cpu time: (user: 2576mss, sys: 161ms), memory:
147mb/188mb\n```\n\nUpon terminating the
process\n\nbefore\n\n```bash\n^C info Tearing down after kill signal\n
debg Ending 28 generator streams\n debg Ending 1 client
streams\n```\n\nafter\n\n```bash\n^C info Tearing down after kill
signal\n debg Ending 1 client streams\n```\n\n### How to test \n\nRun
`node scripts/synthtrace.js slash_logs
--live\n--kibana=http://elastic:changeme@localhost:5601\n--target=http://elastic:changeme@localhost:9200
--liveBucketSize=1000\n--logLevel debug`\n\n---------\n\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"968dd5554e08f5ec70b913fe23bc0b88514cedde","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:skip","Team:obs-ux-infra_services","v9.1.0","v8.19.0"],"title":"Fix
synthtrace OOM
exception","number":216686,"url":"https://github.com/elastic/kibana/pull/216686","mergeCommit":{"message":"Fix
synthtrace OOM exception (#216686)\n\n## Summary\n\nSynthtrace was not
properly freeing memory when `generatorStream` ended.\nThis PR fixes
that.\n\nThe log entries below were printed at 5s
interval\n\n```before\n\n debg cpu time: (user: 718mss, sys: 43ms),
memory: 150mb/194mb\n debg cpu time: (user: 2326mss, sys: 550ms),
memory: 123mb/173mb\n debg cpu time: (user: 1424mss, sys: 114ms),
memory: 167mb/192mb\n debg cpu time: (user: 2829mss, sys: 603ms),
memory: 147mb/201mb\n debg cpu time: (user: 1999mss, sys: 155ms),
memory: 176mb/210mb\n debg cpu time: (user: 3350mss, sys: 651ms),
memory: 192mb/228mb\n debg cpu time: (user: 2517mss, sys: 197ms),
memory: 199mb/242mb\n debg cpu time: (user: 4006mss, sys: 697ms),
memory: 210mb/257mb\n debg cpu time: (user: 2748mss, sys: 216ms),
memory: 238mb/277mb\n debg cpu time: (user: 5557mss, sys: 878ms),
memory: 211mb/250mb\n debg cpu time: (user: 3060mss, sys: 251ms),
memory: 270mb/297mb\n```\nafter \n\n```bash\ndebg cpu time: (user:
725mss, sys: 41ms), memory: 136mb/189mb\ndebg cpu time: (user: 2232mss,
sys: 544ms), memory: 106mb/194mb\ndebg cpu time: (user: 1122mss, sys:
69ms), memory: 117mb/179mb\ndebg cpu time: (user: 2670mss, sys: 577ms),
memory: 117mb/185mb\ndebg cpu time: (user: 1438mss, sys: 82ms), memory:
166mb/193mb\ndebg cpu time: (user: 3148mss, sys: 606ms), memory:
149mb/187mb\ndebg cpu time: (user: 1875mss, sys: 109ms), memory:
129mb/190mb\ndebg cpu time: (user: 3669mss, sys: 629ms), memory:
133mb/178mb\ndebg cpu time: (user: 2252mss, sys: 150ms), memory:
114mb/182mb\ndebg cpu time: (user: 4158mss, sys: 663ms), memory:
104mb/187mb\ndebg cpu time: (user: 2576mss, sys: 161ms), memory:
147mb/188mb\n```\n\nUpon terminating the
process\n\nbefore\n\n```bash\n^C info Tearing down after kill signal\n
debg Ending 28 generator streams\n debg Ending 1 client
streams\n```\n\nafter\n\n```bash\n^C info Tearing down after kill
signal\n debg Ending 1 client streams\n```\n\n### How to test \n\nRun
`node scripts/synthtrace.js slash_logs
--live\n--kibana=http://elastic:changeme@localhost:5601\n--target=http://elastic:changeme@localhost:9200
--liveBucketSize=1000\n--logLevel debug`\n\n---------\n\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"968dd5554e08f5ec70b913fe23bc0b88514cedde"}},"sourceBranch":"main","suggestedTargetBranches":["8.19"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/216686","number":216686,"mergeCommit":{"message":"Fix
synthtrace OOM exception (#216686)\n\n## Summary\n\nSynthtrace was not
properly freeing memory when `generatorStream` ended.\nThis PR fixes
that.\n\nThe log entries below were printed at 5s
interval\n\n```before\n\n debg cpu time: (user: 718mss, sys: 43ms),
memory: 150mb/194mb\n debg cpu time: (user: 2326mss, sys: 550ms),
memory: 123mb/173mb\n debg cpu time: (user: 1424mss, sys: 114ms),
memory: 167mb/192mb\n debg cpu time: (user: 2829mss, sys: 603ms),
memory: 147mb/201mb\n debg cpu time: (user: 1999mss, sys: 155ms),
memory: 176mb/210mb\n debg cpu time: (user: 3350mss, sys: 651ms),
memory: 192mb/228mb\n debg cpu time: (user: 2517mss, sys: 197ms),
memory: 199mb/242mb\n debg cpu time: (user: 4006mss, sys: 697ms),
memory: 210mb/257mb\n debg cpu time: (user: 2748mss, sys: 216ms),
memory: 238mb/277mb\n debg cpu time: (user: 5557mss, sys: 878ms),
memory: 211mb/250mb\n debg cpu time: (user: 3060mss, sys: 251ms),
memory: 270mb/297mb\n```\nafter \n\n```bash\ndebg cpu time: (user:
725mss, sys: 41ms), memory: 136mb/189mb\ndebg cpu time: (user: 2232mss,
sys: 544ms), memory: 106mb/194mb\ndebg cpu time: (user: 1122mss, sys:
69ms), memory: 117mb/179mb\ndebg cpu time: (user: 2670mss, sys: 577ms),
memory: 117mb/185mb\ndebg cpu time: (user: 1438mss, sys: 82ms), memory:
166mb/193mb\ndebg cpu time: (user: 3148mss, sys: 606ms), memory:
149mb/187mb\ndebg cpu time: (user: 1875mss, sys: 109ms), memory:
129mb/190mb\ndebg cpu time: (user: 3669mss, sys: 629ms), memory:
133mb/178mb\ndebg cpu time: (user: 2252mss, sys: 150ms), memory:
114mb/182mb\ndebg cpu time: (user: 4158mss, sys: 663ms), memory:
104mb/187mb\ndebg cpu time: (user: 2576mss, sys: 161ms), memory:
147mb/188mb\n```\n\nUpon terminating the
process\n\nbefore\n\n```bash\n^C info Tearing down after kill signal\n
debg Ending 28 generator streams\n debg Ending 1 client
streams\n```\n\nafter\n\n```bash\n^C info Tearing down after kill
signal\n debg Ending 1 client streams\n```\n\n### How to test \n\nRun
`node scripts/synthtrace.js slash_logs
--live\n--kibana=http://elastic:changeme@localhost:5601\n--target=http://elastic:changeme@localhost:9200
--liveBucketSize=1000\n--logLevel debug`\n\n---------\n\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"968dd5554e08f5ec70b913fe23bc0b88514cedde"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Elastic Machine <[email protected]>

Author: crespocarlos

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts

@@ -9,6 +9,7 @@
 
 import { timerange } from '@kbn/apm-synthtrace-client';
 import { castArray, once } from 'lodash';
+import { memoryUsage } from 'process';
 import { bootstrap } from './bootstrap';
 import { getScenario } from './get_scenario';
 import { RunOptions } from './parse_run_cli_flags';
@@ -34,10 +35,28 @@ export async function startLiveDataUpload({
     teardown: scenarioTearDown,
   } = await scenario({ ...runOptions, logger, from, to });
 
+  function startPeriodicPerfLogging() {
+    let cpuUsage = process.cpuUsage();
+
+    return setInterval(() => {
+      cpuUsage = process.cpuUsage(cpuUsage);
+      const mem = memoryUsage();
+      logger.debug(
+        `cpu time: (user: ${Math.round(cpuUsage.user / 1000)}mss, sys: ${Math.round(
+          cpuUsage.system / 1000
+        )}ms), memory: ${mb(mem.heapUsed)}/${mb(mem.heapTotal)}`
+      );
+    }, 5000);
+  }
+
+  const intervalId = startPeriodicPerfLogging();
+
   const teardown = once(async () => {
     if (scenarioTearDown) {
       await scenarioTearDown(clients);
     }
+
+    clearInterval(intervalId);
   });
 
   const streamManager = new StreamManager(logger, teardown);
@@ -49,6 +68,10 @@ export async function startLiveDataUpload({
   const bucketSizeInMs = runOptions.liveBucketSize;
   let requestedUntil = from;
 
+  function mb(value: number): string {
+    return Math.round(value / 1024 ** 2).toString() + 'mb';
+  }
+
   async function uploadNextBatch() {
     const now = Date.now();
 

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts

@@ -130,6 +130,7 @@ export class StreamManager {
     this.trackedGeneratorStreams.push(generatorStream);
 
     await awaitStream(generatorStream).finally(() => {
+      pull(this.trackedGeneratorStreams, generatorStream);
       pull(streams, generatorStream);
     });
   }