Simplify Worker setup, and support .json.gz inputs in symbolicator-cli (#5556)

The symbolicator-cli script only supported .json files as inputs, not
.json.gz or art traces etc.

In Node, window.Worker is not defined, so making the
symbolicator-cli/index.ts change without the gz.ts changes causes an
error to be shown if .json.gz inputs are given.

I also went ahead and simplified our worker mocking that we used in
tests.

Author: mstange

babel.config.json

@@ -65,7 +65,8 @@
       "module-resolver",
       {
         "alias": {
-          "firefox-profiler": "./src"
+          "firefox-profiler": "./src",
+          "firefox-profiler-res": "./res"
         }
       }
     ]

package.json

@@ -122,6 +122,7 @@
     "@types/common-tags": "^1.8.4",
     "@types/jest": "^30.0.0",
     "@types/minimist": "^1.2.5",
+    "@types/node": "^22.17.2",
     "@types/query-string": "^6.3.0",
     "@types/react": "^18.3.24",
     "@types/react-dom": "^18.3.1",

src/profile-logic/process-profile.ts

@@ -2001,7 +2001,7 @@ export async function unserializeProfileOfArbitraryFormat(
         arbitraryFormat = convertSimpleperfTraceProfile(arrayBuffer);
       } else {
         try {
-          const textDecoder = new TextDecoder();
+          const textDecoder = new TextDecoder(undefined, { fatal: true });
           arbitraryFormat = await textDecoder.decode(arrayBuffer);
         } catch (e) {
           console.error('Source exception:', e);

src/symbolicator-cli/index.ts

@@ -79,8 +79,21 @@ export interface CliOptions {
 
 export async function run(options: CliOptions) {
   console.log(`Loading profile from ${options.input}`);
-  const serializedProfile = JSON.parse(fs.readFileSync(options.input, 'utf8'));
-  const profile = await unserializeProfileOfArbitraryFormat(serializedProfile);
+
+  // Read the raw bytes from the file. It might be a JSON file, but it could also
+  // be a binary file, e.g. a .json.gz file, or any of the binary formats supported
+  // by our importers.
+  const bytes = fs.readFileSync(options.input, null);
+
+  // bytes is a Uint8Array whose underlying ArrayBuffer can be longer than bytes.length.
+  // Copy the contents into a new ArrayBuffer which is sized correctly, so that we
+  // don't include uninitialized data from the extra parts of the underlying buffer.
+  // Alternatively, we could make unserializeProfileOfArbitraryFormat support
+  // Uint8Array or Buffer in addition to ArrayBuffer.
+  const byteBufferCopy = Uint8Array.prototype.slice.call(bytes).buffer;
+
+  // Load the profile.
+  const profile = await unserializeProfileOfArbitraryFormat(byteBufferCopy);
   if (profile === undefined) {
     throw new Error('Unable to parse the profile.');
   }

src/symbolicator-cli/webpack.config.js

@@ -17,12 +17,7 @@ module.exports = {
   module: {
     rules: [
       {
-        test: /\.js$/,
-        use: ['babel-loader'],
-        include: includes,
-      },
-      {
-        test: /\.(ts|tsx)$/,
+        test: /\.(js|ts|tsx)$/,
         use: ['babel-loader'],
         include: includes,
       },

src/utils/__mocks__/worker-factory.ts src/test/fixtures/node-worker.tssrc/utils/__mocks__/worker-factory.ts renamed to src/test/fixtures/node-worker.ts

@@ -2,20 +2,44 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-import { Worker } from 'worker_threads';
+import {
+  Worker as NodeWorkerClass,
+  isMarkedAsUntransferable,
+} from 'worker_threads';
 
-class NodeWorker {
-  _instance: Worker | null;
+function getWorkerScript(file: string): string {
+  return `
+    const { parentPort } = require('worker_threads');
+    const fs = require('fs');
+    const vm = require('vm');
+
+    const scriptContent = fs.readFileSync("${file}", 'utf8');
+
+    const sandbox = {
+      importScripts: function () {
+        throw new Error('The function "importScripts" is not implemented.');
+      },
+      postMessage: parentPort.postMessage.bind(parentPort),
+      onmessage: function () {},
+    };
+
+    vm.runInNewContext(scriptContent, sandbox, { filename: "${file}" });
+
+    parentPort.onmessage = sandbox.onmessage.bind(null);
+  `;
+}
+
+export class NodeWorker {
+  _instance: NodeWorkerClass | null;
   onmessage: ((event: MessageEvent) => unknown) | null;
 
   constructor(file: string) {
-    const worker = new Worker(__dirname + '/node-worker-contents.mjs', {
-      workerData: file,
-    });
+    const worker = new NodeWorkerClass(getWorkerScript(file), { eval: true });
     worker.on('message', this.onMessage);
     worker.on('error', this.onError);
     this._instance = worker;
     this.onmessage = null;
+    workerInstances.push(worker);
   }
 
   postMessage(message: unknown, transfer?: any[]) {
@@ -31,7 +55,11 @@ class NodeWorker {
     if (+major < 11 || (+major === 11 && +minor < 12)) {
       payload = { data: message };
     }
-    this._instance?.postMessage(payload, transfer);
+    // See https://github.com/nodejs/node/issues/55593
+    const actualTransfer = (transfer ?? []).filter(
+      (buf) => !isMarkedAsUntransferable(buf)
+    );
+    this._instance?.postMessage(payload, actualTransfer);
   }
 
   onMessage = (message: unknown) => {
@@ -53,23 +81,13 @@ class NodeWorker {
   }
 }
 
-const workerConfigs: { [key: string]: string } = {
-  'zee-worker': './res/zee-worker.js',
-};
-
-const workerInstances: NodeWorker[] = [];
-
-export default function (file: string): NodeWorker {
-  const path = workerConfigs[file];
-  const worker = new NodeWorker(path);
-  workerInstances.push(worker);
-  return worker;
-}
+const workerInstances: NodeWorkerClass[] = [];
 
-/**
- * This function allows for stopping the workers, and is only part of the mock.
- */
+// Called after running each test (see setup.js), otherwise Jest won't shut down.
 export function __shutdownWorkers() {
-  workerInstances.forEach((worker) => worker.terminate());
+  workerInstances.forEach((worker) => {
+    worker.terminate();
+    worker.unref();
+  });
   workerInstances.length = 0;
 }