Merge pull request #372 from karthik2804/sourcemap_chaining

add ability to generate chained sourcemap for precompiled sources

Author: karthik2804

packages/build-tools/package-lock.json

@@ -29,13 +29,14 @@
     "typescript": "^5.7.3"
   },
   "dependencies": {
+    "@ampproject/remapping": "^2.3.0",
     "@bytecodealliance/componentize-js": "^0.18.1",
     "@bytecodealliance/jco": "^1.10.2",
-    "yargs": "^17.7.2",
     "acorn-walk": "^8.3.4",
     "acron": "^1.0.5",
     "magic-string": "^0.30.17",
-    "regexpu-core": "^6.2.0"
+    "regexpu-core": "^6.2.0",
+    "yargs": "^17.7.2"
   },
   "files": [
     "lib",
@@ -53,4 +54,4 @@
       }
     ]
   }
-}
+}

packages/build-tools/package.json

@@ -0,0 +1,28 @@
+import {
+    getPackagesWithWasiDeps,
+    processWasiDeps
+} from '../../dist/wasiDepsParser.js';
+
+export async function SpinEsbuildPlugin() {
+    const { getWitImports } = await import('../../lib/wit_tools.js');
+
+    // Step 1: Get WIT imports from dependencies
+    const wasiDeps = getPackagesWithWasiDeps(process.cwd(), new Set(), true);
+    const { witPaths, targetWorlds } = processWasiDeps(wasiDeps);
+    const witImports = getWitImports(witPaths, targetWorlds);
+
+    // Store as a Set for fast lookup
+    const externals = new Set(witImports);
+
+    return {
+        name: 'spin-sdk-externals',
+        setup(build) {
+            build.onResolve({ filter: /.*/ }, args => {
+                if (externals.has(args.path)) {
+                    return { path: args.path, external: true };
+                }
+                return null;
+            });
+        }
+    };
+}

packages/build-tools/plugins/esbuild/index.js

@@ -5,6 +5,9 @@ import { version as componentizeVersion } from '@bytecodealliance/componentize-j
 import { getPackagesWithWasiDeps, processWasiDeps } from './wasiDepsParser.js';
 import {
   calculateChecksum,
+  chainSourceMaps,
+  fileExists,
+  getSourceMapFromFile,
   saveBuildData,
 } from './utils.js';
 import { getCliArgs } from './cli.js';
@@ -14,6 +17,8 @@ import { mergeWit } from '../lib/wit_tools.js';
 //@ts-ignore
 import { precompile } from "./precompile.js"
 import path from 'node:path'
+import { SourceMapInput } from '@ampproject/remapping';
+import { get } from 'node:http';
 
 async function main() {
   try {
@@ -65,13 +70,20 @@ async function main() {
     console.log('Componentizing...');
 
     const source = await readFile(src, 'utf8');
-    const precompiledSource = precompile(source, src, true) as string;
+    let { content: precompiledSource, sourceMap: precompiledSourceMap } = precompile(source, src, true, 'precompiled-source.js') as { content: string; sourceMap: SourceMapInput };
+    // Check if input file has a source map because if we does, we need to chain it with the precompiled source map
+    let inputSourceMap = await getSourceMapFromFile(src);
+    if (inputSourceMap) {
+      precompiledSourceMap = chainSourceMaps(precompiledSourceMap, { [src]: inputSourceMap }) as SourceMapInput;
+    }
 
-    // Write precompiled source to disk for debugging purposes In the future we
-    // will also write a source map to make debugging easier
+    // Write precompiled source to disk for debugging purposes.
     let srcDir = path.dirname(src);
     let precompiledSourcePath = path.join(srcDir, 'precompiled-source.js');
     await writeFile(precompiledSourcePath, precompiledSource);
+    if (precompiledSourceMap) {
+      await writeFile(precompiledSourcePath + '.map', JSON.stringify(precompiledSourceMap, null, 2));
+    }
 
     const { component } = await componentize({
       sourcePath: precompiledSourcePath,

packages/build-tools/src/index.ts

@@ -17,7 +17,7 @@ const POSTAMBLE = '}';
 /// will intern regular expressions, duplicating them at the top level and testing them with both
 /// an ascii and utf8 string should ensure that they won't be re-compiled when run in the fetch
 /// handler.
-export function precompile(source, filename = '<input>', moduleMode = false) {
+export function precompile(source, filename = '<input>', moduleMode = false, precompiledFileName = 'precompiled-source.js') {
     const magicString = new MagicString(source, {
         filename,
     });
@@ -51,12 +51,22 @@ export function precompile(source, filename = '<input>', moduleMode = false) {
 
     magicString.prepend(`${PREAMBLE}${precompileCalls.join('\n')}${POSTAMBLE}`);
 
-    // When we're ready to pipe in source maps:
-    // const map = magicString.generateMap({
-    //   source: 'source.js',
-    //   file: 'converted.js.map',
-    //   includeContent: true
-    // });
+    const sourceMapRegex = /\/\/# sourceMappingURL=.*$/gm;
 
-    return magicString.toString();
+    let match;
+    while ((match = sourceMapRegex.exec(source))) {
+        const start = match.index;
+        const end = start + match[0].length;
+        magicString.remove(start, end);
+    }
+
+    const precompiledSource = magicString.toString() + `\n//# sourceMappingURL=${precompiledFileName}.map\n`;
+
+    const map = magicString.generateMap({
+        source: filename,
+        file: `${precompiledFileName}.map`,
+        includeContent: true
+    });
+
+    return { content: precompiledSource, sourceMap: map };
 }

packages/build-tools/src/precompile.js

@@ -1,9 +1,25 @@
 import { readFile } from 'fs/promises';
 import { createHash } from 'node:crypto';
 import { access, writeFile } from 'node:fs/promises';
+import remapping, { SourceMapInput } from '@ampproject/remapping';
+import path from 'path';
 
+type FileName = string;
+type SourceMapLookup = Record<FileName, SourceMapInput>;
+
+export function chainSourceMaps(finalMap: SourceMapInput, sourceMapLookup: SourceMapLookup) {
+  return remapping(finalMap, (source) => {
+    const sourceMap = sourceMapLookup[source];
+    if (sourceMap) {
+      return sourceMap;
+    }
+    // If not the source, we do not want to traverse it further so we return null.
+    // This is because sometimes npm packages have their own source maps but do
+    // not have the original source files.
+    return null;
+  });
+}
 
-// Function to calculate file checksum
 export async function calculateChecksum(content: string | Buffer) {
   try {
     const hash = createHash('sha256');
@@ -58,3 +74,35 @@ export async function saveBuildData(
     throw error;
   }
 }
+
+export async function getSourceMapFromFile(filePath: string): Promise<SourceMapInput | null> {
+  try {
+    const content = await readFile(filePath, 'utf8');
+
+    // Look for the sourceMappingURL comment
+    const sourceMapRegex = /\/\/[#@]\s*sourceMappingURL=(.+)$/m;
+    const match = content.match(sourceMapRegex);
+
+    if (!match) return null;
+
+    const sourceMapUrl = match[1].trim();
+
+    if (sourceMapUrl.startsWith('data:application/json;base64,')) {
+      // Inline base64-encoded source map
+      const base64 = sourceMapUrl.slice('data:application/json;base64,'.length);
+      const rawMap = Buffer.from(base64, 'base64').toString('utf8');
+      return JSON.parse(rawMap);
+    } else {
+      // External .map file
+      const mapPath = path.resolve(path.dirname(filePath), sourceMapUrl);
+      try {
+        const rawMap = await readFile(mapPath, 'utf8');
+        return JSON.parse(rawMap);
+      } catch (e) {
+        return null; // map file not found or invalid
+      }
+    }
+  } catch (err) {
+    return null; // file doesn't exist or can't be read
+  }
+}

packages/build-tools/src/utils.ts

@@ -0,0 +1,43 @@
+// build.mjs
+import { build } from 'esbuild';
+import path from 'path';
+import { SpinEsbuildPlugin } from "@spinframework/build-tools/plugins/esbuild/index.js";
+import fs from 'fs';
+
+const spinPlugin = await SpinEsbuildPlugin();
+
+// plugin to handle vendor files in node_modules that may not be bundled.
+// Instead of generating a real source map for these files, it appends a minimal
+// inline source map pointing to an empty source. This avoids errors and ensures
+// source maps exist even for unbundled vendor code.
+let SourceMapPlugin = {
+    name: 'excludeVendorFromSourceMap',
+    setup(build) {
+        build.onLoad({ filter: /node_modules/ }, args => {
+            return {
+                contents: fs.readFileSync(args.path, 'utf8')
+                    + '\n//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIiJdLCJtYXBwaW5ncyI6IkEifQ==',
+                loader: 'default',
+            }
+        })
+    },
+}
+
+await build({
+    entryPoints: ['./src/index.ts'],
+    outfile: './build/bundle.js',
+    bundle: true,
+    format: 'esm',
+    platform: 'node',
+    sourcemap: true,
+    minify: false,
+    plugins: [spinPlugin, SourceMapPlugin],
+    logLevel: 'error',
+    loader: {
+        '.ts': 'ts',
+        '.tsx': 'tsx',
+    },
+    resolveExtensions: ['.ts', '.tsx', '.js'],
+    // This prevents sourcemaps from traversing into node_modules
+    sourceRoot: path.resolve(process.cwd(), 'src'),
+});