feat: compiling polykey in cjs mode for yao-pkg

Author: aryanjassal

flake.nix

@@ -12,6 +12,7 @@
     let
       # The target systems and their vercel/pkg mapping
       systems = { "x86_64-linux" = [ "linux" "x64" ]; };
+
     in flake-utils.lib.eachSystem (builtins.attrNames systems) (system:
       let
         pkgs = nixpkgs-matrix.legacyPackages.${system};
@@ -60,7 +61,7 @@
           postBuild = ''
             npm run pkg -- \
               --output=out \
-              --bin=dist/polykey.mjs \
+              --bin=dist/polykey.cjs \
               --node-version=${utils.nodeVersion} \
               --platform=${platform} \
               --arch=${arch}

package.json

@@ -102,8 +102,7 @@
       "node_modules/sodium-native/prebuilds/darwin-arm64/node.napi.node",
       "node_modules/sodium-native/prebuilds/darwin-x64/node.napi.node",
       "node_modules/sodium-native/prebuilds/linux-x64/node.napi.node",
-      "node_modules/sodium-native/prebuilds/win32-x64/node.napi.node",
-      "node_modules/sodium-native/index.js"
+      "node_modules/sodium-native/prebuilds/win32-x64/node.napi.node"
     ],
     "scripts": [
       "dist/polykeyWorkerManifest.js"
@@ -121,7 +120,7 @@
     "lintfix": "eslint '{src,tests,scripts,benches}/**/*.{js,mjs,ts,mts,jsx,tsx}' --fix",
     "lint-shell": "find ./src ./tests ./scripts -type f -regextype posix-extended -regex '.*\\.(sh)' -exec shellcheck {} +",
     "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src",
-    "pkg": "node ./scripts/pkg.mjs",
+    "pkg": "node ./scripts/build.mjs --pkg && node ./scripts/pkg.mjs",
     "polykey": "node dist/polykey.mjs",
     "start": "node dist/polykey.mjs -- agent start --verbose"
   },
@@ -173,4 +172,4 @@
     "typedoc": "^0.24.8",
     "typescript": "^5.1.6"
   }
-}
+}

pkg.json

@@ -0,0 +1 @@
+{"assets":["node_modules/@matrixai/db/prebuilds/darwin-x64+arm64/node.napi.node","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/@matrixai/db/prebuilds/win32-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/package.json","node_modules/@matrixai/quic-darwin-universal/node.napi.node","node_modules/@matrixai/quic-darwin-universal/package.json","node_modules/@matrixai/quic-linux-x64/node.napi.node","node_modules/@matrixai/quic-linux-x64/package.json","node_modules/@matrixai/quic-win32-x64/node.napi.node","node_modules/@matrixai/quic-win32-x64/package.json","node_modules/@matrixai/exec-darwin-universal/node.napi.node","node_modules/@matrixai/exec-darwin-universal/package.json","node_modules/@matrixai/exec-linux-x64/node.napi.node","node_modules/@matrixai/exec-linux-x64/package.json","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-arm64/node.napi.node","node_modules/fd-lock/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-arm64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/index.js","dist/**","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node"],"scripts":["dist/polykeyWorkerManifest.js"]}

scripts/build.mjs

@@ -9,16 +9,72 @@ import childProcess from 'node:child_process';
 import esbuild from 'esbuild';
 import config from 'polykey/config.js';
 import packageJSON from '../package.json' assert { type: 'json' };
+import { createRequire } from 'node:module';
 
 const projectPath = path.dirname(
   path.dirname(url.fileURLToPath(import.meta.url)),
 );
 
 const platform = os.platform();
 
+const nativeDirnameOverridePlugin = (nativePackages) => {
+  return {
+    name: 'native-dirname-override',
+    setup(build) {
+      const require = createRequire(import.meta.url);
+
+      for (const pkgName of nativePackages) {
+        let pkgJsonPath;
+        try {
+          pkgJsonPath = require.resolve(`${pkgName}/package.json`);
+        } catch {
+          console.log('Skipping', pkgName);
+          continue;
+        }
+        const pkgRoot = path.dirname(pkgJsonPath);
+
+        // Read package.json and resolve entry point
+        const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8'));
+        let relativeEntry;
+
+        if (pkgJson.exports && typeof pkgJson.exports === 'object') {
+          const mainExport = pkgJson.exports['.'];
+          if (typeof mainExport === 'string') {
+            relativeEntry = mainExport;
+          } else if (mainExport?.require) {
+            relativeEntry = mainExport.require;
+          } else if (mainExport?.import) {
+            relativeEntry = mainExport.import;
+          }
+        }
+
+        if (!relativeEntry) {
+          relativeEntry = 'index.js';
+        }
+
+        const resolvedEntry = path.join(pkgRoot, relativeEntry);
+
+        build.onLoad({ filter: /.*/, namespace: 'file' }, async (args) => {
+          if (path.resolve(args.path) !== path.resolve(resolvedEntry)) return;
+
+          return {
+            contents: ` module.exports = require('node-gyp-build')(${JSON.stringify(
+              pkgRoot,
+            )}); `,
+            loader: 'js',
+          };
+        });
+      }
+    },
+  };
+};
+
 /* eslint-disable no-console */
 async function main(argv = process.argv) {
   argv = argv.slice(2);
+  const pkgIndex = argv.findIndex((v) => v == '--pkg');
+  if (pkgIndex >= 0) argv.splice(pkgIndex, 1);
+  const isPkg = pkgIndex >= 0;
   const buildPath = path.join(projectPath, 'build');
   const distPath = path.join(projectPath, 'dist');
   const gitPath = process.env.GIT_DIR ?? path.join(projectPath, '.git');
@@ -64,9 +120,11 @@ async function main(argv = process.argv) {
     path.join(buildPath, 'build.json'),
     JSON.stringify(buildJSON, null, 2),
   );
+
   // This specifies import paths that is left as an external require
   // This is kept to packages that have a native binding
   const externalDependencies = Object.keys(packageJSON.optionalDependencies);
+  /** @type { import('esbuild').BuildOptions } */
   const esbuildOptions = {
     // 2 entrypoints, the main script and the worker script
     entryPoints: [
@@ -77,26 +135,45 @@ async function main(argv = process.argv) {
     bundle: true,
     platform: 'node',
     outdir: distPath,
-    external: externalDependencies,
+    // external: isPkg ? ['*.node'] : externalDependencies,
+    external: isPkg ? [] : externalDependencies,
+    // external: externalDependencies,
     treeShaking: true,
     // External source map for debugging
     sourcemap: true,
     // Minify and keep the original names
     minify: false,
     keepNames: true,
     // Supporting ESM
-    format: 'esm',
-    inject: [path.join(projectPath, './shims/require-shim.mjs')],
-    outExtension: { '.js': '.mjs' },
+    format: isPkg ? 'cjs' : 'esm',
+    // format: 'esm',
+    inject: isPkg
+      ? [path.join(projectPath, './shims/import-meta-url-shim.mjs')]
+      : [path.join(projectPath, './shims/require-shim.mjs')],
+    // inject: [path.join(projectPath, './shims/require-shim.mjs')],
+    // Fix import.meta.url in CJS output
+    define: isPkg
+      ? {
+          'import.meta.url': '__import_meta_url',
+        }
+      : {},
+    outExtension: isPkg ? { '.js': '.cjs' } : { '.js': '.mjs' },
+    // outExtension: { '.js': '.mjs' },
+    plugins: isPkg ? [nativeDirnameOverridePlugin(externalDependencies)] : [],
   };
   console.error('Running esbuild:');
   console.error(esbuildOptions);
   await esbuild.build(esbuildOptions);
+
   // Rename worker script
   console.error('Renaming worker script');
   childProcess.execFileSync(
     'mv',
-    ['dist/polykeyWorkerManifest.mjs', 'dist/polykeyWorkerManifest.js'],
+    [
+      `dist/polykeyWorkerManifest.${isPkg ? 'cjs' : 'mjs'}`,
+      'dist/polykeyWorkerManifest.js',
+    ],
+    // ['dist/polykeyWorkerManifest.mjs', 'dist/polykeyWorkerManifest.js'],
     {
       stdio: ['inherit', 'inherit', 'inherit'],
       windowsHide: true,
@@ -106,7 +183,11 @@ async function main(argv = process.argv) {
   );
   childProcess.execFileSync(
     'mv',
-    ['dist/polykeyWorkerManifest.mjs.map', 'dist/polykeyWorkerManifest.js.map'],
+    [
+      `dist/polykeyWorkerManifest.${isPkg ? 'cjs' : 'mjs'}.map`,
+      'dist/polykeyWorkerManifest.js.map',
+    ],
+    // ['dist/polykeyWorkerManifest.mjs.map', 'dist/polykeyWorkerManifest.js.map'],
     {
       stdio: ['inherit', 'inherit', 'inherit'],
       windowsHide: true,

scripts/pkg.mjs

@@ -103,6 +103,16 @@ async function main(argv = process.argv) {
   const { default: nodeGypBuild } = await import('node-gyp-build');
   const pkgConfig = packageJSON.pkg ?? {};
   pkgConfig.assets = pkgConfig.assets ?? {};
+
+  // //0---
+  // const nativeAssetDir = path.join(projectPath, 'dist', 'native');
+  // const nativeAssets = await find(nativeAssetDir, /\.node$/);
+
+  // pkgConfig.assets.push(
+  //   ...nativeAssets.map((p) => path.relative(projectPath, p))
+  // );
+  // //0---
+
   const npmLsOut = childProcess.execFileSync(
     'npm',
     ['ls', '--all', '--omit=dev', '--parseable'],
@@ -143,11 +153,11 @@ async function main(argv = process.argv) {
     `--config=${pkgConfigPath}`,
     `--targets=node${nodeVersion}-${pkgPlatform}-${pkgArch}`,
     '--no-bytecode',
-    '--no-native-build',
     '--options=experimental-require-module',
     '--public',
     "--public-packages='*'",
     `--output=${outPath}`,
+    '-d',
     ...restArgs,
   ];
   console.error('Running pkg:');

shims/import-meta-url-shim.mjs

@@ -0,0 +1,10 @@
+let __import_meta_url;
+if (typeof document === 'undefined') {
+  const { pathToFileURL } = require('url');
+  __import_meta_url = pathToFileURL(__filename).href;
+} else {
+  __import_meta_url =
+    (document.currentScript && document.currentScript.src) ||
+    new URL('main.js', document.baseURI).href;
+}
+export { __import_meta_url };

shims/require-shim.mjs

@@ -10,4 +10,3 @@ import path from 'node:path';
 require = createRequire(import.meta.url);
 let __dirname = path.dirname(fileURLToPath(import.meta.url));
 export { __dirname };
-