From 5e464822d9423919c359e9c728896bde0ba16dfc Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Mon, 28 Apr 2025 12:27:05 +1000 Subject: [PATCH 1/2] fix: fixed packaged executable --- flake.nix | 2 +- package.json | 7 +-- pkg.json | 1 + scripts/build.mjs | 100 ++++++++++++++++++++++++++++++--- scripts/pkg.mjs | 2 +- shims/import-meta-url-shim.mjs | 10 ++++ 6 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 pkg.json create mode 100644 shims/import-meta-url-shim.mjs diff --git a/flake.nix b/flake.nix index b88d80f1..676329c4 100644 --- a/flake.nix +++ b/flake.nix @@ -60,7 +60,7 @@ postBuild = '' npm run pkg -- \ --output=out \ - --bin=dist/polykey.mjs \ + --bin=dist/polykey.cjs \ --node-version=${utils.nodeVersion} \ --platform=${platform} \ --arch=${arch} diff --git a/package.json b/package.json index afdedfc3..5a2f7000 100644 --- a/package.json +++ b/package.json @@ -102,8 +102,7 @@ "node_modules/sodium-native/prebuilds/darwin-arm64/node.napi.node", "node_modules/sodium-native/prebuilds/darwin-x64/node.napi.node", "node_modules/sodium-native/prebuilds/linux-x64/node.napi.node", - "node_modules/sodium-native/prebuilds/win32-x64/node.napi.node", - "node_modules/sodium-native/index.js" + "node_modules/sodium-native/prebuilds/win32-x64/node.napi.node" ], "scripts": [ "dist/polykeyWorkerManifest.js" @@ -121,7 +120,7 @@ "lintfix": "eslint '{src,tests,scripts,benches}/**/*.{js,mjs,ts,mts,jsx,tsx}' --fix", "lint-shell": "find ./src ./tests ./scripts -type f -regextype posix-extended -regex '.*\\.(sh)' -exec shellcheck {} +", "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src", - "pkg": "node ./scripts/pkg.mjs", + "pkg": "node ./scripts/build.mjs --pkg && node ./scripts/pkg.mjs", "polykey": "node dist/polykey.mjs", "start": "node dist/polykey.mjs -- agent start --verbose" }, @@ -173,4 +172,4 @@ "typedoc": "^0.24.8", "typescript": "^5.1.6" } -} +} \ No newline at end of file diff --git a/pkg.json b/pkg.json new file mode 100644 index 00000000..e1c393f8 --- /dev/null +++ b/pkg.json @@ -0,0 +1 @@ +{"assets":["node_modules/@matrixai/db/prebuilds/darwin-x64+arm64/node.napi.node","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/@matrixai/db/prebuilds/win32-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/package.json","node_modules/@matrixai/quic-darwin-universal/node.napi.node","node_modules/@matrixai/quic-darwin-universal/package.json","node_modules/@matrixai/quic-linux-x64/node.napi.node","node_modules/@matrixai/quic-linux-x64/package.json","node_modules/@matrixai/quic-win32-x64/node.napi.node","node_modules/@matrixai/quic-win32-x64/package.json","node_modules/@matrixai/exec-darwin-universal/node.napi.node","node_modules/@matrixai/exec-darwin-universal/package.json","node_modules/@matrixai/exec-linux-x64/node.napi.node","node_modules/@matrixai/exec-linux-x64/package.json","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-arm64/node.napi.node","node_modules/fd-lock/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-arm64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/index.js","dist/**","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node"],"scripts":["dist/polykeyWorkerManifest.js"]} \ No newline at end of file diff --git a/scripts/build.mjs b/scripts/build.mjs index 1de1ed5f..069d823b 100755 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -16,9 +16,69 @@ const projectPath = path.dirname( const platform = os.platform(); +/** + * This plugin intercepts loading of certain imports and replaces their contents. + * It overrides the import paths for native imports for rocksdb, fd-lock and sodium-native + */ +const nativeNodeModulesPlugin = { + name: 'native-node-modules', + setup(build) { + build.onLoad({ filter: /\.*/, namespace: 'file' }, (args) => { + const filename = path.basename(args.path); + if (filename === 'rocksdb.js') { + return { + contents: ` + import path from 'node:path'; + import url from 'node:url'; + import nodeGypBuild from 'node-gyp-build'; + const projectPath = path.join(path.dirname(url.fileURLToPath(import.meta.url)), '..', 'node_modules', '@matrixai', 'db'); + const rocksdb = nodeGypBuild(projectPath); + export default rocksdb; + //# sourceMappingURL=rocksdb.js.map + `, + loader: 'js', + }; + } + if (args.path.endsWith('fd-lock/index.js')) { + return { + contents: ` + const path = require('path'); + const binding = require('node-gyp-build')(path.join(__dirname, '..', 'node_modules', 'fd-lock')) + + lock.unlock = unlock + module.exports = lock + + function lock (fd) { + return !!binding.fd_lock(fd) + } + + function unlock (fd) { + return !!binding.fd_unlock(fd) + } + `, + loader: 'js', + }; + } + if (args.path.endsWith('sodium-native/index.js')) { + return { + contents: ` + const path = require('path'); + module.exports = require('node-gyp-build')(path.join(__dirname, '..', 'node_modules', 'sodium-native')) + `, + loader: 'js', + }; + } + return; + }); + }, +}; + /* eslint-disable no-console */ async function main(argv = process.argv) { argv = argv.slice(2); + const pkgIndex = argv.findIndex((v) => v === '--pkg'); + if (pkgIndex >= 0) argv.splice(pkgIndex, 1); + const isPkg = pkgIndex >= 0; const buildPath = path.join(projectPath, 'build'); const distPath = path.join(projectPath, 'dist'); const gitPath = process.env.GIT_DIR ?? path.join(projectPath, '.git'); @@ -64,9 +124,29 @@ async function main(argv = process.argv) { path.join(buildPath, 'build.json'), JSON.stringify(buildJSON, null, 2), ); + // This specifies import paths that is left as an external require // This is kept to packages that have a native binding - const externalDependencies = Object.keys(packageJSON.optionalDependencies); + /** @type { import('esbuild').BuildOptions } */ + const isPkgSwitchedOptions = isPkg + ? { + external: [], + format: 'cjs', + inject: [path.join(projectPath, './shims/import-meta-url-shim.mjs')], + // Fix import.meta.url in CJS output + define: { + 'import.meta.url': '__import_meta_url', + }, + outExtension: { '.js': '.cjs' }, + } + : { + // External: externalDependencies, + format: 'esm', + inject: [path.join(projectPath, './shims/require-shim.mjs')], + outExtension: { '.js': '.mjs' }, + }; + + /** @type { import('esbuild').BuildOptions } */ const esbuildOptions = { // 2 entrypoints, the main script and the worker script entryPoints: [ @@ -77,17 +157,14 @@ async function main(argv = process.argv) { bundle: true, platform: 'node', outdir: distPath, - external: externalDependencies, treeShaking: true, // External source map for debugging sourcemap: true, // Minify and keep the original names minify: false, keepNames: true, - // Supporting ESM - format: 'esm', - inject: [path.join(projectPath, './shims/require-shim.mjs')], - outExtension: { '.js': '.mjs' }, + plugins: [nativeNodeModulesPlugin], + ...isPkgSwitchedOptions, }; console.error('Running esbuild:'); console.error(esbuildOptions); @@ -96,7 +173,11 @@ async function main(argv = process.argv) { console.error('Renaming worker script'); childProcess.execFileSync( 'mv', - ['dist/polykeyWorkerManifest.mjs', 'dist/polykeyWorkerManifest.js'], + [ + `dist/polykeyWorkerManifest.${isPkg ? 'cjs' : 'mjs'}`, + 'dist/polykeyWorkerManifest.js', + ], + // ['dist/polykeyWorkerManifest.mjs', 'dist/polykeyWorkerManifest.js'], { stdio: ['inherit', 'inherit', 'inherit'], windowsHide: true, @@ -106,7 +187,10 @@ async function main(argv = process.argv) { ); childProcess.execFileSync( 'mv', - ['dist/polykeyWorkerManifest.mjs.map', 'dist/polykeyWorkerManifest.js.map'], + [ + `dist/polykeyWorkerManifest.${isPkg ? 'cjs' : 'mjs'}.map`, + 'dist/polykeyWorkerManifest.js.map', + ], { stdio: ['inherit', 'inherit', 'inherit'], windowsHide: true, diff --git a/scripts/pkg.mjs b/scripts/pkg.mjs index 20bd4a30..2b36f9bd 100755 --- a/scripts/pkg.mjs +++ b/scripts/pkg.mjs @@ -103,6 +103,7 @@ async function main(argv = process.argv) { const { default: nodeGypBuild } = await import('node-gyp-build'); const pkgConfig = packageJSON.pkg ?? {}; pkgConfig.assets = pkgConfig.assets ?? {}; + const npmLsOut = childProcess.execFileSync( 'npm', ['ls', '--all', '--omit=dev', '--parseable'], @@ -143,7 +144,6 @@ async function main(argv = process.argv) { `--config=${pkgConfigPath}`, `--targets=node${nodeVersion}-${pkgPlatform}-${pkgArch}`, '--no-bytecode', - '--no-native-build', '--options=experimental-require-module', '--public', "--public-packages='*'", diff --git a/shims/import-meta-url-shim.mjs b/shims/import-meta-url-shim.mjs new file mode 100644 index 00000000..f96b8a12 --- /dev/null +++ b/shims/import-meta-url-shim.mjs @@ -0,0 +1,10 @@ +let __import_meta_url; +if (typeof document === 'undefined') { + const { pathToFileURL } = require('url'); + __import_meta_url = pathToFileURL(__filename).href; +} else { + __import_meta_url = + (document.currentScript && document.currentScript.src) || + new URL('main.js', document.baseURI).href; +} +export { __import_meta_url }; From 4381f83d595b788c82865ef0cb1a20b851fe7a05 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Tue, 29 Apr 2025 15:25:22 +1000 Subject: [PATCH 2/2] fix: general fixes --- .npmignore | 1 + pkg.json | 1 - scripts/build.mjs | 1 - scripts/pkg.mjs | 2 +- 4 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 pkg.json diff --git a/.npmignore b/.npmignore index ea6884b3..9cf3969a 100644 --- a/.npmignore +++ b/.npmignore @@ -15,3 +15,4 @@ /build /builds /dist/tsbuildinfo +pkg.json \ No newline at end of file diff --git a/pkg.json b/pkg.json deleted file mode 100644 index e1c393f8..00000000 --- a/pkg.json +++ /dev/null @@ -1 +0,0 @@ -{"assets":["node_modules/@matrixai/db/prebuilds/darwin-x64+arm64/node.napi.node","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/@matrixai/db/prebuilds/win32-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/node.napi.node","node_modules/@matrixai/mdns-linux-x64/package.json","node_modules/@matrixai/quic-darwin-universal/node.napi.node","node_modules/@matrixai/quic-darwin-universal/package.json","node_modules/@matrixai/quic-linux-x64/node.napi.node","node_modules/@matrixai/quic-linux-x64/package.json","node_modules/@matrixai/quic-win32-x64/node.napi.node","node_modules/@matrixai/quic-win32-x64/package.json","node_modules/@matrixai/exec-darwin-universal/node.napi.node","node_modules/@matrixai/exec-darwin-universal/package.json","node_modules/@matrixai/exec-linux-x64/node.napi.node","node_modules/@matrixai/exec-linux-x64/package.json","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-x64/node.napi.node","node_modules/fd-lock/prebuilds/darwin-arm64/node.napi.node","node_modules/fd-lock/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-arm64/node.napi.node","node_modules/sodium-native/prebuilds/darwin-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/win32-x64/node.napi.node","node_modules/sodium-native/index.js","dist/**","node_modules/@matrixai/db/prebuilds/linux-x64/node.napi.node","node_modules/fd-lock/prebuilds/linux-x64/node.napi.node","node_modules/sodium-native/prebuilds/linux-x64/node.napi.node"],"scripts":["dist/polykeyWorkerManifest.js"]} \ No newline at end of file diff --git a/scripts/build.mjs b/scripts/build.mjs index 069d823b..b5807498 100755 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -177,7 +177,6 @@ async function main(argv = process.argv) { `dist/polykeyWorkerManifest.${isPkg ? 'cjs' : 'mjs'}`, 'dist/polykeyWorkerManifest.js', ], - // ['dist/polykeyWorkerManifest.mjs', 'dist/polykeyWorkerManifest.js'], { stdio: ['inherit', 'inherit', 'inherit'], windowsHide: true, diff --git a/scripts/pkg.mjs b/scripts/pkg.mjs index 2b36f9bd..20bd4a30 100755 --- a/scripts/pkg.mjs +++ b/scripts/pkg.mjs @@ -103,7 +103,6 @@ async function main(argv = process.argv) { const { default: nodeGypBuild } = await import('node-gyp-build'); const pkgConfig = packageJSON.pkg ?? {}; pkgConfig.assets = pkgConfig.assets ?? {}; - const npmLsOut = childProcess.execFileSync( 'npm', ['ls', '--all', '--omit=dev', '--parseable'], @@ -144,6 +143,7 @@ async function main(argv = process.argv) { `--config=${pkgConfigPath}`, `--targets=node${nodeVersion}-${pkgPlatform}-${pkgArch}`, '--no-bytecode', + '--no-native-build', '--options=experimental-require-module', '--public', "--public-packages='*'",