Write our own minimum copy of npm in Python to remove dependency on Node.js during installation

README.md

@@ -74,7 +74,6 @@ Read this if you want to build a local version.
     - rust
     - python3.8 or later with header files (python3-dev)
     - spidermonkey latest from mozilla-central
-    - npm (nodejs)
     - [Poetry](https://python-poetry.org/docs/#installation)
     - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning)
 

python/pminit/pmpm.py

@@ -0,0 +1,67 @@
+# @file     pmpm.py
+#           A minimum copy of npm written in pure Python.
+#           Currently, this can only install dependencies specified by package-lock.json into node_modules.
+# @author   Tom Tang <[email protected]>
+# @date     July 2023
+
+import json
+import io
+import os, shutil
+import tempfile
+import tarfile
+from dataclasses import dataclass
+import urllib.request
+from typing import List, Union
+
+@dataclass
+class PackageItem:
+    installation_path: str
+    tarball_url: str
+    has_install_script: bool
+
+def parse_package_lock_json(json_data: Union[str, bytes]) -> List[PackageItem]:
+    # See https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages
+    packages: dict = json.loads(json_data)["packages"]
+    items: List[PackageItem] = []
+    for key, entry in packages.items():
+        if key == "":
+            # Skip the root project (listed with a key of "")
+            continue
+        items.append(
+            PackageItem(
+                installation_path=key, # relative path from the root project folder
+                                       # The path is flattened for nested node_modules, e.g., "node_modules/create-ecdh/node_modules/bn.js"
+                tarball_url=entry["resolved"], # TODO: handle git dependencies
+                has_install_script=entry.get("hasInstallScript", False) # the package has a preinstall, install, or postinstall script
+            )
+        )
+    return items
+
+def download_package(tarball_url: str) -> bytes:
+    with urllib.request.urlopen(tarball_url) as response:
+        tarball_data: bytes = response.read()
+        return tarball_data
+
+def unpack_package(work_dir:str, installation_path: str, tarball_data: bytes):
+    installation_path = os.path.join(work_dir, installation_path)
+    shutil.rmtree(installation_path, ignore_errors=True)
+
+    with tempfile.TemporaryDirectory(prefix="pmpm_cache-") as tmpdir:
+        with io.BytesIO(tarball_data) as tar_file:
+            with tarfile.open(fileobj=tar_file) as tar:
+                tar.extractall(tmpdir)
+        shutil.move(
+            os.path.join(tmpdir, "package"), # Strip the root folder
+            installation_path
+        )
+
+def main(work_dir: str):
+    with open(os.path.join(work_dir, "package-lock.json"), encoding="utf-8") as f:
+        items = parse_package_lock_json(f.read())
+        for i in items:
+            print("Installing " + i.installation_path)
+            tarball_data = download_package(i.tarball_url)
+            unpack_package(work_dir, i.installation_path, tarball_data)
+
+if __name__ == "__main__":
+    main(os.getcwd())

python/pminit/post-install-hook.py

@@ -1,39 +1,13 @@
-import subprocess
-import sys
-import shutil
+import os
+import pmpm
 
-def execute(cmd: str):
-    popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
-        shell = True, text = True )
-    for stdout_line in iter(popen.stdout.readline, ""):
-        sys.stdout.write(stdout_line)
-        sys.stdout.flush()
-
-    popen.stdout.close()
-    return_code = popen.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, cmd)
+WORK_DIR = os.path.join(
+  os.path.realpath(os.path.dirname(__file__)),
+  "pythonmonkey"
+)
 
 def main():
-    node_package_manager = 'npm'
-    # check if npm is installed on the system
-    if (shutil.which(node_package_manager) is None):
-        print("""
-
-PythonMonkey Build Error:
-
-    
-  *    It appears npm is not installed on this system.
-  *    npm is required for PythonMonkey to build.
-  *    Please install NPM and Node.js before installing PythonMonkey.
-  *    Refer to the documentation for installing NPM and Node.js here: https://nodejs.org/en/download
-
-
-        """)
-        raise Exception("PythonMonkey build error: Unable to find npm on the system.")
-    else:
-        execute(f"cd pythonmonkey && {node_package_manager} i --no-package-lock") # do not update package-lock.json
+  pmpm.main(WORK_DIR) # cd pythonmonkey && npm i
 
 if __name__ == "__main__":
-    main()
-
+  main()

python/pminit/pyproject.toml

@@ -9,6 +9,7 @@ authors = [
   "Hamada Gasmallah <[email protected]>"
 ]
 include = [
+  "pmpm.py",
   # Install extra files into the pythonmonkey package
   "pythonmonkey/package*.json",
   { path = "pythonmonkey/node_modules/**/*", format = "wheel" },