Merge pull request #483 from Distributive-Network/ci-ubuntu20-deprecation

Xmader · web-flow · commit 554e8e05623f · 2025-06-05T03:39:02.000+08:00
CI: Use a Ubuntu 20.04 container to keep Ubuntu 20.04 support
diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml
@@ -49,6 +49,8 @@ env:
   # don't upgrade outdated brew packages because the process is too slow
   HOMEBREW_NO_INSTALL_UPGRADE: 1
   HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
+  # apt-get should run in noninteractive mode
+  DEBIAN_FRONTEND: noninteractive
 
 defaults:
   run:
@@ -69,11 +71,9 @@ jobs:
                                                                            # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available
         python_version: [ '3.10' ]
     runs-on: ${{ matrix.os }}
+    container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python_version }}
       - name: Read the mozilla-central commit hash to be used
         run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
       - name: Cache spidermonkey build
@@ -84,6 +84,17 @@ jobs:
             ./_spidermonkey_install/*
           key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
           lookup-only: true # skip download
+      - name: Setup container
+        if: ${{ matrix.os == 'ubuntu-22.04' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
+        run: |
+          apt-get update -y
+          apt-get install -y sudo libnss3-dev libssl-dev
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+          echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04
+      - uses: actions/setup-python@v5
+        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
+        with:
+          python-version: ${{ matrix.python_version }}
       - name: Setup XCode
         if: ${{ matrix.os == 'macos-13' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
         # SpiderMonkey requires XCode SDK version at least 13.3
@@ -135,7 +146,25 @@ jobs:
         os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ]
         python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
     runs-on: ${{ matrix.os }}
+    container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }}
     steps:
+      - name: Setup container
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
+        run: |
+          apt-get update -y
+          apt-get install -y sudo libnss3-dev libssl-dev
+          apt-get install -y git # required for `actions/checkout`
+          apt-get install -y build-essential
+          apt-get install -y strace # required to run JS tests
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata # tzdata may ask for user interaction if not explicitly installed here
+          echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04
+          git config --global --add safe.directory '*' # silence "git failed because of dubious ownership"
+          
+          # CMake 3.25 or higher is required
+          apt-get install -y ca-certificates gpg wget
+          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
+          apt-get update -y && apt-get install -y cmake
       - uses: actions/checkout@v4
         with:
           submodules: recursive
@@ -219,47 +248,20 @@ jobs:
         with:
           name: wheel-${{ github.run_id }}-${{ github.sha }}-${{ runner.os }}_${{ runner.arch }}_Python${{ matrix.python_version }}
           path: ./dist/
-      - name: Set cores to get stored in /cores
-        if: ${{ matrix.os != 'windows-2022' }}
-        # TODO (Caleb Aikens) figure out how to get Windows core dumps
-        run: |
-          sudo mkdir -p /cores
-          sudo chmod 777 /cores
-          # Core filenames will be of the form osname.pythonversion.executable.pid.timestamp:
-          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-            sudo bash -c 'echo "/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%t" > /proc/sys/kernel/core_pattern'
-          else
-            sudo sysctl kern.corefile="/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%y"
-          fi
       - name: Run Python tests (pytest)
         run: |
-          if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then
-          # TODO (Caleb Aikens) figure out how to get Windows core dumps
-            ulimit -c unlimited
-          fi
           WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
           BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/*
           poetry run python -m pytest tests/python
       - name: Run JS tests (peter-jr)
         if: ${{ (success() || failure()) }}
         run: |
-          if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then
-          # TODO (Caleb Aikens) figure out how to get Windows core dumps
-            ulimit -c unlimited
-          fi
           poetry run bash ./peter-jr ./tests/js/
       - name: SSH debug session
         if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}}
         uses: fawazahmed0/action-debug@main
         with:
           credentials: "admin:admin"
-      - name: Upload core dumps as CI artifacts
-        uses: actions/upload-artifact@v4
-        if: ${{ matrix.os != 'windows-2022' && failure() }}
-        # TODO (Caleb Aikens) figure out how to get Windows core dumps
-        with:
-          name: cores-${{ matrix.os }}-${{ matrix.python_version }}
-          path: /cores
   sdist:
     runs-on: ubuntu-22.04
     steps:
diff --git a/README.md b/README.md
@@ -74,7 +74,6 @@ Read this if you want to build a local version.
     - rust
     - python3.8 or later with header files (python3-dev)
     - spidermonkey latest from mozilla-central
-    - npm (nodejs)
     - [Poetry](https://python-poetry.org/docs/#installation)
     - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning)
 
diff --git a/python/pminit/pmpm.py b/python/pminit/pmpm.py
@@ -0,0 +1,67 @@
+# @file     pmpm.py
+#           A minimum copy of npm written in pure Python.
+#           Currently, this can only install dependencies specified by package-lock.json into node_modules.
+# @author   Tom Tang <xmader@distributive.network>
+# @date     July 2023
+
+import json
+import io
+import os, shutil
+import tempfile
+import tarfile
+from dataclasses import dataclass
+import urllib.request
+from typing import List, Union
+
+@dataclass
+class PackageItem:
+    installation_path: str
+    tarball_url: str
+    has_install_script: bool
+
+def parse_package_lock_json(json_data: Union[str, bytes]) -> List[PackageItem]:
+    # See https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages
+    packages: dict = json.loads(json_data)["packages"]
+    items: List[PackageItem] = []
+    for key, entry in packages.items():
+        if key == "":
+            # Skip the root project (listed with a key of "")
+            continue
+        items.append(
+            PackageItem(
+                installation_path=key, # relative path from the root project folder
+                                       # The path is flattened for nested node_modules, e.g., "node_modules/create-ecdh/node_modules/bn.js"
+                tarball_url=entry["resolved"], # TODO: handle git dependencies
+                has_install_script=entry.get("hasInstallScript", False) # the package has a preinstall, install, or postinstall script
+            )
+        )
+    return items
+
+def download_package(tarball_url: str) -> bytes:
+    with urllib.request.urlopen(tarball_url) as response:
+        tarball_data: bytes = response.read()
+        return tarball_data
+
+def unpack_package(work_dir:str, installation_path: str, tarball_data: bytes):
+    installation_path = os.path.join(work_dir, installation_path)
+    shutil.rmtree(installation_path, ignore_errors=True)
+
+    with tempfile.TemporaryDirectory(prefix="pmpm_cache-") as tmpdir:
+        with io.BytesIO(tarball_data) as tar_file:
+            with tarfile.open(fileobj=tar_file) as tar:
+                tar.extractall(tmpdir)
+        shutil.move(
+            os.path.join(tmpdir, "package"), # Strip the root folder
+            installation_path
+        )
+
+def main(work_dir: str):
+    with open(os.path.join(work_dir, "package-lock.json"), encoding="utf-8") as f:
+        items = parse_package_lock_json(f.read())
+        for i in items:
+            print("Installing " + i.installation_path)
+            tarball_data = download_package(i.tarball_url)
+            unpack_package(work_dir, i.installation_path, tarball_data)
+
+if __name__ == "__main__":
+    main(os.getcwd())
diff --git a/python/pminit/post-install-hook.py b/python/pminit/post-install-hook.py
@@ -1,39 +1,13 @@
-import subprocess
-import sys
-import shutil
+import os
+import pmpm
 
-def execute(cmd: str):
-    popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
-        shell = True, text = True )
-    for stdout_line in iter(popen.stdout.readline, ""):
-        sys.stdout.write(stdout_line)
-        sys.stdout.flush()
-
-    popen.stdout.close()
-    return_code = popen.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, cmd)
+WORK_DIR = os.path.join(
+  os.path.realpath(os.path.dirname(__file__)),
+  "pythonmonkey"
+)
 
 def main():
-    node_package_manager = 'npm'
-    # check if npm is installed on the system
-    if (shutil.which(node_package_manager) is None):
-        print("""
-
-PythonMonkey Build Error:
-
-    
-  *    It appears npm is not installed on this system.
-  *    npm is required for PythonMonkey to build.
-  *    Please install NPM and Node.js before installing PythonMonkey.
-  *    Refer to the documentation for installing NPM and Node.js here: https://nodejs.org/en/download
-
-
-        """)
-        raise Exception("PythonMonkey build error: Unable to find npm on the system.")
-    else:
-        execute(f"cd pythonmonkey && {node_package_manager} i --no-package-lock") # do not update package-lock.json
+  pmpm.main(WORK_DIR) # cd pythonmonkey && npm i
 
 if __name__ == "__main__":
-    main()
-
+  main()
diff --git a/python/pminit/pyproject.toml b/python/pminit/pyproject.toml
@@ -11,6 +11,7 @@ documentation = "https://docs.pythonmonkey.io/"
 repository = "https://github.com/Distributive-Network/PythonMonkey"
 
 include = [
+  "pmpm.py",
   # Install extra files into the pythonmonkey package
   "pythonmonkey/package*.json",
   { path = "pythonmonkey/node_modules/**/*", format = "wheel" },
diff --git a/tests/python/test_dicts.py b/tests/python/test_dicts.py
@@ -367,14 +367,6 @@ def test_toLocaleString():
   pm.eval("(result, obj) => {result[0] = obj.toLocaleString()}")(result, items)
   assert result[0] == '[object Object]'
 
-# repr
-
-
-def test_repr_max_recursion_depth():
-  subprocess.check_call('npm install crypto-js', shell=True)
-  CryptoJS = pm.require('crypto-js')
-  assert str(CryptoJS).__contains__("{'lib': {'Base': {'extend':")
-
 
 # __class__
 def test___class__attribute():
diff --git a/tests/python/test_functions_this.py b/tests/python/test_functions_this.py
@@ -126,14 +126,6 @@ class Class:
 # require
 
 
-def test_require_correct_this():
-  subprocess.check_call('npm install crypto-js', shell=True)
-  CryptoJS = pm.require('crypto-js')
-  cipher = CryptoJS.SHA256("Hello, World!").toString()
-  assert cipher == "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f"
-  subprocess.check_call('npm uninstall crypto-js', shell=True)
-
-
 def test_require_correct_this_old_style_class():
   example = pm.eval("""
   () => {
@@ -207,12 +199,12 @@ def pyFunc():
 def test_method_no_self():
   class What:
     def some_method():
-        return 3
+      return 3
 
   obj = What()
 
   try:
     pm.eval('x => x.some_method()')(obj)
     assert (False)
   except pm.SpiderMonkeyError as e:
-    assert 'takes 0 positional arguments but 1 was given' in str(e)
+    assert 'takes 0 positional arguments but 1 was given' in str(e)
