diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index ab83885a..337e85f6 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -49,6 +49,8 @@ env: # don't upgrade outdated brew packages because the process is too slow HOMEBREW_NO_INSTALL_UPGRADE: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + # apt-get should run in noninteractive mode + DEBIAN_FRONTEND: noninteractive defaults: run: @@ -69,11 +71,9 @@ jobs: # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available python_version: [ '3.10' ] runs-on: ${{ matrix.os }} + container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python_version }} - name: Read the mozilla-central commit hash to be used run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV - name: Cache spidermonkey build @@ -84,6 +84,17 @@ jobs: ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} lookup-only: true # skip download + - name: Setup container + if: ${{ matrix.os == 'ubuntu-22.04' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + run: | + apt-get update -y + apt-get install -y sudo libnss3-dev libssl-dev + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata + echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 + - uses: actions/setup-python@v5 + if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + with: + python-version: ${{ matrix.python_version }} - name: Setup XCode if: ${{ matrix.os == 'macos-13' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} # SpiderMonkey requires XCode SDK version at least 13.3 @@ -135,7 +146,25 @@ jobs: os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ] python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} + container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} steps: + - name: Setup container + if: ${{ matrix.os == 'ubuntu-22.04' }} + run: | + apt-get update -y + apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y git # required for `actions/checkout` + apt-get install -y build-essential + apt-get install -y strace # required to run JS tests + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata # tzdata may ask for user interaction if not explicitly installed here + echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 + git config --global --add safe.directory '*' # silence "git failed because of dubious ownership" + + # CMake 3.25 or higher is required + apt-get install -y ca-certificates gpg wget + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null + apt-get update -y && apt-get install -y cmake - uses: actions/checkout@v4 with: submodules: recursive @@ -219,47 +248,20 @@ jobs: with: name: wheel-${{ github.run_id }}-${{ github.sha }}-${{ runner.os }}_${{ runner.arch }}_Python${{ matrix.python_version }} path: ./dist/ - - name: Set cores to get stored in /cores - if: ${{ matrix.os != 'windows-2022' }} - # TODO (Caleb Aikens) figure out how to get Windows core dumps - run: | - sudo mkdir -p /cores - sudo chmod 777 /cores - # Core filenames will be of the form osname.pythonversion.executable.pid.timestamp: - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - sudo bash -c 'echo "/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%t" > /proc/sys/kernel/core_pattern' - else - sudo sysctl kern.corefile="/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%y" - fi - name: Run Python tests (pytest) run: | - if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then - # TODO (Caleb Aikens) figure out how to get Windows core dumps - ulimit -c unlimited - fi WORKFLOW_BUILD_TYPE=${{ inputs.build_type }} BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/* poetry run python -m pytest tests/python - name: Run JS tests (peter-jr) if: ${{ (success() || failure()) }} run: | - if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then - # TODO (Caleb Aikens) figure out how to get Windows core dumps - ulimit -c unlimited - fi poetry run bash ./peter-jr ./tests/js/ - name: SSH debug session if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}} uses: fawazahmed0/action-debug@main with: credentials: "admin:admin" - - name: Upload core dumps as CI artifacts - uses: actions/upload-artifact@v4 - if: ${{ matrix.os != 'windows-2022' && failure() }} - # TODO (Caleb Aikens) figure out how to get Windows core dumps - with: - name: cores-${{ matrix.os }}-${{ matrix.python_version }} - path: /cores sdist: runs-on: ubuntu-22.04 steps: diff --git a/README.md b/README.md index 1532ae6f..343a73fe 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,6 @@ Read this if you want to build a local version. - rust - python3.8 or later with header files (python3-dev) - spidermonkey latest from mozilla-central - - npm (nodejs) - [Poetry](https://python-poetry.org/docs/#installation) - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning) diff --git a/python/pminit/pmpm.py b/python/pminit/pmpm.py new file mode 100644 index 00000000..59a78a14 --- /dev/null +++ b/python/pminit/pmpm.py @@ -0,0 +1,67 @@ +# @file pmpm.py +# A minimum copy of npm written in pure Python. +# Currently, this can only install dependencies specified by package-lock.json into node_modules. +# @author Tom Tang +# @date July 2023 + +import json +import io +import os, shutil +import tempfile +import tarfile +from dataclasses import dataclass +import urllib.request +from typing import List, Union + +@dataclass +class PackageItem: + installation_path: str + tarball_url: str + has_install_script: bool + +def parse_package_lock_json(json_data: Union[str, bytes]) -> List[PackageItem]: + # See https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages + packages: dict = json.loads(json_data)["packages"] + items: List[PackageItem] = [] + for key, entry in packages.items(): + if key == "": + # Skip the root project (listed with a key of "") + continue + items.append( + PackageItem( + installation_path=key, # relative path from the root project folder + # The path is flattened for nested node_modules, e.g., "node_modules/create-ecdh/node_modules/bn.js" + tarball_url=entry["resolved"], # TODO: handle git dependencies + has_install_script=entry.get("hasInstallScript", False) # the package has a preinstall, install, or postinstall script + ) + ) + return items + +def download_package(tarball_url: str) -> bytes: + with urllib.request.urlopen(tarball_url) as response: + tarball_data: bytes = response.read() + return tarball_data + +def unpack_package(work_dir:str, installation_path: str, tarball_data: bytes): + installation_path = os.path.join(work_dir, installation_path) + shutil.rmtree(installation_path, ignore_errors=True) + + with tempfile.TemporaryDirectory(prefix="pmpm_cache-") as tmpdir: + with io.BytesIO(tarball_data) as tar_file: + with tarfile.open(fileobj=tar_file) as tar: + tar.extractall(tmpdir) + shutil.move( + os.path.join(tmpdir, "package"), # Strip the root folder + installation_path + ) + +def main(work_dir: str): + with open(os.path.join(work_dir, "package-lock.json"), encoding="utf-8") as f: + items = parse_package_lock_json(f.read()) + for i in items: + print("Installing " + i.installation_path) + tarball_data = download_package(i.tarball_url) + unpack_package(work_dir, i.installation_path, tarball_data) + +if __name__ == "__main__": + main(os.getcwd()) diff --git a/python/pminit/post-install-hook.py b/python/pminit/post-install-hook.py index 1b2d26dd..061492c1 100644 --- a/python/pminit/post-install-hook.py +++ b/python/pminit/post-install-hook.py @@ -1,39 +1,13 @@ -import subprocess -import sys -import shutil +import os +import pmpm -def execute(cmd: str): - popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, - shell = True, text = True ) - for stdout_line in iter(popen.stdout.readline, ""): - sys.stdout.write(stdout_line) - sys.stdout.flush() - - popen.stdout.close() - return_code = popen.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, cmd) +WORK_DIR = os.path.join( + os.path.realpath(os.path.dirname(__file__)), + "pythonmonkey" +) def main(): - node_package_manager = 'npm' - # check if npm is installed on the system - if (shutil.which(node_package_manager) is None): - print(""" - -PythonMonkey Build Error: - - - * It appears npm is not installed on this system. - * npm is required for PythonMonkey to build. - * Please install NPM and Node.js before installing PythonMonkey. - * Refer to the documentation for installing NPM and Node.js here: https://nodejs.org/en/download - - - """) - raise Exception("PythonMonkey build error: Unable to find npm on the system.") - else: - execute(f"cd pythonmonkey && {node_package_manager} i --no-package-lock") # do not update package-lock.json + pmpm.main(WORK_DIR) # cd pythonmonkey && npm i if __name__ == "__main__": - main() - + main() diff --git a/python/pminit/pyproject.toml b/python/pminit/pyproject.toml index 09d28174..7c7b84b6 100644 --- a/python/pminit/pyproject.toml +++ b/python/pminit/pyproject.toml @@ -11,6 +11,7 @@ documentation = "https://docs.pythonmonkey.io/" repository = "https://github.com/Distributive-Network/PythonMonkey" include = [ + "pmpm.py", # Install extra files into the pythonmonkey package "pythonmonkey/package*.json", { path = "pythonmonkey/node_modules/**/*", format = "wheel" }, diff --git a/tests/python/test_dicts.py b/tests/python/test_dicts.py index 99959926..b3f481fa 100644 --- a/tests/python/test_dicts.py +++ b/tests/python/test_dicts.py @@ -367,14 +367,6 @@ def test_toLocaleString(): pm.eval("(result, obj) => {result[0] = obj.toLocaleString()}")(result, items) assert result[0] == '[object Object]' -# repr - - -def test_repr_max_recursion_depth(): - subprocess.check_call('npm install crypto-js', shell=True) - CryptoJS = pm.require('crypto-js') - assert str(CryptoJS).__contains__("{'lib': {'Base': {'extend':") - # __class__ def test___class__attribute(): diff --git a/tests/python/test_functions_this.py b/tests/python/test_functions_this.py index d9cbf528..52838781 100644 --- a/tests/python/test_functions_this.py +++ b/tests/python/test_functions_this.py @@ -126,14 +126,6 @@ class Class: # require -def test_require_correct_this(): - subprocess.check_call('npm install crypto-js', shell=True) - CryptoJS = pm.require('crypto-js') - cipher = CryptoJS.SHA256("Hello, World!").toString() - assert cipher == "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f" - subprocess.check_call('npm uninstall crypto-js', shell=True) - - def test_require_correct_this_old_style_class(): example = pm.eval(""" () => { @@ -207,7 +199,7 @@ def pyFunc(): def test_method_no_self(): class What: def some_method(): - return 3 + return 3 obj = What() @@ -215,4 +207,4 @@ def some_method(): pm.eval('x => x.some_method()')(obj) assert (False) except pm.SpiderMonkeyError as e: - assert 'takes 0 positional arguments but 1 was given' in str(e) \ No newline at end of file + assert 'takes 0 positional arguments but 1 was given' in str(e)