diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d143438..a35cf99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,9 @@ on: permissions: contents: read +env: + FORCE_COLOR: 3 + jobs: test: runs-on: ubuntu-latest @@ -20,10 +23,10 @@ jobs: fetch-tags: true persist-credentials: false - - name: Set up Python 3.10 + - name: Set up Python 3.13 uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: - python-version: "3.10" + python-version: "3.13" - name: Install dependencies run: | @@ -35,15 +38,14 @@ jobs: - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 with: - node-version: "20.6.1" + node-version: "22" - name: Install the pyodide npm package - run: | - npm install pyodide@0.24.1 + run: npm install pyodide@0.28.0-alpha.3 - name: Test package + examples run: | - pytest --cov=pyodide_pack -n 2 --cov-report=xml + pytest --cov=pyodide_pack -n auto --cov-report=xml - name: Upload coverage uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32157e6..3144980 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: - python-version: "3.12" + python-version: "3.13" - name: Build the distribution packages run: | diff --git a/.readthedocs.yml b/.readthedocs.yml index 24e9538..86a824d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,6 @@ python: path: . build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.11" + python: "3.13" diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f2b4e..85cd9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `pyproject.toml` files [#35](https://github.com/pyodide/pyodide-pack/pull/35) - - - Add `pyidide minify` command to minify the Python packages with AST rewrites by, + - Add `pyodide minify` command to minify the Python packages with AST rewrites by, removing comments and docstrings [#23](https://github.com/pyodide/pyodide-pack/pull/23) @@ -31,8 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - - Added support for Pyodide 0.24.0. This is now the minimal supported version of Pyodide. - [#26](https://github.com/pyodide/pyodide-pack/pull/26) + - Added support for Pyodide 0.27.3. This is now the minimal supported version of Pyodide. + [#57](https://github.com/pyodide/pyodide-pack/pull/57) ## Fixed diff --git a/README.md b/README.md index bb41ac4..6c11431 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,17 @@ Each of these approaches have different tradeoffs, and can be used separately or ## Install -Pyodide-pack requires Python 3.10+ and can be installed via pip: +`pyodide-pack` requires Python >=3.13 and can be installed via `pip`: ``` pip install pyodide-pack ``` -(optionally) For elimation of unused modules via runtime detection, run NodeJS needs to be installed together with Pyodide 0.24.0+: +(optionally) For elimation of unused modules via runtime detection, run Node.js, needs to be installed together with Pyodide 0.28.0a3 and later: + ```bash -npm install pyodide@">=0.24.0" +npm install pyodide@">=0.28.0a3" ``` ## Usage diff --git a/docs/conf.py b/docs/conf.py index a8ac2db..97830f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ "sphinx_autodoc_typehints", ] intersphinx_mapping = { - "python": ("https://docs.python.org/3.10", None), + "python": ("https://docs.python.org/3.13", None), "pyodide": ("https://pyodide.org/en/stable/", None), } diff --git a/docs/module-elimination-at-runtime.md b/docs/module-elimination-at-runtime.md index 5b37d3f..5217301 100644 --- a/docs/module-elimination-at-runtime.md +++ b/docs/module-elimination-at-runtime.md @@ -17,61 +17,64 @@ 2. Create the package bundle, - ```bash - pyodide pack examples/pandas/app.py - ``` - which would produce the following output + ```bash + pyodide pack examples/pandas/app.py + ``` + + which would produce the following output: + + ``` + Running pyodide-pack on examples/pandas/app.py + + Note: unless otherwise specified all sizes are given for gzip compressed files to be representative of CDN compression. + + Loaded requirements from: examples/pandas/requirements.txt + Running the input code in Node.js to detect used modules.. + + [...] + + Done input code execution in 6.2 s + + Using stdlib (554 files) with a total size of 2.29 MB. + Detected 5 dependencies with a total size of 9.34 MB (uncompressed: 37.62 MB) + In total 487 files and 0 dynamic libraries were accessed. + Total initial size (stdlib + dependencies): 11.63 MB + + + Packing.. + ┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━┓ + ┃ No ┃ Package ┃ All files ┃ .so libs ┃ Size (MB) ┃ Reduction ┃ + ┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━┩ + │ 0 │ stdlib │ 554 → 168 │ │ 2.29 → 0.46 │ 79.8 % │ + │ 1 │ numpy-2.0.2-cp312-cp312-pyodi… │ 338 → 96 │ 19 → 12 │ 3.05 → 2.16 │ 29.3 % │ + │ 2 │ pandas-2.2.3-cp312-cp312-pyod… │ 396 → 295 │ 44 → 42 │ 5.61 → 4.52 │ 19.5 % │ + │ 3 │ python_dateutil-2.9.0.post0-p… │ 25 → 14 │ 0 → 0 │ 0.23 → 0.18 │ 21.6 % │ + │ 4 │ pytz-2024.1-py2.py3-none-any.… │ 615 → 5 │ 0 → 0 │ 0.43 → 0.01 │ 97.8 % │ + │ 5 │ six-1.16.0-py2.py3-none-any.w… │ 6 → 1 │ 0 → 0 │ 0.01 → 0.01 │ 41.8 % │ + └────┴────────────────────────────────┴───────────┴──────────┴─────────────┴───────────┘ + Wrote pyodide-package-bundle.zip with 6.99 MB (25.2% reduction) + + Spawning webserver at http://127.0.0.1:52009 (see logs in /tmp/tmpx0ktv9fw/http-server.log) + Running the input code in Node.js to validate bundle.. + + Validating and benchmarking the output bundle.. + ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓ + ┃ Step ┃ Load time (s) ┃ Fraction of load time ┃ + ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩ + │ loadPyodide │ 0.82 │ 34.6 % │ + │ fetch_unpack_archive │ 0.14 │ 5.9 % │ + │ load_dynamic_libs │ 0.11 │ 4.6 % │ + │ import_run_app │ 1.30 │ 54.9 % │ + │ TOTAL │ 2.36 │ 100 % │ + └──────────────────────┴───────────────┴───────────────────────┘ + + Total output size (stdlib + packages): 7.45 MB (35.9% reduction) + + Bundle validation successful. + ``` - ``` - Running pyodide-pack on examples/pandas/app.py - - Note: unless otherwise specified all sizes are given for gzip compressed files to - be representative of CDN compression. - - Loaded requirements from: examples/pandas/requirements.txt - Running the input code in Node.js to detect used modules.. - - [...] - Done input code execution in 3.8 s - - Using stdlib (547 files) with a total size of 2.25 MB. - Detected 5 dependencies with a total size of 8.92 MB (uncompressed: 35.46 MB) - In total 487 files and 0 dynamic libraries were accessed. - Total initial size (stdlib + dependencies): 11.17 MB - - - Packing.. - ┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━┓ - ┃ No ┃ Package ┃ All files ┃ .so libs ┃ Size (MB) ┃ Reduction ┃ - ┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━┩ - │ 0 │ stdlib │ 547 → 151 │ │ 2.25 → 0.75 │ 66.7 % │ - │ 1 │ numpy-1.25.2-cp311-cp311-emsc… │ 430 → 111 │ 19 → 0 │ 3.06 → 2.36 │ 23.0 % │ - │ 2 │ pandas-1.5.3-cp311-cp311-emsc… │ 462 → 292 │ 42 → 0 │ 5.17 → 4.64 │ 10.3 % │ - │ 3 │ python_dateutil-2.8.2-py2.py3… │ 25 → 15 │ 0 → 0 │ 0.24 → 0.22 │ 9.4 % │ - │ 4 │ pytz-2023.3-py2.py3-none-any.… │ 614 → 5 │ 0 → 0 │ 0.43 → 0.02 │ 96.1 % │ - │ 5 │ six-1.16.0-py2.py3-none-any.w… │ 6 → 1 │ 0 → 0 │ 0.01 → 0.01 │ 18.5 % │ - └────┴────────────────────────────────┴───────────┴──────────┴─────────────┴───────────┘ - Wrote pyodide-package-bundle.zip with 7.37 MB (17.4% reduction) - - Spawning webserver at http://127.0.0.1:52009 (see logs in /tmp/tmpx0ktv9fw/http-server.log) - Running the input code in Node.js to validate bundle.. - - Validating and benchmarking the output bundle.. - ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓ - ┃ Step ┃ Load time (s) ┃ Fraction of load time ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩ - │ loadPyodide │ 1.34 │ 36.1 % │ - │ fetch_unpack_archive │ 0.27 │ 7.4 % │ - │ load_dynamic_libs │ 0.00 │ 0.1 % │ - │ import_run_app │ 2.10 │ 56.5 % │ - │ TOTAL │ 3.72 │ 100 % │ - └──────────────────────┴───────────────┴───────────────────────┘ - - Total output size (stdlib + packages): 8.12 MB (27.3% reduction) - - Bundle validation successful. - ``` 3. Load your Python web application with, + ```js let pyodide = await loadPyodide({fullStdLib: false}); diff --git a/examples/netcdf4/app.py b/examples/netcdf4/app.py new file mode 100644 index 0000000..1bdde07 --- /dev/null +++ b/examples/netcdf4/app.py @@ -0,0 +1,9 @@ +import netCDF4 as nc +import numpy as np + +dataset = nc.Dataset("memory.nc", "w", diskless=True, persist=False) +dataset.createDimension("x", 3) +var = dataset.createVariable("data", np.float32, ("x",)) +var[:] = np.array([1.0, 2.0, 3.0]) +print(f"Data: {var[:]}") +dataset.close() diff --git a/examples/netcdf4/pyproject.toml b/examples/netcdf4/pyproject.toml new file mode 100644 index 0000000..3e45fdc --- /dev/null +++ b/examples/netcdf4/pyproject.toml @@ -0,0 +1,5 @@ +[tool.pyodide_pack] +requires = [ + "numpy", + "netcdf4", +] diff --git a/examples/scikit-learn/pyproject.toml b/examples/scikit-learn/pyproject.toml index 821f766..4f915e5 100644 --- a/examples/scikit-learn/pyproject.toml +++ b/examples/scikit-learn/pyproject.toml @@ -1,5 +1,4 @@ [tool.pyodide_pack] requires = [ - "numpy", "scikit-learn" ] diff --git a/package.json b/package.json index 6c8da3a..1dbfe21 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "dependencies": { "node-fetch": "^3.3.2", - "pyodide": "^0.24.0" + "pyodide": "^0.28.0-alpha.3" } } diff --git a/pyodide_pack/ast_rewrite.py b/pyodide_pack/ast_rewrite.py index 7e4b28c..1a83eb0 100644 --- a/pyodide_pack/ast_rewrite.py +++ b/pyodide_pack/ast_rewrite.py @@ -54,7 +54,7 @@ def _strip_module_docstring(tree: ast.Module) -> ast.Module: if ( tree.body and isinstance(expr := tree.body[0], ast.Expr) - and isinstance(expr.value, ast.Str | ast.Constant) + and isinstance(expr.value, ast.Constant) and isinstance(expr.value.value, str) ): tree.body.pop(0) diff --git a/pyodide_pack/config.py b/pyodide_pack/config.py index 5c5ba54..2aafa74 100644 --- a/pyodide_pack/config.py +++ b/pyodide_pack/config.py @@ -1,16 +1,11 @@ from __future__ import annotations +import tomllib from pathlib import Path from typing import Any from pydantic import BaseModel, ConfigDict -try: - import tomllib -except ImportError: - # Python <3.11 - import tomli as tomllib # type: ignore[no-redef] - def _find_pyproject_toml(input_path: Path) -> Path | None: """Find a `pyproject.toml` in any of the parent dirs""" diff --git a/pyodide_pack/js/discovery.js b/pyodide_pack/js/discovery.js index 471d6aa..fa3775a 100644 --- a/pyodide_pack/js/discovery.js +++ b/pyodide_pack/js/discovery.js @@ -99,7 +99,11 @@ async function main() { await micropip.install({{packages}}); } + // Explicitly import the cp437 encoding to ensure it's included in the bundle. + // It's required for zipfiles to work as decoding can rely on it, see: + // https://github.com/python/cpython/blob/0b05ead877f909b7efe712db758012d9dbece7ce/Lib/zipfile/__init__.py#L1457 await pyodide.runPythonAsync(` +import encodings.cp437 {{ code }} `); // Run code used in the loader diff --git a/pyodide_pack/runtime_detection.py b/pyodide_pack/runtime_detection.py index 53edc7f..f07bb67 100644 --- a/pyodide_pack/runtime_detection.py +++ b/pyodide_pack/runtime_detection.py @@ -23,9 +23,9 @@ def stdlib_prefix(self): Examples -------- - >>> db = RuntimeResults(sys_modules={"pathlib": "/lib/python311.zip/pathlib.py"}) + >>> db = RuntimeResults(sys_modules={"pathlib": "/lib/python312.zip/pathlib.py"}) >>> db.stdlib_prefix - '/lib/python311.zip' + '/lib/python312.zip' """ return self["sys_modules"]["pathlib"].replace("/pathlib.py", "") @@ -37,12 +37,12 @@ def get_imported_paths(self, strip_prefix: str | None = None): Examples -------- >>> db = RuntimeResults(sys_modules={ - ... "pathlib": "/lib/python311.zip/pathlib.py", - ... "os": "/lib/python311.zip/os.py"}, - ... opened_file_names=["/lib/python311.zip/pathlib.py"]) + ... "pathlib": "/lib/python312.zip/pathlib.py", + ... "os": "/lib/python312.zip/os.py"}, + ... opened_file_names=["/lib/python312.zip/pathlib.py"]) >>> db.get_imported_paths() - ['/lib/python311.zip/pathlib.py', '/lib/python311.zip/os.py'] - >>> db.get_imported_paths(strip_prefix="/lib/python311.zip") + ['/lib/python312.zip/pathlib.py', '/lib/python312.zip/os.py'] + >>> db.get_imported_paths(strip_prefix="/lib/python312.zip") ['pathlib.py', 'os.py'] """ imported_paths = list(self["sys_modules"].values()) + self["opened_file_names"] @@ -73,10 +73,11 @@ def from_json(cls, path) -> RuntimeResults: obj["path"], shared=obj.get("global", False), load_order=idx ) for idx, obj in enumerate(db["load_dyn_lib_calls"]) - # Include locally loaded .so by they shared symbols - # or if they are globally loaded + # Include ALL .so libraries regardless of whether they're loaded + # globally or accessed by symbols - this ensures compatibility with + # Pyodide 0.27+ where libraries are being loaded locally by default. + # For more info, see: https://github.com/pyodide/pyodide/pull/4876 if obj["path"].endswith(".so") - and ((obj["path"] in db["dl_accessed_symbols"]) or obj["global"]) } return db @@ -141,13 +142,14 @@ def process_path(self, in_file_name: str) -> str | None: for pattern in self.config.include_paths ): # TODO: this is hack and should be done better - out_file_name = os.path.join("/lib/python3.11/site-utils", in_file_name) + out_file_name = os.path.join("/lib/python3.13/site-utils", in_file_name) match extension: case ".py": stats["py_out"] += 1 case ".so": stats["so_out"] += 1 # Manually included dynamic libraries are going to be loaded first + # and should also be loaded globally dll = DynamicLib(out_file_name, load_order=-1000) self.dynamic_libs.append(dll) return out_file_name diff --git a/pyodide_pack/tests/test_ast_rewrite.py b/pyodide_pack/tests/test_ast_rewrite.py index a036be8..f62ed7f 100644 --- a/pyodide_pack/tests/test_ast_rewrite.py +++ b/pyodide_pack/tests/test_ast_rewrite.py @@ -123,7 +123,7 @@ def test_strip_module_docstrings(): ) -@settings(deadline=300) +@settings(deadline=None) @given(st.sampled_from(_get_stdlib_module_paths())) def test_process_all_stdlib(path): """Check that we can process all of the stdlib without crashing.""" diff --git a/pyodide_pack/tests/test_runtime_detection.py b/pyodide_pack/tests/test_runtime_detection.py index 201b8e9..ce61352 100644 --- a/pyodide_pack/tests/test_runtime_detection.py +++ b/pyodide_pack/tests/test_runtime_detection.py @@ -28,10 +28,9 @@ def test_runtime_results(tmp_path): ] assert res["opened_file_names"] == ["a.py", "b.py"] - # d.so not included as it's not in accessed LDSO symbols - # g.so is include as it's globally loaded assert res["dynamic_libs_map"] == { "c.so": DynamicLib(path="c.so", load_order=0, shared=False), + "d.so": DynamicLib(path="d.so", load_order=1, shared=False), "g.so": DynamicLib(path="g.so", load_order=2, shared=True), } diff --git a/pyodide_pack/tests/test_testing.py b/pyodide_pack/tests/test_testing.py index 40390ad..a2ec508 100644 --- a/pyodide_pack/tests/test_testing.py +++ b/pyodide_pack/tests/test_testing.py @@ -3,5 +3,5 @@ def test_get_stdlib_module_paths(): paths = _get_stdlib_module_paths() - assert len(paths) > 2000 + assert all(p.suffix == ".py" for p in paths) diff --git a/pyproject.toml b/pyproject.toml index dd500ca..6fe5bb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,11 @@ classifiers = [ "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Operating System :: OS Independent", ] -requires-python = ">=3.10" +requires-python = ">=3.13" dependencies = [ "jinja2", "pyodide-cli>=0.2.0", "pyodide-lock", - "tomli; python_version < '3.11'" ] dynamic = ["version"] @@ -58,6 +57,7 @@ doctest_optionflags = [ "ELLIPSIS", ] addopts = [ + "-svra", "--doctest-modules", ] testpaths = [ @@ -79,4 +79,4 @@ lint.select = [ "UP", # pyupgrade ] lint.ignore = ["E402", "E501", "E731", "E741"] -target-version = "py310" +target-version = "py312"