Replace relative URLs to absolute URL when freezing the lockfile (#241)

* Use 0.28.0 for testing

* Replace relative paths to absolute path when freezing the lockfile

* Add test

* Pass base url

* Fix order

* Fix test

* fix test

* syntax

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* changelog

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: ryanking13

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.10.1] - 2024/07/05
+
+### Fixed
+
+- `micropip.freeze()` now updates the URLs inside the lockfile to absolute URLs.
+  This behavior is consistent with the `lockfileURL` behavior change in Pyodide 0.28.0.
+  [#241](https://github.com/pyodide/micropip/pull/241)
+
 ## [0.10.0] - 2024/07/02
 
 ### Added

micropip/_compat/__init__.py

@@ -21,6 +21,8 @@
 
 LOCKFILE_PACKAGES = compatibility_layer.lockfile_packages
 
+lockfile_base_url = compatibility_layer.lockfile_base_url
+
 fetch_bytes = compatibility_layer.fetch_bytes
 
 fetch_string_and_headers = compatibility_layer.fetch_string_and_headers
@@ -40,4 +42,5 @@
     "loadedPackages",
     "loadPackage",
     "to_js",
+    "lockfile_base_url",
 ]

micropip/_compat/_compat_in_pyodide.py

@@ -8,7 +8,7 @@
 
 try:
     import pyodide_js
-    from pyodide_js import loadedPackages, loadPackage
+    from pyodide_js import loadedPackages, loadPackage, lockfileBaseUrl
     from pyodide_js._api import (  # type: ignore[import]
         install,
         loadBinaryFile,
@@ -58,3 +58,5 @@ async def fetch_string_and_headers(
     lockfile_info = LOCKFILE_INFO
 
     lockfile_packages = LOCKFILE_PACKAGES
+
+    lockfile_base_url = lockfileBaseUrl

micropip/_compat/_compat_not_in_pyodide.py

@@ -84,3 +84,5 @@ def to_js(
     lockfile_info = {}
 
     lockfile_packages = {}
+
+    lockfile_base_url = None

micropip/_compat/compatibility_layer.py

@@ -19,6 +19,8 @@ def to_py():
 
     lockfile_packages: dict[str, dict[str, Any]]
 
+    lockfile_base_url: str | None = None
+
     @staticmethod
     @abstractmethod
     async def fetch_bytes(url: str, kwargs: dict[str, str]) -> bytes:

micropip/freeze.py

@@ -4,23 +4,31 @@
 from copy import deepcopy
 from importlib.metadata import Distribution
 from typing import Any
+from urllib.parse import urljoin
 
 from ._utils import get_dist_info
 from ._vendored.packaging.src.packaging.requirements import Requirement
 from ._vendored.packaging.src.packaging.utils import canonicalize_name
 
 
 def freeze_lockfile(
-    lockfile_packages: dict[str, dict[str, Any]], lockfile_info: dict[str, str]
+    lockfile_packages: dict[str, dict[str, Any]],
+    lockfile_info: dict[str, str],
+    lockfile_base_url: str | None = None,
 ) -> str:
-    return json.dumps(freeze_data(lockfile_packages, lockfile_info))
+    return json.dumps(freeze_data(lockfile_packages, lockfile_info, lockfile_base_url))
 
 
 def freeze_data(
-    lockfile_packages: dict[str, dict[str, Any]], lockfile_info: dict[str, str]
+    lockfile_packages: dict[str, dict[str, Any]],
+    lockfile_info: dict[str, str],
+    lockfile_base_url: str | None = None,
 ) -> dict[str, Any]:
     packages = deepcopy(lockfile_packages)
     packages.update(load_pip_packages(lockfile_packages))
+    if lockfile_base_url is not None:
+        # Override the base URL for the packages
+        override_base_url(packages, lockfile_base_url)
 
     # Sort
     packages = dict(sorted(packages.items()))
@@ -30,6 +38,19 @@ def freeze_data(
     }
 
 
+def override_base_url(
+    lockfile_packages: dict[str, dict[str, Any]],
+    lockfile_base_url: str,
+):
+    """
+    Updates the relative URLs in the lockfile packages to absolute URLs by appending the base URL.
+    This assures that when the generated lockfile is deployed separately from the packages,
+    the URLs will still point to the correct location.
+    """
+    for pkg in lockfile_packages.values():
+        pkg["file_name"] = urljoin(lockfile_base_url, pkg["file_name"])
+
+
 def load_pip_packages(
     lockfile_packages: dict[str, dict[str, Any]],
 ) -> Iterator[tuple[str, dict[str, Any]]]:

micropip/package_manager.py

@@ -320,7 +320,9 @@ def freeze(self) -> str:
         ``lockFileURL`` of :js:func:`~globalThis.loadPyodide`.
         """
         return freeze_lockfile(
-            self.compat_layer.lockfile_packages, self.compat_layer.lockfile_info
+            self.compat_layer.lockfile_packages,
+            self.compat_layer.lockfile_info,
+            self.compat_layer.lockfile_base_url,
         )
 
     def add_mock_package(

tests/test_freeze.py

@@ -1,5 +1,6 @@
 import pytest
 from conftest import mock_fetch_cls
+from pytest_pyodide import run_in_pyodide
 
 
 @pytest.mark.asyncio
@@ -102,3 +103,73 @@ def test_freeze_lockfile_compat(
     assert package.install_dir == "site"
     assert not package.unvendored_tests
     assert package.version == wheel.version
+
+
+def test_override_base_url():
+    from micropip.freeze import override_base_url
+
+    lockfile_packages = {
+        "pkg1": {"file_name": "pkg1-1.0.0-py3-none-any.whl"},
+        "pkg2": {"file_name": "pkg2-2.0.0-py3-none-any.whl"},
+        "pkg3": {"file_name": "https://other.com/pkg3-3.0.0-py3-none-any.whl"},
+    }
+    base_url = "https://example.com/packages/"
+
+    override_base_url(lockfile_packages, base_url)
+
+    assert (
+        lockfile_packages["pkg1"]["file_name"]
+        == "https://example.com/packages/pkg1-1.0.0-py3-none-any.whl"
+    )
+    assert (
+        lockfile_packages["pkg2"]["file_name"]
+        == "https://example.com/packages/pkg2-2.0.0-py3-none-any.whl"
+    )
+    assert (
+        lockfile_packages["pkg3"]["file_name"]
+        == "https://other.com/pkg3-3.0.0-py3-none-any.whl"
+    )
+
+
+def test_url_after_freeze_pyodide(selenium_standalone_micropip):
+
+    @run_in_pyodide
+    def _run(selenium, prefix):
+        import json
+
+        from pyodide_js import lockfileBaseUrl
+        from pyodide_js._api import lockfile_packages
+
+        import micropip
+
+        new_lockfile_str = micropip.freeze()
+        new_lockfile_packages = json.loads(new_lockfile_str)["packages"]
+
+        orig_lockfile_packages = lockfile_packages.to_py()
+
+        for orig_pkg_name, orig_pkg in orig_lockfile_packages.items():
+            assert orig_pkg_name in new_lockfile_packages
+
+            new_pkg = new_lockfile_packages[orig_pkg_name]
+
+            assert new_pkg["name"] == orig_pkg["name"]
+            assert new_pkg["version"] == orig_pkg["version"]
+            assert new_pkg["sha256"] == orig_pkg["sha256"]
+            assert new_pkg["imports"] == orig_pkg["imports"]
+            assert new_pkg["depends"] == orig_pkg["depends"]
+            assert new_pkg["install_dir"] == orig_pkg["install_dir"]
+            assert new_pkg["unvendored_tests"] == orig_pkg["unvendored_tests"]
+
+            # original lockfile will have relative URLs
+            # TODO: this might change later if packages are served from PyPI
+            assert not orig_pkg["file_name"].startswith(prefix)
+
+            # new lockfile should have absolute URLs
+            assert new_pkg["file_name"].startswith(prefix)
+            assert new_pkg["file_name"].startswith(lockfileBaseUrl)
+
+            assert orig_pkg["file_name"] in new_pkg["file_name"]
+
+    selenium = selenium_standalone_micropip
+    prefix = ("/",) if selenium.browser == "node" else ("http://", "https://")
+    _run(selenium, prefix)