Implement fetching patches from github

msimacek · msimacek · commit 477d27545b76 · 2024-10-29T14:22:49.000+01:00
diff --git a/graalpython/lib-graalpython/patches/pip-23.2.1.patch b/graalpython/lib-graalpython/patches/pip-23.2.1.patch
@@ -32,6 +32,31 @@ index 02ba608..85c7c22 100644
      help="Don't periodically check PyPI to determine whether a new version "
      "of pip is available for download. Implied with --no-index.",
  )
+diff --git a/pip/_internal/cli/req_command.py b/pip/_internal/cli/req_command.py
+index 86070f1..2c8889a 100644
+--- a/pip/_internal/cli/req_command.py
++++ b/pip/_internal/cli/req_command.py
+@@ -68,6 +68,10 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]:
+     return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ 
+ 
++# GraalPy change: we need the session to fetch remote metadata, but it would be too difficult to pass it down all
++# the possible code paths
++_GRAALPY_SESSION = None
++
+ class SessionCommandMixin(CommandContextMixIn):
+ 
+     """
+@@ -100,6 +104,9 @@ class SessionCommandMixin(CommandContextMixIn):
+             # automatically ContextManager[Any] and self._session becomes Any,
+             # then https://github.com/python/mypy/issues/7696 kicks in
+             assert self._session is not None
++            # GraalPy change
++            global _GRAALPY_SESSION
++            _GRAALPY_SESSION = self._session
+         return self._session
+ 
+     def _build_session(
 diff --git a/pip/_internal/index/package_finder.py b/pip/_internal/index/package_finder.py
 index b6f8d57..6c37e0b 100644
 --- a/pip/_internal/index/package_finder.py
@@ -102,50 +127,76 @@ index a8cd133..20dd1e6 100644
          # file in .data maps to same location as file in wheel root).
 diff --git a/pip/_internal/utils/graalpy.py b/pip/_internal/utils/graalpy.py
 new file mode 100644
-index 0000000..2a67f4b
+index 0000000..375f66e
 --- /dev/null
 +++ b/pip/_internal/utils/graalpy.py
-@@ -0,0 +1,209 @@
-+# ATTENTION: GraalPy uses existence of this module to verify that it is
-+# running a patched pip in pip_hook.py
+@@ -0,0 +1,351 @@
++import abc
++import logging
 +import os
 +import re
++import tempfile
 +import zipfile
++from contextlib import contextmanager
 +from pathlib import Path
-+from urllib.parse import urlparse
++from tomllib import TOMLDecodeError
++from urllib.parse import urlparse, urljoin, urlunparse
 +
 +from pip._internal.models.candidate import InstallationCandidate
 +from pip._internal.models.link import Link
-+from pip._vendor import tomli
++from pip._internal.utils.urls import url_to_path
++from pip._vendor import tomli, requests
 +from pip._vendor.packaging.specifiers import SpecifierSet
++from pip._vendor.packaging.utils import canonicalize_name
 +from pip._vendor.packaging.version import VERSION_PATTERN
 +
-+METADATA_PATH = os.environ.get(
-+    'PIP_GRAALPY_METADATA',
-+    os.path.join(__graalpython__.core_home, 'patches', 'metadata.toml'),
-+)
++MARKER_FILE_NAME = 'GRAALPY_MARKER'
++METADATA_FILENAME = 'metadata.toml'
++DEFAULT_PATCHES_PATH = Path(__graalpython__.core_home) / 'patches'
++VERSION_PARAMETER = '<version>'
++DEFAULT_PATCHES_URL = f'https://raw.githubusercontent.com/oracle/graalpython/refs/heads/github/patches/{VERSION_PARAMETER}/graalpython/lib-graalpython/patches/'
++
++PATCHES_URL = os.environ.get('PIP_GRAALPY_PATCHES_URL', DEFAULT_PATCHES_URL)
 +DISABLE_PATCHING = os.environ.get('PIP_GRAALPY_DISABLE_PATCHING', '').lower() in ('true', '1')
 +DISABLE_VERSION_SELECTION = os.environ.get('PIP_GRAALPY_DISABLE_VERSION_SELECTION', '').lower() in ('true', '1')
 +
++GRAALPY_VERSION = os.environ.get('TEST_PIP_GRAALPY_VERSION', __graalpython__.get_graalvm_version())
++
++logger = logging.getLogger(__name__)
++
 +
-+def normalize_name(name):
-+    return re.sub('[-_.]+', '-', name).lower()
++def url_for_file(patches_url, filename):
++    scheme, netloc, path, params, query, fragment = urlparse(patches_url)
++    path = urljoin(path, filename)
++    return urlunparse((scheme, netloc, path, params, query, fragment))
 +
 +
-+class PatchRepository:
-+    def __init__(self, metadata_path):
-+        self._repository = {}
-+        self.metadata_path = Path(metadata_path)
-+        if self.metadata_path.exists():
-+            with open(self.metadata_path, 'rb') as f:
-+                self._repository = {normalize_name(name): metadata for name, metadata in tomli.load(f).items()}
++class RepositoryException(Exception):
++    pass
++
++
++class RepositoryNotFoundException(RepositoryException):
++    pass
++
++
++class AbstractPatchRepository(metaclass=abc.ABCMeta):
++    def __init__(self, metadata: dict):
++        self._repository = metadata
++
++    @staticmethod
++    def metadata_from_string(metadata_content) -> dict:
++        try:
++            parsed_metadata = tomli.loads(metadata_content)
++            return {canonicalize_name(name): data for name, data in parsed_metadata.items()}
++        except TOMLDecodeError as e:
++            raise RepositoryException(f"'{METADATA_FILENAME}' cannot be parsed: {e}")
 +
 +    def get_rules(self, name):
-+        if metadata := self._repository.get(normalize_name(name)):
++        if metadata := self._repository.get(canonicalize_name(name)):
 +            return metadata.get('rules')
 +
 +    def get_add_sources(self, name):
-+        if metadata := self._repository.get(normalize_name(name)):
++        if metadata := self._repository.get(canonicalize_name(name)):
 +            return metadata.get('add-sources')
 +
 +    def get_priority_for_version(self, name, version):
@@ -176,17 +227,134 @@ index 0000000..2a67f4b
 +                    continue
 +                return rule
 +
-+    def resolve_patch(self, patch_path):
-+        return self.metadata_path.parent / patch_path
++    @abc.abstractmethod
++    def resolve_patch(self, patch_name: str):
++        pass
++
++
++class EmptyRepository(AbstractPatchRepository):
++    def __init__(self):
++        super().__init__({})
++
++    def resolve_patch(self, patch_name: str):
++        raise AssertionError("Invalid call")
++
++
++class LocalPatchRepository(AbstractPatchRepository):
++    def __init__(self, patches_path: Path, repository_data: dict):
++        super().__init__(repository_data)
++        self.patches_path = patches_path
++        logger.debug("Loaded GraalPy patch repository from %s", patches_path)
++
++    @classmethod
++    def from_path(cls, patches_path: Path):
++        try:
++            with open(patches_path / METADATA_FILENAME) as f:
++                metadata_content = f.read()
++        except FileNotFoundError:
++            raise RepositoryNotFoundException(f"'{METADATA_FILENAME}' doesn't exist")
++        except OSError as e:
++            raise RepositoryException(f"'{METADATA_FILENAME}' cannot be read: {e}")
++        return cls(patches_path, cls.metadata_from_string(metadata_content))
++
++    @contextmanager
++    def resolve_patch(self, patch_name: str):
++        yield self.patches_path / patch_name
++
++
++class RemotePatchRepository(AbstractPatchRepository):
++    def __init__(self, patches_url: str, repository_data: dict):
++        super().__init__(repository_data)
++        self.patches_url = patches_url
++        logger.debug("Loaded GraalPy patch repository from %s", patches_url)
++
++    @staticmethod
++    def get_session():
++        from pip._internal.cli.req_command import _GRAALPY_SESSION
++        return _GRAALPY_SESSION or requests.Session()
++
++    @classmethod
++    def from_url(cls, patches_url: str):
++        try:
++            url = url_for_file(patches_url, METADATA_FILENAME)
++            response = cls.get_session().get(url)
++            if response.status_code == 404:
++                raise RepositoryNotFoundException(f"'{METADATA_FILENAME} not found at url: {url}")
++            response.raise_for_status()
++            metadata_content = response.content.decode('utf-8')
++        except RepositoryNotFoundException:
++            raise
++        except Exception as e:
++            raise RepositoryException(f"'{METADATA_FILENAME} cannot be retrieved': {e}")
++        return cls(patches_url, cls.metadata_from_string(metadata_content))
++
++    @contextmanager
++    def resolve_patch(self, patch_name: str):
++        try:
++            response = self.get_session().get(url_for_file(self.patches_url, patch_name))
++            response.raise_for_status()
++        except requests.RequestException as e:
++            logger.warning("Failed to download GraalPy patch '%s': %s", patch_name, e)
++            yield None
++        else:
++            with tempfile.TemporaryDirectory() as tempdir:
++                patch_file = Path(tempdir) / patch_name
++                with open(patch_file, 'wb') as f:
++                    f.write(response.content)
++                yield patch_file
 +
 +
 +__PATCH_REPOSITORY = None
 +
 +
++def repository_from_url_or_path(url_or_path):
++    if '://' not in url_or_path:
++        return LocalPatchRepository.from_path(Path(url_or_path))
++    elif url_or_path.startswith('file:'):
++        patches_path = Path(url_to_path(url_or_path))
++        return LocalPatchRepository.from_path(patches_path)
++    else:
++        patches_url = url_or_path
++        if not patches_url.endswith('/'):
++            patches_url += '/'
++        return RemotePatchRepository.from_url(patches_url)
++
++
++def create_patch_repository(patches_url):
++    if patches_url:
++        if VERSION_PARAMETER in patches_url:
++            if not GRAALPY_VERSION.endswith('-dev'):
++                try_version = GRAALPY_VERSION
++                while '.' in try_version:
++                    url = patches_url.replace(VERSION_PARAMETER, try_version)
++                    try:
++                        return repository_from_url_or_path(url)
++                    except RepositoryNotFoundException:
++                        logger.debug("No patch repository found for version %s at %s", try_version, url)
++                    except RepositoryException as e:
++                        logger.warning("Failed to load GraalPy patch repository from %s: %s", url, e)
++                        logger.warning("Falling back to internal GraalPy patch repository")
++                        break
++                    try_version = try_version.rsplit('.', 1)[0]
++            else:
++                logger.debug("Skipping versioned GraalPy patch repository on snapshot build")
++        else:
++            try:
++                return repository_from_url_or_path(patches_url)
++            except RepositoryException as e:
++                logger.warning("Failed to load GraalPy patch repository from %s: %s", patches_url, e)
++                logger.warning("Falling back to internal GraalPy patch repository")
++    try:
++        return LocalPatchRepository.from_path(DEFAULT_PATCHES_PATH)
++    except RepositoryException as e:
++        logger.warning("Failed to load internal GraalPy patch repository: %s", e)
++    return EmptyRepository()
++
++
 +def get_patch_repository():
 +    global __PATCH_REPOSITORY
 +    if not __PATCH_REPOSITORY:
-+        __PATCH_REPOSITORY = PatchRepository(METADATA_PATH)
++        __PATCH_REPOSITORY = create_patch_repository(PATCHES_URL)
 +    return __PATCH_REPOSITORY
 +
 +
@@ -204,8 +372,8 @@ index 0000000..2a67f4b
 +    name_ver_match = re.match(fr"^(?P<name>.*?)-(?P<version>{VERSION_PATTERN}).*?\.(?P<suffix>tar\.gz|tar|whl|zip)$",
 +                              archive_name, re.VERBOSE | re.I)
 +    if not name_ver_match:
-+        print(f"GraalPy warning: could not parse package name, version, or format from {archive_name!r}.\n"
-+              "Could not determine if any GraalPy specific patches need to be applied.")
++        logger.warning(f"GraalPy warning: could not parse package name, version, or format from {archive_name!r}.\n"
++                       "Could not determine if any GraalPy specific patches need to be applied.")
 +        return
 +
 +    name = name_ver_match.group('name')
@@ -222,7 +390,7 @@ index 0000000..2a67f4b
 +
 +    autopatch_capi.auto_patch_tree(location)
 +
-+    print(f"Looking for GraalPy patches for {name}")
++    logger.info(f"Looking for GraalPy patches for {name}")
 +    repository = get_patch_repository()
 +
 +    if is_wheel:
@@ -240,21 +408,23 @@ index 0000000..2a67f4b
 +            location = os.path.join(location, subdir)
 +    if rule:
 +        if patch := rule.get('patch'):
-+            patch = repository.resolve_patch(patch)
-+            print(f"Patching package {name} using {patch}")
-+            exe = '.exe' if os.name == 'nt' else ''
-+            try:
-+                subprocess.run([f"patch{exe}", "-f", "-d", location, "-p1", "-i", str(patch)], check=True)
-+            except FileNotFoundError:
-+                print(
-+                    "WARNING: GraalPy needs the 'patch' utility to apply compatibility patches. Please install it using your system's package manager.")
-+            except subprocess.CalledProcessError:
-+                print(f"Applying GraalPy patch failed for {name}. The package may still work.")
++            with repository.resolve_patch(patch) as patch_path:
++                if not patch_path:
++                    return
++                logger.info(f"Patching package {name} using {patch}")
++                exe = '.exe' if os.name == 'nt' else ''
++                try:
++                    subprocess.run([f"patch{exe}", "-f", "-d", location, "-p1", "-i", str(patch_path)], check=True)
++                except FileNotFoundError:
++                    logger.warning(
++                        "WARNING: GraalPy needs the 'patch' utility to apply compatibility patches. Please install it using your system's package manager.")
++                except subprocess.CalledProcessError:
++                    logger.warning(f"Applying GraalPy patch failed for {name}. The package may still work.")
 +    elif version_specs := repository.get_suggested_version_specs(name):
-+        print("We have patches to make this package work on GraalVM for some version(s).")
-+        print("If installing or running fails, consider using one of the versions that we have patches for:")
++        logger.info("We have patches to make this package work on GraalVM for some version(s).")
++        logger.info("If installing or running fails, consider using one of the versions that we have patches for:")
 +        for version_spec in version_specs:
-+            print(f'{name} {version_spec}')
++            logger.info(f'{name} {version_spec}')
 +
 +
 +def apply_graalpy_sort_order(sort_key_func):
@@ -294,9 +464,6 @@ index 0000000..2a67f4b
 +    return candidates
 +
 +
-+MARKER_NAME = 'GRAALPY_MARKER'
-+
-+
 +def mark_wheel(path):
 +    if DISABLE_PATCHING:
 +        return
@@ -307,14 +474,14 @@ index 0000000..2a67f4b
 +                dist_info = m.group(1)
 +                break
 +        assert dist_info, "Cannot find .dist_info in built wheel"
-+        marker = f'{dist_info}/{MARKER_NAME}'
++        marker = f'{dist_info}/{MARKER_FILE_NAME}'
 +        with z.open(marker, 'w'):
 +            pass
 +
 +
 +def is_wheel_marked(path):
 +    with zipfile.ZipFile(path) as z:
-+        return any(re.match(rf'[^/]+.dist-info/{MARKER_NAME}$', f) for f in z.namelist())
++        return any(re.match(rf'[^/]+.dist-info/{MARKER_FILE_NAME}$', f) for f in z.namelist())
 diff --git a/pip/_internal/utils/unpacking.py b/pip/_internal/utils/unpacking.py
 index 78b5c13..18a184c 100644
 --- a/pip/_internal/utils/unpacking.py
