[GR-58970] Add ability to ship new patches for a released version

PullRequest: graalpython/3517

Author: msimacek

CHANGELOG.md

@@ -14,6 +14,8 @@ language runtime. The main focus is on user-observable behavior of the engine.
   * See [the documentation](https://github.com/oracle/graalpython/blob/master/docs/user/Interoperability.md#interacting-with-foreign-objects-from-python-scripts) for more information.
 * Remove support for running with Sulong managed both in embeddings as well as through the `graalpy-managed` launcher.
 * Rewrite wheelbuilder to be easier to use and contribute to. This version is now the same we run internally to build publishable wheels for some platforms we support, so the community can build the same wheels on their own hardware easily if desired.
+* `pip` is now able to fetch newer versions of GraalPy patches for third-party packages from `graalpython` GitHub repository, allowing us to add new patches to released versions.
+  * The patch repository can be overridden using `PIP_GRAALPY_PATCHES_URL` environment variable, which can point to a local path or a URL. It can be disabled by setting it to an empty string.
 
 ## Version 24.1.0
 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'.

graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -42,8 +42,12 @@
 import subprocess
 import sys
 import tempfile
+import threading
 import unittest
+from http.server import HTTPServer, SimpleHTTPRequestHandler
 from pathlib import Path
+from urllib.parse import urljoin
+from urllib.request import pathname2url
 
 SETUP_PY_TEMPLATE = '''
 from setuptools import setup, find_packages
@@ -97,7 +101,7 @@ def setUp(self):
             shutil.copytree(self.venv_template_dir, self.venv_dir, symlinks=True)
             self.patch_dir = Path(tempfile.mkdtemp()).resolve()
             self.pip_env = os.environ.copy()
-            self.pip_env['PIPLOADER_PATCHES_BASE_DIRS'] = str(self.patch_dir)
+            self.pip_env['PIP_GRAALPY_PATCHES_URL'] = str(self.patch_dir)
             self.index_dir = Path(tempfile.mkdtemp()).resolve()
 
         def tearDown(self):
@@ -106,18 +110,16 @@ def tearDown(self):
             shutil.rmtree(self.index_dir, ignore_errors=True)
 
         def prepare_config(self, name, rules):
-            package_dir = self.patch_dir / name
-            package_dir.mkdir(exist_ok=True)
             toml_lines = []
             for rule in rules:
-                toml_lines.append('[[rules]]')
+                toml_lines.append(f'[[{name}.rules]]')
                 for k, v in rule.items():
                     if not k.startswith('$'):
                         toml_lines.append(f'{k} = {v!r}')
                 if patch := rule.get('patch'):
-                    with open(package_dir / patch, 'w') as f:
+                    with open(self.patch_dir / patch, 'w') as f:
                         f.write(PATCH_TEMPLATE.format(rule.get('$patch-text', 'Patched')))
-            with open(package_dir / 'metadata.toml', 'w') as f:
+            with open(self.patch_dir / 'metadata.toml', 'w') as f:
                 f.write('\n'.join(toml_lines))
 
         def build_package(self, name, version):
@@ -147,27 +149,37 @@ def add_package_to_index(self, name, version, dist_type):
             package = self.build_package(name, version)[dist_type]
             shutil.copy(package, self.index_dir)
 
-        def run_venv_pip_install(self, package, extra_env=None):
+        def run_venv_pip_install(self, package, extra_env=None, assert_stderr_matches=None):
             env = self.pip_env.copy()
             if extra_env:
                 env.update(extra_env)
-            out = subprocess.check_output([
-                self.venv_python,
-                '--experimental-options', '--python.EnableDebuggingBuiltins=true',
-                '-m', 'pip', 'install', '--force-reinstall',
-                '--find-links', self.index_dir, '--no-index', '--no-cache-dir',
-                package],
-                env=env, universal_newlines=True)
-            assert 'Applying GraalPy patch failed for' not in out
-            return re.findall(r'Successfully installed (\S+)', out)
+            proc = subprocess.run(
+                [
+                    str(self.venv_dir / 'bin' / 'pip'),
+                    '--isolated',
+                    'install',
+                    '--force-reinstall',
+                    '--find-links', self.index_dir,
+                    '--no-index',
+                    '--no-cache-dir',
+                    package,
+                ],
+                check=True,
+                capture_output=True,
+                env=env,
+                universal_newlines=True,
+            )
+            print(proc.stderr)
+            assert 'Applying GraalPy patch failed for' not in proc.stderr
+            if assert_stderr_matches:
+                assert re.search(assert_stderr_matches, proc.stderr), \
+                    f"Didn't match expected stderr.\nExpected (regex): {assert_stderr_matches}\nActual:{proc.stderr}"
+            return re.findall(r'Successfully installed (\S+)', proc.stdout)
 
         def run_test_fun(self):
             code = "import patched_package; print(patched_package.test_fun())"
             return subprocess.check_output([self.venv_python, '-c', code], universal_newlines=True).strip()
 
-        def test_pip_launcher(self):
-            subprocess.check_output([str(self.venv_dir / 'bin' / 'pip'), 'install', '--help'])
-
         def test_wheel_unpatched_version(self):
             self.add_package_to_index('foo', '1.0.0', 'wheel')
             self.prepare_config('foo', [{
@@ -388,3 +400,62 @@ def test_name_with_dashes(self):
             }])
             assert self.run_venv_pip_install('package-with-dashes') == ['package-with-dashes-1.0.0']
             assert self.run_test_fun() == "Patched"
+
+        def check_installing_with_patch_repo(self, url_or_path: str, *, graalpy_version=None, should_be_skipped=False,
+                                             assert_stderr_matches=None):
+            self.pip_env['PIP_GRAALPY_PATCHES_URL'] = url_or_path
+            if graalpy_version:
+                self.pip_env['TEST_PIP_GRAALPY_VERSION'] = graalpy_version
+            self.add_package_to_index('foo', '1.1.0', 'wheel')
+            self.prepare_config('foo', [{'patch': 'foo.patch'}])
+            assert self.run_venv_pip_install('foo', assert_stderr_matches=assert_stderr_matches) == ['foo-1.1.0']
+            assert self.run_test_fun() == ("Unpatched" if should_be_skipped else "Patched")
+
+        def test_broken_patches_path(self):
+            self.check_installing_with_patch_repo(
+                '/tmp/not-there',
+                should_be_skipped=True,
+                assert_stderr_matches="WARNING: Failed to load GraalPy patch repository",
+            )
+
+        def test_patches_file_url(self):
+            self.check_installing_with_patch_repo(urljoin('file:', pathname2url(str(self.patch_dir.absolute()))))
+
+        @unittest.skipIf(
+            __graalpython__.posix_module_backend() == 'java',
+            "Server doesn't work properly under Java posix backend"
+        )
+        def test_patches_http_url(self):
+            patch_dir = self.patch_dir
+
+            class Handler(SimpleHTTPRequestHandler):
+                def __init__(self, *args, **kwargs):
+                    super().__init__(*args, directory=str(patch_dir), **kwargs)
+
+            try:
+                with HTTPServer(('localhost', 0), Handler) as server:
+                    thread = threading.Thread(target=server.serve_forever)
+                    thread.start()
+                    try:
+                        self.check_installing_with_patch_repo(f'http://localhost:{server.server_port}')
+                    finally:
+                        server.shutdown()
+            finally:
+                thread.join()
+
+        def test_patches_repo_version_resolution(self):
+            patch_dir_parent = self.patch_dir
+            graalpy_version = '1.3.2'
+            self.patch_dir = patch_dir_parent / graalpy_version
+            self.patch_dir.mkdir()
+            self.check_installing_with_patch_repo(str(patch_dir_parent / '<version>'), graalpy_version=graalpy_version)
+
+        def test_patches_repo_version_resolution_dev(self):
+            patch_dir_parent = self.patch_dir
+            self.patch_dir = patch_dir_parent / '1.3.2-dev'
+            self.patch_dir.mkdir()
+            self.check_installing_with_patch_repo(
+                str(patch_dir_parent / '<version>'),
+                graalpy_version='1.3.2-dev',
+                should_be_skipped=True,
+            )

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -183,20 +183,21 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final int MICRO = 7;
     public static final int RELEASE_LEVEL_ALPHA = 0xA;
     public static final int RELEASE_LEVEL_BETA = 0xB;
-    public static final int RELEASE_LEVEL_GAMMA = 0xC;
+    public static final int RELEASE_LEVEL_CANDIDATE = 0xC;
     public static final int RELEASE_LEVEL_FINAL = 0xF;
-    public static final int RELEASE_LEVEL = RELEASE_LEVEL_ALPHA;
+    public static final int RELEASE_LEVEL;
     public static final TruffleString RELEASE_LEVEL_STRING;
     public static final String FROZEN_FILENAME_PREFIX = "<frozen ";
     public static final String FROZEN_FILENAME_SUFFIX = ">";
 
     /**
      * GraalVM version. Unfortunately, we cannot just use {@link Version#getCurrent} as it relies on
-     * a GraalVM build, but we may run from Jar files directly during development. We generate the
-     * version during the build that are checked against these constants.
+     * a GraalVM build, but we may run outside GraalVM. We generate the version during the build
+     * that are checked against these constants.
      */
     public static final int GRAALVM_MAJOR;
     public static final int GRAALVM_MINOR;
+    public static final int GRAALVM_MICRO;
     public static final String DEV_TAG;
 
     /**
@@ -207,21 +208,6 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     private static final int VERSION_BASE = '!';
 
     static {
-        switch (RELEASE_LEVEL) {
-            case RELEASE_LEVEL_ALPHA:
-                RELEASE_LEVEL_STRING = tsLiteral("alpha");
-                break;
-            case RELEASE_LEVEL_BETA:
-                RELEASE_LEVEL_STRING = tsLiteral("beta");
-                break;
-            case RELEASE_LEVEL_GAMMA:
-                RELEASE_LEVEL_STRING = tsLiteral("rc");
-                break;
-            case RELEASE_LEVEL_FINAL:
-            default:
-                RELEASE_LEVEL_STRING = tsLiteral("final");
-        }
-
         // The resource file is built by mx from "graalpy-versions" project using mx substitutions.
         // The actual values of the versions are computed by mx helper functions py_version_short,
         // graal_version_short, and dev_tag defined in mx_graalpython.py
@@ -238,7 +224,22 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
             }
             GRAALVM_MAJOR = is.read() - VERSION_BASE;
             GRAALVM_MINOR = is.read() - VERSION_BASE;
-            is.read(); // skip GraalVM micro version
+            GRAALVM_MICRO = is.read() - VERSION_BASE;
+            RELEASE_LEVEL = is.read() - VERSION_BASE;
+            switch (RELEASE_LEVEL) {
+                case RELEASE_LEVEL_ALPHA:
+                    RELEASE_LEVEL_STRING = tsLiteral("alpha");
+                    break;
+                case RELEASE_LEVEL_BETA:
+                    RELEASE_LEVEL_STRING = tsLiteral("beta");
+                    break;
+                case RELEASE_LEVEL_CANDIDATE:
+                    RELEASE_LEVEL_STRING = tsLiteral("candidate");
+                    break;
+                case RELEASE_LEVEL_FINAL:
+                default:
+                    RELEASE_LEVEL_STRING = tsLiteral("final");
+            }
             // see mx.graalpython/mx_graalpython.py:dev_tag
             byte[] rev = new byte[3 /* 'dev' */ + 10 /* revision */];
             if (is.read(rev) == rev.length) {
@@ -254,7 +255,7 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final int VERSION_HEX = MAJOR << 24 |
                     MINOR << 16 |
                     MICRO << 8 |
-                    RELEASE_LEVEL_ALPHA << 4 |
+                    RELEASE_LEVEL << 4 |
                     RELEASE_SERIAL;
     public static final String VERSION = MAJOR + "." + MINOR + "." + MICRO;
     // Rarely updated version of the C API, we should take it from the imported CPython version

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -40,7 +40,12 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
+import static com.oracle.graal.python.PythonLanguage.GRAALVM_MAJOR;
+import static com.oracle.graal.python.PythonLanguage.GRAALVM_MICRO;
+import static com.oracle.graal.python.PythonLanguage.GRAALVM_MINOR;
 import static com.oracle.graal.python.PythonLanguage.J_GRAALPYTHON_ID;
+import static com.oracle.graal.python.PythonLanguage.RELEASE_LEVEL;
+import static com.oracle.graal.python.PythonLanguage.RELEASE_LEVEL_FINAL;
 import static com.oracle.graal.python.nodes.BuiltinNames.J_EXTEND;
 import static com.oracle.graal.python.nodes.BuiltinNames.J___GRAALPYTHON__;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_SHA3;
@@ -78,11 +83,6 @@
 import java.util.List;
 import java.util.logging.Level;
 
-import com.oracle.truffle.api.interop.InvalidArrayIndexException;
-import com.oracle.truffle.api.interop.TruffleObject;
-import com.oracle.truffle.api.library.ExportLibrary;
-import com.oracle.truffle.api.library.ExportMessage;
-import org.graalvm.home.Version;
 import org.graalvm.nativeimage.ImageInfo;
 
 import com.oracle.graal.python.PythonLanguage;
@@ -185,9 +185,13 @@
 import com.oracle.truffle.api.dsl.TypeSystemReference;
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.InvalidArrayIndexException;
+import com.oracle.truffle.api.interop.TruffleObject;
 import com.oracle.truffle.api.interop.UnknownIdentifierException;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
+import com.oracle.truffle.api.library.ExportLibrary;
+import com.oracle.truffle.api.library.ExportMessage;
 import com.oracle.truffle.api.nodes.LanguageInfo;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.nodes.NodeUtil;
@@ -1056,11 +1060,18 @@ static PythonClass createType(VirtualFrame frame, TruffleString name, PTuple bas
     @Builtin(name = "get_graalvm_version", minNumOfPositionalArgs = 0)
     @GenerateNodeFactory
     abstract static class GetGraalVmVersion extends PythonBuiltinNode {
-        @TruffleBoundary
+        private static final TruffleString VERSION_STRING;
+        static {
+            String version = String.format("%d.%d.%d", GRAALVM_MAJOR, GRAALVM_MINOR, GRAALVM_MICRO);
+            if (RELEASE_LEVEL != RELEASE_LEVEL_FINAL) {
+                version += "-dev";
+            }
+            VERSION_STRING = TruffleString.fromJavaStringUncached(version, TS_ENCODING);
+        }
+
         @Specialization
         TruffleString get() {
-            Version current = Version.getCurrent();
-            return TruffleString.fromJavaStringUncached(current.toString(), TS_ENCODING);
+            return VERSION_STRING;
         }
     }