Replace patch license check with patch metadata verification

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/test_patch_metadata.py

@@ -12,7 +12,7 @@ Configuration files are named `metadata.toml` and can contain the following:
 # Optional. Relative path to a patch file. May be omitted when the entry just specifies `install-priority`
 patch = 'foo-1.0.0.patch'
 # Required if 'patch' is specified. SPDX license expression for the package (allows parentheses, 'AND', 'OR', 'WITH').
-# Allowed licenses are enumerated in test_patch_metadata.py
+# Allowed licenses are enumerated in mx.graalpython/verify_patches.py
 license = 'MIT'
 # Optional. Version specifier according to https://peps.python.org/pep-0440/#version-specifiers. If omitted, it will
 # match any version

graalpython/lib-graalpython/patches/README.md

@@ -2099,134 +2099,15 @@ def python_run_mx_filetests(args):
 
 
 def _python_checkpatchfiles():
-    listfilename = tempfile.mktemp()
-    # additionally, we want to check if the packages we are patching all have a permissive license
-    with open(listfilename, "w") as listfile:
-        mx.run(["git", "ls-tree", "-r", "HEAD", "--name-only"], out=listfile)
-    try:
-        # TODO our mirror doesn't handle the json API
-        # pypi_base_url = mx_urlrewrites.rewriteurl("https://pypi.org/packages/").replace("packages/", "")
-        pypi_base_url = "https://pypi.org"
-        with open(listfilename, "r") as listfile:
-            content = listfile.read()
-        patchfile_pattern = re.compile(r"lib-graalpython/patches/([^/]+)/.*?([^/]*\.patch)")
-        checked = {
-            # meson-python puts the whole license text in the field. It's MIT
-            'meson-python.patch',
-            # scipy puts the whole license text in the field, skip it. It's new BSD
-            'scipy-1.9.3.patch',
-            'scipy-1.10.1.patch',
-            # pandas puts the whole license text in the field. Its BSD-3-Clause
-            'pandas-1.5.2.patch',
-            'pandas-2.0.3.patch',
-            'pandas-2.2.2.patch',
-            # numpy started putting the whole license text in the field. Its BSD-3-Clause
-            'numpy-1.23.2.patch',
-            'numpy-1.23.5.patch',
-            'numpy-1.24.3.patch',
-            'numpy-1.26.4.patch',
-            'numpy-2.0.0.patch',
-            # libcst is MIT
-            'libcst-1.0.1.patch',
-            # Empty license field, skip it. It's MIT
-            'setuptools-60.patch',
-            'setuptools-60.9.patch',
-            'setuptools-63.patch',
-            'setuptools-65.patch',
-            # Empty license field. It's MIT
-            'urllib3-2.patch',
-            # Empty license field. It's MIT
-            'wheel-0.43.patch',
-            'wheel-0.41.2.patch',
-            'wheel-0.40.patch',
-            'wheel-0.38.patch',
-            'wheel-0.37.patch',
-            'wheel-0.35.patch',
-            'wheel-pre-0.35.patch',
-            # Empty license field. It's ASL 2.0 or BSD 2-Clause
-            'packaging.patch',
-            # pymongo puts the whole license in the field. It's ASL 2.0
-            'pymongo.patch',
-            # Empty license field. It's ASL 2.0
-            'tokenizers-0.13.3.patch',
-            'tokenizers-0.15.0.patch',
-            # Empty license field. It's Apache 2
-            'safetensors-0.3.3.patch',
-            'tensorflow-io-0.34.0.patch',
-            'tensorflow-io-gcs-filesystem-0.34.0.patch',
-            # Puts whole license into the field. It's BSD 3-Clause
-            'pythran-0.12.0.patch',
-            'pythran-0.13.patch',
-            'pyzmq.patch',
-            'jupyter_server.patch',
-            # Empty license field. It's BSD-3-Clause
-            'prompt_toolkit.patch',
-            # Whole license in the field. It's BSD-3-Clause
-            'pyzmq.patch',
-            # Whole license in the field. It's PSF
-            'matplotlib-3.5.3.patch',
-            'matplotlib-3.6.3.patch',
-            'matplotlib-3.7.0.patch',
-            # Empty license field. It's MIT
-            'autopep8.patch',
-            # Whole license in the field. It's MIT
-            'tiktoken-0.7.0.patch',
-            # License field is null. It's PSF
-            'typing_extensions.patch',
-        }
-        allowed_licenses = [
-            "MIT",
-            "BSD",
-            "BSD-3-Clause",
-            "3-Clause BSD License",
-            "Apache License, Version 2.0",
-            "http://www.apache.org/licenses/LICENSE-2.0",
-            "PSF",
-            "Apache",
-            "new BSD",
-            "Apache-2.0",
-            "MPL-2.0",
-            "LGPL",
-        ]
-
-        def as_license_regex(name):
-            subregex = re.escape(name).replace(r'\-', '[- ]')
-            return f'(?:{subregex}(?: license)?)'
-
-        allowed_licenses_regex = re.compile('|'.join(map(as_license_regex, allowed_licenses)), re.IGNORECASE)
-
-        for line in content.split("\n"):
-            if not line or os.stat(line).st_size == 0:
-                # empty files are just markers and do not need to be license checked
-                continue
-            match = patchfile_pattern.search(line)
-            if match:
-                package_name = match.group(1)
-                patch_name = match.group(2)
-                if patch_name in checked:
-                    continue
-                checked.add(patch_name)
-                package_url = "/".join([pypi_base_url, "pypi", package_name, "json"])
-                mx.log("Checking license of patchfile for " + package_url)
-                response = urllib_request.urlopen(package_url)
-                try:
-                    data = json.loads(response.read())
-                    license_field = data["info"]["license"]
-                    license_field_no_parens = re.sub(r'[()]', '', license_field)
-                    license_tokens = re.split(r' AND | OR ', license_field_no_parens)
-                    for license_token in license_tokens:
-                        if not allowed_licenses_regex.match(license_token):
-                            mx.abort(
-                                f"The license for the original project of patch file {patch_name!r} is {license_field!r}. "
-                                f"We cannot include a patch for it. Allowed licenses are: {allowed_licenses}"
-                            )
-                except Exception as e: # pylint: disable=broad-except;
-                    mx.abort("Error getting %r.\n%r" % (package_url, e))
-                finally:
-                    if response:
-                        response.close()
-    finally:
-        os.unlink(listfilename)
+    exe = os.environ.get("PYTHON3_HOME", sys.executable)
+    env = os.environ.copy()
+    env['PYTHONPATH'] = 'graalpython/lib-python/3/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl'
+    mx_dir = Path(__file__).parent
+    mx.run(
+        [exe, str(mx_dir / 'verify_patches.py'), str(mx_dir.parent / 'graalpython/lib-graalpython/patches')],
+        env=env,
+        nonZeroIsFatal=True,
+    )
 
 
 # ----------------------------------------------------------------------------------------------------------------------