[GR-68746] Fix loading delvewheel wheels

PullRequest: graalpython/3958

Author: msimacek

.gitignore

@@ -21,7 +21,6 @@ workingsets.xml
 GRAALPYTHON.dist
 GRAALPYTHON_UNIT_TESTS.dist
 mx.graalpython/eclipse-launches
-*.json
 !jbang-catalog.json
 !**/resources/*.json
 !**/META-INF/**/*.json

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -1167,11 +1167,6 @@ protected void printHelp(OptionCategory maxCategory) {
                                         "\nEnvironment variables specific to the Graal Python launcher:\n" : ""));
     }
 
-    @Override
-    protected String[] getDefaultLanguages() {
-        return new String[]{getLanguageId(), "llvm", "regex"};
-    }
-
     @Override
     protected void collectArguments(Set<String> options) {
         // This list of arguments is used when we are launched through the Polyglot

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -41,7 +41,6 @@
 import enum
 import fnmatch
 import json
-import math
 import os
 import pickle
 import platform
@@ -60,6 +59,7 @@
 import typing
 import unittest
 import unittest.loader
+import urllib.request
 from abc import abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -81,6 +81,18 @@
 CURRENT_PLATFORM = f'{sys.platform}-{platform.machine()}'
 CURRENT_PLATFORM_KEYS = frozenset({CURRENT_PLATFORM})
 
+RUNNER_ENV = {}
+DISABLE_JIT_ENV = {'GRAAL_PYTHON_VM_ARGS': '--experimental-options --engine.Compilation=false'}
+
+# We leave the JIT enabled for the tests themselves, but disable it for subprocesses
+# noinspection PyUnresolvedReferences
+if IS_GRAALPY and __graalpython__.is_native and 'GRAAL_PYTHON_VM_ARGS' not in os.environ:
+    try:
+        subprocess.check_output([sys.executable, '--version'], env={**os.environ, **DISABLE_JIT_ENV})
+        RUNNER_ENV = DISABLE_JIT_ENV
+    except subprocess.CalledProcessError:
+        pass
+
 
 class Logger:
     report_incomplete = sys.stdout.isatty()
@@ -335,16 +347,16 @@ def __init__(self, *, failfast: bool, report_durations: int | None):
         self.total_duration = 0.0
 
     @staticmethod
-    def report_start(test_id: TestId):
-        log(f"{test_id} ... ", incomplete=True)
+    def report_start(test_id: TestId, prefix=''):
+        log(f"{prefix}{test_id} ... ", incomplete=True)
 
-    def report_result(self, result: TestResult):
+    def report_result(self, result: TestResult, prefix=''):
         self.results.append(result)
         message = f"{result.test_id} ... {result.status}"
         if result.status == TestStatus.SKIPPED and result.param:
-            message = f"{message} {result.param!r}"
+            message = f"{prefix}{message} {result.param!r}"
         else:
-            message = f"{message} ({result.duration:.2f}s)"
+            message = f"{prefix}{message} ({result.duration:.2f}s)"
         log(message)
 
     def tests_failed(self):
@@ -532,10 +544,10 @@ def __init__(self, *, num_processes, subprocess_args, separate_workers, timeout_
         self.crashes = []
         self.default_test_timeout = 600
 
-    def report_result(self, result: TestResult):
+    def report_result(self, result: TestResult, prefix=''):
         if self.failfast and result.status in FAILED_STATES:
             self.stop_event.set()
-        super().report_result(result)
+        super().report_result(result, prefix=prefix)
 
     def tests_failed(self):
         return super().tests_failed() or bool(self.crashes)
@@ -550,10 +562,36 @@ def partition_tests_into_processes(self, suites: list['TestSuite']) -> list[list
                 lambda suite: suite.test_file.config.new_worker_per_file,
             )
         partitions = [suite.collected_tests for suite in per_file_suites]
-        per_partition = int(math.ceil(len(unpartitioned) / max(1, self.num_processes)))
-        while unpartitioned:
-            partitions.append([test for suite in unpartitioned[:per_partition] for test in suite.collected_tests])
-            unpartitioned = unpartitioned[per_partition:]
+
+        # Use timings if available to partition unpartitioned optimally
+        timings = {}
+        if unpartitioned and self.num_processes:
+            configdir = unpartitioned[0].test_file.config.configdir if unpartitioned else None
+            if configdir:
+                timing_path = configdir / f"timings-{sys.platform.lower()}.json"
+                if timing_path.exists():
+                    with open(timing_path, "r", encoding="utf-8") as f:
+                        timings = json.load(f)
+
+        timed_files = []
+        for suite in unpartitioned:
+            file_path = str(suite.test_file.path).replace("\\", "/")
+            total = timings.get(file_path, 20.0)
+            timed_files.append((total, suite))
+
+        # Sort descending by expected time
+        timed_files.sort(reverse=True, key=lambda x: x[0])
+
+        # Greedily assign to balance by timing sum
+        process_loads = [[] for _ in range(self.num_processes)]
+        process_times = [0.0] * self.num_processes
+        for t, suite in timed_files:
+            i = process_times.index(min(process_times))
+            process_loads[i].append(suite)
+            process_times[i] += t
+        for group in process_loads:
+            partitions.append([test for suite in group for test in suite.collected_tests])
+
         return partitions
 
     def run_tests(self, tests: list['TestSuite']):
@@ -582,7 +620,7 @@ def run_tests(self, tests: list['TestSuite']):
                 log(crash)
 
     def run_partitions_in_subprocesses(self, executor, partitions: list[list['Test']]):
-        workers = [SubprocessWorker(self, partition) for i, partition in enumerate(partitions)]
+        workers = [SubprocessWorker(i, self, partition) for i, partition in enumerate(partitions)]
         futures = [executor.submit(worker.run_in_subprocess_and_watch) for worker in workers]
 
         def dump_worker_status():
@@ -626,7 +664,8 @@ def sigterm_handler(_signum, _frame):
 
 
 class SubprocessWorker:
-    def __init__(self, runner: ParallelTestRunner, tests: list['Test']):
+    def __init__(self, worker_id: int, runner: ParallelTestRunner, tests: list['Test']):
+        self.prefix = f'[worker-{worker_id + 1}] '
         self.runner = runner
         self.stop_event = runner.stop_event
         self.lock = threading.RLock()
@@ -649,7 +688,7 @@ def process_event(self, event):
                 except ValueError:
                     # It executed something we didn't ask for. Not sure why this happens
                     log(f'WARNING: unexpected test started {test_id}')
-                self.runner.report_start(test_id)
+                self.runner.report_start(test_id, prefix=self.prefix)
                 with self.lock:
                     self.last_started_test_id = test_id
                     self.last_started_time = time.time()
@@ -668,7 +707,7 @@ def process_event(self, event):
                     output=test_output,
                     duration=event.get('duration'),
                 )
-                self.runner.report_result(result)
+                self.runner.report_result(result, prefix=self.prefix)
                 with self.lock:
                     self.last_started_test_id = None
                     self.last_started_time = time.time()  # Starts timeout for the following teardown/setup
@@ -820,7 +859,7 @@ def run_in_subprocess_and_watch(self):
                             param=message,
                             output=output,
                             duration=(time.time() - self.last_started_time),
-                        ))
+                        ), prefix=self.prefix)
                         if blame_id is not self.last_started_test_id:
                             # If we're here, it means we didn't know exactly which test we were executing, we were
                             # somewhere in between
@@ -899,6 +938,7 @@ def parse_config(cls, config_path: Path):
             if config_tags_dir := settings.get('tags_dir'):
                 tags_dir = (config_path.parent / config_tags_dir).resolve()
             # Temporary hack for Bytecode DSL development in master branch:
+            # noinspection PyUnresolvedReferences
             if IS_GRAALPY and getattr(__graalpython__, 'is_bytecode_dsl_interpreter', False) and tags_dir:
                 new_tags_dir = (config_path.parent / (config_tags_dir + '_bytecode_dsl')).resolve()
                 if new_tags_dir.exists():
@@ -972,6 +1012,7 @@ class TestSuite:
     collected_tests: list['Test']
 
     def run(self, result):
+        os.environ.update(RUNNER_ENV)
         saved_path = sys.path[:]
         sys.path[:] = self.pythonpath
         try:
@@ -1299,6 +1340,31 @@ def get_bool_env(name: str):
     return os.environ.get(name, '').lower() in ('true', '1')
 
 
+def main_extract_test_timings(args):
+    """
+    Fetches a test log from the given URL, extracts per-file test timings, and writes the output as JSON.
+    """
+
+    # Download the log file
+    with urllib.request.urlopen(args.url) as response:
+        log_content = response.read().decode("utf-8", errors="replace")
+
+    pattern = re.compile(
+        r"^(?P<path>[^\s:]+)::\S+ +\.\.\. (?:ok|FAIL|ERROR|SKIPPED|expected failure|unexpected success|\S+) \((?P<time>[\d.]+)s\)",
+        re.MULTILINE,
+    )
+
+    timings = {}
+    for match in pattern.finditer(log_content):
+        raw_path = match.group("path").replace("\\", "/")
+        t = float(match.group("time"))
+        timings.setdefault(raw_path, 0.0)
+        timings[raw_path] += t
+
+    with open(args.output, "w", encoding="utf-8") as f:
+        json.dump(timings, f, indent=2, sort_keys=True)
+
+
 def main():
     is_mx_graalpytest = get_bool_env('MX_GRAALPYTEST')
     parent_parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
@@ -1428,6 +1494,20 @@ def main():
     merge_tags_parser.add_argument('report_path')
 
     # run the appropriate command
+
+    # extract-test-timings command declaration
+    extract_parser = subparsers.add_parser(
+        "extract-test-timings",
+        help="Extract per-file test timings from a test log URL and write them as JSON"
+    )
+    extract_parser.add_argument(
+        "url", help="URL of the test log file"
+    )
+    extract_parser.add_argument(
+        "output", help="Output JSON file for per-file timings"
+    )
+    extract_parser.set_defaults(main=main_extract_test_timings)
+
     args = parent_parser.parse_args()
     args.main(args)
 

graalpython/com.oracle.graal.python.test/src/tests/conftest.toml

@@ -47,6 +47,7 @@ exclude_on = ['jvm']
 selector = [
     # These test would work on JVM too, but they are prohibitively slow due to a large amount of subprocesses
     "test_patched_pip.py",
+    "test_wheel.py",
 ]
 
 [[test_rules]]

graalpython/com.oracle.graal.python.test/src/tests/test_ctypes.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+from tests.testlib_helper import build_testlib
+
+
+class TestCtypesInterop(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        orig_root = Path(__file__).parent.resolve()
+        orig_testlib = orig_root / "testlib"
+        cls.tmpdir = Path(tempfile.mkdtemp(prefix="testctypes_tmp_"))
+        cls.lib_path = build_testlib(cls.tmpdir, orig_testlib)
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmpdir, ignore_errors=True)
+
+    def run_in_subprocess(self, code, *args):
+        proc = subprocess.run(
+            [sys.executable, "-c", code, *args],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        if proc.returncode != 0:
+            self.fail(
+                "Subprocess failed with exit code {}\nstdout:\n{}\nstderr:\n{}".format(
+                    proc.returncode, proc.stdout, proc.stderr
+                )
+            )
+
+    def test_ctypes_load_and_call(self):
+        # Pass the library path as an argument
+        code = textwrap.dedent(
+            """
+            import sys
+            from ctypes import CDLL, c_int
+            lib_path = sys.argv[1]
+            lib = CDLL(lib_path)
+            get_answer = lib.get_answer
+            get_answer.restype = c_int
+            result = get_answer()
+            assert result == 42, f'expected 42, got {result}'
+            """
+        )
+        self.run_in_subprocess(code, str(self.lib_path))
+
+    @unittest.skipIf(sys.platform != "win32", "Windows-only test")
+    def test_os_add_dll_directory_and_unload(self):
+        # Pass the library dir as argument
+        code = textwrap.dedent(
+            """
+            import os
+            import sys
+            from pathlib import Path
+            from ctypes import CDLL, c_int
+            lib_dir = Path(sys.argv[1])
+            dll_dir = lib_dir.parent
+            dll_name = lib_dir.name
+            # Should fail to load when DLL dir not added
+            try:
+                CDLL(dll_name)
+            except OSError:
+                pass
+            else:
+                raise AssertionError("CDLL(dll_name) should fail outside dll dir context")
+            # Should succeed when DLL dir is temporarily added
+            with os.add_dll_directory(dll_dir):
+                lib = CDLL(dll_name)
+                get_answer = lib.get_answer
+                get_answer.restype = c_int
+                result = get_answer()
+                assert result == 42, f'expected 42, got {result}'
+            """
+        )
+        self.run_in_subprocess(code, str(self.lib_path))
+
+
+if __name__ == "__main__":
+    unittest.main()