[GR-68157] Allow users to opt-out of early failure on unsupported platforms.

PullRequest: graalpython/3940

Author: timfel

.github/workflows/platforms-tests.yml

@@ -0,0 +1,61 @@
+name: Test on exotic platforms
+on:
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+jobs:
+  centos-ppc64le-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build maven artifacts
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git cmake build-essential python3 maven
+          git clone --depth=1 https://github.com/graalvm/mx.git
+          export PATH=$PWD/mx:$PATH
+          export NATIVE_IMAGES=''
+          mx sforceimport
+          mx -p ../graal/vm fetch-jdk -A --jdk-id labsjdk-ce-latest
+          export JAVA_HOME="$HOME/.mx/jdks/labsjdk-ce-latest/"
+          mx deploy-local-maven-repo
+          mv mxbuild/jdk*/mx.graalpython/public-maven-repo m2repo
+
+      - name: Test on OpenJ9
+        run: |
+          # Install IBM Semeru OpenJ9 Java 21
+          wget https://github.com/ibmruntimes/semeru21-binaries/releases/download/jdk-21.0.8%2B9_openj9-0.53.0/ibm-semeru-open-jdk_x64_linux_21.0.8_9_openj9-0.53.0.tar.gz
+          mkdir -p /opt/java/openjdk-21-openj9
+          tar -C /opt/java/openjdk-21-openj9 --strip-components=1 -xzf ibm-semeru*.tar.gz
+          export JAVA_HOME=/opt/java/openjdk-21-openj9
+
+          export PATH=$JAVA_HOME/bin:$PATH
+          mkdir $(pwd)/user_resource_cache
+          mvn -f graalpython/com.oracle.graal.python.test.integration/pom.xml -Dcom.oracle.graal.python.test.polyglot.version=26.0.0 -Dcom.oracle.graal.python.test.polyglot_repo=file:///$(pwd)/m2repo --batch-mode -U -Dtruffle.UseFallbackRuntime=true -Dpolyglot.engine.allowUnsupportedPlatform=true -Dpolyglot.engine.userResourceCache=/$(pwd)/user_resource_cache -Dpolyglot.python.UnsupportedPlatformEmulates=linux -Dorg.graalvm.python.resources.exclude=native.files test -Dtest=HelloWorldTests,AttributeTests,BuiltinSubclassTest,ComplexTexts,CreateClassTest,AsyncActionThreadingTest,JavaInteropTest
+          rm -rf $(pwd)/user_resource_cache
+
+      - name: Test on ppc64le
+        uses: uraimo/run-on-arch-action@v3
+        with:
+          arch: ppc64le
+          distro: ubuntu_latest
+          dockerRunArgs: |
+            --volume "${GITHUB_WORKSPACE}:/workspace"
+            --memory-reservation 6G
+          run: |
+            apt-get update
+            apt-get install -y python3 git wget tar gzip cmake build-essential maven openjdk-21-jdk
+
+            export JAVA_HOME=/usr/lib/jvm/java-21-openjdk-ppc64el
+            export PATH=$JAVA_HOME/bin:$PATH
+
+            echo "Using $JAVA_HOME"
+            $JAVA_HOME/bin/java -version
+
+            # Run some basic smoke tests
+            cd /workspace
+            mkdir $(pwd)/user_resource_cache
+            mvn -f graalpython/com.oracle.graal.python.test.integration/pom.xml -Dcom.oracle.graal.python.test.polyglot.version=26.0.0 -Dcom.oracle.graal.python.test.polyglot_repo=file:///$(pwd)/m2repo --batch-mode -U -Dtruffle.UseFallbackRuntime=true -Dpolyglot.engine.allowUnsupportedPlatform=true -Dpolyglot.engine.userResourceCache=/$(pwd)/user_resource_cache -Dpolyglot.python.UnsupportedPlatformEmulates=linux -Dorg.graalvm.python.resources.exclude=native.files test -Dtest=HelloWorldTests,AttributeTests,BuiltinSubclassTest,ComplexTexts,CreateClassTest,AsyncActionThreadingTest,JavaInteropTest
+            rm -rf $(pwd)/user_resource_cache

CHANGELOG.md

@@ -3,6 +3,9 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 25.0.1
+* Allow users to keep going on unsupported JDK/OS/ARCH combinations at their own risk by opting out of early failure using `-Dtruffle.UseFallbackRuntime=true`, `-Dpolyglot.engine.userResourceCache=/set/to/a/writeable/dir`, `-Dpolyglot.engine.allowUnsupportedPlatform=true`, and `-Dpolyglot.python.UnsupportedPlatformEmulates=[linux|macos|windows]` and `-Dorg.graalvm.python.resources.exclude=native.files`.
+
 ## Version 25.0.0
 * `sys.implementation.version` now returns the GraalPy version instead of the Python version it implements. Also available as `sys.graalpy_version_info` for better discoverability by people already familiar with PyPy and its `sys.pypy_version_info`.
 * `GRAALPY_VERSION_NUM` C macro now inlcudes the release level and serial number at the end to conform to the `hexversion` format. This shouldn't break any existing comparisons.

graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java

@@ -105,7 +105,8 @@ public final class PythonResource implements InternalResource {
     @Override
     public void unpackFiles(Env env, Path targetDirectory) throws IOException {
         OS os = env.getOS();
-        Path osArch = Path.of(os.toString()).resolve(env.getCPUArchitecture().toString());
+        CPUArchitecture cpuArchitecture = env.getCPUArchitecture();
+        Path osArch = Path.of(os.toString()).resolve(cpuArchitecture.toString());
         ResourcesFilter filter = new ResourcesFilter();
         if (os.equals(OS.WINDOWS)) {
             env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("Lib"), BASE_PATH.resolve(LIBPYTHON), filter);
@@ -121,7 +122,23 @@ public void unpackFiles(Env env, Path targetDirectory) throws IOException {
         // ni files are in the same place on all platforms
         env.unpackResourceFiles(BASE_PATH.resolve(NI_FILES), targetDirectory, BASE_PATH, filter);
         // native files already have the correct structure
-        env.unpackResourceFiles(BASE_PATH.resolve(osArch).resolve(NATIVE_FILES), targetDirectory, BASE_PATH.resolve(osArch), filter);
+        if (!os.equals(OS.UNSUPPORTED) && !cpuArchitecture.equals(CPUArchitecture.UNSUPPORTED)) {
+            env.unpackResourceFiles(BASE_PATH.resolve(osArch).resolve(NATIVE_FILES), targetDirectory, BASE_PATH.resolve(osArch), filter);
+        } else {
+            // Native resources are not available for unsupported operating systems.
+            if (!filter.test(Path.of(NATIVE_FILES))) {
+                throw new IOException("Extracting native resources failed for " +
+                                osArch.toString() +
+                                ". This means that GraalPy cannot use native extensions and some built-in features that rely on native code will be unavailable. " +
+                                "Currently supported platforms are linux/amd64, linux/aarch64, macos/amd64, macos/aarch64, and windows/amd64. " +
+                                "If you are running on one of these platforms and are receiving this error, that indicates a bug in this build of GraalPy. " +
+                                "If you are running on a different platform and want to keep going with this unsupported configuration, you can specify the system property \"" +
+                                ResourcesFilter.EXCLUDE_PROP +
+                                "\" with a pattern to exclude \"" +
+                                NATIVE_FILES +
+                                "\" to continue past this point. ");
+            }
+        }
 
         if (filter.log) {
             System.out.println("unpacked python resources:");
@@ -153,8 +170,7 @@ private static class ResourcesFilter implements Predicate<Path> {
         private final List<String> included;
         private static final String INCLUDE_PROP = "org.graalvm.python.resources.include";
         private static final String EXCLUDE_PROP = "org.graalvm.python.resources.exclude";
-
-        private static final String LOG_PROP = "org.graalvm.python.resources.exclude";
+        private static final String LOG_PROP = "org.graalvm.python.resources.log";
 
         private ResourcesFilter() {
             include = getProperty(INCLUDE_PROP);
@@ -211,7 +227,14 @@ private static String getProperty(String prop) {
     @Override
     public String versionHash(Env env) {
         StringBuilder sb = new StringBuilder();
-        for (var s : List.of(LIBGRAALPY_SHA256, LIBPYTHON_SHA256, NI_SHA256, INCLUDE_SHA256, Path.of(env.getOS().toString()).resolve(env.getCPUArchitecture().toString()).resolve(NATIVE_SHA256))) {
+        List<Path> al = List.of(LIBGRAALPY_SHA256, LIBPYTHON_SHA256, NI_SHA256, INCLUDE_SHA256);
+        OS os = env.getOS();
+        CPUArchitecture cpuArchitecture = env.getCPUArchitecture();
+        if (!os.equals(OS.UNSUPPORTED) && !cpuArchitecture.equals(CPUArchitecture.UNSUPPORTED)) {
+            al = new ArrayList<>(al);
+            al.add(Path.of(os.toString()).resolve(cpuArchitecture.toString()).resolve(NATIVE_SHA256));
+        }
+        for (var s : al) {
             try {
                 sb.append(env.readResourceLines(BASE_PATH.resolve(s)).get(0).substring(0, 8));
             } catch (IOException | IndexOutOfBoundsException | InvalidPathException e) {

graalpython/com.oracle.graal.python.test/src/tests/test_posix.py

@@ -912,14 +912,18 @@ def test_sysconf(self):
         else:
             assert False
 
+        def sysconf_max(name):
+            value = os.sysconf(name)
+            return sys.maxsize - value if value < 0 else value
+
         # constants taken from POSIX where defined
-        self.assertGreaterEqual(os.sysconf('SC_ARG_MAX'), 4096)
-        self.assertGreaterEqual(os.sysconf('SC_CHILD_MAX'), 25)
-        self.assertGreaterEqual(os.sysconf('SC_LOGIN_NAME_MAX'), 9)
+        self.assertGreaterEqual(sysconf_max('SC_ARG_MAX'), 4096)
+        self.assertGreaterEqual(sysconf_max('SC_CHILD_MAX'), 25)
+        self.assertGreaterEqual(sysconf_max('SC_LOGIN_NAME_MAX'), 9)
         self.assertGreaterEqual(os.sysconf('SC_CLK_TCK'), 0)
-        self.assertGreaterEqual(os.sysconf('SC_OPEN_MAX'), 20)
+        self.assertGreaterEqual(sysconf_max('SC_OPEN_MAX'), 20)
         self.assertGreaterEqual(os.sysconf('SC_PAGESIZE'), 1)
-        os.sysconf('SC_SEM_NSEMS_MAX') # returns -1 on my linux box, just check it's there
+        self.assertGreaterEqual(sysconf_max('SC_SEM_NSEMS_MAX'), 32)
         self.assertGreaterEqual(os.sysconf('SC_PHYS_PAGES'), 1)
         self.assertGreaterEqual(os.sysconf('SC_NPROCESSORS_CONF'), 1)
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -59,6 +59,7 @@
 
 import com.oracle.graal.python.builtins.Python3Core;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
+import com.oracle.graal.python.builtins.PythonOS;
 import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.SignalModuleBuiltins;
 import com.oracle.graal.python.builtins.objects.PNone;
@@ -460,7 +461,6 @@ protected boolean patchContext(PythonContext context, Env newEnv) {
     @Override
     protected PythonContext createContext(Env env) {
         final PythonContext context = new PythonContext(this, env);
-        context.initializeHomeAndPrefixPaths(env, getLanguageHome());
 
         Object[] engineOptionsUnroll = this.engineOptionsStorage;
         if (engineOptionsUnroll == null) {
@@ -476,13 +476,6 @@ protected PythonContext createContext(Env env) {
             assert areOptionsCompatible(options, PythonOptions.createEngineOptions(env)) : "invalid engine options";
         }
 
-        if (allocationReporter == null) {
-            allocationReporter = env.lookup(AllocationReporter.class);
-        } else {
-            // GR-61960
-            // assert allocationReporter == env.lookup(AllocationReporter.class);
-        }
-
         return context;
     }
 
@@ -509,8 +502,22 @@ protected OptionDescriptors getOptionDescriptors() {
     @Override
     protected void initializeContext(PythonContext context) {
         if (!isLanguageInitialized) {
+            if (allocationReporter == null) {
+                allocationReporter = context.getEnv().lookup(AllocationReporter.class);
+            } else {
+                // GR-61960
+                // assert allocationReporter == env.lookup(AllocationReporter.class);
+            }
             initializeLanguage();
         }
+        if (PythonOS.isUnsupported() && context.getEnv().isNativeAccessAllowed()) {
+            LOGGER.log(Level.WARNING, "Loading native libraries into GraalPy on unsupported platforms. " +
+                            "You can ensure that native access is disallowed for this context and configure GraalPy to use Java backends where possible. " +
+                            "Refer to https://www.graalvm.org/python/docs/ for more information on native and Java module backends. " +
+                            "This is not fatal, because other languages may allow native libraries to run on this platform in the same context, " +
+                            "but attempting to load a native library on GraalPy will fail.");
+        }
+        context.initializeHomeAndPrefixPaths(context.getEnv(), getLanguageHome());
         context.initialize();
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -430,9 +430,8 @@ public abstract class Python3Core {
     private static final TruffleString T__FROZEN_IMPORTLIB_EXTERNAL = tsLiteral("_frozen_importlib_external");
     private static final TruffleString T__FROZEN_IMPORTLIB = tsLiteral("_frozen_importlib");
     private static final TruffleString T_IMPORTLIB_BOOTSTRAP = tsLiteral("importlib._bootstrap");
-    private final TruffleString[] coreFiles;
 
-    private static TruffleString[] initializeCoreFiles() {
+    private static TruffleString[] getCoreFiles() {
         // Order matters!
         List<TruffleString> coreFiles = List.of(
                         T___GRAALPYTHON__,
@@ -450,7 +449,7 @@ private static TruffleString[] initializeCoreFiles() {
         return coreFiles.toArray(new TruffleString[0]);
     }
 
-    private final PythonBuiltins[] builtins;
+    private PythonBuiltins[] builtins;
 
     public static final boolean HAS_PROFILER_TOOL;
     static {
@@ -865,8 +864,6 @@ private static PythonBuiltins[] initializeBuiltins(TruffleLanguage.Env env) {
 
     public Python3Core(PythonLanguage language, TruffleLanguage.Env env) {
         this.language = language;
-        this.builtins = initializeBuiltins(env);
-        this.coreFiles = initializeCoreFiles();
     }
 
     @CompilerDirectives.ValueType
@@ -944,6 +941,8 @@ public final boolean isCoreInitialized() {
      * Load the core library and prepare all builtin classes and modules.
      */
     public final void initialize(PythonContext context) {
+        assert this.builtins == null;
+        this.builtins = initializeBuiltins(context.getEnv());
         initializeJavaCore();
         initializeImportlib();
         context.applyModuleOptions();
@@ -1042,7 +1041,7 @@ private void initializeImportlib() {
     }
 
     private void initializePython3Core(TruffleString coreHome) {
-        for (TruffleString s : coreFiles) {
+        for (TruffleString s : getCoreFiles()) {
             loadFile(s, coreHome);
         }
         initialized = true;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonOS.java

@@ -42,18 +42,23 @@
 
 import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
 
+import java.util.Locale;
+
+import org.graalvm.nativeimage.ImageInfo;
+
+import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.runtime.PythonOptions;
+import com.oracle.truffle.api.exception.AbstractTruffleException;
 import com.oracle.truffle.api.strings.TruffleString;
 
 public enum PythonOS {
-    PLATFORM_JAVA("java", "Java"),
-    PLATFORM_CYGWIN("cygwin", "CYGWIN"),
     PLATFORM_LINUX("linux", "Linux"),
     PLATFORM_DARWIN("darwin", "Darwin"),
     PLATFORM_WIN32("win32", "Windows"),
-    PLATFORM_SUNOS("sunos", "SunOS"),
-    PLATFORM_FREEBSD("freebsd", "FreeBSD"),
     PLATFORM_ANY(null, null);
 
+    public static final String SUPPORTED_PLATFORMS = "linux/amd64, linux/aarch64, macos/amd64, macos/aarch64, and windows/amd64";
+
     private final TruffleString name;
     private final TruffleString uname;
 
@@ -73,28 +78,66 @@ public TruffleString getUname() {
     private static final PythonOS current;
 
     static {
-        String property = System.getProperty("os.name");
-        PythonOS os = PLATFORM_JAVA;
-        if (property != null) {
-            property = property.toLowerCase();
-            if (property.contains("cygwin")) {
-                os = PLATFORM_CYGWIN;
-            } else if (property.contains("linux")) {
-                os = PLATFORM_LINUX;
-            } else if (property.contains("mac")) {
-                os = PLATFORM_DARWIN;
-            } else if (property.contains("windows")) {
-                os = PLATFORM_WIN32;
-            } else if (property.contains("sunos")) {
-                os = PLATFORM_SUNOS;
-            } else if (property.contains("freebsd")) {
-                os = PLATFORM_FREEBSD;
-            }
+        String property = System.getProperty("os.name", "").toLowerCase(Locale.ENGLISH);
+        if (property.contains("linux")) {
+            current = PLATFORM_LINUX;
+        } else if (property.contains("mac") || property.contains("darwin")) {
+            current = PLATFORM_DARWIN;
+        } else if (property.contains("windows")) {
+            current = PLATFORM_WIN32;
+        } else {
+            current = PLATFORM_ANY;
         }
-        current = os;
     }
 
     public static PythonOS getPythonOS() {
+        if (current == PLATFORM_ANY) {
+            if (ImageInfo.inImageBuildtimeCode()) {
+                throw new RuntimeException("Native images with GraalPy are only supported on " + SUPPORTED_PLATFORMS + ".");
+            }
+            String emulated = PythonLanguage.get(null).getEngineOption(PythonOptions.UnsupportedPlatformEmulates);
+            if (!emulated.isEmpty()) {
+                switch (emulated) {
+                    case "linux":
+                        return PLATFORM_LINUX;
+                    case "macos":
+                        return PLATFORM_DARWIN;
+                    case "windows":
+                        return PLATFORM_WIN32;
+                    default:
+                        throw new UnsupportedPlatform("UnsupportedPlatformEmulates must be exactly one of \"linux\", \"macos\", or \"windows\"");
+                }
+            } else {
+                throw new UnsupportedPlatform("This platform is not currently supported. " +
+                                "Currently supported platforms are " + SUPPORTED_PLATFORMS + ". " +
+                                "If you are running on one of these platforms and are receiving this error, that indicates a bug in this build of GraalPy. " +
+                                "If you are running on a different platform and accept that any functionality that interacts with the system may be " +
+                                "incorrect and Python native extensions will not work, you can specify the system property \"UnsupportedPlatformEmulates\" " +
+                                "with a value of either \"linux\", \"macos\", or \"windows\" to continue further and have GraalPy behave as if it were running on " +
+                                "the OS specified. Loading native libraries will not work and must be disabled using the context options. " +
+                                "See https://www.graalvm.org/python/docs/ for details on GraalPy modules with both native and Java backends, " +
+                                "and https://www.graalvm.org/truffle/javadoc/org/graalvm/polyglot/Context.Builder.html to learn about disallowing native access.");
+            }
+        }
         return current;
     }
+
+    public static void throwIfUnsupported(String msg) {
+        if (isUnsupported()) {
+            throw new UnsupportedPlatform(msg +
+                            "\nThis point was reached as earlier platform checks were overridden using the system property UnsupportedPlatformEmulates");
+        }
+    }
+
+    public static boolean isUnsupported() {
+        return current == PLATFORM_ANY;
+    }
+
+    public static final class UnsupportedPlatform extends AbstractTruffleException {
+        public UnsupportedPlatform(String msg) {
+            super(msg);
+        }
+
+        private static final long serialVersionUID = 1L;
+    }
 }