Use truffle resource api when getLanguageHome cannot find valid home

timfel · timfel · commit b25271b4af66 · 2023-08-01T17:27:19.000+02:00
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java
@@ -146,6 +146,7 @@
                 interactive = true, internal = false, //
                 contextPolicy = TruffleLanguage.ContextPolicy.SHARED, //
                 fileTypeDetectors = PythonFileDetector.class, //
+                internalResources = PythonResource.class, //
                 website = "https://www.graalvm.org/python/")
 @ProvidedTags({
                 StandardTags.CallTag.class,
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonResource.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonResource.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python;
+
+import java.io.IOException;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.util.List;
+
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.InternalResource;
+import com.oracle.truffle.api.InternalResource.Id;
+
+/**
+ * This code needs to be kept in sync somehow with the suite.py code. In particular, the layouts
+ * produced by the unpacking logic in {@link #unpackFiles} should produce the same layout as the
+ * GRAALPYTHON_GRAALVM_SUPPORT distribution in the suite.py.
+ */
+@Id("python-home")
+public final class PythonResource implements InternalResource {
+    private static final Path BASE_PATH = Path.of("META-INF", "resources");
+    private static final String LIBGRAALPY = "libgraalpy";
+    private static final String LIBPYTHON = "libpython";
+    private static final String LIBGRAALPY_FILES = LIBGRAALPY + ".files";
+    private static final String LIBPYTHON_FILES = LIBPYTHON + ".files";
+    private static final String INCLUDE_FILES = "include.files";
+    private static final String NATIVE_FILES = "native.files";
+    private static final Path LIBGRAALPY_SHA256 = Path.of(LIBGRAALPY + ".sha256");
+    private static final Path LIBPYTHON_SHA256 = Path.of(LIBPYTHON + ".sha256");
+    private static final Path INCLUDE_SHA256 = Path.of("include.sha256");
+    private static final Path NATIVE_SHA256 = Path.of("native.sha256");
+
+    @Override
+    public void unpackFiles(Env env, Path targetDirectory) throws IOException {
+        OS os = env.getOS();
+        Path osArch = Path.of(os.toString()).resolve(env.getCPUArchitecture().toString());
+        if (os.equals(OS.WINDOWS)) {
+            env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("Lib"), BASE_PATH.resolve(LIBPYTHON));
+            env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib-graalpython"), BASE_PATH.resolve(LIBGRAALPY));
+        } else {
+            env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("lib").resolve("python" + PythonLanguage.MAJOR + "." + PythonLanguage.MINOR), BASE_PATH.resolve(LIBPYTHON));
+            env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib").resolve("graalpy" + PythonLanguage.GRAALVM_MAJOR + "." + PythonLanguage.GRAALVM_MINOR), BASE_PATH.resolve(LIBGRAALPY));
+        }
+        // include files are in the same place on all platforms
+        env.unpackResourceFiles(BASE_PATH.resolve(INCLUDE_FILES), targetDirectory, BASE_PATH);
+        // native files already have the correct structure
+        env.unpackResourceFiles(BASE_PATH.resolve(osArch).resolve(NATIVE_FILES), targetDirectory, BASE_PATH.resolve(osArch));
+    }
+
+    @Override
+    public String versionHash(Env env) {
+        StringBuilder sb = new StringBuilder();
+        for (var s : List.of(LIBGRAALPY_SHA256, LIBPYTHON_SHA256, INCLUDE_SHA256, Path.of(env.getOS().toString()).resolve(env.getCPUArchitecture().toString()).resolve(NATIVE_SHA256))) {
+            try {
+                sb.append(env.readResourceLines(BASE_PATH.resolve(s)).get(0).substring(0, 8));
+            } catch (IOException | IndexOutOfBoundsException | InvalidPathException e) {
+                throw CompilerDirectives.shouldNotReachHere(e);
+            }
+        }
+        return sb.toString();
+    }
+}
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java
@@ -226,7 +226,7 @@ public void postInitialize(Python3Core core) {
         PythonModule mod = core.lookupBuiltinModule(T___GRAALPYTHON__);
         PythonLanguage language = context.getLanguage();
         if (!ImageInfo.inImageBuildtimeCode()) {
-            mod.setAttribute(tsLiteral("home"), toTruffleStringUncached(language.getHome()));
+            mod.setAttribute(tsLiteral("home"), context.getLanguageHome());
         }
         mod.setAttribute(tsLiteral("in_image_buildtime"), ImageInfo.inImageBuildtimeCode());
         mod.setAttribute(tsLiteral("in_image"), ImageInfo.inImageCode());
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java
@@ -98,6 +98,7 @@
 import org.graalvm.options.OptionKey;
 
 import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.PythonResource;
 import com.oracle.graal.python.builtins.Python3Core;
 import com.oracle.graal.python.builtins.PythonOS;
 import com.oracle.graal.python.builtins.modules.ImpModuleBuiltins;
@@ -1678,7 +1679,7 @@ private void initializePosixSupport() {
         }
     }
 
-    private TruffleString sysPrefix, basePrefix, coreHome, capiHome, jniHome, stdLibHome;
+    private TruffleString langHome, sysPrefix, basePrefix, coreHome, capiHome, jniHome, stdLibHome;
 
     public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
         sysPrefix = newEnv.getOptions().get(PythonOptions.SysPrefix);
@@ -1721,25 +1722,44 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
             home = null;
         }
 
-        if (home != null) {
+        Supplier<?>[] homeCandidates = new Supplier<?>[] {
+            () -> { return home; },
+            () -> {
+                try {
+                    return newEnv.getInternalResource(PythonResource.class).getAbsoluteFile();
+                } catch (IOException e) {
+                    return null;
+                }
+            }
+        };
+        for (Supplier<?> homeCandidateSupplier : homeCandidates) {
+            final TruffleFile homeCandidate = (TruffleFile) homeCandidateSupplier.get();
+            if (homeCandidate == null) {
+                continue;
+            }
+            boolean homeSeemsValid = !coreHome.isEmpty() && !stdLibHome.isEmpty();
+
+            langHome = toTruffleStringUncached(homeCandidate.toString());
             if (sysPrefix.isEmpty()) {
-                sysPrefix = toTruffleStringUncached(home.getAbsoluteFile().getPath());
+                sysPrefix = toTruffleStringUncached(homeCandidate.getAbsoluteFile().getPath());
             }
 
             if (basePrefix.isEmpty()) {
-                basePrefix = toTruffleStringUncached(home.getAbsoluteFile().getPath());
+                basePrefix = toTruffleStringUncached(homeCandidate.getAbsoluteFile().getPath());
             }
 
             if (coreHome.isEmpty()) {
                 try {
-                    outer: for (TruffleFile f : home.list()) {
+                    outer: for (TruffleFile f : homeCandidate.list()) {
                         if (f.getName().equals("lib-graalpython") && f.isDirectory()) {
                             coreHome = toTruffleStringUncached(f.getPath());
+                            homeSeemsValid = true;
                             break;
                         } else if (f.getName().equals("lib") && f.isDirectory()) {
                             for (TruffleFile f2 : f.list()) {
                                 if (f2.getName().equals("graalpy" + PythonLanguage.GRAALVM_MAJOR + "." + PythonLanguage.GRAALVM_MINOR) && f.isDirectory()) {
                                     coreHome = toTruffleStringUncached(f2.getPath());
+                                    homeSeemsValid = true;
                                     break outer;
                                 }
                             }
@@ -1752,7 +1772,7 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
             if (stdLibHome.isEmpty()) {
                 // try stdlib layouts per sysconfig or our sources
                 try {
-                    outer: for (TruffleFile f : home.list()) {
+                    outer: for (TruffleFile f : homeCandidate.list()) {
                         if (getPythonOS() == PLATFORM_WIN32 && (f.getName().equals("Lib") || f.getName().equals("lib")) && f.isDirectory()) {
                             // nt stdlib layout
                             stdLibHome = toTruffleStringUncached(f.getPath());
@@ -1761,6 +1781,7 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
                             for (TruffleFile f2 : f.list()) {
                                 if (f2.getName().equals("python" + PythonLanguage.MAJOR + "." + PythonLanguage.MINOR) && f.isDirectory()) {
                                     stdLibHome = toTruffleStringUncached(f2.getPath());
+                                    homeSeemsValid = true;
                                     break outer;
                                 }
                             }
@@ -1769,6 +1790,7 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
                             for (TruffleFile f2 : f.list()) {
                                 if (f2.getName().equals("3") && f.isDirectory()) {
                                     stdLibHome = toTruffleStringUncached(f2.getPath());
+                                    homeSeemsValid = true;
                                     break outer;
                                 }
                             }
@@ -1785,6 +1807,18 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
             if (jniHome.isEmpty()) {
                 jniHome = coreHome;
             }
+
+            if (homeSeemsValid) {
+                break;
+            } else {
+                // reset values
+                sysPrefix = newEnv.getOptions().get(PythonOptions.SysPrefix);
+                basePrefix = newEnv.getOptions().get(PythonOptions.SysBasePrefix);
+                coreHome = newEnv.getOptions().get(PythonOptions.CoreHome);
+                stdLibHome = newEnv.getOptions().get(PythonOptions.StdLibHome);
+                capiHome = newEnv.getOptions().get(PythonOptions.CAPI);
+                jniHome = newEnv.getOptions().get(PythonOptions.JNIHome);
+            }
         }
 
         if (ImageInfo.inImageBuildtimeCode()) {
@@ -1810,10 +1844,18 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
                         "\n\tStdLibHome: {4}" +
                         "\n\tExecutable: {5}" +
                         "\n\tCAPI: {6}" +
-                        "\n\tJNI library: {7}", home != null ? home.getPath() : "", sysPrefix, basePrefix, coreHome, stdLibHome, newEnv.getOptions().get(PythonOptions.Executable), capiHome,
+                        "\n\tJNI library: {7}", langHome, sysPrefix, basePrefix, coreHome, stdLibHome, newEnv.getOptions().get(PythonOptions.Executable), capiHome,
                         jniHome));
     }
 
+    @TruffleBoundary
+    public TruffleString getLanguageHome() {
+        if (langHome == null || langHome.isEmpty()) {
+            langHome = T_PREFIX;
+        }
+        return langHome;
+    }
+
     @TruffleBoundary
     public TruffleString getSysPrefix() {
         if (sysPrefix.isEmpty()) {
@@ -1826,13 +1868,7 @@ public TruffleString getSysPrefix() {
     @TruffleBoundary
     public TruffleString getSysBasePrefix() {
         if (basePrefix.isEmpty()) {
-            String homePrefix = getLanguage().getHome();
-            if (homePrefix == null || homePrefix.isEmpty()) {
-                basePrefix = T_PREFIX;
-            } else {
-                basePrefix = toTruffleStringUncached(homePrefix);
-            }
-
+            basePrefix = getLanguageHome();
         }
         return basePrefix;
     }
@@ -2294,15 +2330,13 @@ private static boolean hasAllowedSuffix(TruffleString path, TruffleString[] allo
     @TruffleBoundary
     public boolean isPyFileInLanguageHome(TruffleFile path) {
         assert !ImageInfo.inImageBuildtimeCode() : "language home won't be available during image build time";
-        String languageHome = getLanguage().getHome();
-
         // The language home may be 'null' if an embedder uses Python. In this case, IO must just be
         // allowed.
-        if (languageHome != null) {
+        if (langHome != null) {
             // This deliberately uses 'getAbsoluteFile' and not 'getCanonicalFile' because if, e.g.,
             // 'path' is a symlink outside of the language home, the user should not be able to read
             // the symlink if 'allowIO' is false.
-            TruffleFile coreHomePath = env.getInternalTruffleFile(languageHome).getAbsoluteFile();
+            TruffleFile coreHomePath = getEnv().getInternalTruffleFile(langHome.toJavaStringUncached()).getAbsoluteFile();
             TruffleFile absolutePath = path.getAbsoluteFile();
             return absolutePath.startsWith(coreHomePath);
         }
diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py

Original file line number	Diff line number	Diff line change
`@@ -226,7 +226,7 @@ public void postInitialize(Python3Core core) {`
`226`	`226`	`PythonModule mod = core.lookupBuiltinModule(T___GRAALPYTHON__);`
`227`	`227`	`PythonLanguage language = context.getLanguage();`
`228`	`228`	`if (!ImageInfo.inImageBuildtimeCode()) {`
`229`		`- mod.setAttribute(tsLiteral("home"), toTruffleStringUncached(language.getHome()));`
	`229`	`+ mod.setAttribute(tsLiteral("home"), context.getLanguageHome());`
`230`	`230`	`}`
`231`	`231`	`mod.setAttribute(tsLiteral("in_image_buildtime"), ImageInfo.inImageBuildtimeCode());`
`232`	`232`	`mod.setAttribute(tsLiteral("in_image"), ImageInfo.inImageCode());`