Skip to content

Commit 5093af9

Browse files
tomasstupkatomasstupka
committed
do not copy python home into vfs
1 parent e6b565c commit 5093af9

File tree

12 files changed

+334
-540
lines changed

12 files changed

+334
-540
lines changed

‎graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,16 @@
4040
*/
4141
package com.oracle.graal.python.resources;
4242

43+
import java.io.File;
4344
import java.io.IOException;
4445
import java.io.InputStream;
4546
import java.nio.file.InvalidPathException;
4647
import java.nio.file.Path;
48+
import java.util.ArrayList;
49+
import java.util.Collections;
4750
import java.util.List;
51+
import java.util.function.Predicate;
52+
import java.util.regex.Pattern;
4853

4954
import com.oracle.truffle.api.CompilerDirectives;
5055
import com.oracle.truffle.api.InternalResource;
@@ -100,20 +105,94 @@ public final class PythonResource implements InternalResource {
100105
public void unpackFiles(Env env, Path targetDirectory) throws IOException {
101106
OS os = env.getOS();
102107
Path osArch = Path.of(os.toString()).resolve(env.getCPUArchitecture().toString());
108+
ResourcesFilter filter = new ResourcesFilter();
103109
if (os.equals(OS.WINDOWS)) {
104-
env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("Lib"), BASE_PATH.resolve(LIBPYTHON));
105-
env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib-graalpython"), BASE_PATH.resolve(LIBGRAALPY));
106-
env.unpackResourceFiles(BASE_PATH.resolve(INCLUDE_FILES), targetDirectory.resolve("Include"), BASE_PATH.resolve(INCLUDE));
110+
env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("Lib"), BASE_PATH.resolve(LIBPYTHON), filter);
111+
env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib-graalpython"), BASE_PATH.resolve(LIBGRAALPY), filter);
112+
env.unpackResourceFiles(BASE_PATH.resolve(INCLUDE_FILES), targetDirectory.resolve("Include"), BASE_PATH.resolve(INCLUDE), filter);
107113
} else {
108114
String pythonMajMin = "python" + PYTHON_MAJOR + "." + PYTHON_MINOR;
109-
env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("lib").resolve(pythonMajMin), BASE_PATH.resolve(LIBPYTHON));
110-
env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib").resolve("graalpy" + GRAALVM_MAJOR + "." + GRAALVM_MINOR), BASE_PATH.resolve(LIBGRAALPY));
111-
env.unpackResourceFiles(BASE_PATH.resolve(INCLUDE_FILES), targetDirectory.resolve("include").resolve(pythonMajMin), BASE_PATH.resolve(INCLUDE));
115+
env.unpackResourceFiles(BASE_PATH.resolve(LIBPYTHON_FILES), targetDirectory.resolve("lib").resolve(pythonMajMin), BASE_PATH.resolve(LIBPYTHON), filter);
116+
env.unpackResourceFiles(BASE_PATH.resolve(LIBGRAALPY_FILES), targetDirectory.resolve("lib").resolve("graalpy" + GRAALVM_MAJOR + "." + GRAALVM_MINOR), BASE_PATH.resolve(LIBGRAALPY),
117+
filter);
118+
env.unpackResourceFiles(BASE_PATH.resolve(INCLUDE_FILES), targetDirectory.resolve("include").resolve(pythonMajMin), BASE_PATH.resolve(INCLUDE), filter);
112119
}
113120
// ni files are in the same place on all platforms
114-
env.unpackResourceFiles(BASE_PATH.resolve(NI_FILES), targetDirectory, BASE_PATH);
121+
env.unpackResourceFiles(BASE_PATH.resolve(NI_FILES), targetDirectory, BASE_PATH, filter);
115122
// native files already have the correct structure
116-
env.unpackResourceFiles(BASE_PATH.resolve(osArch).resolve(NATIVE_FILES), targetDirectory, BASE_PATH.resolve(osArch));
123+
env.unpackResourceFiles(BASE_PATH.resolve(osArch).resolve(NATIVE_FILES), targetDirectory, BASE_PATH.resolve(osArch), filter);
124+
125+
if (filter.log) {
126+
System.out.println("unpacked python resources:");
127+
System.out.println("include pattern: '" + filter.include + "'");
128+
System.out.println("exclude pattern: '" + filter.exclude + "'");
129+
listFiles("included files:", filter.included);
130+
listFiles("excluded files:", filter.excluded);
131+
}
132+
}
133+
134+
private static void listFiles(String header, List<String> l) {
135+
if (!l.isEmpty()) {
136+
Collections.sort(l);
137+
System.out.println(header);
138+
for (String s : l) {
139+
System.out.println("\t" + s);
140+
}
141+
}
142+
}
143+
144+
private static class ResourcesFilter implements Predicate<Path> {
145+
146+
private final boolean log;
147+
private final Pattern includePattern;
148+
private final Pattern excludePattern;
149+
private final List<String> excluded;
150+
private final String exclude;
151+
private final String include;
152+
private final ArrayList<String> included;
153+
154+
private ResourcesFilter() {
155+
include = getProperty("org.graalvm.python.resources.include");
156+
exclude = getProperty("org.graalvm.python.resources.exclude");
157+
log = Boolean.parseBoolean(getProperty("org.graalvm.python.resources.log"));
158+
includePattern = include != null ? Pattern.compile(include) : null;
159+
excludePattern = exclude != null ? Pattern.compile(exclude) : null;
160+
included = log ? new ArrayList<>() : null;
161+
excluded = log ? new ArrayList<>() : null;
162+
}
163+
164+
@Override
165+
public boolean test(Path path) {
166+
String absolutePath = path.toAbsolutePath().toString();
167+
if (File.separator.equals("\\")) {
168+
absolutePath = absolutePath.replaceAll("\\\\", "/");
169+
}
170+
if ((includePattern != null && !includePattern.matcher(absolutePath).matches()) ||
171+
(excludePattern != null && excludePattern.matcher(absolutePath).matches())) {
172+
if (log) {
173+
excluded.add(absolutePath);
174+
}
175+
return false;
176+
} else {
177+
if (log) {
178+
included.add(absolutePath);
179+
}
180+
return true;
181+
}
182+
}
183+
184+
private static String getProperty(String prop) {
185+
String s = System.getProperty(prop);
186+
if (s != null) {
187+
if (s.isEmpty()) {
188+
s = null;
189+
} else if (s.startsWith("\"") && s.endsWith("\"")) {
190+
// native-gradle-plugin sends system properties wrapped in "
191+
s = s.substring(1, s.length() - 1);
192+
}
193+
}
194+
return s;
195+
}
117196
}
118197

119198
@Override

‎graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/integration/VirtualFileSystemIntegrationTest.java

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -545,7 +545,7 @@ public void pythonPathsTest() throws IOException {
545545
defaultMountPoint = vfs.getMountPoint();
546546
}
547547

548-
String getPathsSource = "import sys; [__graalpython__.get_python_home_paths(), sys.path, sys.executable]";
548+
String getPathsSource = "import sys; [sys.path, sys.executable]";
549549
try (Context ctx = GraalPyResources.createContext()) {
550550
Value paths = ctx.eval("python", getPathsSource);
551551

@@ -563,7 +563,7 @@ public void pythonPathsTest() throws IOException {
563563
assertEquals(VFS_MOUNT_POINT, vfs.getMountPoint());
564564
try (Context ctx = GraalPyResources.contextBuilder(vfs).build()) {
565565
Value paths = ctx.eval("python", getPathsSource);
566-
checkPaths(paths.as(List.class), vfs.getMountPoint(), true);
566+
checkPaths(paths.as(List.class), vfs.getMountPoint());
567567
}
568568
Path resourcesDir = Files.createTempDirectory("python-resources");
569569

@@ -573,31 +573,12 @@ public void pythonPathsTest() throws IOException {
573573
}
574574
}
575575

576-
private static void checkPaths(List<Object> l, String pathPrefix) {
577-
checkPaths(l, pathPrefix, false);
578-
}
579-
580576
@SuppressWarnings("unchecked")
581-
private static void checkPaths(List<Object> l, String pathPrefix, boolean checkHome) {
582-
// python.PythonHome
583-
// TODO how to check python.PythonHome?
584-
// /org.graalvm.python.vfs/home has to be in fileslist.txt
585-
// - if it is in fileslist.txt, than it also has to contain the whole stdlib,
586-
// or other tests will fail when trying to load python
587-
// - the pythonHome value is implicitly covered in maven and gradle plugin tests
588-
// if (checkHome) {
589-
// assertTrue(((String) l.get(0)).contains(pathPrefix + File.separator + "home" +
590-
// File.separator +
591-
// "lib-graalpython"));
592-
// assertTrue(((String) l.get(0)).contains(pathPrefix + File.separator + "home" +
593-
// File.separator +
594-
// "lib-python" + File.separator + "3"));
595-
// }
596-
577+
private static void checkPaths(List<Object> l, String pathPrefix) {
597578
// option python.PythonPath
598-
assertTrue(((List<Object>) l.get(1)).contains(pathPrefix + File.separator + "src"));
579+
assertTrue(((List<Object>) l.get(0)).contains(pathPrefix + File.separator + "src"));
599580
// option python.Executable
600-
assertEquals(l.get(2), pathPrefix + (IS_WINDOWS ? "\\venv\\Scripts\\python.exe" : "/venv/bin/python"));
581+
assertEquals(l.get(1), pathPrefix + (IS_WINDOWS ? "\\venv\\Scripts\\python.exe" : "/venv/bin/python"));
601582
}
602583

603584
private static Builder addTestOptions(Builder builder) {

0 commit comments

Comments
 (0)