Skip to content

Commit 6d96ab7

Browse files
tomasstupkatomasstupka
committed
[GR-52200] [GR-52201] We should make embedding resources optional.
PullRequest: graalpython/3357
2 parents 17dcb78 + 24274fb commit 6d96ab7

File tree

18 files changed

+711
-1047
lines changed

18 files changed

+711
-1047
lines changed

‎graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/VirtualFileSystemTest.java

Lines changed: 74 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,11 @@
4545
import static org.junit.Assert.assertEquals;
4646
import static org.junit.Assert.assertTrue;
4747

48+
import java.io.BufferedReader;
4849
import java.io.File;
4950
import java.io.IOException;
51+
import java.io.InputStream;
52+
import java.io.InputStreamReader;
5053
import java.nio.file.Files;
5154
import java.nio.file.Path;
5255
import java.util.List;
@@ -55,8 +58,10 @@
5558
import org.graalvm.polyglot.Context;
5659
import org.graalvm.polyglot.HostAccess;
5760
import org.graalvm.polyglot.PolyglotException;
61+
import org.graalvm.polyglot.io.FileSystem;
5862
import org.graalvm.polyglot.io.IOAccess;
59-
import org.graalvm.python.embedding.vfs.VirtualFileSystem;
63+
import org.graalvm.python.embedding.utils.GraalPyResources;
64+
import org.graalvm.python.embedding.utils.VirtualFileSystem;
6065
import org.junit.Test;
6166

6267
public class VirtualFileSystemTest {
@@ -69,71 +74,72 @@ public class VirtualFileSystemTest {
6974

7075
@Test
7176
public void defaultValues() throws Exception {
72-
VirtualFileSystem vfs = VirtualFileSystem.create();
73-
VirtualFileSystem vfs2 = VirtualFileSystem.newBuilder().build();
77+
VirtualFileSystem fs = VirtualFileSystem.create();
78+
VirtualFileSystem fs2 = VirtualFileSystem.create();
7479

75-
assertEquals(vfs.getMountPoint(), vfs2.getMountPoint());
80+
assertEquals(fs.getMountPoint(), fs2.getMountPoint());
7681

77-
assertEquals(IS_WINDOWS ? "X:\\graalpy_vfs" : "/graalpy_vfs", vfs.getMountPoint());
82+
assertEquals(IS_WINDOWS ? "X:\\graalpy_vfs" : "/graalpy_vfs", fs.getMountPoint());
7883
}
7984

8085
@Test
8186
public void mountPoints() throws Exception {
82-
VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
87+
VirtualFileSystem fs = VirtualFileSystem.newBuilder().//
8388
unixMountPoint(VFS_MOUNT_POINT).//
8489
windowsMountPoint(VFS_WIN_MOUNT_POINT).build();
8590

86-
assertEquals(VFS_MOUNT_POINT, vfs.getMountPoint());
91+
assertEquals(VFS_MOUNT_POINT, fs.getMountPoint());
8792
}
8893

8994
@Test
9095
public void toRealPath() throws Exception {
91-
VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
96+
FileSystem fs = VirtualFileSystem.newBuilder().//
9297
unixMountPoint(VFS_MOUNT_POINT).//
9398
windowsMountPoint(VFS_WIN_MOUNT_POINT).//
9499
extractFilter(p -> p.getFileName().toString().equals("file1")).//
95100
resourceLoadingClass(VirtualFileSystemTest.class).build();
96101
// check regular resource dir
97-
assertEquals("dir1", vfs.toRealPath(Path.of("dir1")).toString());
98-
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", vfs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1")).toString());
102+
assertEquals("dir1", fs.toRealPath(Path.of("dir1")).toString());
103+
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", fs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1")).toString());
99104
// check regular resource file
100-
assertEquals("SomeFile", vfs.toRealPath(Path.of("SomeFile")).toString());
101-
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", vfs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "SomeFile")).toString());
105+
assertEquals("SomeFile", fs.toRealPath(Path.of("SomeFile")).toString());
106+
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", fs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "SomeFile")).toString());
102107
// check to be extracted file
103-
checkExtractedFile(vfs.toRealPath(Path.of("file1")), new String[]{"text1", "text2"});
104-
checkExtractedFile(vfs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "file1")), new String[]{"text1", "text2"});
108+
checkExtractedFile(fs.toRealPath(Path.of("file1")), new String[]{"text1", "text2"});
109+
checkExtractedFile(fs.toRealPath(Path.of(VFS_MOUNT_POINT + File.separator + "file1")), new String[]{"text1", "text2"});
105110
}
106111

107112
@Test
108113
public void toAbsolutePath() throws Exception {
109-
VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
114+
FileSystem fs = VirtualFileSystem.newBuilder().//
110115
unixMountPoint(VFS_MOUNT_POINT).//
111116
windowsMountPoint(VFS_WIN_MOUNT_POINT).//
112117
extractFilter(p -> p.getFileName().toString().equals("file1") || p.getFileName().toString().equals("file2")).//
113118
resourceLoadingClass(VirtualFileSystemTest.class).build();
114119
// check regular resource dir
115-
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", vfs.toAbsolutePath(Path.of("dir1")).toString());
116-
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", vfs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1")).toString());
120+
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", fs.toAbsolutePath(Path.of("dir1")).toString());
121+
assertEquals(VFS_MOUNT_POINT + File.separator + "dir1", fs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1")).toString());
117122
// check regular resource file
118-
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", vfs.toAbsolutePath(Path.of("SomeFile")).toString());
119-
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", vfs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "SomeFile")).toString());
123+
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", fs.toAbsolutePath(Path.of("SomeFile")).toString());
124+
assertEquals(VFS_MOUNT_POINT + File.separator + "SomeFile", fs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "SomeFile")).toString());
120125
// check to be extracted file
121-
checkExtractedFile(vfs.toAbsolutePath(Path.of("file1")), new String[]{"text1", "text2"});
122-
checkExtractedFile(vfs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "file1")), new String[]{"text1", "text2"});
123-
checkExtractedFile(vfs.toAbsolutePath(Path.of("dir1/file2")), null);
124-
checkExtractedFile(vfs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1/file2")), null);
126+
checkExtractedFile(fs.toAbsolutePath(Path.of("file1")), new String[]{"text1", "text2"});
127+
checkExtractedFile(fs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "file1")), new String[]{"text1", "text2"});
128+
checkExtractedFile(fs.toAbsolutePath(Path.of("dir1/file2")), null);
129+
checkExtractedFile(fs.toAbsolutePath(Path.of(VFS_MOUNT_POINT + File.separator + "dir1/file2")), null);
125130
}
126131

127132
@Test
128133
public void libsExtract() throws Exception {
129-
VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
134+
FileSystem fs = VirtualFileSystem.newBuilder().//
130135
unixMountPoint(VFS_MOUNT_POINT).//
131136
windowsMountPoint(VFS_WIN_MOUNT_POINT).//
132137
extractFilter(p -> p.getFileName().toString().endsWith(".tso")).//
133138
resourceLoadingClass(VirtualFileSystemTest.class).build();
134-
Path p = vfs.toAbsolutePath(Path.of("site-packages/testpkg/file.tso"));
139+
Path p = fs.toAbsolutePath(Path.of("site-packages/testpkg/file.tso"));
135140
checkExtractedFile(p, null);
136141
Path extractedRoot = p.getParent().getParent().getParent();
142+
137143
checkExtractedFile(extractedRoot.resolve("site-packages/testpkg.libs/file1.tso"), null);
138144
checkExtractedFile(extractedRoot.resolve("site-packages/testpkg.libs/file2.tso"), null);
139145
checkExtractedFile(extractedRoot.resolve("site-packages/testpkg.libs/dir/file1.tso"), null);
@@ -142,7 +148,7 @@ public void libsExtract() throws Exception {
142148
checkExtractedFile(extractedRoot.resolve("site-packages/testpkg.libs/dir/dir/file1.tso"), null);
143149
checkExtractedFile(extractedRoot.resolve("site-packages/testpkg.libs/dir/dir/file2.tso"), null);
144150

145-
p = vfs.toAbsolutePath(Path.of("site-packages/testpkg-nolibs/file.tso"));
151+
p = fs.toAbsolutePath(Path.of("site-packages/testpkg-nolibs/file.tso"));
146152
checkExtractedFile(p, null);
147153
}
148154

@@ -433,24 +439,25 @@ public Context getContext(Function<VirtualFileSystem.Builder, VirtualFileSystem.
433439
VirtualFileSystem.Builder builder = VirtualFileSystem.newBuilder().//
434440
unixMountPoint(VFS_MOUNT_POINT).//
435441
windowsMountPoint(VFS_WIN_MOUNT_POINT).//
442+
extractFilter(p -> p.getFileName().toString().endsWith(".tso")).//
436443
resourceLoadingClass(VirtualFileSystemTest.class);
437444
if (builderFunction != null) {
438445
builder = builderFunction.apply(builder);
439446
}
440-
VirtualFileSystem vfs = builder.build();
441-
Context context = VirtualFileSystem.contextBuilder(vfs).build();
447+
VirtualFileSystem fs = builder.build();
448+
Context context = GraalPyResources.contextBuilder(fs).build();
442449
if (builderFunction == null) {
443450
cachedContext = context;
444451
}
445452
return context;
446453
}
447454

448455
@Test
449-
public void builderTest() {
450-
Context context = VirtualFileSystem.contextBuilder().allowAllAccess(true).allowHostAccess(HostAccess.ALL).build();
456+
public void vfsBuilderTest() {
457+
Context context = GraalPyResources.contextBuilder().allowAllAccess(true).allowHostAccess(HostAccess.ALL).build();
451458
context.eval(PYTHON, "import java; java.type('java.lang.String')");
452459

453-
context = VirtualFileSystem.contextBuilder().allowAllAccess(false).allowHostAccess(HostAccess.NONE).build();
460+
context = GraalPyResources.contextBuilder().allowAllAccess(false).allowHostAccess(HostAccess.NONE).build();
454461
context.eval(PYTHON, """
455462
import java
456463
try:
@@ -461,17 +468,17 @@ public void builderTest() {
461468
assert False, 'expected NotImplementedError'
462469
""");
463470

464-
VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
471+
VirtualFileSystem fs = VirtualFileSystem.newBuilder().//
465472
unixMountPoint(VFS_MOUNT_POINT).//
466473
windowsMountPoint(VFS_WIN_MOUNT_POINT).//
467474
resourceLoadingClass(VirtualFileSystemTest.class).build();
468-
context = VirtualFileSystem.contextBuilder(vfs).build();
475+
context = GraalPyResources.contextBuilder(fs).build();
469476
context.eval(PYTHON, patchMountPoint("from os import listdir; listdir('/test_mount_point')"));
470477

471-
context = VirtualFileSystem.contextBuilder().build();
478+
context = GraalPyResources.createContext();
472479
context.eval(PYTHON, "from os import listdir; listdir('.')");
473480

474-
context = VirtualFileSystem.contextBuilder().allowIO(IOAccess.NONE).build();
481+
context = GraalPyResources.contextBuilder().allowIO(IOAccess.NONE).build();
475482
boolean gotPE = false;
476483
try {
477484
context.eval(PYTHON, "from os import listdir; listdir('.')");
@@ -480,4 +487,36 @@ public void builderTest() {
480487
}
481488
assert gotPE : "expected PolyglotException";
482489
}
490+
491+
@Test
492+
public void externalResourcesBuilderTest() throws IOException {
493+
VirtualFileSystem fs = VirtualFileSystem.newBuilder().resourceLoadingClass(VirtualFileSystemTest.class).build();
494+
Path resourcesDir = Files.createTempDirectory("vfs-test-resources");
495+
496+
// extract VFS
497+
GraalPyResources.extractVirtualFileSystemResources(fs, resourcesDir);
498+
499+
// check extracted contents
500+
InputStream stream = VirtualFileSystemTest.class.getResourceAsStream("/org.graalvm.python.vfs/fileslist.txt");
501+
BufferedReader br = new BufferedReader(new InputStreamReader(stream));
502+
String line;
503+
while ((line = br.readLine()) != null) {
504+
line = line.substring("/org.graalvm.python.vfs/".length());
505+
if (line.length() == 0) {
506+
continue;
507+
}
508+
Path extractedFile = resourcesDir.resolve(line);
509+
assert Files.exists(extractedFile);
510+
if (line.endsWith("/")) {
511+
assert Files.isDirectory(extractedFile);
512+
}
513+
}
514+
checkExtractedFile(resourcesDir.resolve(Path.of("file1")), new String[]{"text1", "text2"});
515+
516+
// create context with extracted resource dir and check if we can see the extracted file
517+
try (Context context = GraalPyResources.contextBuilder(resourcesDir).build()) {
518+
context.eval("python", "import os; assert os.path.exists('" + resourcesDir.resolve("file1").toString().replace("\\", "\\\\") + "')");
519+
}
520+
}
521+
483522
}

‎graalpython/com.oracle.graal.python.test/src/tests/standalone/micronaut/hello/src/main/java/example/micronaut/ContextFactory.j

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import org.graalvm.polyglot.HostAccess;
1010
import org.graalvm.polyglot.PolyglotAccess;
1111
import org.graalvm.polyglot.io.IOAccess;
1212
import org.graalvm.python.embedding.micronaut.GraalPyContextFactory;
13-
import org.graalvm.python.embedding.vfs.VirtualFileSystem;
13+
import org.graalvm.python.embedding.utils.GraalPyResources;
1414

1515
import java.io.IOException;
1616
import java.time.Duration;
@@ -25,7 +25,7 @@ class ContextFactory {
2525
@Singleton
2626
@Replaces(value = Context.class, factory = GraalPyContextFactory.class)
2727
public Context createContext() {
28-
context = VirtualFileSystem.contextBuilder().build();
28+
context = GraalPyResources.createContext();
2929
context.initialize("python");
3030
System.out.println("=== CREATED REPLACE CONTEXT ===");
3131
return context;

‎graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ def test_generated_app(self):
211211
self.env["MAVEN_OPTS"] = "-ea -esa"
212212

213213
# import struct from python file triggers extract of native extension files in VirtualFileSystem
214-
hello_src = os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "proj", "hello.py")
214+
hello_src = os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py")
215215
contents = open(hello_src, 'r').read()
216216
with open(hello_src, 'w') as f:
217217
f.write("import struct\n" + contents)
@@ -225,6 +225,60 @@ def test_generated_app(self):
225225
#GR-51132 - NoClassDefFoundError when running polyglot app in java mode
226226
util.check_ouput("java.lang.NoClassDefFoundError", out, False)
227227

228+
@unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
229+
def test_generated_app_external_resources(self):
230+
with tempfile.TemporaryDirectory() as tmpdir:
231+
target_name = "generated_app_ext_resources_test"
232+
target_dir = os.path.join(str(tmpdir), target_name)
233+
self.generate_app(tmpdir, target_dir, target_name)
234+
235+
# patch project to use external directory for resources
236+
resources_dir = os.path.join(target_dir, "python")
237+
os.makedirs(resources_dir, exist_ok=True)
238+
src_dir = os.path.join(resources_dir, "src")
239+
os.makedirs(src_dir, exist_ok=True)
240+
# copy hello.py
241+
shutil.copyfile(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py"), os.path.join(src_dir, "hello.py"))
242+
shutil.rmtree(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs"))
243+
# patch GraalPy.java
244+
replace_in_file(os.path.join(target_dir, "src", "main", "java", "it", "pkg", "GraalPy.java"),
245+
"package it.pkg;",
246+
"package it.pkg;\nimport java.nio.file.Path;")
247+
replace_in_file(os.path.join(target_dir, "src", "main", "java", "it", "pkg", "GraalPy.java"),
248+
"GraalPyResources.createContext()",
249+
"GraalPyResources.contextBuilder(Path.of(\"python\")).build()")
250+
251+
# patch pom.xml
252+
replace_in_file(os.path.join(target_dir, "pom.xml"),
253+
"<packages>",
254+
"<pythonResourcesDirectory>${project.basedir}/python</pythonResourcesDirectory>\n<packages>")
255+
256+
mvnw_cmd = util.get_mvn_wrapper(target_dir, self.env)
257+
258+
# build
259+
cmd = mvnw_cmd + ["package", "-Pnative", "-DmainClass=it.pkg.GraalPy"]
260+
out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir)
261+
util.check_ouput("BUILD SUCCESS", out)
262+
263+
# execute and check native image
264+
cmd = [os.path.join(target_dir, "target", target_name)]
265+
out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir)
266+
util.check_ouput("hello java", out)
267+
268+
# 2.) check java build and exec
269+
# run with java asserts on
270+
if self.env.get("MAVEN_OPTS"):
271+
self.env["MAVEN_OPTS"] = self.env.get("MAVEN_OPTS") + " -ea -esa"
272+
else:
273+
self.env["MAVEN_OPTS"] = "-ea -esa"
274+
275+
# build and exec
276+
cmd = mvnw_cmd + ["package", "exec:java", "-Dexec.mainClass=it.pkg.GraalPy"]
277+
out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir)
278+
util.check_ouput("BUILD SUCCESS", out)
279+
util.check_ouput("hello java", out)
280+
281+
228282
@unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
229283
def test_fail_without_graalpy_dep(self):
230284
with tempfile.TemporaryDirectory() as tmpdir:

‎graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<properties>
1313
<graalpy.version>24.2.0</graalpy.version>
1414
<graalpy.edition>python-community</graalpy.edition>
15+
<native-maven-plugin.version>0.9.28</native-maven-plugin.version>
1516
<maven.compiler.target>17</maven.compiler.target>
1617
<maven.compiler.source>17</maven.compiler.source>
1718
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -52,12 +53,12 @@
5253
</packages>
5354
<pythonHome>
5455
<includes>
55-
<!-- java-like regular expression what file paths should be included into venv/home;
56+
<!-- java-like regular expression what file paths should be included;
5657
if empty than default is all -->
5758
<include>.*</include>
5859
</includes>
5960
<excludes>
60-
<!-- java-like regular expression what file paths should be excluded from venv/home;
61+
<!-- java-like regular expression what file paths should be excluded;
6162
default is none -->
6263
<!-- <exclude></exclude> -->
6364
</excludes>
@@ -80,7 +81,7 @@
8081
<plugin>
8182
<groupId>org.graalvm.buildtools</groupId>
8283
<artifactId>native-maven-plugin</artifactId>
83-
<version>0.9.27</version>
84+
<version>${symbol_dollar}{native-maven-plugin.version}</version>
8485
<extensions>true</extensions>
8586
<executions>
8687
<execution>

‎graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/src/main/java/GraalPy.java

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,25 +46,20 @@
4646
import org.graalvm.polyglot.Source;
4747
import org.graalvm.polyglot.Value;
4848
import java.io.IOException;
49-
import org.graalvm.python.embedding.vfs.VirtualFileSystem;
49+
import org.graalvm.python.embedding.utils.GraalPyResources;
5050

5151
public class GraalPy {
5252
private static final String PYTHON = "python";
5353

54-
public static Context getContext() {
55-
return VirtualFileSystem.contextBuilder().build();
56-
}
57-
5854
public static void main(String[] args) {
59-
try (Context context = getContext()) {
55+
try (Context context = GraalPyResources.createContext()) {
6056
Source source;
6157
try {
6258
source = Source.newBuilder(PYTHON, "import hello", "<internal>").internal(true).build();
6359
} catch (IOException e) {
6460
throw new RuntimeException(e);
6561
}
66-
67-
// eval the snipet __graalpython__.run_path() which executes what the option python.InputFilePath points to
62+
6863
context.eval(source);
6964

7065
// retrieve the python PyHello class

‎graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/src/main/resources/org.graalvm.python.vfs/proj/hello.py renamed to ‎graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/src/main/resources/org.graalvm.python.vfs/src/hello.py

File renamed without changes.

0 commit comments

Comments
 (0)