Skip to content

Commit 39f227e

Browse files
tomasstupkatomasstupka
committed
added functionality to extract VFS resources into an external directory
1 parent e78fe8d commit 39f227e

File tree

3 files changed

+113
-29
lines changed

3 files changed

+113
-29
lines changed

‎graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/VirtualFileSystemTest.java

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,11 @@
4545
import static org.junit.Assert.assertEquals;
4646
import static org.junit.Assert.assertTrue;
4747

48+
import java.io.BufferedReader;
4849
import java.io.File;
4950
import java.io.IOException;
51+
import java.io.InputStream;
52+
import java.io.InputStreamReader;
5053
import java.nio.file.Files;
5154
import java.nio.file.Path;
5255
import java.util.List;
@@ -450,7 +453,7 @@ public Context getContext(Function<VirtualFileSystemBuilder, VirtualFileSystemBu
450453
}
451454

452455
@Test
453-
public void builderTest() {
456+
public void vfsBuilderTest() {
454457
Context context = GraalPyResources.contextBuilder().allowAllAccess(true).allowHostAccess(HostAccess.ALL).build();
455458
context.eval(PYTHON, "import java; java.type('java.lang.String')");
456459

@@ -484,4 +487,36 @@ public void builderTest() {
484487
}
485488
assert gotPE : "expected PolyglotException";
486489
}
490+
491+
@Test
492+
public void externalResourcesBuilderTest() throws IOException {
493+
FileSystem fs = GraalPyResources.virtualFileSystemBuilder().resourceLoadingClass(VirtualFileSystemTest.class).build();
494+
Path resourcesDir = Files.createTempDirectory("vfs-test-resources");
495+
496+
// extract VFS
497+
GraalPyResources.extractVirtualFileSystemResources(fs, resourcesDir);
498+
499+
// check extracted contents
500+
InputStream stream = VirtualFileSystemTest.class.getResourceAsStream("/org.graalvm.python.vfs/fileslist.txt");
501+
BufferedReader br = new BufferedReader(new InputStreamReader(stream));
502+
String line;
503+
while ((line = br.readLine()) != null) {
504+
line = line.substring("/org.graalvm.python.vfs/".length());
505+
if (line.length() == 0) {
506+
continue;
507+
}
508+
Path extractedFile = resourcesDir.resolve(line);
509+
assert Files.exists(extractedFile);
510+
if (line.endsWith("/")) {
511+
assert Files.isDirectory(extractedFile);
512+
}
513+
}
514+
checkExtractedFile(resourcesDir.resolve(Path.of("file1")), new String[]{"text1", "text2"});
515+
516+
// create context with extracted resource dir and check if we can see the extracted file
517+
try (Context context = GraalPyResources.contextBuilder(resourcesDir).build()) {
518+
context.eval("python", "import os; assert os.path.exists('" + resourcesDir.resolve("file1").toString().replace("\\", "\\\\") + "')");
519+
}
520+
}
521+
487522
}

‎graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/utils/GraalPyResources.java

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151

5252
import java.io.File;
5353
import java.io.IOException;
54+
import java.nio.file.Files;
5455
import java.nio.file.Path;
5556
import java.nio.file.Paths;
5657
import java.util.function.Predicate;
@@ -125,7 +126,7 @@
125126
* directory:
126127
*
127128
* <pre>
128-
* try (Context context = GraalPyResources.contextBuilder(Path.of("python")).build()) {
129+
* try (Context context = GraalPyResources.contextBuilder(Path.of("python-resources")).build()) {
129130
* context.eval("python", "import mymodule; mymodule.print_hello_world()");
130131
* } catch (PolyglotException e) {
131132
* if (e.isExit()) {
@@ -205,9 +206,9 @@ public static Context.Builder contextBuilder(Path resourcesPath) {
205206
allowIO(IOAccess.ALL).
206207
// The sys.executable path, a virtual path that is used by the interpreter
207208
// to discover packages
208-
option("python.Executable", execPath).//
209+
option("python.Executable", execPath).
209210
// Set the python home to be read from the embedded resources
210-
option("python.PythonHome", homePath).//
211+
option("python.PythonHome", homePath).
211212
// Set python path to point to sources stored in
212213
// src/main/resources/org.graalvm.python.vfs/src
213214
option("python.PythonPath", srcPath).
@@ -379,14 +380,15 @@ public static String getMountPoint(FileSystem fs) {
379380
* located next to a native image executable.
380381
*
381382
* <pre>
382-
* Path resourcesDir = GraalPyResources.getNativeExecutablePath().getParent().resolve("python");
383+
* Path resourcesDir = GraalPyResources.getNativeExecutablePath().getParent().resolve("python-resources");
383384
* try (Context context = GraalPyResources.contextBuilder(resourcesDir).build()) {
384385
* context.eval("python", "print('hello world')");
385386
* }
386387
* </pre>
387388
* </p>
388389
*
389390
* @return the native executable path if it could be retrieved, otherwise <code>null</code>.
391+
* @see #contextBuilder(Path)
390392
*/
391393
public static Path getNativeExecutablePath() {
392394
if (ImageInfo.inImageRuntimeCode()) {
@@ -402,4 +404,31 @@ public static Path getNativeExecutablePath() {
402404
}
403405
return null;
404406
}
407+
408+
/**
409+
* Extract the contents of the given virtual filesystem into a directory. This can be useful to
410+
* manage and ship resources with the Maven workflow, but use them (cached) from the real
411+
* filesystem for better compatibility.
412+
* <p>
413+
* <b>Example</b>
414+
*
415+
* <pre>
416+
* Path resourcesDir = Path.of(System.getProperty("user.home"), ".cache", "my.java.python.app.resources");
417+
* FileSystem fs = GraalPyResources.createVirtualFileSystem();
418+
* GraalPyResources.extractVirtualFileSystemResources(fs, resourcesDir);
419+
* try (Context context = GraalPyResources.contextBuilder(resourcesDir).build()) {
420+
* context.eval("python", "print('hello world')");
421+
* }
422+
* </pre>
423+
* </p>
424+
*
425+
* @see #getNativeExecutablePath()
426+
*/
427+
public static void extractVirtualFileSystemResources(FileSystem fs, Path destDir) throws IOException {
428+
assert fs instanceof VirtualFileSystemImpl : "can extract resources only from filessytems created with VirtualFileSystemBuilder";
429+
if (Files.exists(destDir) && !Files.isDirectory(destDir)) {
430+
throw new IOException(String.format("%s has to be a directory", destDir.toString()));
431+
}
432+
((VirtualFileSystemImpl) fs).extractResources(destDir);
433+
}
405434
}

‎graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/utils/VirtualFileSystemImpl.java

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -78,17 +78,27 @@
7878

7979
final class VirtualFileSystemImpl implements FileSystem, AutoCloseable {
8080

81-
static final String VFS_ROOT = "org.graalvm.python.vfs";
82-
static final String VFS_FILESLIST = "fileslist.txt";
81+
/*
82+
* Root of the virtual filesystem in the resources.
83+
*/
84+
static final String VFS_ROOT = "/org.graalvm.python.vfs";
85+
8386
static final String VFS_HOME = "home";
8487
static final String VFS_VENV = "venv";
8588
static final String VFS_PROJ = "proj";
8689
static final String VFS_SRC = "src";
8790

88-
private static final String VENV_PREFIX = "/" + VFS_ROOT + "/" + VFS_VENV;
89-
private static final String HOME_PREFIX = "/" + VFS_ROOT + "/" + VFS_HOME;
90-
private static final String PROJ_PREFIX = "/" + VFS_ROOT + "/" + VFS_PROJ;
91-
private static final String SRC_PREFIX = "/" + VFS_ROOT + "/" + VFS_SRC;
91+
/*
92+
* Index of all files and directories available in the resources at runtime. - paths are
93+
* absolute - directory paths end with a '/' - uses '/' separator regardless of platform. Used
94+
* to determine directory entries, if an entry is a file or a directory, etc.
95+
*/
96+
private static final String FILES_LIST_PATH = VFS_ROOT + "/fileslist.txt";
97+
98+
private static final String VENV_PREFIX = VFS_ROOT + "/" + VFS_VENV;
99+
private static final String HOME_PREFIX = VFS_ROOT + "/" + VFS_HOME;
100+
private static final String PROJ_PREFIX = VFS_ROOT + "/" + VFS_PROJ;
101+
private static final String SRC_PREFIX = VFS_ROOT + "/" + VFS_SRC;
92102
private boolean extractOnStartup = "true".equals(System.getProperty("graalpy.vfs.extractOnStartup"));
93103

94104
public static enum HostIO {
@@ -97,18 +107,6 @@ public static enum HostIO {
97107
READ_WRITE,
98108
}
99109

100-
/*
101-
* Root of the virtual filesystem in the resources.
102-
*/
103-
private final String vfsPrefix;
104-
105-
/*
106-
* Index of all files and directories available in the resources at runtime. - paths are
107-
* absolute - directory paths end with a '/' - uses '/' separator regardless of platform. Used
108-
* to determine directory entries, if an entry is a file or a directory, etc.
109-
*/
110-
private final String filesListPath;
111-
112110
/*
113111
* Maps platform-specific paths to entries.
114112
*/
@@ -241,8 +239,6 @@ private void removeExtractDir() {
241239
} else {
242240
this.resourceLoadingClass = VirtualFileSystemImpl.class;
243241
}
244-
this.vfsPrefix = "/" + VirtualFileSystemImpl.VFS_ROOT;
245-
this.filesListPath = vfsPrefix + "/" + VirtualFileSystemImpl.VFS_FILESLIST;
246242

247243
this.caseInsensitive = caseInsensitive;
248244
String mp = System.getenv("GRAALPY_VFS_MOUNT_POINT");
@@ -302,8 +298,8 @@ String vfsVenvPath() {
302298
}
303299

304300
private String resourcePathToPlatformPath(String inputPath) {
305-
assert inputPath.length() > vfsPrefix.length() && inputPath.startsWith(vfsPrefix) : "inputPath expected to start with '" + vfsPrefix + "' but was '" + inputPath + "'";
306-
var path = inputPath.substring(vfsPrefix.length() + 1);
301+
assert inputPath.length() > VFS_ROOT.length() && inputPath.startsWith(VFS_ROOT) : "inputPath expected to start with '" + VFS_ROOT + "' but was '" + inputPath + "'";
302+
var path = inputPath.substring(VFS_ROOT.length() + 1);
307303
if (!PLATFORM_SEPARATOR.equals(RESOURCE_SEPARATOR)) {
308304
path = path.replace(RESOURCE_SEPARATOR, PLATFORM_SEPARATOR);
309305
}
@@ -327,7 +323,7 @@ private String platformPathToResourcePath(String inputPath) {
327323
if (path.endsWith(RESOURCE_SEPARATOR)) {
328324
path = path.substring(0, path.length() - RESOURCE_SEPARATOR.length());
329325
}
330-
path = vfsPrefix + path;
326+
path = VFS_ROOT + path;
331327
return path;
332328
}
333329

@@ -337,7 +333,7 @@ private String toCaseComparable(String file) {
337333

338334
private void initEntries() throws IOException {
339335
vfsEntries = new HashMap<>();
340-
try (InputStream stream = this.resourceLoadingClass.getResourceAsStream(filesListPath)) {
336+
try (InputStream stream = this.resourceLoadingClass.getResourceAsStream(FILES_LIST_PATH)) {
341337
if (stream == null) {
342338
return;
343339
}
@@ -506,6 +502,30 @@ private void extract(Path path) throws IOException {
506502
}
507503
}
508504

505+
public void extractResources(Path destRootDir) throws IOException {
506+
InputStream stream = this.resourceLoadingClass.getResourceAsStream(FILES_LIST_PATH);
507+
if (stream == null) {
508+
return;
509+
}
510+
BufferedReader br = new BufferedReader(new InputStreamReader(stream));
511+
String resourcePath;
512+
while ((resourcePath = br.readLine()) != null) {
513+
Path destFile = destRootDir.resolve(Path.of(resourcePath.substring(VFS_ROOT.length() + 1)));
514+
if (destFile == null) {
515+
continue;
516+
}
517+
if (resourcePath.endsWith(RESOURCE_SEPARATOR)) {
518+
Files.createDirectories(destFile);
519+
} else {
520+
Path parent = destFile.getParent();
521+
if (parent != null) {
522+
Files.createDirectories(parent);
523+
}
524+
Files.write(destFile, readResource(resourcePath));
525+
}
526+
}
527+
}
528+
509529
@Override
510530
public Path parsePath(URI uri) {
511531
if (uri.getScheme().equals("file")) {

0 commit comments

Comments
 (0)