[WIP] [GR-52202] Hardcode the path to the vfs.

PullRequest: graalpython/3252

Author: tomasstupka

graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/vfs/SomeFile renamed to graalpython/com.oracle.graal.python.test.integration/src/org.graalvm.python.vfs/SomeFile

@@ -0,0 +1,8 @@
+/org.graalvm.python.vfs/
+/org.graalvm.python.vfs/dir1/
+/org.graalvm.python.vfs/dir1/dir2/
+/org.graalvm.python.vfs/dir1/file2
+/org.graalvm.python.vfs/emptydir/
+/org.graalvm.python.vfs/file1
+/org.graalvm.python.vfs/fileslist.txt
+/org.graalvm.python.vfs/SomeFile

graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/vfs/dir1/file2 renamed to graalpython/com.oracle.graal.python.test.integration/src/org.graalvm.python.vfs/dir1/file2

@@ -61,8 +61,6 @@
 
 public class VirtualFileSystemTest {
 
-    private static final String VFS_PREFIX = "/org/graalvm/python/embedding/utils/test/vfs";
-
     private static final String VFS_UNIX_MOUNT_POINT = "/test_mount_point";
     private static final String VFS_WIN_MOUNT_POINT = "X:\\test_win_mount_point";
     private static final String VFS_MOUNT_POINT = IS_WINDOWS ? VFS_WIN_MOUNT_POINT : VFS_UNIX_MOUNT_POINT;
@@ -81,7 +79,6 @@ public void defaultValues() throws Exception {
     @Test
     public void toRealPath() throws Exception {
         VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
-                        vfsPrefix(VFS_PREFIX).//
                         unixMountPoint(VFS_MOUNT_POINT).//
                         windowsMountPoint(VFS_WIN_MOUNT_POINT).//
                         extractFilter(p -> p.getFileName().toString().equals("file1")).//
@@ -97,7 +94,6 @@ public void toRealPath() throws Exception {
     @Test
     public void toAbsolutePath() throws Exception {
         VirtualFileSystem vfs = VirtualFileSystem.newBuilder().//
-                        vfsPrefix(VFS_PREFIX).//
                         unixMountPoint(VFS_MOUNT_POINT).//
                         windowsMountPoint(VFS_WIN_MOUNT_POINT).//
                         extractFilter(p -> p.getFileName().toString().equals("file1")).//
@@ -384,7 +380,7 @@ public Context getContext(Function<VirtualFileSystem.Builder, VirtualFileSystem.
             return cachedContext;
         }
         VirtualFileSystem.Builder builder = VirtualFileSystem.newBuilder().//
-                        vfsPrefix(VFS_PREFIX).//
+// vfsPrefix(VFS_PREFIX).//
                         unixMountPoint(VFS_MOUNT_POINT).//
                         windowsMountPoint(VFS_WIN_MOUNT_POINT).//
                         resourceLoadingClass(VirtualFileSystemTest.class);

graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/vfs/file1 renamed to graalpython/com.oracle.graal.python.test.integration/src/org.graalvm.python.vfs/file1

@@ -196,20 +196,20 @@ def test_graal_version(self):
             script_path = os.path.normpath(os.path.join(os.path.dirname(self.catalog_file), script_ref))
             with open(script_path, 'r') as script_file:
                 content = script_file.read()
-            self.assertIn (f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION} was not foudn in {script_path}")
-            self.assertIn (f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION} was not foudn in {script_path}")
-            self.assertIn (f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION} was not foudn in {script_path}")
-            self.assertIn (f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION} was not foudn in {script_path}")
+            self.assertIn (f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION} was not found in {script_path}")
+            self.assertIn (f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION} was not found in {script_path}")
+            self.assertIn (f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION} was not found in {script_path}")
+            self.assertIn (f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION} was not found in {script_path}")
 
         for template in json_data.get("templates", {}).values():
             for file_ref in template.get("file-refs", {}).values():
                 file_path = os.path.normpath(os.path.join(os.path.dirname(self.catalog_file), file_ref))
                 with open(script_path, 'r') as script_file:
                     content = script_file.read()
-                self.assertIn (f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION} was not foudn in {script_path}")
-                self.assertIn (f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION} was not foudn in {script_path}")
-                self.assertIn (f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION} was not foudn in {script_path}")
-                self.assertIn (f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION} was not foudn in {script_path}")
+                self.assertIn (f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-language:{GRAAL_VERSION} was not found in {script_path}")
+                self.assertIn (f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-resources:{GRAAL_VERSION} was not found in {script_path}")
+                self.assertIn (f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-launcher:{GRAAL_VERSION} was not found in {script_path}")
+                self.assertIn (f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION}", content, f"//DEPS org.graalvm.python:python-embedding:{GRAAL_VERSION} was not found in {script_path}")
 
     @unittest.skipUnless(is_enabled, "ENABLE_JBANG_INTEGRATION_UNITTESTS is not true")
     def test_graalpy_template(self):

graalpython/com.oracle.graal.python.test.integration/src/org.graalvm.python.vfs/fileslist.txt

@@ -49,13 +49,17 @@
 
 MVN_CMD = [shutil.which('mvn'), "--batch-mode"]
 
+VFS_PREFIX = "org.graalvm.python.vfs"
+
 def run_cmd(cmd, env, cwd=None):
     print(f"Executing:\n    {cmd=}\n")
     process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True, errors='backslashreplace')
     out = []
+    print("============== output =============")
     for line in iter(process.stdout.readline, ""):
         print(line, end="")
         out.append(line)
+    print("\n========== end of output ==========")
     return "".join(out), process.wait()
 
 def get_executable(file):
@@ -259,28 +263,28 @@ def test_generated_app(self):
             self.generate_app(tmpdir, target_dir, target_name)
 
             # build
-            cmd = MVN_CMD + ["package", "-Pnative", "-DmainClass=it.pkg.GraalPy"] #, f"-Dgraalpy.version={self.graalvmVersion}", "-Dgraalpy.edition=python-community"]
+            cmd = MVN_CMD + ["package", "-Pnative", "-DmainClass=it.pkg.GraalPy"]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "BUILD SUCCESS" in out
+            assert "BUILD SUCCESS" in out, "unexpected output from " + str(cmd)
 
             # check fileslist.txt
-            fl_path = os.path.join(target_dir, "target", "classes", "vfs", "fileslist.txt")
+            fl_path = os.path.join(target_dir, "target", "classes", VFS_PREFIX, "fileslist.txt")
             with open(fl_path) as f:
                 lines = f.readlines()
-            assert "/vfs/\n" in lines
-            assert "/vfs/home/\n" in lines
-            assert "/vfs/home/lib-graalpython/\n" in lines
-            assert "/vfs/home/lib-python/\n" in lines
+            assert "/" + VFS_PREFIX + "/\n" in lines, "unexpected output from " + str(cmd)
+            assert "/" + VFS_PREFIX + "/home/\n" in lines, "unexpected output from " + str(cmd)
+            assert "/" + VFS_PREFIX + "/home/lib-graalpython/\n" in lines, "unexpected output from " + str(cmd)
+            assert "/" + VFS_PREFIX + "/home/lib-python/\n" in lines, "unexpected output from " + str(cmd)
 
             # execute and check native image
             cmd = [os.path.join(target_dir, "target", target_name)]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "hello java" in out
+            assert "hello java" in out, "unexpected output from " + str(cmd)
 
             # execute with java and check
             cmd = MVN_CMD + ["exec:java", "-Dexec.mainClass=it.pkg.GraalPy"]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "hello java" in out
+            assert "hello java" in out, "unexpected output from " + str(cmd)
 
             #GR-51132 - NoClassDefFoundError when running polyglot app in java mode
             assert "java.lang.NoClassDefFoundError" not in out
@@ -307,18 +311,18 @@ def test_gen_launcher_and_venv(self):
 
             cmd = MVN_CMD + ["process-resources"]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "-m venv" in out
-            assert "-m ensurepip" in out
-            assert "ujson" in out
-            assert "termcolor" in out
+            assert "-m venv" in out, "unexpected output from " + str(cmd)
+            assert "-m ensurepip" in out, "unexpected output from " + str(cmd)
+            assert "ujson" in out, "unexpected output from " + str(cmd)
+            assert "termcolor" in out, "unexpected output from " + str(cmd)
 
             # run again and assert that we do not regenerate the venv
             cmd = MVN_CMD + ["process-resources"]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "-m venv" not in out
-            assert "-m ensurepip" not in out
-            assert "ujson" not in out
-            assert "termcolor" not in out
+            assert "-m venv" not in out, "unexpected output from " + str(cmd)
+            assert "-m ensurepip" not in out, "unexpected output from " + str(cmd)
+            assert "ujson" not in out, "unexpected output from " + str(cmd)
+            assert "termcolor" not in out, "unexpected output from " + str(cmd)
 
             # remove ujson pkg from plugin config and check if unistalled
             with open(os.path.join(target_dir, "pom.xml"), "r") as f:
@@ -329,10 +333,10 @@ def test_gen_launcher_and_venv(self):
 
             cmd = MVN_CMD + ["process-resources"]
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
-            assert "-m venv" not in out
-            assert "-m ensurepip" not in out
-            assert "Uninstalling ujson" in out
-            assert "termcolor" not in out
+            assert "-m venv" not in out, "unexpected output from " + str(cmd)
+            assert "-m ensurepip" not in out, "unexpected output from " + str(cmd)
+            assert "Uninstalling ujson" in out, "unexpected output from " + str(cmd)
+            assert "termcolor" not in out, "unexpected output from " + str(cmd)
 
     @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
     def test_check_home(self):
@@ -346,13 +350,13 @@ def test_check_home(self):
             out, return_code = run_cmd(cmd, self.env, cwd=target_dir)
 
             # check fileslist.txt
-            fl_path = os.path.join(target_dir, "target", "classes", "vfs", "fileslist.txt")
+            fl_path = os.path.join(target_dir, "target", "classes", VFS_PREFIX, "fileslist.txt")
             with open(fl_path) as f:
                 for line in f:
                     line = f.readline()
                     # string \n
                     line = line[:len(line)-1]
-                    if line.endswith("/") or line == "/vfs/home/tagfile" or line == "/vfs/proj/hello.py":
+                    if line.endswith("/") or line == "/" + VFS_PREFIX + "/home/tagfile" or line == "/" + VFS_PREFIX + "/proj/hello.py":
                         continue
                     assert line.endswith("/__init__.py")
                     assert not line.endswith("html/__init__.py")
@@ -376,11 +380,11 @@ def test_native_executable_one_file():
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-m", source_file, "-o", target_file]
 
         out, return_code = run_cmd(cmd, env)
-        assert "Bundling Python resources into" in out
+        assert "Bundling Python resources into" in out, "unexpected output from " + str(cmd)
 
         cmd = [target_file, "arg1", "arg2"]
         out, return_code = run_cmd(cmd, env)
-        assert "hello world, argv[1:]: " + str(cmd[1:]) in out
+        assert "hello world, argv[1:]: " + str(cmd[1:]) in out, "unexpected output from " + str(cmd)
 
 @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
 def test_native_executable_venv_and_one_file():
@@ -411,13 +415,12 @@ def test_native_executable_venv_and_one_file():
         target_file = os.path.join(target_dir, "hello")
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-Os", "-m", source_file, "--venv", venv_dir, "-o", target_file]
         out, return_code = run_cmd(cmd, env)
-        assert "Bundling Python resources into" in out
+        assert "Bundling Python resources into" in out, "unexpected output from " + str(cmd)
 
         cmd = [target_file]
         out, return_code = run_cmd(cmd, env)
-
-        assert "hello standalone world" in out
-        assert "key=value" in out
+        assert "hello standalone world" in out, "unexpected output from " + str(cmd)
+        assert "key=value" in out, "unexpected output from " + str(cmd)
 
 @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
 def test_native_executable_module():
@@ -444,13 +447,12 @@ def test_native_executable_module():
 
         target_file = os.path.join(tmp_dir, "hello")
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-Os", "-m", module_dir, "-o", target_file]
-
         out, return_code = run_cmd(cmd, env)
-        assert "Bundling Python resources into" in out
+        assert "Bundling Python resources into" in out, "unexpected output from " + str(cmd)
 
         cmd = [target_file]
         out, return_code = run_cmd(cmd, env)
-        assert "hello standalone world" in out
+        assert "hello standalone world" in out, "unexpected output from " + str(cmd)
 
 
 unittest.skip_deselected_test_functions(globals())

graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/VirtualFileSystemTest.java

@@ -54,10 +54,6 @@
 import org.graalvm.python.embedding.vfs.VirtualFileSystem;
 
 public class GraalPy {
-    private static final String VENV_PREFIX = "/vfs/venv";
-    private static final String HOME_PREFIX = "/vfs/home";
-    private static final String PROJ_PREFIX = "/vfs/proj";
-    
     private static final String PYTHON = "python";
 
     public static Context getContext() {
@@ -102,14 +98,14 @@ public static Context getContext() {
             // Force to automatically import site.py module, to make Python packages available
             .option("python.ForceImportSite", "true")
             // The sys.executable path, a virtual path that is used by the interpreter to discover packages
-            .option("python.Executable", vfs.resourcePathToPlatformPath(VENV_PREFIX) + (VirtualFileSystem.isWindows() ? "${symbol_escape}${symbol_escape}Scripts${symbol_escape}${symbol_escape}python.exe" : "/bin/python"))
+            .option("python.Executable", vfs.vfsVenvPath() + (VirtualFileSystem.isWindows() ? "${symbol_escape}${symbol_escape}Scripts${symbol_escape}${symbol_escape}python.exe" : "/bin/python"))
             // Set the python home to be read from the embedded resources
-            .option("python.PythonHome", vfs.resourcePathToPlatformPath(HOME_PREFIX))
+            .option("python.PythonHome", vfs.vfsHomePath())
             // Do not warn if running without JIT. This can be desirable for short running scripts
             // to reduce memory footprint.
             .option("engine.WarnInterpreterOnly", "false")
-            // Set python path to point to sources stored in src/main/resources/vfs/proj
-            .option("python.PythonPath", vfs.resourcePathToPlatformPath(PROJ_PREFIX))
+            // Set python path to point to sources stored in src/main/resources/org.graalvm.python.vfs/proj
+            .option("python.PythonPath", vfs.vfsProjPath())
             .build();
         return context;
     }

graalpython/com.oracle.graal.python.test.integration/src/org/graalvm/python/embedding/utils/test/vfs/fileslist.txt

@@ -1,7 +1,7 @@
 {
   "resources": {
     "includes": [
-      {"pattern": "vfs/.*"}
+      {"pattern": "org.graalvm.python.vfs/.*"}
     ]
   }
 }