minor edits and cleanups for running standalone tests

timfel · timfel · commit aba5cddd8297 · 2023-09-07T22:53:44.000+02:00
(cherry picked from commit 999b6b3)
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_standalone.py b/graalpython/com.oracle.graal.python.test/src/tests/test_standalone.py
@@ -48,6 +48,14 @@
 if 'MAVEN_REPO_OVERRIDE' in os.environ:
     MVN_CMD += ['-Dmaven.repo.remote=' + os.environ['MAVEN_REPO_OVERRIDE']]
 
+def run_cmd(cmd, env, cwd=None):
+    process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True)
+    out = []
+    for line in iter(process.stdout.readline, ""):
+        print(line, end="")
+        out.append(line)
+    return "".join(out)
+
 def get_executable(file):
     if os.path.isfile(file):
         return file
@@ -99,24 +107,15 @@ def test_polyglot_app():
         target_dir = os.path.join(tmpdir, "polyglot_app_test")
 
         cmd = [graalpy, "-m", "standalone", "--verbose", "polyglot_app", "-o", target_dir]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(p.stdout.decode(errors='backslashreplace'))
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "Creating polyglot java python application in directory " + target_dir in out
 
         cmd = MVN_CMD + ["package", "-Pnative"]
-        p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env, cwd=target_dir)
         assert "BUILD SUCCESS" in out
 
         cmd = [os.path.join(target_dir, "target", "polyglot_app")]
-        p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env, cwd=target_dir)
         assert out.endswith("hello java\n")
 
 
@@ -137,21 +136,15 @@ def test_native_executable_one_file():
         target_file = os.path.join(tmpdir, "hello")
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-m", source_file, "-o", target_file]
 
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out
 
         cmd = [target_file, "arg1", "arg2"]
-        p = subprocess.run(" ".join(cmd), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "hello world, argv[1:]: " + str(cmd[1:]) in out
 
 @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true")
-def test_native_executable_one_file_venv():
+def test_native_executable_venv_and_one_file():
     graalpy = get_gp()
     if graalpy is None:
         return
@@ -166,31 +159,19 @@ def test_native_executable_one_file_venv():
 
         venv_dir = os.path.join(target_dir, "venv")
         cmd = [graalpy, "-m", "venv", venv_dir]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
 
         venv_python = os.path.join(venv_dir, "Scripts", "python.cmd") if os.name == "nt" else os.path.join(venv_dir, "bin", "python")
         cmd = [venv_python, "-m", "pip", "--no-cache-dir", "install", "termcolor"]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
 
         target_file = os.path.join(target_dir, "hello")
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", source_file, "--venv", venv_dir, "-o", target_file]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out
 
         cmd = [target_file]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
 
         assert "hello standalone world" in out
 
@@ -219,15 +200,9 @@ def test_native_executable_module():
         target_file = os.path.join(tmp_dir, "hello")
         cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", module_dir, "-o", target_file]
 
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out
 
         cmd = [target_file]
-        p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = p.stdout.decode(errors='backslashreplace')
-        print(out)
-        print(p.stderr.decode(errors='backslashreplace'))
+        out = run_cmd(cmd, env)
         assert "hello standalone world" in out
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java
@@ -1075,25 +1075,6 @@ abstract static class GetGraalVmVersion extends PythonBuiltinNode {
         @Specialization
         TruffleString get() {
             Version current = Version.getCurrent();
-            // TODO according to Version java doc, the output of toString is not standardized and may change without notice.
-            // Unfortunately, we have no way to tell what is the minimum count of version components and the snapshot suffix
-            // and have to guess is. For now, we can either do the same as version.toString does, or simply call it.
-//            String snapshotSuffix = "-dev";
-//            int min_version_components = 3;
-//
-//            int i = 1;
-//            String format = "";
-//            while (true) {
-//                if (current.format("%[" + i + "XX]").isEmpty() && i > min_version_components) {
-//                    break;
-//                }
-//                format += (format.isEmpty() ? "%d" : ".%d");
-//                i++;
-//            }
-//            String ret = current.format(format);
-//            if(current.isSnapshot()) {
-//                ret += snapshotSuffix;
-//            }
             return TruffleString.fromJavaStringUncached(current.toString(), TS_ENCODING);
         }
     }
diff --git a/graalpython/lib-graalpython/modules/standalone/__main__.py b/graalpython/lib-graalpython/modules/standalone/__main__.py
@@ -449,11 +449,10 @@ def download_python(self):
         if self.parsed_args.verbose:
             print(f"downloading graalpython maven artefacts: {' '.join(cmd)}")
 
-        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
         if self.parsed_args.verbose:
-            print(p.stdout.decode())
-            print(p.stderr.decode())
+            p = subprocess.run(cmd)
+        else:
+            p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
         if p.returncode != 0:
             if not self.parsed_args.verbose:
@@ -467,9 +466,7 @@ def build_binary(self, ni, jc):
         os.chdir(self.target_dir)
 
         try:
-            modulepath = f"{self.modules_path}/org.graalvm.polyglot-polyglot-{self.graalvm_version}.jar"
-            # it would seem it is enough to compile the launcher file, but on some linux setups it isn't
-            cmd = [jc, "--module-path", modulepath, os.path.join(self.target_dir,MODULE_INFO_FILE), os.path.join(self.target_dir, MODULE_NAME, "VirtualFileSystem.java"), self.launcher_file]
+            cmd = [jc, "--module-path", self.modules_path, os.path.join(self.target_dir,MODULE_INFO_FILE), os.path.join(self.target_dir, MODULE_NAME, "VirtualFileSystem.java"), self.launcher_file]
             if self.parsed_args.verbose:
                 print(f"Compiling code for Python standalone entry point: {' '.join(cmd)}")
             p = subprocess.run(cmd, cwd=self.target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -478,15 +475,8 @@ def build_binary(self, ni, jc):
                 print(p.stderr.decode())
                 exit(1)
 
-            modules = []
-            for f in os.listdir(self.modules_path):
-                if f.endswith("jar"):
-                    modules.append(f)
-            mp = f":{self.modules_path}/".join(modules)
-            modulepath = f"{self.modules_path}/{mp}:."
-
-            add_modules = "standalone"
-            cmd = [ni, "--module-path", modulepath, "--add-modules", add_modules] + self.parsed_args.ni_args[:]
+            ni_modules = ":".join([os.path.join(self.modules_path, f) for f in os.listdir(self.modules_path) if f.endswith(".jar")] + [self.target_dir])
+            cmd = [ni, "--module-path", ni_modules] + self.parsed_args.ni_args[:]
 
             if self.parsed_args.Os:
                 cmd +=[
@@ -496,7 +486,7 @@ def build_binary(self, ni, jc):
             cmd += [
                 "--no-fallback",
                 "-H:-CopyLanguageResources",
-                "-H:ResourceConfigurationFiles=native-image-resources.json",
+                f"-H:ResourceConfigurationFiles={self.target_dir}/native-image-resources.json",
                 "-o",
                 output,
                 f"{MODULE_NAME}.{NATIVE_EXEC_LAUNCHER}",
diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py
@@ -31,6 +31,7 @@
 import itertools
 import json
 import os
+import pathlib
 import re
 import shlex
 import shutil
@@ -1503,25 +1504,53 @@ def graalpython_gate_runner(args, tasks):
 
     with Task('GraalPython standalone module tests', tasks, tags=[GraalPythonTags.unittest_standalone]) as task:
         if task:
+            env = {}
+            env['ENABLE_STANDALONE_UNITTESTS'] = 'true'
+            env['MAVEN_REPO_OVERRIDE'] = mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/')
             # build graalvm jdk
             mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', 'vm'), '--env', 'ce']
-            mx.run_mx(mx_args + ["build", "--dep", f"GRAALVM_COMMUNITY_JAVA{get_jdk().javaCompliance}"])
+            if not DISABLE_REBUILD:
+                mx.run_mx(mx_args + ["build", "--dep", f"GRAALVM_COMMUNITY_JAVA{get_jdk().javaCompliance}"])
             out = mx.OutputCapture()
             mx.run_mx(mx_args + ["graalvm-home"], out=out)
             home = out.data.splitlines()[-1].strip()
-            env = os.environ.copy()
-            env['ENABLE_STANDALONE_UNITTESTS'] = 'true'
-            env['MAVEN_REPO_OVERRIDE'] = mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/')
             env['JAVA_HOME'] = home
             # build python standalone
             mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', 'vm'), '--env', 'ce-python']
-            mx.run_mx(mx_args + ["--native-images=", "build", "--dep", f"PYTHON_JAVA_STANDALONE_SVM_JAVA{get_jdk().javaCompliance}"])
+            if not DISABLE_REBUILD:
+                mx.run_mx(mx_args + ["build", "--dep", f"PYTHON_JAVA_STANDALONE_SVM_JAVA{get_jdk().javaCompliance}"])
             out = mx.OutputCapture()
             mx.run_mx(mx_args + ["standalone-home", "--type", "jvm", "python"], out=out)
             python_home = out.data.splitlines()[-1].strip()
             env['PYTHON_STANDALONE_HOME'] = python_home
+            # deploy maven artifacts
+            import mx_sdk_vm_impl
+            version = mx_sdk_vm_impl.graalvm_version('graalvm')
+            path = os.path.join(SUITE.get_mx_output_dir(), 'public-maven-repo')
+            licenses = ['EPL-2.0', 'PSF-License', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-simplified', 'BSD-new', 'UPL', 'MIT']
+            deploy_args = [
+                '--tags=public',
+                '--all-suites',
+                '--all-distribution-types',
+                f'--version-string={version}',
+                '--validate=none',
+                '--licenses', ','.join(licenses),
+                '--suppress-javadoc',
+                'local',
+                pathlib.Path(path).as_uri(),
+            ]
+            if not DISABLE_REBUILD:
+                mx.rmtree(path, ignore_errors=True)
+                os.mkdir(path)
+                mx.maven_deploy(deploy_args)
+            # setup maven downloader overrides
+            env["org.graalvm.maven.downloader.version"] = version
+            env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(path).as_uri()}/"
             # run the test
-            run_python_unittests(python_gvm(), paths=["test_standalone.py"], javaAsserts=True, report=report(), env=env)
+            mx.logv(f"running with {env=}")
+            full_env = os.environ.copy()
+            full_env.update(env)
+            mx.run([sys.executable, _graalpytest_driver(), "-v", "graalpython/com.oracle.graal.python.test/src/tests/test_standalone.py"], env=full_env)
 
     with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.tagged]) as task:
         if task:
