Skip to content

Commit d25e526

Browse files
tomasstupkatomasstupka
committed
[GR-44824] Provide a tool to create self-contained executables from Python programs.
PullRequest: graalpython/2833
2 parents b67781f + 321b99b commit d25e526

File tree

18 files changed

+1867
-770
lines changed

18 files changed

+1867
-770
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ mx.graalpython/eclipse-launches
2424
asv
2525
*.json
2626
!**/resources/*.json
27+
!graalpython/lib-graalpython/modules/standalone/shared/native-image-resources.json
28+
!graalpython/lib-graalpython/modules/standalone/shared/native-image-proxy-configuration.json
2729
language
2830
*.bc
2931
*.iml

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "overlay": "32dabcc2b777b4bbd934f634b3a07ad5ce4e8725" }
1+
{ "overlay": "fdf83c219692bf4a4d0637e68b7ae9399a546297" }

‎graalpython/com.oracle.graal.python.test/src/tests/test_standalone.py

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
2+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+
#
4+
# The Universal Permissive License (UPL), Version 1.0
5+
#
6+
# Subject to the condition set forth below, permission is hereby granted to any
7+
# person obtaining a copy of this software, associated documentation and/or
8+
# data (collectively the "Software"), free of charge and under any and all
9+
# copyright rights in the Software, and any and all patent rights owned or
10+
# freely licensable by each licensor hereunder covering either (i) the
11+
# unmodified Software as contributed to or provided by such licensor, or (ii)
12+
# the Larger Works (as defined below), to deal in both
13+
#
14+
# (a) the Software, and
15+
#
16+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
17+
# one is included with the Software each a "Larger Work" to which the Software
18+
# is contributed by such licensors),
19+
#
20+
# without restriction, including without limitation the rights to copy, create
21+
# derivative works of, display, perform, and distribute the Software and make,
22+
# use, sell, offer for sale, import, export, have made, and have sold the
23+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
24+
# either these or other terms.
25+
#
26+
# This license is subject to the following condition:
27+
#
28+
# The above copyright notice and either this complete permission notice or at a
29+
# minimum a reference to the UPL must be included in all copies or substantial
30+
# portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38+
# SOFTWARE.
39+
40+
import os
41+
import subprocess
42+
import tempfile
43+
import unittest
44+
45+
is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true"
46+
47+
def get_executable(file):
48+
if os.path.isfile(file):
49+
return file
50+
exe = f"{file}.exe"
51+
if os.path.isfile(exe):
52+
return exe
53+
exe = f"{file}.cmd"
54+
if os.path.isfile(exe):
55+
return exe
56+
return None
57+
58+
def get_gp():
59+
java_home = os.path.join(__graalpython__.home, "..", "..")
60+
61+
ni = get_executable(os.path.join(java_home, "bin", "native-image"))
62+
jc = get_executable(os.path.join(java_home, "bin", "javac"))
63+
graalpy = get_executable(os.path.join(java_home, "bin", "graalpy"))
64+
java = get_executable(os.path.join(java_home, "bin", "java"))
65+
66+
if not os.path.isfile(graalpy) or not os.path.isfile(java) or not os.path.isfile(jc) or not os.path.isfile(ni):
67+
print(
68+
"Standalone module tests require a GraalVM installation including graalpy, java, javac and native-image",
69+
"Please point the JAVA_HOME environment variable to such a GraalVM root.",
70+
"__graalpython__.home : " + java_home,
71+
"native-image exists: " + str(os.path.exists(ni)),
72+
"javac exists: " + str(os.path.exists(jc)),
73+
"graalpy exits: " + str(os.path.exists(graalpy)),
74+
"java exists: " + str(os.path.exists(java)),
75+
sep="\n",
76+
)
77+
assert False
78+
return java_home, graalpy, java
79+
80+
def get_env(java_home):
81+
env = os.environ.copy()
82+
env.update({"JAVA_HOME" : java_home})
83+
84+
to_be_removed = []
85+
for k in env:
86+
# subprocess complaining about key names with "=" in them
87+
if "=" in k:
88+
to_be_removed.append(k)
89+
for k in to_be_removed:
90+
del env[k]
91+
if len(to_be_removed) > 0:
92+
print("\ntest_standalone: removed keys from subprocess environment :", to_be_removed)
93+
94+
return env
95+
96+
@unittest.skipUnless(is_enabled)
97+
def test_polyglot_app():
98+
99+
java_home, graalpy, java = get_gp()
100+
env = get_env(java_home)
101+
102+
with tempfile.TemporaryDirectory() as tmpdir:
103+
104+
target_dir = os.path.join(tmpdir, "polyglot_jp_app")
105+
106+
cmd = [graalpy, "-m", "standalone", "--verbose", "polyglot_app", "-o", target_dir]
107+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
108+
out = p.stdout.decode()
109+
print(p.stdout.decode())
110+
print(p.stderr.decode())
111+
assert "Creating polyglot java python application in directory " + target_dir in out
112+
113+
cmd = ["mvn", "package", "-Pnative"]
114+
p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
115+
out = p.stdout.decode()
116+
print(out)
117+
print(p.stderr.decode())
118+
assert "BUILD SUCCESS" in out
119+
120+
cmd = [os.path.join(target_dir, "target", "java_python_app")]
121+
p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
122+
out = p.stdout.decode()
123+
print(out)
124+
print(p.stderr.decode())
125+
assert out.endswith("hello java\n")
126+
127+
cmd = ["mvn", "package", "-Pjar"]
128+
p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
129+
out = p.stdout.decode()
130+
print(out)
131+
print(p.stderr.decode())
132+
assert "BUILD SUCCESS" in out
133+
134+
cmd = [java, "-jar", os.path.join(target_dir, "target", "java_python_app-1.0-SNAPSHOT.jar")]
135+
p = subprocess.run(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
136+
out = p.stdout.decode()
137+
print(out)
138+
print(p.stderr.decode())
139+
assert out.endswith("hello java\n")
140+
141+
@unittest.skipUnless(is_enabled)
142+
def test_native_executable_one_file():
143+
java_home, graalpy, java = get_gp()
144+
if graalpy is None or java is None:
145+
return
146+
147+
env = get_env(java_home)
148+
149+
with tempfile.TemporaryDirectory() as tmpdir:
150+
151+
source_file = os.path.join(tmpdir, "hello.py")
152+
with open(source_file, 'w') as f:
153+
f.write("print('hello world')")
154+
155+
target_file = os.path.join(tmpdir, "hello")
156+
cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-m", source_file, "-o", target_file]
157+
158+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
159+
out = p.stdout.decode()
160+
print(out)
161+
print(p.stderr.decode())
162+
assert "Bundling Python resources into" in out
163+
164+
cmd = [target_file]
165+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
166+
out = p.stdout.decode()
167+
print(out)
168+
print(p.stderr.decode())
169+
assert "hello world" in out
170+
171+
@unittest.skipUnless(is_enabled)
172+
def test_native_executable_one_file_venv():
173+
java_home, graalpy, java = get_gp()
174+
if graalpy is None or java is None:
175+
return
176+
177+
env = get_env(java_home)
178+
179+
with tempfile.TemporaryDirectory() as target_dir:
180+
source_file = os.path.join(target_dir, "hello.py")
181+
with open(source_file, 'w') as f:
182+
f.write("from termcolor import colored, cprint\n")
183+
f.write("colored_text = colored('hello standalone world', 'red', attrs=['reverse', 'blink'])\n")
184+
f.write("print(colored_text)\n")
185+
186+
venv_dir = os.path.join(target_dir, "venv")
187+
cmd = [graalpy, "-m", "venv", venv_dir]
188+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
189+
out = p.stdout.decode()
190+
print(out)
191+
print(p.stderr.decode())
192+
193+
pip = os.path.join(venv_dir, "bin", "pip")
194+
cmd = [pip, "install", "termcolor"]
195+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
196+
out = p.stdout.decode()
197+
print(out)
198+
print(p.stderr.decode())
199+
200+
target_file = os.path.join(target_dir, "hello")
201+
cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", source_file, "--venv", venv_dir, "-o", target_file]
202+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
203+
out = p.stdout.decode()
204+
print(out)
205+
print(p.stderr.decode())
206+
assert "Bundling Python resources into" in out
207+
208+
cmd = [target_file]
209+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
210+
out = p.stdout.decode()
211+
print(out)
212+
print(p.stderr.decode())
213+
214+
assert "hello standalone world" in out
215+
216+
@unittest.skipUnless(is_enabled)
217+
def test_native_executable_module():
218+
java_home, graalpy, java = get_gp()
219+
if graalpy is None or java is None:
220+
return
221+
222+
env = get_env(java_home)
223+
224+
with tempfile.TemporaryDirectory() as tmp_dir:
225+
226+
module_dir = os.path.join(tmp_dir, "hello_app")
227+
os.makedirs(module_dir, exist_ok=True)
228+
229+
source_file = os.path.join(module_dir, "hello.py")
230+
with open(source_file, 'w') as f:
231+
f.write("def print_hello():\n")
232+
f.write(" print('hello standalone world')\n")
233+
234+
source_file = os.path.join(module_dir, "__main__.py")
235+
with open(source_file, 'w') as f:
236+
f.write("import hello\n")
237+
f.write("hello.print_hello()\n")
238+
239+
target_file = os.path.join(tmp_dir, "hello")
240+
cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", module_dir, "-o", target_file]
241+
242+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
243+
out = p.stdout.decode()
244+
print(out)
245+
print(p.stderr.decode())
246+
assert "Bundling Python resources into" in out
247+
248+
cmd = [target_file]
249+
p = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
250+
out = p.stdout.decode()
251+
print(out)
252+
print(p.stderr.decode())
253+
assert "hello standalone world" in out

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,21 @@
6060
import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
6161
import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
6262

63+
import java.io.BufferedWriter;
6364
import java.io.File;
65+
import java.io.FileOutputStream;
6466
import java.io.IOException;
6567
import java.io.InputStreamReader;
68+
import java.io.OutputStream;
69+
import java.io.OutputStreamWriter;
6670
import java.io.PrintWriter;
6771
import java.nio.charset.StandardCharsets;
6872
import java.nio.file.Files;
6973
import java.nio.file.Path;
7074
import java.nio.file.Paths;
75+
import java.util.ArrayList;
7176
import java.util.Arrays;
77+
import java.util.Collection;
7278
import java.util.List;
7379
import java.util.logging.Level;
7480

@@ -250,6 +256,9 @@ public void postInitialize(Python3Core core) {
250256
mod.setAttribute(tsLiteral("dump_heap"), PNone.NO_VALUE);
251257
mod.setAttribute(tsLiteral("is_native_object"), PNone.NO_VALUE);
252258
}
259+
if (!context.getOption(PythonOptions.RunViaLauncher)) {
260+
mod.setAttribute(tsLiteral("list_files"), PNone.NO_VALUE);
261+
}
253262
}
254263

255264
@TruffleBoundary
@@ -943,4 +952,61 @@ static PythonClass createType(VirtualFrame frame, TruffleString name, PTuple bas
943952
return createType.execute(frame, namespaceOrig, name, bases, metaclass, PKeyword.EMPTY_KEYWORDS);
944953
}
945954
}
955+
956+
@Builtin(name = "list_files", minNumOfPositionalArgs = 2)
957+
@GenerateNodeFactory
958+
abstract static class ListFiles extends PythonBinaryBuiltinNode {
959+
@TruffleBoundary
960+
@Specialization
961+
Object list(TruffleString dirPath, TruffleString filesListPath) {
962+
print(getContext().getStandardOut(), String.format("listing files from '%s' to '%s'\n", dirPath, filesListPath));
963+
964+
TruffleFile dir = getContext().getPublicTruffleFileRelaxed(dirPath);
965+
if (!dir.exists() || !dir.isDirectory()) {
966+
print(getContext().getStandardErr(), String.format("'%s' has to exist and be a directory.\n", dirPath));
967+
}
968+
969+
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filesListPath.toJavaStringUncached())))) {
970+
getContext().getPublicTruffleFileRelaxed(filesListPath).getParent().createDirectories();
971+
List<String> ret = list(dir);
972+
String parentPathString = dir.getParent().getAbsoluteFile().getPath();
973+
for (String f : ret) {
974+
bw.write(f.substring(parentPathString.length()));
975+
bw.write("\n");
976+
}
977+
} catch (IOException e) {
978+
String msg = String.format("error while creating '%s': %s \n", filesListPath, e);
979+
print(getContext().getStandardErr(), msg);
980+
}
981+
return PNone.NONE;
982+
}
983+
984+
private static List<String> list(TruffleFile dir) throws IOException {
985+
List<String> ret = new ArrayList<>();
986+
Collection<TruffleFile> files = dir.list();
987+
String dirPath = dir.getAbsoluteFile().getPath();
988+
if (!dirPath.endsWith("/")) {
989+
dirPath = dirPath + "/";
990+
}
991+
ret.add(dirPath);
992+
if (files != null) {
993+
for (TruffleFile f : files) {
994+
if (f.isRegularFile()) {
995+
ret.add(f.getAbsoluteFile().getPath());
996+
} else {
997+
ret.addAll(list(f));
998+
}
999+
}
1000+
}
1001+
return ret;
1002+
}
1003+
1004+
private void print(OutputStream out, String msg) {
1005+
try {
1006+
out.write(String.format("%s: %s", getContext().getOption(PythonOptions.Executable), msg).getBytes(StandardCharsets.UTF_8));
1007+
} catch (IOException ioException) {
1008+
// Ignore
1009+
}
1010+
}
1011+
}
9461012
}

0 commit comments

Comments
 (0)