Skip to content

Commit 3530f2f

Browse files
timfeltimfel
committed
[GR-12617] Second set of changes required for building numpy on graalpython
PullRequest: graalpython/304
2 parents c99a799 + 6bd0c0a commit 3530f2f

File tree

4 files changed

+144
-27
lines changed

4 files changed

+144
-27
lines changed

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonCC.java

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,22 @@ public class GraalPythonCC extends GraalPythonCompiler {
5555
private String outputFilename;
5656
private boolean linkExecutable;
5757
private boolean link;
58+
private boolean linkLLI;
5859
private Boolean compile;
5960
private List<String> clangArgs;
61+
private List<String> execLinkArgs;
6062
private List<String> fileInputs;
6163

6264
GraalPythonCC() {
6365
}
6466

67+
private static List<String> execLinkPrefix = Arrays.asList(new String[]{
68+
"clang",
69+
"-fembed-bitcode",
70+
"-fPIC",
71+
"-ggdb",
72+
"-O1",
73+
});
6574
private static List<String> clangPrefix = Arrays.asList(new String[]{
6675
"clang",
6776
"-emit-llvm",
@@ -100,10 +109,12 @@ private void run(String[] args) {
100109
private void parseOptions(String[] args) {
101110
outputFilename = A_OUT;
102111
linkExecutable = true;
112+
linkLLI = false;
103113
link = true;
104114
verbose = false;
105115
compile = null;
106116
clangArgs = new ArrayList<>(clangPrefix);
117+
execLinkArgs = new ArrayList<>(execLinkPrefix);
107118
fileInputs = new ArrayList<>();
108119
for (int i = 0; i < args.length; i++) {
109120
String arg = args[i];
@@ -123,6 +134,9 @@ private void parseOptions(String[] args) {
123134
link = false;
124135
linkExecutable = false;
125136
break;
137+
case "--link-lli-scripts":
138+
linkLLI = true;
139+
continue; // skip adding this to clang args
126140
case "-v":
127141
if (!verbose) {
128142
verbose = true;
@@ -144,6 +158,8 @@ private void parseOptions(String[] args) {
144158
throw new RuntimeException("cannot mix source and compiled sources");
145159
}
146160
fileInputs.add(arg);
161+
} else {
162+
execLinkArgs.add(arg);
147163
}
148164
}
149165
clangArgs.add(arg);
@@ -173,6 +189,13 @@ private void launchCC() {
173189
String bcFile = bcFileFromFilename(f);
174190
assert Files.exists(Paths.get(bcFile));
175191
optFile(bcFile);
192+
try {
193+
String objFile = objectFileFromFilename(f);
194+
logV("Optimized:", bcFile, "->", objFile);
195+
Files.move(Paths.get(bcFile), Paths.get(objFile));
196+
} catch (IOException e) {
197+
throw new RuntimeException(e);
198+
}
176199
}
177200
} else {
178201
optFile(outputFilename);
@@ -192,15 +215,21 @@ private void launchCC() {
192215
}
193216
}
194217

195-
private static void linkExecutable(String executableScript, String linkedBcFile) throws IOException {
196-
List<String> cmdline = GraalPythonMain.getCmdline(Arrays.asList(), Arrays.asList());
197-
cmdline.add("-LLI");
198-
cmdline.add(linkedBcFile);
199-
Path executablePath = Paths.get(executableScript);
200-
Files.write(executablePath, String.join(" ", cmdline).getBytes());
201-
HashSet<PosixFilePermission> perms = new HashSet<>(Arrays.asList(new PosixFilePermission[]{PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE}));
202-
perms.addAll(Files.getPosixFilePermissions(executablePath));
203-
Files.setPosixFilePermissions(executablePath, perms);
218+
private void linkExecutable(String executableScript, String linkedBcFile) throws IOException {
219+
if (linkLLI) {
220+
List<String> cmdline = GraalPythonMain.getCmdline(Arrays.asList(), Arrays.asList());
221+
cmdline.add("-LLI");
222+
cmdline.add(linkedBcFile);
223+
Path executablePath = Paths.get(executableScript);
224+
Files.write(executablePath, String.join(" ", cmdline).getBytes());
225+
HashSet<PosixFilePermission> perms = new HashSet<>(Arrays.asList(new PosixFilePermission[]{PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE}));
226+
perms.addAll(Files.getPosixFilePermissions(executablePath));
227+
} else {
228+
execLinkArgs.add(linkedBcFile);
229+
execLinkArgs.add("-o");
230+
execLinkArgs.add(executableScript);
231+
exec(execLinkArgs);
232+
}
204233
}
205234

206235
private void linkShared(List<String> bitcodeFiles) {
@@ -221,4 +250,17 @@ private void optFile(String bcFile) {
221250
opt.add(bcFile);
222251
exec(opt);
223252
}
253+
254+
private static String bcFileFromFilename(String f) {
255+
int dotIdx = f.lastIndexOf('.');
256+
if (dotIdx > 1) {
257+
return f.substring(0, dotIdx + 1) + "bc";
258+
} else {
259+
return f + ".bc";
260+
}
261+
}
262+
263+
private static String objectFileFromFilename(String f) {
264+
return bcFileFromFilename(f).replaceAll("\\.bc$", ".o");
265+
}
224266
}

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonCompiler.java

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,20 +41,12 @@
4141
package com.oracle.graal.python.shell;
4242

4343
import java.io.IOException;
44+
import java.util.Arrays;
4445
import java.util.List;
4546

4647
public class GraalPythonCompiler {
4748
protected static final String A_OUT = "a.out";
4849

49-
protected static String bcFileFromFilename(String f) {
50-
int dotIdx = f.lastIndexOf('.');
51-
if (dotIdx > 1) {
52-
return f.substring(0, dotIdx + 1) + "bc";
53-
} else {
54-
return f + ".bc";
55-
}
56-
}
57-
5850
protected boolean verbose;
5951

6052
public GraalPythonCompiler() {
@@ -91,4 +83,7 @@ protected void logV(List<String> args) {
9183
logV(null, args);
9284
}
9385

86+
protected void logV(String... args) {
87+
logV(null, Arrays.asList(args));
88+
}
9489
}

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonLD.java

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,20 @@
4040
*/
4141
package com.oracle.graal.python.shell;
4242

43+
import java.io.BufferedReader;
44+
import java.io.File;
45+
import java.io.IOException;
46+
import java.lang.ProcessBuilder.Redirect;
4347
import java.nio.file.Files;
48+
import java.nio.file.Path;
4449
import java.nio.file.Paths;
4550
import java.util.ArrayList;
4651
import java.util.Arrays;
52+
import java.util.Collection;
4753
import java.util.List;
4854

55+
import jline.internal.InputStreamReader;
56+
4957
public class GraalPythonLD extends GraalPythonCompiler {
5058
private static List<String> linkPrefix = Arrays.asList(new String[]{
5159
"llvm-link",
@@ -70,6 +78,7 @@ private void parseOptions(String[] args) {
7078
ldArgs = new ArrayList<>(linkPrefix);
7179
fileInputs = new ArrayList<>();
7280
List<String> droppedArgs = new ArrayList<>();
81+
List<String> libraryDirs = new ArrayList<>();
7382
for (int i = 0; i < args.length; i++) {
7483
String arg = args[i];
7584
switch (arg) {
@@ -84,15 +93,24 @@ private void parseOptions(String[] args) {
8493
verbose = true;
8594
break;
8695
default:
87-
if (arg.endsWith(".o")) {
88-
String bc = bcFileFromFilename(arg);
89-
if (Files.exists(Paths.get(bc))) {
90-
fileInputs.add(bc);
91-
} else {
92-
fileInputs.add(arg);
93-
}
94-
} else if (arg.endsWith(".bc")) {
96+
if (arg.endsWith(".o") || arg.endsWith(".bc")) {
9597
fileInputs.add(arg);
98+
} else if (arg.startsWith("-L")) {
99+
libraryDirs.add(arg.substring(2));
100+
} else if (arg.startsWith("-l")) {
101+
List<String> bcFiles = searchLib(libraryDirs, arg.substring(2));
102+
for (String bcFile : bcFiles) {
103+
try {
104+
if (Files.probeContentType(Paths.get(bcFile)).contains("llvm-ir-bitcode")) {
105+
logV("library input:", bcFile);
106+
fileInputs.add(bcFile);
107+
} else {
108+
droppedArgs.add(bcFile + "(dropped as library input)");
109+
}
110+
} catch (IOException e) {
111+
throw new RuntimeException(e);
112+
}
113+
}
96114
} else {
97115
droppedArgs.add(arg);
98116
}
@@ -104,6 +122,66 @@ private void parseOptions(String[] args) {
104122
logV("Dropped args: ", droppedArgs);
105123
}
106124

125+
private List<String> searchLib(List<String> libraryDirs, String lib) {
126+
List<String> bcFiles = new ArrayList<>();
127+
String[] suffixes = new String[]{".bc", ".o", ".a", ".so"};
128+
String[] prefixes = new String[]{"lib", ""};
129+
for (String libdir : libraryDirs) {
130+
for (String prefix : prefixes) {
131+
for (String suffix : suffixes) {
132+
Path path = Paths.get(libdir, prefix + lib + suffix);
133+
String pathString = path.toAbsolutePath().toString();
134+
logV("Checking for library:", pathString);
135+
if (Files.exists(path)) {
136+
if (suffix.equals(".a")) {
137+
// extract members
138+
try {
139+
bcFiles.addAll(arMembers(pathString));
140+
} catch (IOException | InterruptedException e) {
141+
throw new RuntimeException(e);
142+
}
143+
} else {
144+
bcFiles.add(pathString);
145+
break;
146+
}
147+
}
148+
}
149+
}
150+
}
151+
logV("Found bitcode files for library:", bcFiles);
152+
return bcFiles;
153+
}
154+
155+
private Collection<? extends String> arMembers(String path) throws IOException, InterruptedException {
156+
List<String> members = new ArrayList<>();
157+
File temp = File.createTempFile(path, Long.toString(System.nanoTime()));
158+
temp.delete();
159+
temp.mkdir();
160+
temp.deleteOnExit();
161+
162+
ProcessBuilder extractAr = new ProcessBuilder();
163+
extractAr.redirectInput(Redirect.INHERIT);
164+
extractAr.redirectError(Redirect.INHERIT);
165+
extractAr.redirectOutput(Redirect.PIPE);
166+
extractAr.directory(temp);
167+
logV("ar", path);
168+
// "ar t" lists the members one per line
169+
extractAr.command("ar", "t", path);
170+
Process start = extractAr.start();
171+
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(start.getInputStream()))) {
172+
String line = null;
173+
while ((line = buffer.readLine()) != null) {
174+
members.add(temp.getAbsolutePath() + File.separator + line);
175+
}
176+
}
177+
start.waitFor();
178+
// actually extract them now
179+
extractAr.redirectOutput(Redirect.INHERIT);
180+
extractAr.command("ar", "xv", path);
181+
extractAr.start().waitFor();
182+
return members;
183+
}
184+
107185
private void launchLD() {
108186
ldArgs.add(outputFilename);
109187
ldArgs.addAll(fileInputs);

‎graalpython/lib-graalpython/_codecs.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,9 @@ def utf_32_ex_decode(data, errors=None, byteorder=0, final=False):
247247

248248
@__builtin__
249249
def unicode_escape_encode(string, errors=None):
250-
raise NotImplementedError("unicode_escape_encode")
250+
if not isinstance(string, str):
251+
raise TypeError("unicode_escape_encode() argument 1 must be str, not %s", type(string))
252+
return __truffle_encode(repr(string)[1:-1], "latin-1", errors)
251253

252254

253255
@__builtin__

0 commit comments

Comments
 (0)