Fix copying c extensions on darwin

Author: timfel

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java

@@ -109,4 +109,9 @@ public void changeOrAddDependency(String oldName, String newName) throws IOExcep
     public byte[] write() throws IOException {
         return tempfile.readAllBytes();
     }
+
+    @Override
+    protected void fixup(TruffleFile copy) {
+        // Nothing to do
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/MachODylibCommand.java

@@ -114,7 +114,7 @@ public void setName(String name) {
         int nameLen = nameBytes.length;
         int paddedLen = (nameLen & ~(Long.BYTES - 1)) + Long.BYTES;
         byte[] paddedNameBytes = new byte[paddedLen];
-        System.arraycopy(nameBytes, 0, paddedNameBytes, 0, paddedLen);
+        System.arraycopy(nameBytes, 0, paddedNameBytes, 0, nameLen);
         this.paddedName = paddedNameBytes;
         this.cmdSize = MachODylibCommand.SIZE + paddedLen;
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/MachOFile.java

@@ -46,14 +46,18 @@
 import java.util.List;
 
 import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.truffle.api.TruffleFile;
+import com.oracle.truffle.api.TruffleLanguage.Env;
 
 final class MachOFile extends SharedObject {
+    private final PythonContext context;
     private final ByteBuffer buffer;
     private final MachOHeader mh;
     private final List<MachOLoadCommand> loadCommands;
     private int emptySpace;
 
     MachOFile(byte[] f, PythonContext context) throws IOException {
+        this.context = context;
         this.buffer = ByteBuffer.wrap(f);
         this.mh = MachOHeader.read(buffer);
         this.loadCommands = new ArrayList<>();
@@ -68,93 +72,111 @@ final class MachOFile extends SharedObject {
         this.emptySpace = zeroBytes;
     }
 
-    public void removeCodeSignature() {
+    private void removeCommand(MachOLoadCommand cmd) {
+        loadCommands.remove(cmd);
+        mh.nCmds -= 1;
+        mh.sizeOfCmds -= cmd.cmdSize;
+        emptySpace += cmd.cmdSize;
+    }
+
+    private void addCommand(MachODylibCommand cmd) throws IOException {
+        if (cmd.cmdSize > emptySpace) {
+            throw new IOException(String.format("Not enough empty space add new cmd with string %s.", cmd.getName()));
+        }
+        var newBuffer = ByteBuffer.allocate(cmd.cmdSize);
+        newBuffer.order(buffer.order());
+        cmd.put(newBuffer);
+        newBuffer.position(0);
+        loadCommands.add(MachOLoadCommand.get(newBuffer));
+        mh.nCmds += 1;
+        mh.sizeOfCmds += cmd.cmdSize;
+        emptySpace -= cmd.cmdSize;
+    }
+
+    private void removeCodeSignature() {
         for (int i = 0; i < loadCommands.size(); ++i) {
             var cmd = loadCommands.get(i);
             if (cmd.cmd == MachOLoadCommand.LC_CODE_SIGNATURE) {
-                loadCommands.remove(i);
-                emptySpace += cmd.cmdSize;
-                break;
+                removeCommand(cmd);
+                LOGGER.fine(() -> String.format("Removing code LC_CODE_SIGNATURE. New empty space is %d", emptySpace));
             }
         }
     }
 
-    @Override
-    public void setId(String newId) throws IOException {
-        removeCodeSignature();
-        MachOLoadCommand oldIdCommand = null;
+    private void removeId() {
         for (int i = 0; i < loadCommands.size(); ++i) {
             var cmd = loadCommands.get(i);
             if (cmd.cmd == MachODylibCommand.LC_ID_DYLIB) {
-                oldIdCommand = cmd;
-                break;
+                removeCommand(cmd);
             }
         }
-
-        MachODylibCommand newCmd;
-        if (oldIdCommand != null) {
-            newCmd = MachODylibCommand.get(oldIdCommand.content);
-        } else {
-            newCmd = new MachODylibCommand(MachODylibCommand.LC_ID_DYLIB, MachODylibCommand.SIZE, new byte[0], MachODylibCommand.SIZE, 0, 0, 0);
-        }
-        newCmd.setName(newId);
-
-        if ((oldIdCommand != null && newCmd.cmdSize - oldIdCommand.cmdSize > emptySpace) || newCmd.cmdSize > emptySpace) {
-            throw new IOException("Not enough empty space to change ID");
-        }
-
-        loadCommands.remove(oldIdCommand);
-
-        var newBuffer = ByteBuffer.allocate(newCmd.cmdSize);
-        newBuffer.order(buffer.order());
-        newCmd.put(newBuffer);
-        loadCommands.add(MachOLoadCommand.get(newBuffer));
-
-        if (oldIdCommand != null) {
-            mh.sizeOfCmds -= oldIdCommand.cmdSize;
-            emptySpace += oldIdCommand.cmdSize;
-        }
-        emptySpace -= newCmd.cmdSize;
-        mh.sizeOfCmds += newCmd.cmdSize;
     }
 
-    @Override
-    public void changeOrAddDependency(String oldName, String newName) throws IOException {
-        removeCodeSignature();
-
+    private void removeLoad(String oldName) {
         for (int i = 0; i < loadCommands.size(); ++i) {
             var cmd = loadCommands.get(i);
             if (cmd.cmd == MachODylibCommand.LC_LOAD_DYLIB) {
                 var loadCmd = MachODylibCommand.get(cmd.content);
                 if (loadCmd.getName().equals(oldName)) {
-                    loadCommands.remove(i);
-                    emptySpace += cmd.cmdSize;
-                    break;
+                    removeCommand(cmd);
+                    LOGGER.fine(() -> String.format("Removing LC_LOAD_DYLIB %s. New empty space is %d.", oldName, emptySpace));
                 }
             }
         }
+    }
 
-        var newCmd = new MachODylibCommand(MachODylibCommand.LC_LOAD_DYLIB, MachODylibCommand.SIZE, new byte[0], MachODylibCommand.SIZE, 0, 0, 0);
-        newCmd.setName(newName);
+    @Override
+    public void setId(String newId) throws IOException {
+        removeCodeSignature();
+        removeId();
 
-        if (newCmd.cmdSize > emptySpace) {
-            throw new IOException("Not enough empty space to add dependency");
-        }
+        var newCmd = new MachODylibCommand(MachODylibCommand.LC_ID_DYLIB, MachODylibCommand.SIZE, new byte[0], MachODylibCommand.SIZE, 0, 0, 0);
+        newCmd.setName(newId);
+        addCommand(newCmd);
 
-        var newBuffer = ByteBuffer.allocate(newCmd.cmdSize);
-        newBuffer.order(buffer.order());
-        newCmd.put(newBuffer);
-        loadCommands.add(MachOLoadCommand.get(newBuffer));
+        LOGGER.fine(() -> String.format("Added LC_ID_DYLIB %s. New empty space is %d.", newId, emptySpace));
+    }
 
-        mh.nCmds += 1;
-        mh.sizeOfCmds += newCmd.cmdSize;
-        emptySpace -= newCmd.cmdSize;
+    @Override
+    public void changeOrAddDependency(String oldName, String newName) throws IOException {
+        removeCodeSignature();
+        removeLoad(oldName);
+
+        var newCmd = new MachODylibCommand(MachODylibCommand.LC_LOAD_DYLIB, MachODylibCommand.SIZE, new byte[0], MachODylibCommand.SIZE, 0, 0, 0);
+        newCmd.setName(newName);
+        addCommand(newCmd);
+
+        LOGGER.fine(() -> String.format("Added LC_LOAD_DYLIB %s. New empty space is %d.", newName, emptySpace));
     }
 
     @Override
     public byte[] write() {
         buffer.position(0);
         mh.put(buffer);
+        for (var cmd : loadCommands) {
+            cmd.put(buffer);
+        }
         return buffer.array();
     }
+
+    private String getCodesign() {
+        Env env = context.getEnv();
+        var path = env.getEnvironment().getOrDefault("PATH", "").split(env.getPathSeparator());
+        var i = 0;
+        TruffleFile codesign;
+        do {
+            codesign = env.getPublicTruffleFile(path[i++]).resolve("codesign");
+        } while (!codesign.isExecutable() && i < path.length);
+        return codesign.toString();
+    }
+
+    @Override
+    protected void fixup(TruffleFile copy) throws IOException, InterruptedException {
+        var pb = newProcessBuilder(context);
+        pb.command(getCodesign(), "--force", "--sign", "-", copy.getAbsoluteFile().getPath());
+        var proc = pb.start();
+        if (proc.waitFor() != 0) {
+            throw new IOException("Failed to run `codesign` command. Make sure you have it on your PATH.");
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java

@@ -213,6 +213,7 @@ private static void replicate(TruffleFile original, TruffleFile copy, PythonCont
         try (var os = copy.newOutputStream(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
             os.write(o.write());
         }
+        o.fixup(copy);
     }
 
     private static void walk(TruffleFile dir, String suffix, String capiOriginalName, PythonContext context, int capiSlot, BiFunction<TruffleFile, String, TruffleFile> f)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java

@@ -89,4 +89,9 @@ public void changeOrAddDependency(String oldName, String newName) throws IOExcep
     public byte[] write() throws IOException {
         return tempfile.readAllBytes();
     }
+
+    @Override
+    protected void fixup(TruffleFile copy) {
+        // TODO: Maybe this should be signed again?
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/SharedObject.java

@@ -70,8 +70,9 @@ static SharedObject open(TruffleFile file, PythonContext context) throws IOExcep
         }
     }
 
+    protected static final TruffleLogger LOGGER = PythonLanguage.getLogger("NativeLibraryLocator");
+
     protected static final class LoggingOutputStream extends OutputStream {
-        private static final TruffleLogger LOGGER = PythonLanguage.getLogger("NativeLibraryLocator");
         private final StringBuilder sb = new StringBuilder();
 
         @Override
@@ -97,4 +98,6 @@ protected static TruffleProcessBuilder newProcessBuilder(PythonContext context)
         pb.redirectError(pb.createRedirectToStream(new LoggingOutputStream()));
         return pb;
     }
+
+    protected abstract void fixup(TruffleFile copy) throws IOException, InterruptedException;
 }