Merge pull request #22 from TheBlueMatt/main

Support building on OSX

Author: TheBlueMatt

.github/workflows/build.yml

@@ -51,13 +51,9 @@ jobs:
         run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false
       - name: Run Java Tests against Debug Bindings
         run: |
-          rm liblightningjni.so
-          ln -s liblightningjni_debug.so liblightningjni.so
-          export LD_LIBRARY_PATH=.
-          export LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so
+          mv liblightningjni_debug_Linux-amd64.so liblightningjni.so
           export ASAN_OPTIONS=detect_leaks=0
-          mvn test
-          git checkout liblightningjni.so
+          LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test
       - name: Build Java/TS Release Bindings
         run: |
           # We assume the top commit is just a bindings update commit, so we
@@ -73,8 +69,113 @@ jobs:
           fi
           echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version"
           ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
-      - name: Check latest headers are in git
+      - name: Check latest headers and release lib are in git
         run: |
-          # For some reason the debug library is not deterministic, this may be fixed in a future rustc
-          git checkout liblightningjni_debug.so
           git diff --exit-code
+      - name: Run Java Tests against built release jar
+        run: |
+          mvn -DskipTests=true package
+          mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+          cd javatester
+          mvn package
+          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
+          cd ..
+
+  osx:
+    strategy:
+      matrix:
+        include:
+          - platform: macos-10.15
+          # MacOS 11 is currently in private preview, we've applied for access
+          # - platform: macos-11
+    runs-on: ${{ matrix.platform }}
+    env:
+      TOOLCHAIN: stable
+    steps:
+      - name: Install other Rust platforms
+        run: rustup target install aarch64-apple-darwin
+      - name: Fetch upstream LLVM/clang snapshot
+        run: |
+          wget -O clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
+          if [ "$(shasum -a 256 clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "7bc2259bf75c003f644882460fc8e844ddb23b27236fe43a2787870a4cd8ab50" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+      - name: Unpack upstream LLVM+clang and use it by default
+        run: |
+          tar xvvf clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
+      - name: Checkout source code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
+      - name: Checkout Rust-Lightning and LDK-C-Bindings git
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "LDK CI"
+          git clone https://github.com/rust-bitcoin/rust-lightning
+          cd rust-lightning
+          git remote add matt https://git.bitcoin.ninja/rust-lightning
+          git fetch matt
+          git merge matt/2021-03-java-bindings-base
+          cd ..
+          git clone https://github.com/lightningdevkit/ldk-c-bindings
+      - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
+        run: |
+          cd ldk-c-bindings
+          export PATH=`pwd`/clang+llvm-12.0.0-x86_64-apple-darwin/bin:$PATH
+          CC=clang ./genbindings.sh ../rust-lightning true
+          cd ..
+      - name: Fetch OpenJDK 16
+        run: |
+          wget -O openjdk-16.0.1_osx-x64_bin.tar.gz https://download.java.net/java/GA/jdk16.0.1/7147401fd7354114ac51ef3e1328291f/9/GPL/openjdk-16.0.1_osx-x64_bin.tar.gz
+          if [ "$(shasum -a 256 openjdk-16.0.1_osx-x64_bin.tar.gz | awk '{ print $1 }')" != "6098f839954439d4916444757c542c1b8778a32461706812d41cc8bbefce7f2f" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+          tar xvvf openjdk-16.0.1_osx-x64_bin.tar.gz
+          export JAVA_HOME=`pwd`/jdk-16.0.1.jdk/Contents/Home
+          export PATH=$JAVA_HOME/bin:$PATH
+      - name: Build Java/TS Release Bindings
+        run: |
+          # We assume the top commit is just a bindings update commit, so we
+          # check out the previous commit to use as the commit we git describe.
+          # If the top commit is a merge commit, we need to get the last merge
+          # head and assume the latest bindings are built against its parent.
+          COMMIT_PARENTS=$(git show -s --pretty=format:%P HEAD)
+          if [ "${#COMMIT_PARENTS}" = 40 ]; then
+            export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD^1)"
+          else
+            MERGE_HEAD=$(git show --pretty=format:%P HEAD | ( for last in $(cat /dev/stdin); do true; done; echo $last ))
+            export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag $MERGE_HEAD^1)"
+          fi
+          echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version"
+          # genbindings.sh always fails as there is no wasm32-wasi library
+          # available, so instead we delete the expected JNI library and check
+          # that it was created.
+          rm src/main/resources/liblightningjni_MacOSX-x86_64.nativelib
+          ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo
+          cat src/main/resources/liblightningjni_MacOSX-x86_64.nativelib > /dev/null
+      - name: Fetch Maven 3.8.1
+        run: |
+          wget -O apache-maven-3.8.1-bin.tar.gz https://apache.osuosl.org/maven/maven-3/3.8.1/binaries/apache-maven-3.8.1-bin.tar.gz
+          if [ "$(shasum -a 256 apache-maven-3.8.1-bin.tar.gz | awk '{ print $1 }')" != "b98a1905eb554d07427b2e5509ff09bd53e2f1dd7a0afa38384968b113abef02" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+          tar xvvf apache-maven-3.8.1-bin.tar.gz
+          export PATH=apache-maven-3.8.1/bin:$PATH
+      - name: Run Java Tests against built jar
+        run: |
+          mvn -DskipTests=true package
+          mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+          cd javatester
+          mvn package
+          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
+          cd ..
+      - name: Check latest headers and release lib are in git
+        run: |
+          if [ "${{ matrix.platform }}" = "macos-11" ]; then
+            git diff --exit-code
+          fi

README.md

@@ -25,6 +25,17 @@ bindings should be built with workarounds required for Android. JNI CFLAGS on de
 When running a program linking against the library in debug mode, LD_PRELOAD should likely include
 the relevant `libclang_rt.asan-platform.so` path.
 
+Note that building with address sanitizer is only supported on MacOS with upstream clang (ie where
+the LLVM version in use matches the LLVM version against which rustc was built), not with Apple clang.
+Builds with Apple clang will work fine, but largely only be useful in a release context.
+To build on Mac with address sanitizer, you will need to run `ldk-c-bindings`' `genbindings.sh`
+script with upstream clang in your PATH and likely replace your $JAVA_HOME/bin/java with a simple
+wrapper which calls java after an export like:
+`export DYLD_INSERT_LIBRARIES=/path/to/upstream/llvm/lib/clang/12.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib`
+
+To build for Apple M1 (ie aarch64-apple-darwin), you probably want something like
+`CC="clang --target=aarch64-apple-darwin" LDK_TARGET=aarch64-apple-darwin LDK_TARGET_CPU=generic ./genbindings.sh ...`
+
 Status
 ======
 

genbindings.sh

@@ -10,15 +10,30 @@ usage() {
 [ "$3" != "true" -a "$3" != "false" ] && usage
 [ "$4" != "true" -a "$4" != "false" ] && usage
 
+set -x
+
 if [ "$CC" != "" ]; then
 	COMMON_COMPILE="$CC -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)="
 else
+	CC=clang
 	COMMON_COMPILE="clang -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)="
 fi
 
-if [ "$LDK_TARGET" != "" ]; then
-	LDK_TARGET_SUFFIX="_$LDK_TARGET"
+TARGET_STRING="$LDK_TARGET"
+if [ "$TARGET_STRING" = "" ]; then
+	# We assume clang-style $CC --version here, but worst-case we just get an empty suffix
+	TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')"
 fi
+case "$TARGET_STRING" in
+	"x86_64-pc-linux"*)
+		LDK_TARGET_SUFFIX="_Linux-amd64" ;;
+	"x86_64-apple-darwin"*)
+		LDK_TARGET_SUFFIX="_MacOSX-x86_64" ;;
+	"aarch64-apple-darwin"*)
+		LDK_TARGET_SUFFIX="_MacOSX-aarch64" ;;
+	*)
+		LDK_TARGET_SUFFIX=""
+esac
 if [ "$LDK_TARGET_CPU" = "" ]; then
 	LDK_TARGET_CPU="sandybridge"
 fi
@@ -52,12 +67,23 @@ cat src/main/jni/bindings.c.body >> src/main/jni/bindings.c
 javac -h src/main/jni src/main/java/org/ldk/enums/*.java src/main/java/org/ldk/impl/bindings.java
 rm src/main/java/org/ldk/enums/*.class src/main/java/org/ldk/impl/bindings*.class
 
+IS_MAC=false
+[ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true
+
 echo "Building Java bindings..."
-COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -Wl,--no-undefined -shared -fPIC"
+COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -shared -fPIC"
+[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined"
+[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9"
 if [ "$3" = "true" ]; then
-	$COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
+	[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free"
+	$COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
 else
-	$COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=libcode.version -flto -fuse-ld=lld -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
+	[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld"
+	$COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
+	if [ "$LDK_TARGET_SUFFIX" != "" ]; then
+		# Copy to JNI native directory for inclusion in JARs
+		cp liblightningjni_release$LDK_TARGET_SUFFIX.so src/main/resources/liblightningjni$LDK_TARGET_SUFFIX.nativelib
+	fi
 fi
 
 echo "Creating TS bindings..."

java_strings.py

@@ -1,5 +1,6 @@
 from bindingstypes import *
 from enum import Enum
+import sys
 
 class Target(Enum):
     JAVA = 1,
@@ -23,6 +24,12 @@ def __init__(self, DEBUG: bool, target: Target, **kwargs):
 
         self.bindings_header = """package org.ldk.impl;
 import org.ldk.enums.*;
+import java.io.File;
+import java.io.InputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 
 public class bindings {
 	public static class VecOrSliceDef {
@@ -34,7 +41,24 @@ def __init__(self, DEBUG: bool, target: Target, **kwargs):
 		}
 	}
 	static {
-		System.loadLibrary(\"lightningjni\");
+		try {
+			// Try to load natively first, this works on Android and in testing.
+			System.loadLibrary(\"lightningjni\");
+		} catch (UnsatisfiedLinkError _ignored) {
+			// Otherwise try to load from the library jar.
+			File tmpdir = new File(System.getProperty("java.io.tmpdir"), "ldk-java-nativelib");
+			tmpdir.mkdir(); // If it fails to create, assume it was there already
+			tmpdir.deleteOnExit();
+			String libname = "liblightningjni_" + System.getProperty("os.name").replaceAll(" ", "") +
+				"-" + System.getProperty("os.arch").replaceAll(" ", "") + ".nativelib";
+			try (InputStream is = bindings.class.getResourceAsStream("/" + libname)) {
+				Path libpath = new File(tmpdir.toPath().toString(), "liblightningjni.so").toPath();
+				Files.copy(is, libpath, StandardCopyOption.REPLACE_EXISTING);
+				Runtime.getRuntime().load(libpath.toString());
+			} catch (IOException e) {
+				throw new IllegalArgumentException(e);
+			}
+		}
 		init(java.lang.Enum.class, VecOrSliceDef.class);
 		init_class_cache();
 		if (!get_lib_version_string().equals(get_ldk_java_bindings_version()))
@@ -83,11 +107,13 @@ class CommonBase {
 	long ptr;
 	LinkedList<Object> ptrs_to = new LinkedList();
 	protected CommonBase(long ptr) { this.ptr = ptr; }
-	public long _test_only_get_ptr() { return this.ptr; }
 }
 """
 
-        self.c_file_pfx = """#include \"org_ldk_impl_bindings.h\"
+        self.c_file_pfx = """#include <jni.h>
+// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here
+#define int64_t jlong
+#include \"org_ldk_impl_bindings.h\"
 #include <lightning.h>
 #include <string.h>
 #include <stdatomic.h>
@@ -124,13 +150,15 @@ class CommonBase {
         else:
             self.c_file_pfx = self.c_file_pfx + "#define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)\n"
 
-        if not DEBUG:
+        if not DEBUG or sys.platform == "darwin":
             self.c_file_pfx = self.c_file_pfx + """#define MALLOC(a, _) malloc(a)
 #define FREE(p) if ((uint64_t)(p) > 1024) { free(p); }
-#define DO_ASSERT(a) (void)(a)
+"""
+        if not DEBUG:
+            self.c_file_pfx += """#define DO_ASSERT(a) (void)(a)
 #define CHECK(a)
 """
-        else:
+        if DEBUG:
             self.c_file_pfx = self.c_file_pfx + """#include <assert.h>
 // Always run a, then assert it is true:
 #define DO_ASSERT(a) do { bool _assert_val = (a); assert(_assert_val); } while(0)
@@ -144,16 +172,19 @@ class CommonBase {
 		DEBUG_PRINT("LDK C Bindings version did not match the header we built against\\n");
 	DEBUG_PRINT("Loaded LDK-Java Bindings with LDK %s and LDK-C-Bindings %s\\n", check_get_ldk_version(), check_get_ldk_bindings_version());
 }
+"""
 
+            if sys.platform != "darwin":
+                self.c_file_pfx += """
 // Running a leak check across all the allocations and frees of the JDK is a mess,
 // so instead we implement our own naive leak checker here, relying on the -wrap
 // linker option to wrap malloc/calloc/realloc/free, tracking everyhing allocated
 // and free'd in Rust or C across the generated bindings shared library.
 #include <threads.h>
 """
 
-            if self.target == Target.ANDROID:
-                self.c_file_pfx = self.c_file_pfx + """
+                if self.target == Target.ANDROID:
+                    self.c_file_pfx = self.c_file_pfx + """
 #include <unwind.h>
 #include <dlfcn.h>
 
@@ -194,9 +225,9 @@ class CommonBase {
 	}
 }
 """
-            else:
-                self.c_file_pfx = self.c_file_pfx + "#include <execinfo.h>\n"
-            self.c_file_pfx = self.c_file_pfx + """
+                else:
+                    self.c_file_pfx = self.c_file_pfx + "#include <execinfo.h>\n"
+                self.c_file_pfx = self.c_file_pfx + """
 #include <unistd.h>
 static mtx_t allocation_mtx;
 

javatester/pom.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.ldk</groupId>
+    <artifactId>ldk-java-tests</artifactId>
+    <version>1.0-SNAPSHOT</version>
+	<dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>RELEASE</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.bitcoinj</groupId>
+            <artifactId>bitcoinj-core</artifactId>
+            <version>0.15.10</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains</groupId>
+            <artifactId>annotations</artifactId>
+            <version>RELEASE</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.ldk</groupId>
+            <artifactId>ldk-java</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <archive>
+                                <manifest>
+                                    <mainClass>org.ldk.HumanObjectPeerTest</mainClass>
+                                </manifest>
+                            </archive>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                        </configuration>
+                    </execution>
+                 </executions>
+             </plugin>
+        </plugins>
+    </build>
+</project>

javatester/src/main/java/org/ldk/HumanObjectPeerTest.java

@@ -0,0 +1 @@
+../../../../../../src/test/java/org/ldk/HumanObjectPeerTest.java