Merge pull request #128 from TheBlueMatt/main

Update to LDK 0.0.115

Author: TheBlueMatt

.github/workflows/build.yml

@@ -40,11 +40,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2023-02-0.0.114-java-bindings
+          git checkout origin/2023-04-0.0.115-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.114
+          git checkout 0.0.115
       - name: Rebuild C bindings without STD
         run: |
           cd ldk-c-bindings
@@ -134,11 +134,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2023-02-0.0.114-java-bindings
+          git checkout origin/2023-04-0.0.115-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.114
+          git checkout 0.0.115
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -190,11 +190,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2023-02-0.0.114-java-bindings
+          git checkout origin/2023-04-0.0.115-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.114
+          git checkout 0.0.115
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -297,11 +297,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2023-02-0.0.114-java-bindings
+          git checkout origin/2023-04-0.0.115-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.114
+          git checkout 0.0.115
       - name: Checkout Android AAR binaries and artifacts
         run: |
           # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
@@ -378,11 +378,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2023-02-0.0.114-java-bindings
+          git checkout origin/2023-04-0.0.115-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.114
+          git checkout 0.0.115
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
           export PATH=`pwd`/clang+llvm-15.0.7-x86_64-apple-darwin21.0/bin:$PATH

genbindings.py

@@ -30,8 +30,12 @@
     import csharp_strings
     from csharp_strings import Consts
     target = csharp_strings.Target.CSHARP
+elif sys.argv[6] == "python":
+    import python_strings
+    from python_strings import Consts
+    target = python_strings.Target.PYTHON
 else:
-    print("Only java, typescript, or c_sharp can be set for lang")
+    print("Only java, typescript, python, or c_sharp can be set for lang")
     sys.exit(1)
 
 
@@ -128,13 +132,15 @@ def java_c_types(fn_arg, ret_arr_len):
     rust_obj = None
     arr_access = None
     java_hu_ty = None
-    if fn_arg.startswith("LDKPaymentPreimage") or fn_arg.startswith("LDKPaymentSecret") or fn_arg.startswith("LDKPaymentHash"):
+    if fn_arg.startswith("LDKPaymentPreimage") or fn_arg.startswith("LDKPaymentSecret") or fn_arg.startswith("LDKPaymentHash") or fn_arg.startswith("LDKChainHash"):
         if fn_arg.startswith("LDKPaymentPreimage"):
             fn_arg = "uint8_t (*" + fn_arg[19:] + ")[32]"
         elif fn_arg.startswith("LDKPaymentSecret"):
             fn_arg = "uint8_t (*" + fn_arg[17:] + ")[32]"
         elif fn_arg.startswith("LDKPaymentHash"):
             fn_arg = "uint8_t (*" + fn_arg[15:] + ")[32]"
+        elif fn_arg.startswith("LDKChainHash"):
+            fn_arg = "uint8_t (*" + fn_arg[13:] + ")[32]"
         assert var_is_arr_regex.match(fn_arg[8:])
         rust_obj = "LDKThirtyTwoBytes"
         arr_access = "data"
@@ -339,14 +345,17 @@ def java_c_types(fn_arg, ret_arr_len):
         arr_ty = "LDKStr"
         fn_ty_arg = "Ljava/lang/String;"
         fn_arg = fn_arg[6:].strip()
-    elif fn_arg.startswith("LDKStr"):
+    elif fn_arg.startswith("LDKStr") or fn_arg.startswith("LDKAddress"):
         rust_obj = "LDKStr"
         arr_ty = "LDKStr"
         java_ty = consts.java_type_map["String"]
         java_hu_ty = consts.java_hu_type_map["String"]
         c_ty = "jstring"
         fn_ty_arg = "Ljava/lang/String;"
-        fn_arg = fn_arg[6:].strip()
+        if fn_arg.startswith("LDKAddress"):
+            fn_arg = fn_arg[10:].strip()
+        else:
+            fn_arg = fn_arg[6:].strip()
         arr_access = "chars"
         arr_len = "len"
     elif fn_arg.startswith("LDKError ") or fn_arg == "LDKError":

genbindings.sh

@@ -121,9 +121,74 @@ if [ "$2" = "c_sharp" ]; then
 		$COMPILE $LINK -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
 	else
 		$COMPILE -o bindings.o -c -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c
-		$COMPILE $LINK -o liblightningjni_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=c_sharp/libcode.version -flto -O3 -Wl,--lto-O3 -Wl,-O3 -Wl,--version-script=c_sharp/libcode.version -I"$1"/lightning-c-bindings/include/ $2 bindings.o "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a -lm
+		$COMPILE $LINK -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -Wl,--lto-O3 -Wl,-O3 -Wl,--version-script=c_sharp/libcode.version -I"$1"/lightning-c-bindings/include/ $2 bindings.o "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a -lm
 		llvm-strip liblightningjni_release$LDK_TARGET_SUFFIX.so
 	fi
+elif [ "$2" = "python" ]; then
+	TARGET_STRING="$LDK_TARGET"
+	if [ "$TARGET_STRING" = "" ]; then
+		# We assume clang-style $CC --version here, but worst-case we just get an empty suffix
+		TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')"
+	fi
+	case "$TARGET_STRING" in
+		"x86_64-pc-linux"*)
+			LDK_TARGET_SUFFIX="_Linux-amd64"
+			LDK_JAR_TARGET=true
+			;;
+		"x86_64-apple-darwin"*)
+			LDK_TARGET_SUFFIX="_MacOSX-x86_64"
+			LDK_JAR_TARGET=true
+			;;
+		"aarch64-apple-darwin"*)
+			LDK_TARGET_CPU="apple-a14"
+			LDK_TARGET_SUFFIX="_MacOSX-aarch64"
+			LDK_JAR_TARGET=true
+			;;
+		*)
+			LDK_TARGET_SUFFIX="_${TARGET_STRING}"
+	esac
+	if [ "$LDK_TARGET_CPU" = "" ]; then
+		LDK_TARGET_CPU="sandybridge"
+	fi
+
+	echo "Creating Python bindings..."
+	mkdir -p python/src/{enums,structs,impl}
+	rm -f python/src/{enums,structs,impl}/*.py
+	./genbindings.py "./lightning.h" python/src/impl python/src python/ $DEBUG_ARG python $4
+	rm -f python/bindings.c
+	if [ "$3" = "true" ]; then
+		echo "#define LDK_DEBUG_BUILD" > python/bindings.c
+	elif [ "$3" = "leaks" ]; then
+		# For leak checking we use release libldk which doesn't expose
+		# __unmangle_inner_ptr, but the C code expects to be able to call it.
+		echo "#define __unmangle_inner_ptr(a) (a)" > python/bindings.c
+	fi
+	echo "#define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ" >> python/bindings.c
+	echo "#define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free" >> python/bindings.c
+	cat python/bindings.c.body >> python/bindings.c
+
+	IS_MAC=false
+	[ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true
+	IS_APPLE_CLANG=false
+	[ "$($CC --version | grep "Apple clang version")" != "" ] && IS_APPLE_CLANG=true
+
+	echo "Building Python bindings..."
+	COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -fPIC"
+	LINK="-ldl -shared"
+	[ "$IS_MAC" = "false" ] && LINK="$LINK -Wl,--no-undefined"
+	[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9"
+	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && LINK="$LINK -fuse-ld=lld"
+	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && echo "WARNING: Need at least upstream clang 13!"
+	[ "$IS_MAC" = "false" -a "$3" != "false" ] && LINK="$LINK -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free"
+
+	exit 0 # Sadly compilation doesn't currently work
+	if [ "$3" = "true" ]; then
+		$COMPILE $LINK -o liblightningpython_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
+	else
+		$COMPILE -o bindings.o -c -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c
+		$COMPILE $LINK -o liblightningpython_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=python/libcode.version -flto -O3 -Wl,--lto-O3 -Wl,-O3 -I"$1"/lightning-c-bindings/include/ $2 bindings.o "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a -lm
+		llvm-strip liblightningpython_release$LDK_TARGET_SUFFIX.so
+	fi
 elif [ "$2" = "wasm" ]; then
 	echo "Creating TS bindings..."
 	mkdir -p ts/{enums,structs}

java_strings.py

@@ -521,6 +521,8 @@ def create_native_arr_call(self, arr_len, ty_info):
             clz_var = ty_info.java_fn_ty_arg[1:].replace("[", "arr_of_")
             self.c_array_class_caches.add(clz_var)
             return "(*env)->NewObjectArray(env, " + arr_len + ", " + clz_var + "_clz, NULL);\n"
+        elif ty_info.subty.c_ty == "jstring":
+            return "(*env)->NewObjectArray(env, " + arr_len + ", String_clz, NULL);\n"
         else:
             return "(*env)->New" + ty_info.java_ty.strip("[]").title() + "Array(env, " + arr_len + ")"
     def set_native_arr_contents(self, arr_name, arr_len, ty_info):
@@ -531,6 +533,8 @@ def set_native_arr_contents(self, arr_name, arr_len, ty_info):
         else:
             assert False
     def get_native_arr_contents(self, arr_name, dest_name, arr_len, ty_info, copy):
+        if "String" in ty_info.java_ty:
+            return None
         if ty_info.c_ty == "int8_tArray" or ty_info.c_ty == "int16_tArray":
             fn_ty = "Byte" if ty_info.c_ty == "int8_tArray" else "Short"
             if copy:
@@ -590,11 +594,16 @@ def init_str(self):
         res = ""
         for ty in sorted(self.c_array_class_caches):
             res = res + "static jclass " + ty + "_clz = NULL;\n"
+        res = res + "static jclass String_clz = NULL;\n"
         res = res + "JNIEXPORT void Java_org_ldk_impl_bindings_init_1class_1cache(JNIEnv * env, jclass clz) {\n"
         for ty in sorted(self.c_array_class_caches):
             res = res + "\t" + ty + "_clz = (*env)->FindClass(env, \"" + ty.replace("arr_of_", "[") + "\");\n"
             res = res + "\tCHECK(" + ty + "_clz != NULL);\n"
             res = res + "\t" + ty + "_clz = (*env)->NewGlobalRef(env, " + ty + "_clz);\n"
+        res = res + "\tString_clz = (*env)->FindClass(env, \"Ljava/lang/String;\");\n"
+        res = res + "\tCHECK(String_clz != NULL);\n"
+        res = res + "\tString_clz = (*env)->NewGlobalRef(env, String_clz);\n"
+
         res = res + "}\n"
         return res
 
@@ -1331,6 +1340,8 @@ def map_function(self, argument_types, c_call_string, method_name, meth_n, retur
                     extra_java_struct_out += "\t\tif (!(o instanceof " + struct_meth + ")) return false;\n"
                     extra_java_struct_out += "\t\treturn this.eq((" + struct_meth + ")o);\n"
                     extra_java_struct_out += "\t}\n"
+                if meth_n == "wait":
+                    meth_n = "wait_indefinite"
                 out_java_struct += ("\tpublic " + return_type_info.java_hu_ty + " " + meth_n + "(")
             for idx, arg in enumerate(argument_types):
                 if idx != 0:

javatester/pom.xml

@@ -17,7 +17,7 @@
         <dependency>
             <groupId>org.bitcoinj</groupId>
             <artifactId>bitcoinj-core</artifactId>
-            <version>0.15.10</version>
+            <version>0.17-alpha1</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

pom.xml

@@ -52,7 +52,7 @@
         <dependency>
             <groupId>org.bitcoinj</groupId>
             <artifactId>bitcoinj-core</artifactId>
-            <version>RELEASE</version>
+            <version>0.17-alpha1</version>
             <scope>test</scope>
         </dependency>
         <dependency>