Merge pull request #126 from TheBlueMatt/main

0.0.114.0

Author: TheBlueMatt

.github/workflows/build.yml

@@ -34,16 +34,17 @@ jobs:
           cargo install --locked --path .
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
+          git config --global safe.directory '*'
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2022-12-0.0.113-java-bindings
+          git checkout origin/2023-02-0.0.114-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.113
+          git checkout 0.0.114
       - name: Rebuild C bindings without STD
         run: |
           cd ldk-c-bindings
@@ -120,6 +121,7 @@ jobs:
           fetch-depth: 0
       - name: Install cbindgen
         run: |
+          git config --global safe.directory '*'
           git clone https://github.com/eqrion/cbindgen
           cd cbindgen/
           git checkout v0.20.0
@@ -132,11 +134,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2022-12-0.0.113-java-bindings
+          git checkout origin/2023-02-0.0.114-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.113
+          git checkout 0.0.114
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -175,6 +177,7 @@ jobs:
           fetch-depth: 0
       - name: Install cbindgen
         run: |
+          git config --global safe.directory '*'
           git clone https://github.com/eqrion/cbindgen
           cd cbindgen/
           git checkout v0.20.0
@@ -187,11 +190,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2022-12-0.0.113-java-bindings
+          git checkout origin/2023-02-0.0.114-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.113
+          git checkout 0.0.114
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -263,6 +266,9 @@ jobs:
           chmod +x ./rustup.sh
           ./rustup.sh -y
           . $HOME/.cargo/env
+          # Temporarily use 1.67.1 to fix https://github.com/rust-lang/rust/issues/108943
+          rustup install 1.67.1
+          rustup default 1.67.1
           rustup target add armv7-linux-androideabi
           rustup target add aarch64-linux-android
           rustup target add i686-linux-android
@@ -273,6 +279,7 @@ jobs:
           fetch-depth: 0
       - name: Install android NDK compilers
         run: |
+          git config --global safe.directory '*'
           curl https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip > android-ndk-r22b-linux-x86_64.zip
           if [ "$(sha256sum android-ndk-r22b-linux-x86_64.zip | awk '{ print $1 }')" != "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b" ]; then
             echo "Bad hash"
@@ -290,11 +297,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2022-12-0.0.113-java-bindings
+          git checkout origin/2023-02-0.0.114-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.113
+          git checkout 0.0.114
       - name: Checkout Android AAR binaries and artifacts
         run: |
           # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
@@ -351,16 +358,21 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Temporarily use rustc from yesterday
+        run: |
+          rustup install 1.67.1
+          rustup default 1.67.1
       - name: Fetch upstream LLVM/clang snapshot
         run: |
-          wget -O clang+llvm-15.0.3-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.3/clang+llvm-15.0.3-x86_64-apple-darwin.tar.xz
-          if [ "$(shasum -a 256 clang+llvm-15.0.3-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "ac668586b2b3d068f1e43520a3ef0b1592e5dc3eff1a4a4b772e29803b428a69" ]; then
+          git config --global safe.directory '*'
+          wget -O clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz
+          if [ "$(shasum -a 256 clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz | awk '{ print $1 }')" != "d16b6d536364c5bec6583d12dd7e6cf841b9f508c4430d9ee886726bd9983f1c" ]; then
             echo "Bad hash"
             exit 1
           fi
       - name: Unpack upstream LLVM+clang and use it by default
         run: |
-          tar xvvf clang+llvm-15.0.3-x86_64-apple-darwin.tar.xz
+          tar xvvf clang+llvm-15.0.7-x86_64-apple-darwin21.0.tar.xz
       - name: Install cbindgen
         run: cargo install cbindgen
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
@@ -370,15 +382,18 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2022-12-0.0.113-java-bindings
+          git checkout origin/2023-02-0.0.114-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.113
+          git checkout 0.0.114
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
-          export PATH=`pwd`/clang+llvm-15.0.3-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-15.0.7-x86_64-apple-darwin21.0/bin:$PATH
           cd ldk-c-bindings
+          export CC=clang
+          export CC_x86_64_apple_darwin=clang
+          export CC_aarch64_apple_darwin=clang
           CC=clang ./genbindings.sh ../rust-lightning true
       - name: Fetch OpenJDK 18
         run: |
@@ -416,7 +431,7 @@ jobs:
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
           export JAVA_HOME=`pwd`/jdk-18.0.1.1.jdk/Contents/Home
           export PATH=$JAVA_HOME/bin:$PATH
-          export PATH=`pwd`/clang+llvm-15.0.3-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-15.0.7-x86_64-apple-darwin21.0/bin:$PATH
           ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false
 
           if [ "${{ matrix.platform }}" = "macos-11" ]; then

gen_type_mapping.py

@@ -43,14 +43,14 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
             else:
                 arr_name = "ret"
                 arr_len = ret_arr_len
-            if ty_info.c_ty == "int8_tArray":
+            if ty_info.c_ty == "int8_tArray" or ty_info.c_ty == "int16_tArray":
                 (set_pfx, set_sfx) = self.consts.set_native_arr_contents(arr_name + "_arr", arr_len, ty_info)
-                ret_conv = ("int8_tArray " + arr_name + "_arr = " + self.consts.create_native_arr_call(arr_len, ty_info) + ";\n" + set_pfx, "")
+                ret_conv = (ty_info.c_ty + " " + arr_name + "_arr = " + self.consts.create_native_arr_call(arr_len, ty_info) + ";\n" + set_pfx, "")
                 arg_conv_cleanup = None
                 if not arr_len.isdigit():
                     arg_conv = ty_info.rust_obj + " " + arr_name + "_ref;\n"
                     arg_conv = arg_conv + arr_name + "_ref." + arr_len + " = " +  self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + ";\n"
-                    if (not ty_info.is_ptr or not holds_ref) and ty_info.rust_obj != "LDKu8slice":
+                    if (not ty_info.is_ptr or not holds_ref) and (ty_info.rust_obj != "LDKu8slice" and ty_info.rust_obj != "LDKu16slice"):
                         arg_conv = arg_conv + arr_name + "_ref." + ty_info.arr_access + " = MALLOC(" + arr_name + "_ref." + arr_len + ", \"" + ty_info.rust_obj + " Bytes\");\n"
                         arg_conv = arg_conv + self.consts.get_native_arr_contents(arr_name, arr_name + "_ref." + ty_info.arr_access, arr_name + "_ref." + arr_len, ty_info, True) + ";"
                     else:
@@ -59,10 +59,10 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     if ty_info.rust_obj == "LDKTransaction" or ty_info.rust_obj == "LDKWitness":
                         arg_conv = arg_conv + "\n" + arr_name + "_ref.data_is_owned = " + str(holds_ref).lower() + ";"
                     ret_conv = (ty_info.rust_obj + " " + arr_name + "_var = ", "")
-                    ret_conv = (ret_conv[0], ";\nint8_tArray " + arr_name + "_arr = " + self.consts.create_native_arr_call(arr_name + "_var." + arr_len, ty_info) + ";\n")
+                    ret_conv = (ret_conv[0], ";\n" + ty_info.c_ty + " " + arr_name + "_arr = " + self.consts.create_native_arr_call(arr_name + "_var." + arr_len, ty_info) + ";\n")
                     (pfx, sfx) = self.consts.set_native_arr_contents(arr_name + "_arr", arr_name + "_var." + arr_len, ty_info)
                     ret_conv = (ret_conv[0], ret_conv[1] + pfx + arr_name + "_var." + ty_info.arr_access + sfx + ";")
-                    if not holds_ref and ty_info.rust_obj != "LDKu8slice":
+                    if not holds_ref and (ty_info.rust_obj != "LDKu8slice" and ty_info.rust_obj != "LDKu16slice"):
                         ret_conv = (ret_conv[0], ret_conv[1] + "\n" + ty_info.rust_obj.replace("LDK", "") + "_free(" + arr_name + "_var);")
                     from_hu_conv = self.consts.primitive_arr_from_hu(ty_info, None, arr_name)
                     to_hu_conv = self.consts.primitive_arr_to_hu(ty_info, None, arr_name, arr_name + "_conv")
@@ -74,10 +74,11 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     from_hu_conv = self.consts.primitive_arr_from_hu(ty_info, arr_len, arr_name)
                     to_hu_conv = self.consts.primitive_arr_to_hu(ty_info, None, arr_name, arr_name + "_conv")
                 else:
-                    arg_conv = "unsigned char " + arr_name + "_arr[" + arr_len + "];\n"
+                    # Note that we just blindly assume we should be using unsigned integers here.
+                    arg_conv = "u" + ty_info.subty.c_ty + " " + arr_name + "_arr[" + arr_len + "];\n"
                     arg_conv = arg_conv + "CHECK(" + self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + " == " + arr_len + ");\n"
                     arg_conv = arg_conv + self.consts.get_native_arr_contents(arr_name, arr_name + "_arr", arr_len, ty_info, True) + ";\n"
-                    arg_conv = arg_conv + "unsigned char (*" + arr_name + "_ref)[" + arr_len + "] = &" + arr_name + "_arr;"
+                    arg_conv = arg_conv + "u" + ty_info.subty.c_ty + " (*" + arr_name + "_ref)[" + arr_len + "] = &" + arr_name + "_arr;"
                     ret_conv = (ret_conv[0] + "*", set_sfx + ";")
                     from_hu_conv = self.consts.primitive_arr_from_hu(ty_info, arr_len, arr_name)
                     to_hu_conv = self.consts.primitive_arr_to_hu(ty_info, None, arr_name, arr_name + "_conv")
@@ -429,15 +430,13 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     # underlying unlike Vecs, and it gives Java more freedom.
                     base_conv = base_conv + "\nFREE(untag_ptr(" + ty_info.var_name + "));"
                 if ty_info.rust_obj in self.complex_enums:
-                    to_hu_conv_sfx = ""
                     if needs_full_clone and (ty_info.rust_obj.replace("LDK", "") + "_clone") not in self.clone_fns:
                         # We really need a full clone here, but for now we just implement
                         # a manual clone explicitly for Option<Trait>s
                         if ty_info.contains_trait:
                             assert ty_info.rust_obj.startswith("LDKCOption") # We don't support contained traits for anything else yet
                             optional_ty = ty_info.rust_obj[11:-1]
                             assert "LDK" + optional_ty in self.trait_structs # We don't support contained traits for anything else yet
-                            to_hu_conv_sfx = self.consts.add_ref("this", ty_info.var_name)
                             base_conv += "\nif (" + ty_info.var_name + "_conv.tag == " + ty_info.rust_obj + "_Some) {"
                             base_conv += "\n\t// Manually implement clone for Java trait instances"
                             optional_ty_info = self.java_c_types("LDK" + optional_ty + " " + ty_info.var_name, None)
@@ -451,7 +450,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                         ret_conv = (ret_conv[0], ";\n" + self.consts.ptr_c_ty + " " + ty_info.var_name + "_ref = tag_ptr(" + ty_info.var_name + "_copy, true);")
                     if from_hu_conv is None:
                         from_hu_conv = (self.consts.get_ptr(ty_info.var_name), "")
-                    from_hu_conv = (from_hu_conv[0], to_hu_conv_sfx)
+                    from_hu_conv = (from_hu_conv[0], self.consts.add_ref("this", ty_info.var_name))
                     fully_qualified_ty = self.consts.fully_qualified_hu_ty_path(ty_info)
                     to_hu_call = fully_qualified_ty + ".constr_from_ptr(" + ty_info.var_name + ")"
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,

genbindings.py

@@ -143,6 +143,11 @@ def java_c_types(fn_arg, ret_arr_len):
         assert var_is_arr_regex.match(fn_arg[8:])
         rust_obj = "LDKThirtyTwoBytes"
         arr_access = "data"
+    elif fn_arg.startswith("LDKEightU16s"):
+        fn_arg = "uint16_t (*" + fn_arg[13:] + ")[8]"
+        assert var_is_arr_regex.match(fn_arg[9:])
+        rust_obj = "LDKEightU16s"
+        arr_access = "data"
     elif fn_arg.startswith("LDKU128"):
         if fn_arg == "LDKU128":
             fn_arg = "LDKU128 arg"
@@ -760,13 +765,20 @@ def map_trait(struct_name, field_var_lines, trait_fn_lines, trait_doc_comment):
             for var_line in field_var_lines:
                 if var_line.group(1) in trait_structs:
                     field_var_convs.append((var_line.group(1), var_line.group(2), trait_structs[var_line.group(1)]))
-                    flattened_field_var_convs.append((var_line.group(1), var_line.group(2), ))
-                    flattened_field_var_convs.extend(trait_structs[var_line.group(1)])
+                    flattened_field_var_convs.append((var_line.group(1), var_line.group(2), var_line.group(2)))
+                    for field_var in trait_structs[var_line.group(1)]:
+                        if isinstance(field_var, ConvInfo):
+                            flattened_field_var_convs.append(field_var)
+                        else:
+                            path = var_line.group(2)
+                            if len(field_var) > 2:
+                                path = var_line.group(2) + "." + field_var[2]
+                            flattened_field_var_convs.append((field_var[0], field_var[1], path))
                 else:
                     mapped = type_mapping_generator.map_type(var_line.group(1) + " " + var_line.group(2), False, None, False, False)
                     field_var_convs.append(mapped)
                     flattened_field_var_convs.append(mapped)
-            trait_structs[struct_name] = field_var_convs
+            trait_structs[struct_name] = flattened_field_var_convs
 
             field_fns = []
             for fn_docs, fn_line in trait_fn_lines:

genbindings.sh

@@ -110,18 +110,18 @@ if [ "$2" = "c_sharp" ]; then
 	echo "Building C# bindings..."
 	COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -fPIC"
 	LINK="-ldl -shared"
-	[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined"
+	[ "$IS_MAC" = "false" ] && LINK="$LINK -Wl,--no-undefined"
 	[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9"
 	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && LINK="$LINK -fuse-ld=lld"
 	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && echo "WARNING: Need at least upstream clang 13!"
-	[ "$IS_MAC" = "false" -a "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free"
+	[ "$IS_MAC" = "false" -a "$3" != "false" ] && LINK="$LINK -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free"
 
 	exit 0 # Sadly compilation doesn't currently work
 	if [ "$3" = "true" ]; then
 		$COMPILE $LINK -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
 	else
-		$COMPILE -o bindings.o -c -flto -O3 -Wl,--version-script=c_sharp/libcode.version -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c
-		$COMPILE $LINK -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -Wl,--lto-O3 -Wl,-O3 -Wl,--version-script=c_sharp/libcode.version -I"$1"/lightning-c-bindings/include/ $2 bindings.o "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a -lm
+		$COMPILE -o bindings.o -c -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 c_sharp/bindings.c
+		$COMPILE $LINK -o liblightningjni_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=c_sharp/libcode.version -flto -O3 -Wl,--lto-O3 -Wl,-O3 -Wl,--version-script=c_sharp/libcode.version -I"$1"/lightning-c-bindings/include/ $2 bindings.o "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a -lm
 		llvm-strip liblightningjni_release$LDK_TARGET_SUFFIX.so
 	fi
 elif [ "$2" = "wasm" ]; then
@@ -249,17 +249,17 @@ else
 	echo "Building Java bindings..."
 	COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -fPIC"
 	LINK="-ldl -shared"
-	[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined"
+	[ "$IS_MAC" = "false" ] && LINK="$LINK -Wl,--no-undefined"
 	[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9"
 	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && LINK="$LINK -fuse-ld=lld"
 	[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && echo "WARNING: Need at least upstream clang 13!"
-	[ "$IS_MAC" = "false" -a "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free"
+	[ "$IS_MAC" = "false" -a "$3" != "false" ] && LINK="$LINK -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free"
 	if [ "$3" = "true" ]; then
 		$COMPILE $LINK -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
 	else
 		LDK_LIB="$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
 		if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then
-			COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld"
+			LINK="$LINK -Wl,--version-script=libcode.version -fuse-ld=lld"
 			# __cxa_thread_atexit_impl is used to more effeciently cleanup per-thread local storage by rust libstd.
 			# However, it is not available on glibc versions 2.17 or earlier, and rust libstd has a null-check and
 			# fallback in case it is missing.