Merge pull request #103 from TheBlueMatt/main

0.0.108.0

Author: TheBlueMatt

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install cargo libstd-rust-dev-wasm32 lld git g++ clang curl node-typescript npm python3
+          apt-get -y install cargo libstd-rust-dev-wasm32 lld git g++ clang curl node-typescript npm python3 wasi-libc
           echo "int main() {}" > genbindings_wasm_test_file.c
           clang -nostdlib -o /dev/null --target=wasm32-wasi -Wl,--no-entry genbindings_wasm_test_file.c
           rm genbindings_wasm_test_file.c
@@ -26,19 +26,24 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install cbindgen
-        run: cargo install --version 0.20.0 cbindgen
+        run: |
+          git clone https://github.com/eqrion/cbindgen
+          cd cbindgen/
+          git checkout v0.20.0
+          cargo update -p indexmap --precise "1.6.2" --verbose
+          cargo install --locked --path .
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2021-03-java-bindings-base
+          git checkout origin/2022-06-108-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.106
+          git checkout 0.0.108
       - name: Rebuild C bindings without STD
         run: |
           cd ldk-c-bindings
@@ -61,7 +66,7 @@ jobs:
           export HOME=/root/ # Github actions is apparently broken
           export PATH=$(pwd)/$(echo node-*/bin):$PATH
           # npx playwright install-deps is broken so we do it manually, see https://github.com/microsoft/playwright/issues/11165
-          apt-get install -y --no-install-recommends fonts-liberation libenchant-2-2 libicu67 libjpeg62-turbo libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libdrm2 libegl1 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb fonts-noto-color-emoji fonts-unifont libfontconfig libfreetype6 xfonts-scalable fonts-ipafont-gothic fonts-wqy-zenhei fonts-tlwg-loma-otf ffmpeg libcairo-gobject2 libdbus-glib-1-2 libfontconfig1 libgdk-pixbuf2.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 libxcb-shm0 libxcursor1 libxi6 libxrender1 libxt6 gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good libepoxy0 libevdev2 libgl1 libgles2 libgstreamer-gl1.0-0 libgstreamer1.0-0 libharfbuzz-icu0 libharfbuzz0b libhyphen0 libnotify4 libopenjp2-7 libopus0 libpng16-16 libsecret-1-0 libsoup2.4-1 libwayland-client0 libwayland-egl1 libwayland-server0 libwebp7 libwebpdemux2 libwoff1 libxkbcommon0 libxml2 libxslt1.1
+          apt-get install -y --no-install-recommends fonts-liberation libenchant-2-2 libicu71 libjpeg62-turbo libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libdrm2 libegl1 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb fonts-noto-color-emoji fonts-unifont libfontconfig libfreetype6 xfonts-scalable fonts-ipafont-gothic fonts-wqy-zenhei fonts-tlwg-loma-otf ffmpeg libcairo-gobject2 libdbus-glib-1-2 libfontconfig1 libgdk-pixbuf2.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 libxcb-shm0 libxcursor1 libxi6 libxrender1 libxt6 gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good libepoxy0 libevdev2 libgl1 libgles2 libgstreamer-gl1.0-0 libgstreamer1.0-0 libharfbuzz-icu0 libharfbuzz0b libhyphen0 libnotify4 libopenjp2-7 libopus0 libpng16-16 libsecret-1-0 libsoup2.4-1 libwayland-client0 libwayland-egl1 libwayland-server0 libwebp7 libwebpdemux2 libwoff1 libxkbcommon0 libxml2 libxslt1.1
           mkdir -p $HOME/.cache/ms-playwright # `playwright install` is too dumb to create this for us
           chmod -R 777 $HOME/
           npm i -D playwright
@@ -110,19 +115,24 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install cbindgen
-        run: cargo install --version 0.20.0 cbindgen
+        run: |
+          git clone https://github.com/eqrion/cbindgen
+          cd cbindgen/
+          git checkout v0.20.0
+          cargo update -p indexmap --precise "1.6.2" --verbose
+          cargo install --locked --path .
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2021-03-java-bindings-base
+          git checkout origin/2022-06-108-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.106
+          git checkout 0.0.108
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -160,19 +170,24 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install cbindgen
-        run: cargo install --version 0.20.0 cbindgen
+        run: |
+          git clone https://github.com/eqrion/cbindgen
+          cd cbindgen/
+          git checkout v0.20.0
+          cargo update -p indexmap --precise "1.6.2" --verbose
+          cargo install --locked --path .
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2021-03-java-bindings-base
+          git checkout origin/2022-06-108-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.106
+          git checkout 0.0.108
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
@@ -263,19 +278,19 @@ jobs:
       - name: Install cbindgen
         run: |
           . $HOME/.cargo/env
-          cargo install --version 0.20.0 cbindgen
+          cargo install cbindgen
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2021-03-java-bindings-base
+          git checkout origin/2022-06-108-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.106
+          git checkout 0.0.108
       - name: Checkout Android AAR binaries and artifacts
         run: |
           # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
@@ -330,35 +345,35 @@ jobs:
         run: rustup target install aarch64-apple-darwin
       - name: Fetch upstream LLVM/clang snapshot
         run: |
-          wget -O clang+llvm-14.0.1-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.1/clang+llvm-14.0.1-x86_64-apple-darwin.tar.xz
-          if [ "$(shasum -a 256 clang+llvm-14.0.1-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "43149390e95b1cdbf1d4ef2e9d214bbb6d35858ceb2df27245868e06bc4fc44c" ]; then
+          wget -O clang+llvm-14.0.5-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.5/clang+llvm-14.0.5-x86_64-apple-darwin.tar.xz
+          if [ "$(shasum -a 256 clang+llvm-14.0.5-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "66cf1b8e00289a567b2f5f740f068b7682e27ccf048647b836d3624376a64705" ]; then
             echo "Bad hash"
             exit 1
           fi
       - name: Unpack upstream LLVM+clang and use it by default
         run: |
-          tar xvvf clang+llvm-14.0.1-x86_64-apple-darwin.tar.xz
+          tar xvvf clang+llvm-14.0.5-x86_64-apple-darwin.tar.xz
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - name: Install cbindgen
-        run: cargo install --version 0.20.0 cbindgen
+        run: cargo install cbindgen
       - name: Checkout Rust-Lightning and LDK-C-Bindings git
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "LDK CI"
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git checkout origin/2021-03-java-bindings-base
+          git checkout origin/2022-06-108-java-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.106
+          git checkout 0.0.108
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
-          export PATH=`pwd`/clang+llvm-14.0.1-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-14.0.5-x86_64-apple-darwin/bin:$PATH
           cd ldk-c-bindings
           CC=clang ./genbindings.sh ../rust-lightning true
       - name: Fetch OpenJDK 18
@@ -397,7 +412,7 @@ jobs:
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
           export JAVA_HOME=`pwd`/jdk-18.0.1.1.jdk/Contents/Home
           export PATH=$JAVA_HOME/bin:$PATH
-          export PATH=`pwd`/clang+llvm-14.0.1-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-14.0.5-x86_64-apple-darwin/bin:$PATH
           ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false
 
           if [ "${{ matrix.platform }}" = "macos-11" ]; then

gen_type_mapping.py

@@ -337,20 +337,21 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 to_hu_conv_sfx = ""
                 if not ty_info.is_ptr or holds_ref:
                     to_hu_conv_sfx = "\n" + self.consts.add_ref(ty_info.var_name + "_hu_conv", "this") + ";"
+                qualified_hu_ty = self.consts.fully_qualified_hu_ty_path(ty_info)
                 if ty_info.is_ptr:
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = opaque_arg_conv, arg_conv_name = "&" + ty_info.var_name + "_conv", arg_conv_cleanup = None,
                         ret_conv = (ty_info.rust_obj + " " + ty_info.var_name + "_var = *", opaque_ret_conv_suf),
                         ret_conv_name = ty_info.var_name + "_ref",
-                        to_hu_conv = self.consts.to_hu_conv_templates['ptr'].replace('{human_type}', ty_info.java_hu_ty).replace('{var_name}', ty_info.var_name) + to_hu_conv_sfx,
+                        to_hu_conv = self.consts.to_hu_conv_templates['ptr'].replace('{human_type}', qualified_hu_ty).replace('{var_name}', ty_info.var_name) + to_hu_conv_sfx,
                         to_hu_conv_name = ty_info.var_name + "_hu_conv",
                         from_hu_conv = from_hu_conv)
                 else:
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = opaque_arg_conv, arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
                         ret_conv = (ty_info.rust_obj + " " + ty_info.var_name + "_var = ", opaque_ret_conv_suf),
                         ret_conv_name = ty_info.var_name + "_ref",
-                        to_hu_conv = self.consts.to_hu_conv_templates['default'].replace('{human_type}', ty_info.java_hu_ty).replace('{var_name}', ty_info.var_name) + to_hu_conv_sfx,
+                        to_hu_conv = self.consts.to_hu_conv_templates['default'].replace('{human_type}', qualified_hu_ty).replace('{var_name}', ty_info.var_name) + to_hu_conv_sfx,
                         to_hu_conv_name = ty_info.var_name + "_hu_conv",
                         from_hu_conv = from_hu_conv)
 
@@ -503,10 +504,10 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                         to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, ty_info.var_name + "_conv", "new " + ty_info.java_hu_ty + "(" + ty_info.var_name + ")") + ";",
                         to_hu_conv_name = ty_info.var_name + "_conv", from_hu_conv = ("0", ""))
 
-                if ty_info.rust_obj == "LDKu5":
+                if ty_info.rust_obj == "LDKu5" or ty_info.rust_obj == "LDKWitnessVersion":
                     assert from_hu_conv is None
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
-                        arg_conv = "", arg_conv_name = "(LDKu5){ ._0 = " + ty_info.var_name + " }", arg_conv_cleanup = None,
+                        arg_conv = "", arg_conv_name = "(" + ty_info.rust_obj + "){ ._0 = " + ty_info.var_name + " }", arg_conv_cleanup = None,
                         ret_conv = ("uint8_t " + ty_info.var_name + "_val = ", "._0;"), ret_conv_name = ty_info.var_name + "_val",
                         to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, ty_info.var_name + "_conv", "new " + ty_info.java_hu_ty + "(" + ty_info.var_name + ")") + ";",
                         to_hu_conv_name = ty_info.var_name + "_conv", from_hu_conv = (ty_info.var_name + ".getVal()", ""))

genbindings.py

@@ -263,14 +263,19 @@ def java_c_types(fn_arg, ret_arr_len):
         arr_ty = "uint8_t"
         fn_arg = fn_arg[7:].strip()
         is_primitive = True
-    elif fn_arg.startswith("LDKu5"):
+    elif fn_arg.startswith("LDKu5") or fn_arg.startswith("LDKWitnessVersion"):
         java_ty = consts.c_type_map['uint8_t'][0]
-        java_hu_ty = "UInt5"
-        rust_obj = "LDKu5"
+        if fn_arg.startswith("LDKu5"):
+            java_hu_ty = "UInt5"
+            rust_obj = "LDKu5"
+            fn_arg = fn_arg[6:].strip()
+        else:
+            java_hu_ty = "WitnessVersion"
+            rust_obj = "LDKWitnessVersion"
+            fn_arg = fn_arg[18:].strip()
         c_ty = "int8_t"
         arr_ty = "uint8_t"
         fn_ty_arg = "B"
-        fn_arg = fn_arg[6:].strip()
     elif fn_arg.startswith("uint16_t"):
         mapped_type = consts.c_type_map['uint16_t']
         java_ty = mapped_type[0]
@@ -648,7 +653,13 @@ def map_complex_enum(struct_name, union_enum_items, inline_enum_variants, enum_d
                 elif camel_to_snake(variant_name) in inline_enum_variants:
                     # TODO: If we ever have a rust enum Variant(Option<Struct>) we need to pipe
                     # docs through to there, and then potentially mark the field nullable.
-                    mapped = type_mapping_generator.map_type(inline_enum_variants[camel_to_snake(variant_name)] + " " + camel_to_snake(variant_name), False, None, False, True)
+                    (variant_info, variant_field_docs) = inline_enum_variants[camel_to_snake(variant_name)]
+                    variant_ty_text = variant_info + " " + camel_to_snake(variant_name)
+                    variant_ty_info = type_mapping_generator.java_c_types(variant_ty_text, None)
+                    if variant_field_docs is not None and doc_to_field_nullable(variant_field_docs):
+                        mapped = type_mapping_generator.map_type_with_info(variant_ty_info, False, None, False, True, True)
+                    else:
+                        mapped = type_mapping_generator.map_type_with_info(variant_ty_info, False, None, False, True, False)
                     contains_trait |= mapped.ty_info.contains_trait
                     fields.append((mapped, None))
                     enum_variants.append(ComplexEnumVariantInfo(variant_name, variant_docs, fields, True))
@@ -939,7 +950,7 @@ def map_tuple(struct_name, field_lines):
                 elif struct_name in union_enum_items:
                     tuple_variants = {}
                     elem_items = -1
-                    for line, _ in field_lines:
+                    for line, field_block_comment in field_lines:
                         if line == "      struct {":
                             elem_items = 0
                         elif line == "      };":
@@ -952,7 +963,7 @@ def map_tuple(struct_name, field_lines):
                                 line = line[5:]
                             split = line.split(" ")
                             assert len(split) == 2
-                            tuple_variants[split[1].strip(";")] = split[0]
+                            tuple_variants[split[1].strip(";")] = (split[0], field_block_comment)
                             elem_items += 1
                             if elem_items > 1:
                                 # We don't currently support tuple variant with more than one element

genbindings.sh

@@ -144,17 +144,17 @@ if [ "$2" != "wasm" ]; then
 				echo "Archive contained non-object files!"
 				exit 1
 			fi
-			if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep ldk.ldk.*-cgu.*.rcgu.o | wc -l)" != "1" ]; then
+			if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep ldk.*-cgu.*.rcgu.o | wc -l)" != "1" ]; then
 				echo "Archive contained more than one LDK object file"
 				exit 1
 			fi
 			mkdir -p tmp
 			rm -f tmp/*
 			ar x --output=tmp "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
 			pushd tmp
-			llvm-dis ldk.ldk.*-cgu.*.rcgu.o
-			sed -i 's/br i1 icmp eq (i8\* @__cxa_thread_atexit_impl, i8\* null)/br i1 icmp eq (i8* null, i8* null)/g' ldk.ldk.*-cgu.*.rcgu.o.ll
-			llvm-as ldk.ldk.*-cgu.*.rcgu.o.ll -o ./libldk.bc
+			llvm-dis ldk*-cgu.*.rcgu.o
+			sed -i 's/br i1 icmp eq (i8\* @__cxa_thread_atexit_impl, i8\* null)/br i1 icmp eq (i8* null, i8* null)/g' ldk*-cgu.*.rcgu.o.ll
+			llvm-as ldk*-cgu.*.rcgu.o.ll -o ./libldk.bc
 			ar q libldk.a *.o
 			popd
 			LDK_LIB="tmp/libldk.bc tmp/libldk.a"
@@ -203,8 +203,7 @@ else
 	echo "Building TS bindings..."
 	COMPILE="$COMMON_COMPILE -flto -Wl,--no-entry -nostdlib --target=wasm32-wasi -Wl,-z -Wl,stack-size=$((8*1024*1024)) -Wl,--initial-memory=$((16*1024*1024)) -Wl,--max-memory=$((1024*1024*1024)) -Wl,--global-base=4096"
 	# We only need malloc and assert/abort, but for now just use WASI for those:
-	#EXTRA_LINK=/usr/lib/wasm32-wasi/libc.a
-	EXTRA_LINK=
+	EXTRA_LINK=/usr/lib/wasm32-wasi/libc.a
 	[ "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,aligned_alloc -Wl,-wrap,free"
 	if [ "$3" = "true" ]; then
 		WASM_FILE=liblightningjs_debug.wasm