Merge pull request #74 from TheBlueMatt/2022-01-ts-hu-files

[TypeScript] Make Basic "Human" Object Construction Work

Author: TheBlueMatt

.github/workflows/build.yml

@@ -50,7 +50,7 @@ jobs:
           # Thus, we install the binary nodejs from nodejs.org and test with that.
           curl https://nodejs.org/dist/v16.13.1/node-v16.13.1-linux-x64.tar.xz > nodejs.tar.xz
           tar xvvf nodejs.tar.xz
-          export PATH=$(echo node-*/bin):$PATH
+          export PATH=$(pwd)/$(echo node-*/bin):$PATH
           ./genbindings.sh ./ldk-c-bindings/ wasm false false
       - name: Build and Test TS Bindings for Web
         run: |
@@ -63,9 +63,10 @@ jobs:
           mkdir -p $HOME/.cache/ms-playwright # `playwright install` is too dumb to create this for us
           chmod -R 777 $HOME/
           npx playwright install
-          export PATH=$(echo node-*/bin):$PATH
+          export PATH=$(pwd)/$(echo node-*/bin):$PATH
+          cd ts
           python3 -m http.server &
-          node ts/test/browser.mjs
+          node test/browser.mjs
       - name: Check latest TS files are in git
         run: |
           git diff --exit-code
@@ -104,8 +105,6 @@ jobs:
       - name: Rebuild C bindings, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
-          # Reset the Cargo.toml file so that git describe doesn't think we're "-dirty"
-          git checkout lightning-c-bindings/Cargo.toml
           ./genbindings.sh ../rust-lightning true
       - name: Build Java Debug Bindings
         run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false
@@ -114,6 +113,49 @@ jobs:
           mv liblightningjni_debug_Linux-amd64.so liblightningjni.so
           export ASAN_OPTIONS=detect_leaks=0
           LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test
+      - name: Build Java Release Bindings
+        run: |
+          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
+      - name: Check latest headers are in git
+        run: |
+          git checkout pom.xml
+          git diff --exit-code
+
+  java_determinism:
+    runs-on: ubuntu-latest
+    # Ubuntu's version of rustc uses its own LLVM instead of being a real native package.
+    # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS.
+    container: debian:bullseye
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Install native Rust toolchain, Valgrind, and build utilitis
+        run: |
+          apt-get update
+          apt-get -y dist-upgrade
+          apt-get -y install cargo valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl
+      - name: Checkout source code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
+      - name: Checkout Rust-Lightning and LDK-C-Bindings git
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "LDK CI"
+          # Note this is a different endpoint, as we need one non-upstream commit!
+          git clone https://git.bitcoin.ninja/rust-lightning
+          cd rust-lightning
+          git checkout origin/2021-03-java-bindings-base
+          cd ..
+          git clone https://github.com/lightningdevkit/ldk-c-bindings
+          cd ldk-c-bindings
+          git checkout 0.0.104
+      - name: Rebuild C bindings, and check the sample app builds + links
+        run: |
+          cd ldk-c-bindings
+          ./genbindings.sh ../rust-lightning true
       - name: Checkout latest MacOS binaries
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
@@ -135,10 +177,6 @@ jobs:
           ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
       - name: Build deterministic release jar
         run: ./build-release-jar.sh
-      - name: Check latest headers are in git
-        run: |
-          git checkout pom.xml
-          git diff --exit-code
       - name: Check latest library and jars are in bins repo
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"

README.md

@@ -39,7 +39,7 @@ To build for Apple M1 (ie aarch64-apple-darwin), you probably want something lik
 Status
 ======
 
-The TypeScript Bindings are still in early development and generated code contains syntax errors.
+## Java
 
 While the underlying library and C bindings are relatively mature, the Java bindings should be
 considered beta quality and some issues may still appear. Specifically, because the Java bindings
@@ -53,6 +53,17 @@ exit there will likely be many false positives. While it will require some compl
 are likely some use-after-free or unkonwn-free bugs remaining. The debug-mode build links LLVM
 address sanitizer and will print diagnostic information in case of such issues.
 
+## TypeScript
+
+The TypeScript bindings are functionally complete, but should be considered early alpha quality.
+Some functions may error spuriously due to oversights or missing implementations.
+
+The TypeScript bindings require modern web standards, including support for `FinalizationRegistry`
+and `WeakRef` (Chrome 84, Firefox 79, Safari 14.1/iOS 14.5 and Node 14.6) and WASM BigInt support
+(Chrome 85, Firefox 78, Safari 14.1/iOS 14.5, and Node ??).
+
+## General
+
 The only known issue resulting in a use-after-free bug requires custom a custom ChannelKeys instance
 created as a part of a new channel. After the channel is created, the ChannelKeys object will not
 be freed while the parent ChannelManager exists, however if the ChannelManager is garbage collected

gen_type_mapping.py

@@ -47,7 +47,6 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 (set_pfx, set_sfx) = self.consts.set_native_arr_contents(arr_name + "_arr", arr_len, ty_info)
                 ret_conv = ("int8_tArray " + arr_name + "_arr = " + self.consts.create_native_arr_call(arr_len, ty_info) + ";\n" + set_pfx, "")
                 arg_conv_cleanup = None
-                from_hu_conv = None
                 if not arr_len.isdigit():
                     arg_conv = ty_info.rust_obj + " " + arr_name + "_ref;\n"
                     arg_conv = arg_conv + arr_name + "_ref." + arr_len + " = " +  self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + ";\n"
@@ -65,26 +64,34 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     ret_conv = (ret_conv[0], ret_conv[1] + pfx + arr_name + "_var." + ty_info.arr_access + sfx + ";")
                     if not holds_ref and ty_info.rust_obj != "LDKu8slice":
                         ret_conv = (ret_conv[0], ret_conv[1] + "\n" + ty_info.rust_obj.replace("LDK", "") + "_free(" + arr_name + "_var);")
+                    from_hu_conv = self.consts.primitive_arr_from_hu(ty_info.subty, None, arr_name)
+                    to_hu_conv = self.consts.primitive_arr_to_hu(ty_info.subty, None, arr_name, arr_name + "_conv")
                 elif ty_info.rust_obj is not None:
                     arg_conv = ty_info.rust_obj + " " + arr_name + "_ref;\n"
                     arg_conv = arg_conv + "CHECK(" + self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + " == " + arr_len + ");\n"
                     arg_conv = arg_conv + self.consts.get_native_arr_contents(arr_name, arr_name + "_ref." + ty_info.arr_access, arr_len, ty_info, True) + ";"
                     ret_conv = (ret_conv[0], "." + ty_info.arr_access + set_sfx + ";")
-                    from_hu_conv = ("InternalUtils.check_arr_len(" + arr_name + ", " + arr_len + ")", "")
+                    from_hu_conv = self.consts.primitive_arr_from_hu(ty_info.subty, arr_len, arr_name)
+                    to_hu_conv = self.consts.primitive_arr_to_hu(ty_info.subty, None, arr_name, arr_name + "_conv")
                 else:
                     arg_conv = "unsigned char " + arr_name + "_arr[" + arr_len + "];\n"
                     arg_conv = arg_conv + "CHECK(" + self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + " == " + arr_len + ");\n"
                     arg_conv = arg_conv + self.consts.get_native_arr_contents(arr_name, arr_name + "_arr", arr_len, ty_info, True) + ";\n"
                     arg_conv = arg_conv + "unsigned char (*" + arr_name + "_ref)[" + arr_len + "] = &" + arr_name + "_arr;"
                     ret_conv = (ret_conv[0] + "*", set_sfx + ";")
-                    from_hu_conv = ("InternalUtils.check_arr_len(" + arr_name + ", " + arr_len + ")", "")
+                    from_hu_conv = self.consts.primitive_arr_from_hu(ty_info.subty, arr_len, arr_name)
+                    to_hu_conv = self.consts.primitive_arr_to_hu(ty_info.subty, None, arr_name, arr_name + "_conv")
+                to_hu_conv_name = None
+                if to_hu_conv is not None:
+                    to_hu_conv_name = arr_name + "_conv"
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arr_name + "_ref", arg_conv_cleanup = arg_conv_cleanup,
-                    ret_conv = ret_conv, ret_conv_name = arr_name + "_arr", to_hu_conv = None, to_hu_conv_name = None,
+                    ret_conv = ret_conv, ret_conv_name = arr_name + "_arr",
+                    to_hu_conv = to_hu_conv, to_hu_conv_name = to_hu_conv_name,
                     from_hu_conv = from_hu_conv)
             else:
                 assert not arr_len.isdigit() # fixed length arrays not implemented
-                assert ty_info.java_ty[len(ty_info.java_ty) - 2:] == "[]"
+                assert ty_info.java_hu_ty[len(ty_info.java_hu_ty) - 2:] == "[]"
                 if arr_name == "":
                     arr_name = "ret"
                 conv_name = arr_name + "_conv_" + str(len(ty_info.java_hu_ty))
@@ -183,24 +190,30 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 to_hu_conv = None
                 to_hu_conv_name = None
                 if subty.to_hu_conv is not None:
-                    to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, conv_name + "_arr", self.consts.constr_hu_array(ty_info, arr_name + ".length"))
-                    to_hu_conv += ";\n" + self.consts.for_n_in_range(idxc, "0", arr_name + ".length") + "\n"
-                    to_hu_conv += "\t" + self.consts.var_decl_statement(subty.java_ty, conv_name, arr_name + "[" + idxc + "]") + ";\n"
+                    to_hu_conv = self.consts.var_decl_statement(self.consts.c_type_map["uint32_t"][0], conv_name + "_len", self.consts.get_java_arr_len(arr_name)) + ";\n"
+                    to_hu_conv += self.consts.var_decl_statement(ty_info.java_hu_ty, conv_name + "_arr", self.consts.constr_hu_array(ty_info, conv_name + "_len"))
+                    to_hu_conv += ";\n" + self.consts.for_n_in_range(idxc, "0", conv_name + "_len") + "\n"
+                    to_hu_conv += "\t" + self.consts.var_decl_statement(subty.java_ty, conv_name, self.consts.get_java_arr_elem(subty, arr_name, idxc)) + ";\n"
                     to_hu_conv += "\t" + subty.to_hu_conv.replace("\n", "\n\t") + "\n"
                     to_hu_conv += "\t" + conv_name + "_arr[" + idxc + "] = " + subty.to_hu_conv_name + ";\n}"
                     to_hu_conv_name = conv_name + "_arr"
-                from_hu_conv = None
+                from_hu_conv = self.consts.primitive_arr_from_hu(ty_info.subty, None, arr_name)
                 if subty.from_hu_conv is not None:
                     hu_conv_b = ""
                     if subty.from_hu_conv[1] != "":
                         iterator = self.consts.for_n_in_arr(conv_name, arr_name, subty)
                         hu_conv_b = iterator[0] + subty.from_hu_conv[1] + ";" + iterator[1]
-                    from_hu_conv = (self.consts.map_hu_array_elems(arr_name, conv_name, ty_info, subty), hu_conv_b)
+                    if from_hu_conv is not None:
+                        arr_conv = self.consts.primitive_arr_from_hu(ty_info.subty, None, self.consts.map_hu_array_elems(arr_name, conv_name, ty_info, subty))
+                        assert arr_conv[1] == ""
+                        from_hu_conv = (arr_conv[0], hu_conv_b)
+                    else:
+                        from_hu_conv = (self.consts.map_hu_array_elems(arr_name, conv_name, ty_info, subty), hu_conv_b)
 
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = arg_conv_cleanup,
                     ret_conv = ret_conv, ret_conv_name = arr_name + "_arr", to_hu_conv = to_hu_conv, to_hu_conv_name = to_hu_conv_name, from_hu_conv = from_hu_conv)
-        elif ty_info.java_ty == "String":
+        elif ty_info.java_fn_ty_arg == "Ljava/lang/String;":
             assert not is_nullable
             if not is_free:
                 arg_conv = "LDKStr " + ty_info.var_name + "_conv = " + self.consts.str_ref_to_c_call(ty_info.var_name) + ";"
@@ -209,6 +222,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 arg_conv = "LDKStr dummy = { .chars = NULL, .len = 0, .chars_is_owned = false };"
                 arg_conv_name = "dummy"
             if ty_info.arr_access is None:
+                assert False
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = None,
                     ret_conv = ("const char* " + ty_info.var_name + "_str = ",
@@ -219,11 +233,17 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 free_str = ""
                 if not holds_ref:
                     free_str = "\nStr_free(" + ty_info.var_name + "_str);"
+                to_hu_conv = self.consts.str_to_hu_conv(ty_info.var_name)
+                to_hu_conv_name = None
+                if to_hu_conv is not None:
+                    to_hu_conv_name = ty_info.var_name + "_conv"
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = None,
                     ret_conv = ("LDKStr " + ty_info.var_name + "_str = ",
                         ";\njstring " + ty_info.var_name + "_conv = " + self.consts.str_ref_to_native_call(ty_info.var_name + "_str." + ty_info.arr_access, ty_info.var_name + "_str." + ty_info.arr_len) + ";" + free_str),
-                    ret_conv_name = ty_info.var_name + "_conv", to_hu_conv = None, to_hu_conv_name = None, from_hu_conv = None)
+                    ret_conv_name = ty_info.var_name + "_conv",
+                    to_hu_conv=to_hu_conv, to_hu_conv_name=to_hu_conv_name,
+                    from_hu_conv = self.consts.str_from_hu_conv(ty_info.var_name))
         elif ty_info.var_name == "" and not print_void:
             # We don't have a parameter name, and want one, just call it arg
             if not ty_info.is_native_primitive:
@@ -534,9 +554,12 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                             to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, "ret_hu_conv", "new " + ty_info.java_hu_ty + "(null, " + ty_info.var_name + ")") + ";\n" + self.consts.add_ref("ret_hu_conv", "this") + ";",
                             to_hu_conv_name = "ret_hu_conv",
                             from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), self.consts.add_ref("this", ty_info.var_name)))
+                ret_conv = ("uint64_t ret_" + ty_info.var_name + " = (uint64_t)", ";")
+                if holds_ref:
+                    ret_conv = (ret_conv[0], " | 1;")
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = ty_info.rust_obj + "* " + ty_info.var_name + "_conv = (" + ty_info.rust_obj + "*)(" + ty_info.var_name + " & ~1);",
                     arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
-                    ret_conv = ("uint64_t ret_" + ty_info.var_name + " = (uint64_t)", ";"), ret_conv_name = "ret_" + ty_info.var_name,
+                    ret_conv = ret_conv, ret_conv_name = "ret_" + ty_info.var_name,
                     to_hu_conv = "TODO 3", to_hu_conv_name = None, from_hu_conv = None) # its a pointer, no conv needed
             assert False # We should have handled every case by now.