Merge bitcoin/bitcoin#31741: multiprocess: Add libmultiprocess git subtree

babb9f5 depends: remove non-native libmultiprocess build (Cory Fields)
5d105fb depends: Switch libmultiprocess packages to use local git subtree (Ryan Ofsky)
9b35518 depends, moveonly: split up int_get_build_id function (Ryan Ofsky)
2d373e2 lint: Add exclusions for libmultiprocess subtree (Ryan Ofsky)
e88ab39 doc: Update documentation to explain libmultiprocess subtree (Ryan Ofsky)
d4bc563 cmake: Fix clang-tidy "no input files" errors (Ryan Ofsky)
abdf3cb cmake: Fix warnings from boost headers (Ryan Ofsky)
8532fcb cmake: Fix ctest mptest "Unable to find executable" errors (Ryan Ofsky)
d597ab1 cmake: Support building with libmultiprocess subtree (Ryan Ofsky)
69f0d4a scripted-diff: s/WITH_MULTIPROCESS/ENABLE_IPC/ in cmake (Ryan Ofsky)
a2f28e4 Squashed 'src/ipc/libmultiprocess/' content from commit 35944ffd23fa (Ryan Ofsky)
d6244f8 depends: Update libmultiprocess library to simplify cmake subtree build (Ryan Ofsky)

Pull request description:

  This adds the [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library and code generator as a subtree in `src/ipc/libmultiprocess` and allows it to be built with the cmake `-DENABLE_IPC` option, which is disabled by default.

  This PR does not entirely remove the depends system [libmultiprocess package](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/native_libmultiprocess.mk) because the package is useful when cross compiling. (A cross-compiling cmake build cannot easily build and run a native code generation tool.) However, it does update the depends package to build from the new git subtree, instead of being downloaded separately from github, so the same sources are used to build both the runtime library and the code generator.

  This PR includes the following manual changes (not created automatically with `git subtree add`) which just update the build system and documentation:

  - [`d6244f85c509` depends: Update libmultiprocess library to simplify cmake subtree build](bitcoin/bitcoin@d6244f8)
  - [`69f0d4adb72c` scripted-diff: s/WITH_MULTIPROCESS/ENABLE_IPC/ in cmake](bitcoin/bitcoin@69f0d4a)
  - [`d597ab1dee6b` cmake: Support building with libmultiprocess subtree](bitcoin/bitcoin@d597ab1)
  - [`8532fcb1c30d` cmake: Fix ctest mptest "Unable to find executable" errors](bitcoin/bitcoin@8532fcb)
  - [`abdf3cb6456f` cmake: Fix warnings from boost headers](bitcoin/bitcoin@abdf3cb)
  - [`d4bc5639829f` cmake: Fix clang-tidy "no input files" errors](bitcoin/bitcoin@d4bc563)
  - [`e88ab394c163` doc: Update documentation to explain libmultiprocess subtree](bitcoin/bitcoin@e88ab39)
  - [`2d373e27071f` lint: Add exclusions for libmultiprocess subtree](bitcoin/bitcoin@2d373e2)
  - [`9b35518d2f3f` depends, moveonly: split up int_get_build_id function](bitcoin/bitcoin@9b35518)
  - [`5d105fb8c3ff` depends: Switch libmultiprocess packages to use local git subtree](bitcoin/bitcoin@5d105fb)
  - [`babb9f5db641` depends: remove non-native libmultiprocess build](bitcoin/bitcoin@babb9f5)

  ---

  Previous minisketch subtree PR #23114 may be useful for comparison

  Instructions for subtree verification can be found:

  - https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#subtrees
  - https://github.com/bitcoin/bitcoin/tree/master/test/lint#git-subtree-checksh

  TL&DR:

  ```sh
  git remote add --fetch libmultiprocess https://github.com/chaincodelabs/libmultiprocess.git
  test/lint/git-subtree-check.sh -r src/ipc/libmultiprocess
  ```

  ---

  This PR is part of the [process separation project](bitcoin/bitcoin#28722).

ACKs for top commit:
  Sjors:
    re-ACK babb9f5
  TheCharlatan:
    tACK babb9f5
  vasild:
    ACK babb9f5

Tree-SHA512: 43d4eecca5aab63e55c613de935965666eaced327f9fe859a0e9c9b85f7685dc16c5c8d6e03e09ca998628c5d468633f4f743529930b037049abe8e0101e0143

Author: fanquake

CMakeLists.txt

@@ -94,7 +94,7 @@ option(BUILD_DAEMON "Build bitcoind executable." ON)
 option(BUILD_GUI "Build bitcoin-qt executable." OFF)
 option(BUILD_CLI "Build bitcoin-cli executable." ON)
 
-option(BUILD_TESTS "Build test_bitcoin executable." ON)
+option(BUILD_TESTS "Build test_bitcoin and other unit test executables." ON)
 option(BUILD_TX "Build bitcoin-tx executable." ${BUILD_TESTS})
 option(BUILD_UTIL "Build bitcoin-util executable." ${BUILD_TESTS})
 
@@ -153,8 +153,9 @@ endif()
 
 cmake_dependent_option(WITH_DBUS "Enable DBus support." ON "CMAKE_SYSTEM_NAME STREQUAL \"Linux\" AND BUILD_GUI" OFF)
 
-option(WITH_MULTIPROCESS "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental." OFF)
-if(WITH_MULTIPROCESS)
+option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental." OFF)
+cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful when cross-compiling or making changes to the upstream project." OFF "ENABLE_IPC" OFF)
+if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS)
   find_package(Libmultiprocess REQUIRED COMPONENTS Lib)
   find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin
     NAMES Libmultiprocess
@@ -648,14 +649,14 @@ message("Configure summary")
 message("=================")
 message("Executables:")
 message("  bitcoind ............................ ${BUILD_DAEMON}")
-if(BUILD_DAEMON AND WITH_MULTIPROCESS)
+if(BUILD_DAEMON AND ENABLE_IPC)
   set(bitcoin_daemon_status ON)
 else()
   set(bitcoin_daemon_status OFF)
 endif()
 message("  bitcoin-node (multiprocess) ......... ${bitcoin_daemon_status}")
 message("  bitcoin-qt (GUI) .................... ${BUILD_GUI}")
-if(BUILD_GUI AND WITH_MULTIPROCESS)
+if(BUILD_GUI AND ENABLE_IPC)
   set(bitcoin_gui_status ON)
 else()
   set(bitcoin_gui_status OFF)
@@ -674,6 +675,16 @@ if(ENABLE_WALLET)
 endif()
 message("  external signer ..................... ${ENABLE_EXTERNAL_SIGNER}")
 message("  ZeroMQ .............................. ${WITH_ZMQ}")
+if(ENABLE_IPC)
+  if (WITH_EXTERNAL_LIBMULTIPROCESS)
+    set(ipc_status "ON (with external libmultiprocess)")
+  else()
+    set(ipc_status ON)
+  endif()
+else()
+  set(ipc_status OFF)
+endif()
+message("  IPC ................................. ${ipc_status}")
 message("  USDT tracing ........................ ${WITH_USDT}")
 message("  QR code (GUI) ....................... ${WITH_QRENCODE}")
 message("  DBus (GUI, Linux only) .............. ${WITH_DBUS}")

CMakePresets.json

@@ -81,7 +81,7 @@
         "ENABLE_WALLET": "ON",
         "WARN_INCOMPATIBLE_BDB": "OFF",
         "WITH_BDB": "ON",
-        "WITH_MULTIPROCESS": "ON",
+        "ENABLE_IPC": "ON",
         "WITH_QRENCODE": "ON",
         "WITH_USDT": "ON",
         "WITH_ZMQ": "ON"

cmake/libmultiprocess.cmake

@@ -0,0 +1,38 @@
+# Copyright (c) 2025 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+function(add_libmultiprocess subdir)
+  # Set BUILD_TESTING to match BUILD_TESTS. BUILD_TESTING is a standard cmake
+  # option that controls whether enable_testing() is called, but in the bitcoin
+  # build a BUILD_TESTS option is used instead.
+  set(BUILD_TESTING "${BUILD_TESTS}")
+  add_subdirectory(${subdir} EXCLUDE_FROM_ALL)
+  # Apply core_interface compile options to libmultiprocess runtime library.
+  target_link_libraries(multiprocess PUBLIC $<BUILD_INTERFACE:core_interface>)
+  target_link_libraries(mputil PUBLIC $<BUILD_INTERFACE:core_interface>)
+  target_link_libraries(mpgen PUBLIC $<BUILD_INTERFACE:core_interface>)
+  # Mark capproto options as advanced to hide by default from cmake UI
+  mark_as_advanced(CapnProto_DIR)
+  mark_as_advanced(CapnProto_capnpc_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_capnp_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_capnp-json_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_capnp-rpc_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_capnp-websocket_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj-async_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj-gzip_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj-http_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj-test_IMPORTED_LOCATION)
+  mark_as_advanced(CapnProto_kj-tls_IMPORTED_LOCATION)
+  if(BUILD_TESTS)
+    # Add tests to "all" target so ctest can run them
+    set_target_properties(mptests PROPERTIES EXCLUDE_FROM_ALL OFF)
+  endif()
+  # Exclude examples from compilation database, because the examples are not
+  # built by default, and they contain generated c++ code. Without this
+  # exclusion, tools like clang-tidy and IWYU that make use of compilation
+  # database would complain that the generated c++ source files do not exist. An
+  # alternate fix could build "mpexamples" by default like "mptests" above.
+  set_target_properties(mpcalculator mpprinter mpexample PROPERTIES EXPORT_COMPILE_COMMANDS OFF)
+endfunction()

depends/funcs.mk

@@ -39,9 +39,33 @@ define fetch_file
       $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5))))
 endef
 
+# Shell script to create a source tarball in $(1)_source from local directory
+# $(1)_local_dir instead of downloading remote sources. Tarball is recreated if
+# any paths in the local directory have a newer mtime, and checksum of the
+# tarball is saved to $(1)_fetched and returned as output.
+define fetch_local_dir_sha256
+    if ! [ -f $($(1)_source) ] || [ -n "$$(find $($(1)_local_dir) -newer $($(1)_source) | head -n1)" ]; then \
+        mkdir -p $(dir $($(1)_source)) && \
+        $(build_TAR) -c -f $($(1)_source) -C $($(1)_local_dir) . && \
+        rm -f $($(1)_fetched); \
+    fi && \
+    if ! [ -f $($(1)_fetched) ] || [ -n "$$(find $($(1)_source) -newer $($(1)_fetched))" ]; then \
+        mkdir -p $(dir $($(1)_fetched)) && \
+        cd $($(1)_source_dir) && \
+        $(build_SHA256SUM) $($(1)_all_sources) > $($(1)_fetched); \
+    fi && \
+    cut -d" " -f1 $($(1)_fetched)
+endef
+
 define int_get_build_recipe_hash
 $(eval $(1)_patches_path?=$(PATCHES_PATH)/$(1))
 $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $($(1)_patches_path)/,$($(1)_patches)) | cut -d" " -f1))
+# If $(1)_local_dir is set, create a tarball of the local directory contents to
+# use as the source of the package, and include a hash of the tarball in the
+# package id, so if directory contents change, the package and packages
+# depending on it will be rebuilt.
+$(if $($(1)_local_dir),$(eval $(1)_sha256_hash:=$(shell $(call fetch_local_dir_sha256,$(1)))))
+$(if $($(1)_local_dir),$(eval $(1)_all_file_checksums+=$($(1)_sha256_hash)))
 $(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1))
 endef
 
@@ -54,20 +78,32 @@ $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SU
 final_build_id_long+=$($(package)_build_id_long)
 
 #compute package-specific paths
-$(1)_build_subdir?=.
-$(1)_download_file?=$($(1)_file_name)
-$(1)_source_dir:=$(SOURCES_PATH)
-$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name)
 $(1)_staging_dir=$(base_staging_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
 $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix)
 $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
-$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
 $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir)
 $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash
 $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id)
-$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
 $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz
 $(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log
+endef
+
+# Convert a string to a human-readable filename, replacing dot, slash, and space
+# characters that could cause problems with dashes and collapsing them.
+define int_friendly_file_name
+$(subst $(null) $(null),-,$(strip $(subst ., ,$(subst /, ,$(1)))))
+endef
+
+define int_get_build_properties
+$(1)_build_subdir?=.
+$(1)_download_file?=$($(1)_file_name)
+$(1)_source_dir:=$(SOURCES_PATH)
+# If $(1)_file_name is empty and $(1)_local_dir is nonempty, set file name to a
+# .tar file with a friendly filename named after the directory path.
+$(if $($(1)_file_name),,$(if $($(1)_local_dir),$(eval $(1)_file_name:=$(call int_friendly_file_name,$($(1)_local_dir)).tar)))
+$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name)
+$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
+$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
 $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources)
 
 #stamps
@@ -288,6 +324,9 @@ $(foreach package,$(all_packages),$(eval $(call int_vars,$(package))))
 $(foreach native_package,$(native_packages),$(eval include packages/$(native_package).mk))
 $(foreach package,$(packages),$(eval include packages/$(package).mk))
 
+#set build properties for included package files
+$(foreach package,$(all_packages),$(eval $(call int_get_build_properties,$(package))))
+
 #compute a hash of all files that comprise this package's build recipe
 $(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package))))
 

depends/packages.md

@@ -11,7 +11,8 @@ General tips:
   [below](#secondary-dependencies) for more details.
 
 ## Identifiers
-Each package is required to define at least these variables:
+If package does not define a `$(package)_local_dir` variable, it is required to
+define these variables:
 
     $(package)_version:
     Version of the upstream library or program. If there is no version, a
@@ -28,6 +29,9 @@ Each package is required to define at least these variables:
     $(package)_sha256_hash:
     The sha256 hash of the upstream file
 
+If a package does define a `$(package)_local_dir` variable, the above variables
+are not required and will be ignored.
+
 These variables are optional:
 
     $(package)_build_subdir:
@@ -48,6 +52,18 @@ These variables are optional:
     Any extra files that will be fetched via $(package)_fetch_cmds. These are
     specified so that they can be fetched and verified via 'make download'.
 
+## Local packages
+
+If a package defines a `$(package)_local_dir` variable, the specified directory
+will be treated as a download source, and a tarball of its contents will be
+saved to `sources/`. A hash of the tarball will also become part of the package
+build id, so if the directory contents change, the package and everything
+depending on it will be rebuilt. For efficiency, the tarball is cached once it
+has been created, but if the local directory is touched, it will be rebuilt.
+
+Local packages can be useful for using git submodules or subtrees to manage
+package sources, or for testing local changes that are not available to
+download from an external source.
 
 ## Build Variables:
 After defining the main identifiers, build variables may be added or customized

depends/packages/libmultiprocess.mk

@@ -1,8 +1,5 @@
 package=native_libmultiprocess
-$(package)_version=1954f7f65661d49e700c344eae0fc8092decf975
-$(package)_download_path=https://github.com/bitcoin-core/libmultiprocess/archive
-$(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=fc014bd74727c1d5d30b396813685012c965d079244dd07b53bc1c75c610a2cb
+$(package)_local_dir=../src/ipc/libmultiprocess
 $(package)_dependencies=native_capnp
 
 define $(package)_config_cmds

depends/packages/native_libmultiprocess.mk

@@ -20,7 +20,7 @@ sqlite_packages=sqlite
 
 zmq_packages=zeromq
 
-multiprocess_packages = libmultiprocess capnp
+multiprocess_packages = capnp
 multiprocess_native_packages = native_libmultiprocess native_capnp
 
 usdt_linux_packages=systemtap

depends/packages/packages.mk

@@ -159,9 +159,10 @@ else()
 endif()
 
 if("@multiprocess@" STREQUAL "1")
-  set(WITH_MULTIPROCESS ON CACHE BOOL "")
-  set(Libmultiprocess_ROOT "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
-  set(LibmultiprocessNative_ROOT "${CMAKE_CURRENT_LIST_DIR}/native" CACHE PATH "")
+  set(ENABLE_IPC ON CACHE BOOL "")
+  set(MPGEN_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/mpgen" CACHE FILEPATH "")
+  set(CAPNP_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnp" CACHE FILEPATH "")
+  set(CAPNPC_CXX_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnpc-c++" CACHE FILEPATH "")
 else()
-  set(WITH_MULTIPROCESS OFF CACHE BOOL "")
+  set(ENABLE_IPC OFF CACHE BOOL "")
 endif()

depends/toolchain.cmake.in

@@ -8,7 +8,7 @@
 | *libbitcoin_crypto*      | Hardware-optimized functions for data encryption, hashing, message authentication, and key derivation. |
 | *libbitcoin_kernel*      | Consensus engine and support library used for validation by *libbitcoin_node*. |
 | *libbitcoinqt*           | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. |
-| *libbitcoin_ipc*         | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`-DWITH_MULTIPROCESS=ON`](multiprocess.md) is used. |
+| *libbitcoin_ipc*         | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`-DENABLE_IPC=ON`](multiprocess.md) is used. |
 | *libbitcoin_node*        | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. |
 | *libbitcoin_util*        | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). |
 | *libbitcoin_wallet*      | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. |