Skip to content

Commit 97b21b3

Browse files
committed
Merge #18677: Multiprocess build support
e2bab2a multiprocess: add multiprocess travis configuration (Russell Yanofsky) 603fd6a depends: add MULTIPROCESS depends option (Russell Yanofsky) 5d1377b build: multiprocess autotools changes (Russell Yanofsky) Pull request description: This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/projects/10). --- This PR consists of build changes only. It adds an `--enable-multiprocess` autoconf option (off by default and marked experimental), that builds new `bitcoin-node` and `bitcoin-gui` binaries. These currently function the same as existing `bitcoind` and `bitcoin-qt` binaries, but are extended in #10102 with IPC features to execute node, wallet, and gui functions in separate processes. In addition to adding the `--enable-multiprocess` config flag, it also adds a depends package and autoconf rules to build with the [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library, and it adds new travis configuration to exercise the build code and run functional tests with the new binaries. The changes in this PR were originally part of #10102 but were moved into #16367 to be able to develop and review the multiprocess build changes independently of the code changes. #16367 was briefly merged and then reverted in #18588. Only change since #16367 has been dropping the `native_boost.mk` depends package which was pointed out to be no longer necessary in bitcoin/bitcoin#16367 (comment) and bitcoin/bitcoin#18588 (review) ACKs for top commit: practicalswift: ACK e2bab2a Sjors: tACK e2bab2a on macOS 10.15.4 hebasto: ACK e2bab2a, tested on Linux Mint 19.3 (x86_64): Tree-SHA512: b5a76eab5abf63d9d8b6d628cbdff4cc1888eef15cafa0a5d56369e2f9d02595fed623f4b74b2cf2830c42c05a774f0943e700f9c768a82d9d348cad199e135c
2 parents 17df15e + e2bab2a commit 97b21b3

23 files changed

+289
-44
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
src/bitcoin
66
src/bitcoind
77
src/bitcoin-cli
8+
src/bitcoin-gui
9+
src/bitcoin-node
810
src/bitcoin-tx
911
src/bitcoin-wallet
1012
src/test/fuzz

.travis.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,12 @@ jobs:
128128
env: >-
129129
FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh"
130130
131+
- stage: test
132+
name: 'x86_64 Linux [GOAL: install] [bionic] [multiprocess]'
133+
if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ # Skip on non-depends, non-multiprocess PRs
134+
env: >-
135+
FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh"
136+
131137
- stage: test
132138
name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]'
133139
env: >-

ci/test/00_setup_env.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
3434
export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
3535
export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
3636
export TEST_PREVIOUS_RELEASES=${TEST_PREVIOUS_RELEASES:-false}
37+
export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
3738
export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
3839
export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}
3940
export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2020 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C.UTF-8
8+
9+
export CONTAINER_NAME=ci_native_multiprocess
10+
export PACKAGES="cmake python3"
11+
export DEP_OPTS="MULTIPROCESS=1"
12+
export GOAL="install"
13+
export BITCOIN_CONFIG=""
14+
export TEST_RUNNER_ENV="BITCOIND=bitcoin-node"

ci/test/06_script_b.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ fi
3737

3838
if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
3939
BEGIN_FOLD functional-tests
40-
DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast
40+
DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast
4141
END_FOLD
4242
fi
4343

configure.ac

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,24 @@ if test x$enable_bip70 != xno; then
242242
AC_MSG_ERROR([BIP70 is no longer supported!])
243243
fi
244244

245+
AC_ARG_WITH([libmultiprocess],
246+
[AS_HELP_STRING([--with-libmultiprocess=yes|no|auto],
247+
[Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])],
248+
[with_libmultiprocess=$withval],
249+
[with_libmultiprocess=auto])
250+
251+
AC_ARG_WITH([mpgen],
252+
[AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX],
253+
[Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])],
254+
[with_mpgen=$withval],
255+
[with_mpgen=auto])
256+
257+
AC_ARG_ENABLE([multiprocess],
258+
[AS_HELP_STRING([--enable-multiprocess],
259+
[build multiprocess bitcoin-node, bitcoin-wallet, and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental (default is no)])],
260+
[enable_multiprocess=$enableval],
261+
[enable_multiprocess=no])
262+
245263
AC_ARG_ENABLE(man,
246264
[AS_HELP_STRING([--disable-man],
247265
[do not install man pages (default is to install)])],,
@@ -1389,6 +1407,50 @@ AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes])
13891407
AC_SUBST(UNIVALUE_CFLAGS)
13901408
AC_SUBST(UNIVALUE_LIBS)
13911409

1410+
dnl libmultiprocess library check
1411+
1412+
libmultiprocess_found=no
1413+
if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then
1414+
if test "x$use_pkgconfig" = xyes; then
1415+
m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [
1416+
libmultiprocess_found=yes;
1417+
libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`;
1418+
], [true])])
1419+
fi
1420+
elif test "x$with_libmultiprocess" != xno; then
1421+
AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no])
1422+
fi
1423+
AC_SUBST(LIBMULTIPROCESS_CFLAGS)
1424+
AC_SUBST(LIBMULTIPROCESS_LIBS)
1425+
1426+
dnl Enable multiprocess check
1427+
1428+
if test "x$enable_multiprocess" = xyes; then
1429+
if test "x$libmultiprocess_found" != xyes; then
1430+
AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.])
1431+
fi
1432+
build_multiprocess=yes
1433+
elif test "x$enable_multiprocess" = xauto; then
1434+
build_multiprocess=$libmultiprocess_found
1435+
else
1436+
build_multiprocess=no
1437+
fi
1438+
1439+
AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes])
1440+
AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes])
1441+
AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes])
1442+
1443+
dnl codegen tools check
1444+
1445+
if test x$build_multiprocess != xno; then
1446+
if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then
1447+
MPGEN_PREFIX="$libmultiprocess_prefix"
1448+
elif test "x$with_mpgen" != xno; then
1449+
MPGEN_PREFIX="$with_mpgen";
1450+
fi
1451+
AC_SUBST(MPGEN_PREFIX)
1452+
fi
1453+
13921454
AC_MSG_CHECKING([whether to build bitcoind])
13931455
AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes])
13941456
AC_MSG_RESULT($build_bitcoind)
@@ -1667,6 +1729,7 @@ esac
16671729

16681730
echo
16691731
echo "Options used to compile and link:"
1732+
echo " multiprocess = $build_multiprocess"
16701733
echo " with wallet = $enable_wallet"
16711734
echo " with gui / qt = $bitcoin_enable_qt"
16721735
if test x$bitcoin_enable_qt != xno; then

depends/Makefile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ NO_QR ?=
1313
NO_WALLET ?=
1414
NO_ZMQ ?=
1515
NO_UPNP ?=
16+
MULTIPROCESS ?=
1617
FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
1718

1819
BUILD = $(shell ./config.guess)
@@ -107,6 +108,7 @@ qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch
107108
wallet_packages_$(NO_WALLET) = $(wallet_packages)
108109
upnp_packages_$(NO_UPNP) = $(upnp_packages)
109110
zmq_packages_$(NO_ZMQ) = $(zmq_packages)
111+
multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
110112

111113
packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_)
112114
native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
@@ -115,6 +117,11 @@ ifneq ($(zmq_packages_),)
115117
packages += $(zmq_packages)
116118
endif
117119

120+
ifeq ($(multiprocess_packages_),)
121+
packages += $(multiprocess_packages)
122+
native_packages += $(multiprocess_native_packages)
123+
endif
124+
118125
all_packages = $(packages) $(native_packages)
119126

120127
meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
@@ -155,6 +162,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
155162
-e 's|@no_zmq@|$(NO_ZMQ)|' \
156163
-e 's|@no_wallet@|$(NO_WALLET)|' \
157164
-e 's|@no_upnp@|$(NO_UPNP)|' \
165+
-e 's|@multiprocess@|$(MULTIPROCESS)|' \
158166
-e 's|@debug@|$(DEBUG)|' \
159167
$< > $@
160168
$(AT)touch $@

depends/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ The following can be set when running make: make FOO=bar
9191
NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq
9292
NO_WALLET: Don't download/build/cache libs needed to enable the wallet
9393
NO_UPNP: Don't download/build/cache packages needed for enabling upnp
94+
MULTIPROCESS: build libmultiprocess (experimental, requires cmake)
9495
DEBUG: disable some optimizations and enable more runtime checking
9596
HOST_ID_SALT: Optional salt to use when generating host package ids
9697
BUILD_ID_SALT: Optional salt to use when generating build package ids

depends/config.site.in

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ fi
1616
if test -z $with_qt_bindir && test -z "@no_qt@"; then
1717
with_qt_bindir=$depends_prefix/native/bin
1818
fi
19+
if test -z $with_mpgen && test -n "@multiprocess@"; then
20+
with_mpgen=$depends_prefix/native
21+
fi
1922

2023
if test -z $with_qrencode && test -n "@no_qr@"; then
2124
with_qrencode=no
@@ -25,6 +28,10 @@ if test -z $enable_wallet && test -n "@no_wallet@"; then
2528
enable_wallet=no
2629
fi
2730

31+
if test -z $enable_multiprocess && test -n "@multiprocess@"; then
32+
enable_multiprocess=yes
33+
fi
34+
2835
if test -z $with_miniupnpc && test -n "@no_upnp@"; then
2936
with_miniupnpc=no
3037
fi

depends/funcs.mk

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$(
130130

131131
$(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig
132132
$(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
133+
$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake
133134
$(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH)
134135
$(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH)
135136
$(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH)
136137
$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
137-
138138
ifneq ($($(1)_nm),)
139139
$(1)_autoconf += NM="$$($(1)_nm)"
140140
endif
@@ -156,6 +156,15 @@ endif
156156
ifneq ($($(1)_ldflags),)
157157
$(1)_autoconf += LDFLAGS="$$($(1)_ldflags)"
158158
endif
159+
160+
$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix)
161+
ifneq ($($(1)_type),build)
162+
ifneq ($(host),$(build))
163+
$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix)
164+
$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))"
165+
$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))"
166+
endif
167+
endif
159168
endef
160169

161170
define int_add_cmds

0 commit comments

Comments
 (0)