build: multiprocess autotools changes

autoconf and automake changes to support multiprocess gui/node/wallet execution.

This adds a new --enable-multiprocess flag, and build configuration code to
detect libraries needed for multiprocess support. The --enable-multiprocess
flag builds new bitcoin-node and bitcoin-gui executables, which are updated in
https://github.com/bitcoin/bitcoin/pull/10102 to communicate across processes.
But for now they are functionally equivalent to existing bitcoind and
bitcoin-qt executables.
pull/18677/head
Russell Yanofsky 5 years ago
parent 8da1e43b63
commit 5d1377b52b

2
.gitignore vendored

@ -5,6 +5,8 @@
src/bitcoin
src/bitcoind
src/bitcoin-cli
src/bitcoin-gui
src/bitcoin-node
src/bitcoin-tx
src/bitcoin-wallet
src/test/fuzz

@ -242,6 +242,24 @@ if test x$enable_bip70 != xno; then
AC_MSG_ERROR([BIP70 is no longer supported!])
fi
AC_ARG_WITH([libmultiprocess],
[AS_HELP_STRING([--with-libmultiprocess=yes|no|auto],
[Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])],
[with_libmultiprocess=$withval],
[with_libmultiprocess=auto])
AC_ARG_WITH([mpgen],
[AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX],
[Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])],
[with_mpgen=$withval],
[with_mpgen=auto])
AC_ARG_ENABLE([multiprocess],
[AS_HELP_STRING([--enable-multiprocess],
[build multiprocess bitcoin-node, bitcoin-wallet, and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental (default is no)])],
[enable_multiprocess=$enableval],
[enable_multiprocess=no])
AC_ARG_ENABLE(man,
[AS_HELP_STRING([--disable-man],
[do not install man pages (default is to install)])],,
@ -1426,6 +1444,50 @@ AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes])
AC_SUBST(UNIVALUE_CFLAGS)
AC_SUBST(UNIVALUE_LIBS)
dnl libmultiprocess library check
libmultiprocess_found=no
if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then
if test "x$use_pkgconfig" = xyes; then
m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [
libmultiprocess_found=yes;
libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`;
], [true])])
fi
elif test "x$with_libmultiprocess" != xno; then
AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no])
fi
AC_SUBST(LIBMULTIPROCESS_CFLAGS)
AC_SUBST(LIBMULTIPROCESS_LIBS)
dnl Enable multiprocess check
if test "x$enable_multiprocess" = xyes; then
if test "x$libmultiprocess_found" != xyes; then
AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.])
fi
build_multiprocess=yes
elif test "x$enable_multiprocess" = xauto; then
build_multiprocess=$libmultiprocess_found
else
build_multiprocess=no
fi
AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes])
AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes])
AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes])
dnl codegen tools check
if test x$build_multiprocess != xno; then
if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then
MPGEN_PREFIX="$libmultiprocess_prefix"
elif test "x$with_mpgen" != xno; then
MPGEN_PREFIX="$with_mpgen";
fi
AC_SUBST(MPGEN_PREFIX)
fi
AC_MSG_CHECKING([whether to build bitcoind])
AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes])
AC_MSG_RESULT($build_bitcoind)
@ -1704,6 +1766,7 @@ esac
echo
echo "Options used to compile and link:"
echo " multiprocess = $build_multiprocess"
echo " with wallet = $enable_wallet"
echo " with gui / qt = $bitcoin_enable_qt"
if test x$bitcoin_enable_qt != xno; then

@ -0,0 +1,35 @@
# Multiprocess Bitcoin
On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables alongside existing `bitcoind` and `bitcoin-qt` executables.
`bitcoin-node` is a drop-in replacement for `bitcoind`, and `bitcoin-gui` is a drop-in replacement for `bitcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally (after [#10102](https://github.com/bitcoin/bitcoin/pull/10102)), `bitcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `bitcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments.
## Next steps
Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102) will be:
- [ ] Adding `-ipcbind` and `-ipcconnect` options to `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables so they can listen and connect to TCP ports and unix socket paths. This will allow separate processes to be started and stopped any time and connect to each other.
- [ ] Adding `-server` and `-rpcbind` options to the `bitcoin-wallet` executable so wallet processes can handle RPC requests directly without going through the node.
- [ ] Supporting windows, not just unix systems. The existing socket code is already cross-platform, so the only windows-specific code that needs to be written is code spawning a process and passing a socket descriptor. This can be implemented with `CreateProcess` and `WSADuplicateSocket`. Example: https://memset.wordpress.com/2010/10/13/win32-api-passing-socket-with-ipc-method/.
- [ ] Adding sandbox features, restricting subprocess access to resources and data. See [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin).
## Debugging
After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes.
## Installation
The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make:
```
cd <BITCOIN_SOURCE_DIRECTORY>
make -C depends NO_QT=1 MULTIPROCESS=1
./configure --prefix=$PWD/depends/x86_64-pc-linux-gnu
make
src/bitcoin-node -regtest -printtoconsole -debug=ipc
BITCOIND=bitcoin-node test/functional/test_runner.py
```
The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option).
Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general.

@ -85,6 +85,10 @@ if BUILD_BITCOIND
bin_PROGRAMS += bitcoind
endif
if BUILD_BITCOIN_NODE
bin_PROGRAMS += bitcoin-node
endif
if BUILD_BITCOIN_CLI
bin_PROGRAMS += bitcoin-cli
endif
@ -547,22 +551,21 @@ libbitcoin_cli_a_SOURCES = \
nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h
#
# bitcoind binary #
bitcoind_SOURCES = bitcoind.cpp
bitcoind_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
bitcoind_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
# bitcoind & bitcoin-node binaries #
bitcoin_daemon_sources = bitcoind.cpp
bitcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
bitcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS)
bitcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
if TARGET_WINDOWS
bitcoind_SOURCES += bitcoind-res.rc
bitcoin_daemon_sources += bitcoind-res.rc
endif
bitcoind_LDADD = \
$(LIBBITCOIN_SERVER) \
bitcoin_bin_ldadd = \
$(LIBBITCOIN_WALLET) \
$(LIBBITCOIN_COMMON) \
$(LIBUNIVALUE) \
$(LIBBITCOIN_UTIL) \
$(LIBUNIVALUE) \
$(LIBBITCOIN_ZMQ) \
$(LIBBITCOIN_CONSENSUS) \
$(LIBBITCOIN_CRYPTO) \
@ -571,7 +574,19 @@ bitcoind_LDADD = \
$(LIBMEMENV) \
$(LIBSECP256K1)
bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
bitcoind_SOURCES = $(bitcoin_daemon_sources)
bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags)
bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags)
bitcoind_LDFLAGS = $(bitcoin_bin_ldflags)
bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd)
bitcoin_node_SOURCES = $(bitcoin_daemon_sources)
bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags)
bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags)
bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags)
bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd)
# bitcoin-cli binary #
bitcoin_cli_SOURCES = bitcoin-cli.cpp
@ -615,29 +630,14 @@ bitcoin_tx_LDADD += $(BOOST_LIBS)
# bitcoin-wallet binary #
bitcoin_wallet_SOURCES = bitcoin-wallet.cpp
bitcoin_wallet_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
bitcoin_wallet_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
bitcoin_wallet_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags)
bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags)
bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags)
bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd)
if TARGET_WINDOWS
bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc
endif
bitcoin_wallet_LDADD = \
$(LIBBITCOIN_WALLET_TOOL) \
$(LIBBITCOIN_WALLET) \
$(LIBBITCOIN_COMMON) \
$(LIBBITCOIN_CONSENSUS) \
$(LIBBITCOIN_UTIL) \
$(LIBBITCOIN_CRYPTO) \
$(LIBBITCOIN_ZMQ) \
$(LIBLEVELDB) \
$(LIBLEVELDB_SSE42) \
$(LIBMEMENV) \
$(LIBSECP256K1) \
$(LIBUNIVALUE)
bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS)
#
# bitcoinconsensus library #

@ -3,6 +3,11 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
bin_PROGRAMS += qt/bitcoin-qt
if BUILD_BITCOIN_GUI
bin_PROGRAMS += bitcoin-gui
endif
EXTRA_LIBRARIES += qt/libbitcoinqt.a
# bitcoin qt core #
@ -294,29 +299,43 @@ QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI:
# Most files will depend on the forms and moc files as includes. Generate them
# before anything else.
$(QT_MOC): $(QT_FORMS_H)
$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) : | $(QT_MOC)
$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) $(bitcoin_gui_OBJECTS) : | $(QT_MOC)
# bitcoin-qt binary #
qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \
# bitcoin-qt and bitcoin-gui binaries #
bitcoin_qt_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \
$(QT_INCLUDES) $(QR_CFLAGS)
qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS)
bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS)
qt_bitcoin_qt_SOURCES = qt/main.cpp
bitcoin_qt_sources = qt/main.cpp
if TARGET_WINDOWS
qt_bitcoin_qt_SOURCES += $(BITCOIN_RC)
bitcoin_qt_sources += $(BITCOIN_RC)
endif
qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER)
bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER)
if ENABLE_WALLET
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET)
bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET)
endif
if ENABLE_ZMQ
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX
bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX
qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags)
qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags)
qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources)
qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd)
qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags)
qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags)
bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags)
bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags)
bitcoin_gui_SOURCES = $(bitcoin_qt_sources)
bitcoin_gui_LDADD = $(bitcoin_qt_ldadd)
bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags)
bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags)
#locale/foo.ts -> locale/foo.qm
QT_QM=$(QT_TS:.ts=.qm)

Loading…
Cancel
Save