Merge #14328: [0.17] Backports

542651cfb4 travis: Remove deprecated sudo (MarcoFalke)
ec71f06a8d build: Add bitcoin-tx.exe into Windows installer (Chun Kuan Lee)
7edebedef1 build: Remove illegal spacing in darwin.mk (Jon Layton)
fb9ad043f8 Fix listreceivedbyaddress not taking address as a string (Eric Scrivner)
91fa15aaeb wallet: Avoid potential use of unitialized value bnb_used in CWallet::CreateTransaction(...) (practicalswift)
96f15e8bb3 Tests: Fix a comment (fridokus)
60f7a97930 qa: Add test to ensure node can generate all help texts at runtime (MarcoFalke)
2f9fd29321 disallow oversized CBlockHeaderAndShortTxIDs (Kaz Wesley)
5331ad0506 fix a deserialization overflow edge case (Kaz Wesley)
94065024c7 add a test demonstrating an overflow in a deserialization edge case (Kaz Wesley)
85aacc41ba Add autogen.sh in ARM Cross-compilation (Walter)
bb90695551 [wallet] Ensure wallet is unlocked before signing (gustavonalle)

Pull request description:

Tree-SHA512: d82813134e5fc5437fe690127a4701d7ba66bf27799d7ecb1fbc2cc4dd81b6b3f708c1f314b725e8a3a6525ffa388299e277157f784f762256e01afb24822b25

.travis.yml

@@ -1,4 +1,3 @@
-sudo: required
 dist: trusty
 os: linux
 language: minimal
@@ -143,7 +142,6 @@ jobs:
         BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
     - stage: lint
       env:
-      sudo: false
       cache: false
       language: python
       python: '3.6'

Makefile.am

@@ -19,6 +19,7 @@ endif
 BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT)
 BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT)
 BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT)
+BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT)
 BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT)
 
 empty :=
@@ -74,6 +75,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release
+	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release
 	@test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \
 	  echo error: could not build $@
 	@echo built $@
@@ -167,6 +169,9 @@ $(BITCOIND_BIN): FORCE
 $(BITCOIN_CLI_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
+$(BITCOIN_TX_BIN): FORCE
+	$(MAKE) -C src $(@F)
+
 if USE_LCOV
 LCOV_FILTER_PATTERN=-p "/usr/include/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1"
 

depends/builders/darwin.mk

@@ -1,13 +1,13 @@
-build_darwin_CC: = $(shell xcrun -f clang)
-build_darwin_CXX: = $(shell xcrun -f clang++)
-build_darwin_AR: = $(shell xcrun -f ar)
-build_darwin_RANLIB: = $(shell xcrun -f ranlib)
-build_darwin_STRIP: = $(shell xcrun -f strip)
-build_darwin_OTOOL: = $(shell xcrun -f otool)
-build_darwin_NM: = $(shell xcrun -f nm)
+build_darwin_CC:=$(shell xcrun -f clang)
+build_darwin_CXX:=$(shell xcrun -f clang++)
+build_darwin_AR:=$(shell xcrun -f ar)
+build_darwin_RANLIB:=$(shell xcrun -f ranlib)
+build_darwin_STRIP:=$(shell xcrun -f strip)
+build_darwin_OTOOL:=$(shell xcrun -f otool)
+build_darwin_NM:=$(shell xcrun -f nm)
 build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
-build_darwin_SHA256SUM = shasum -a 256
-build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
+build_darwin_SHA256SUM=shasum -a 256
+build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
 darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION)

doc/build-unix.md

@@ -279,6 +279,7 @@ To build executables for ARM:
     cd depends
     make HOST=arm-linux-gnueabihf NO_QT=1
     cd ..
+    ./autogen.sh
     ./configure --prefix=$PWD/depends/arm-linux-gnueabihf --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++
     make
 

share/setup.nsi.in

@@ -80,6 +80,7 @@ Section -Main SEC0000
     SetOutPath $INSTDIR\daemon
     File @abs_top_srcdir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@
     File @abs_top_srcdir@/release/@BITCOIN_CLI_NAME@@EXEEXT@
+    File @abs_top_srcdir@/release/@BITCOIN_TX_NAME@@EXEEXT@
     SetOutPath $INSTDIR\doc
     File /r /x Makefile* @abs_top_srcdir@/doc\*.*
     SetOutPath $INSTDIR

src/blockencodings.h

@@ -52,12 +52,12 @@ public:
                 }
             }
 
-            uint16_t offset = 0;
+            int32_t offset = 0;
             for (size_t j = 0; j < indexes.size(); j++) {
-                if (uint64_t(indexes[j]) + uint64_t(offset) > std::numeric_limits<uint16_t>::max())
+                if (int32_t(indexes[j]) + offset > std::numeric_limits<uint16_t>::max())
                     throw std::ios_base::failure("indexes overflowed 16 bits");
                 indexes[j] = indexes[j] + offset;
-                offset = indexes[j] + 1;
+                offset = int32_t(indexes[j]) + 1;
             }
         } else {
             for (size_t i = 0; i < indexes.size(); i++) {
@@ -186,6 +186,9 @@ public:
 
         READWRITE(prefilledtxn);
 
+        if (BlockTxCount() > std::numeric_limits<uint16_t>::max())
+            throw std::ios_base::failure("indexes overflowed 16 bits");
+
         if (ser_action.ForRead())
             FillShortTxIDSelector();
     }

src/rpc/client.cpp

@@ -45,7 +45,6 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "listreceivedbyaddress", 0, "minconf" },
     { "listreceivedbyaddress", 1, "include_empty" },
     { "listreceivedbyaddress", 2, "include_watchonly" },
-    { "listreceivedbyaddress", 3, "address_filter" },
     { "listreceivedbyaccount", 0, "minconf" },
     { "listreceivedbyaccount", 1, "include_empty" },
     { "listreceivedbyaccount", 2, "include_watchonly" },

src/test/blockencodings_tests.cpp

@@ -344,4 +344,49 @@ BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) {
     BOOST_CHECK_EQUAL(req1.indexes[3], req2.indexes[3]);
 }
 
+BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationMaxTest) {
+    // Check that the highest legal index is decoded correctly
+    BlockTransactionsRequest req0;
+    req0.blockhash = InsecureRand256();
+    req0.indexes.resize(1);
+    req0.indexes[0] = 0xffff;
+    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
+    stream << req0;
+
+    BlockTransactionsRequest req1;
+    stream >> req1;
+    BOOST_CHECK_EQUAL(req0.indexes.size(), req1.indexes.size());
+    BOOST_CHECK_EQUAL(req0.indexes[0], req1.indexes[0]);
+}
+
+BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationOverflowTest) {
+    // Any set of index deltas that starts with N values that sum to (0x10000 - N)
+    // causes the edge-case overflow that was originally not checked for. Such
+    // a request cannot be created by serializing a real BlockTransactionsRequest
+    // due to the overflow, so here we'll serialize from raw deltas.
+    BlockTransactionsRequest req0;
+    req0.blockhash = InsecureRand256();
+    req0.indexes.resize(3);
+    req0.indexes[0] = 0x7000;
+    req0.indexes[1] = 0x10000 - 0x7000 - 2;
+    req0.indexes[2] = 0;
+    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
+    stream << req0.blockhash;
+    WriteCompactSize(stream, req0.indexes.size());
+    WriteCompactSize(stream, req0.indexes[0]);
+    WriteCompactSize(stream, req0.indexes[1]);
+    WriteCompactSize(stream, req0.indexes[2]);
+
+    BlockTransactionsRequest req1;
+    try {
+        stream >> req1;
+        // before patch: deserialize above succeeds and this check fails, demonstrating the overflow
+        BOOST_CHECK(req1.indexes[1] < req1.indexes[2]);
+        // this shouldn't be reachable before or after patch
+        BOOST_CHECK(0);
+    } catch(std::ios_base::failure &) {
+        // deserialize should fail
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/wallet/rpcwallet.cpp

@@ -3732,6 +3732,8 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request)
 
     // Sign the transaction
     LOCK2(cs_main, pwallet->cs_wallet);
+    EnsureWalletIsUnlocked(pwallet);
+
     return SignTransaction(mtx, request.params[1], pwallet, false, request.params[2]);
 }
 

src/wallet/wallet.cpp

@@ -2846,6 +2846,8 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransac
                             return false;
                         }
                     }
+                } else {
+                    bnb_used = false;
                 }
 
                 const CAmount nChange = nValueIn - nValueToSelect;

test/functional/mempool_resurrect.py

@@ -47,12 +47,11 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
             tx = self.nodes[0].gettransaction(txid)
             assert(tx["confirmations"] > 0)
 
-        # Use invalidateblock to re-org back; all transactions should
-        # end up unconfirmed and back in the mempool
+        # Use invalidateblock to re-org back
         for node in self.nodes:
             node.invalidateblock(blocks[0])
 
-        # mempool should be empty, all txns confirmed
+        # All txns should be back in mempool with 0 confirmations
         assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id))
         for txid in spends1_id+spends2_id:
             tx = self.nodes[0].gettransaction(txid)

test/functional/rpc_help.py

@@ -7,12 +7,18 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_raises_rpc_error
 
+import os
+
 
 class HelpRpcTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
 
     def run_test(self):
+        self.test_categories()
+        self.dump_help()
+
+    def test_categories(self):
         node = self.nodes[0]
 
         # wrong argument count
@@ -37,6 +43,15 @@ class HelpRpcTest(BitcoinTestFramework):
 
         assert_equal(titles, components)
 
+    def dump_help(self):
+        dump_dir = os.path.join(self.options.tmpdir, 'rpc_help_dump')
+        os.mkdir(dump_dir)
+        calls = [line.split(' ', 1)[0] for line in self.nodes[0].help().splitlines() if line and not line.startswith('==')]
+        for call in calls:
+            with open(os.path.join(dump_dir, call), 'w', encoding='utf-8') as f:
+                # Make sure the node can generate the help at runtime without crashing
+                f.write(self.nodes[0].help(call))
+
 
 if __name__ == '__main__':
     HelpRpcTest().main()

test/functional/rpc_signrawtransaction.py

@@ -49,6 +49,14 @@ class SignRawTransactionsTest(BitcoinTestFramework):
         rawTxSigned2 = self.nodes[0].signrawtransaction(rawTx, inputs, privKeys)
         assert_equal(rawTxSigned, rawTxSigned2)
 
+    def test_with_lock_outputs(self):
+        """Test correct error reporting when trying to sign a locked output"""
+        self.nodes[0].encryptwallet("password")
+        self.restart_node(0)
+        rawTx = '020000000156b958f78e3f24e0b2f4e4db1255426b0902027cb37e3ddadb52e37c3557dddb0000000000ffffffff01c0a6b929010000001600149a2ee8c77140a053f36018ac8124a6ececc1668a00000000'
+
+        assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signrawtransactionwithwallet, rawTx)
+
     def script_verification_error_test(self):
         """Create and sign a raw transaction with valid (vin 0), invalid (vin 1) and one missing (vin 2) input script.
 
@@ -150,6 +158,7 @@ class SignRawTransactionsTest(BitcoinTestFramework):
     def run_test(self):
         self.successful_signing_test()
         self.script_verification_error_test()
+        self.test_with_lock_outputs()
 
 
 if __name__ == '__main__':

test/functional/wallet_listreceivedby.py

@@ -68,6 +68,10 @@ class ReceivedByTest(BitcoinTestFramework):
         res = self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True, address_filter=addr)
         assert_array_result(res, {"address": addr}, expected)
         assert_equal(len(res), 1)
+        # Test for regression on CLI calls with address string (#14173)
+        cli_res = self.nodes[1].cli.listreceivedbyaddress(0, True, True, addr)
+        assert_array_result(cli_res, {"address": addr}, expected)
+        assert_equal(len(cli_res), 1)
         # Error on invalid address
         assert_raises_rpc_error(-4, "address_filter parameter was invalid", self.nodes[1].listreceivedbyaddress, minconf=0, include_empty=True, include_watchonly=True, address_filter="bamboozling")
         # Another address receive money