Added "mwebheader" getdata type and CMerkleBlockWithMWEB response

src/merkleblock.cpp

@@ -178,3 +178,14 @@ uint256 CPartialMerkleTree::ExtractMatches(std::vector<uint256> &vMatch, std::ve
         return uint256();
     return hashMerkleRoot;
 }
+
+CMerkleBlockWithMWEB::CMerkleBlockWithMWEB(const CBlock& block)
+{
+    hogex = block.GetHogEx();
+    assert(hogex != nullptr);
+
+    mweb_header = block.mweb_block.GetMWEBHeader();
+    assert(mweb_header != nullptr);
+
+    merkle = CMerkleBlock(block, std::set<uint256>{hogex->GetHash()});
+}

src/merkleblock.h

@@ -155,4 +155,53 @@ private:
     CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids);
 };
 
+
+/**
+ * Can be requested by MWEB light clients to retrieve and verify MWEB headers.
+ * 
+ * Serializes as merkleblock + tx (HogEx) + MWEB header:
+ * 
+ * Merkle Block
+ *  - uint32         version (4 bytes)
+ *  - uint256        prev_block (32 bytes)
+ *  - uint256        merkle_root (32 bytes)
+ *  - uint32         timestamp (4 bytes)
+ *  - uint32         bits (4 bytes)
+ *  - uint32         nonce (4 bytes)
+ *  - uint32         total_transactions (4 bytes)
+ *  - varint         number of hashes   (1-3 bytes)
+ *  - uint256[]      hashes in depth-first order (<= 32*N bytes)
+ *  - varint         number of bytes of flag bits (1-3 bytes)
+ *  - byte[]         flag bits, packed per 8 in a byte, least significant bit first (<= 2*N-1 bits)
+ * 
+ * HogEx Transaction
+ *  - CTransaction   hogex transaction (? bytes)
+ * 
+ * MWEB Header
+ *  - int32          height (4 bytes)
+ *  - uint256        output_root (32 bytes)
+ *  - uint256        kernel_root (32 bytes)
+ *  - uint256        leafset_root (32 bytes)
+ *  - uint256        kernel_offset (32 bytes)
+ *  - uint256        stealth_offset (32 bytes)
+ *  - uint64         output_mmr_size (8 bytes)
+ *  - uint64         kernel_mmr_size (8 bytes)
+ * 
+ * NOTE: The class assumes that the given CBlock contains a hogex transaction and MWEB block data.
+ * If it does not, an assertion will be hit.
+ */
+class CMerkleBlockWithMWEB
+{
+private:
+    CMerkleBlock merkle;
+    CTransactionRef hogex;
+    mw::Header::CPtr mweb_header;
+
+public:
+    CMerkleBlockWithMWEB() { }
+    CMerkleBlockWithMWEB(const CBlock& block);
+
+    SERIALIZE_METHODS(CMerkleBlockWithMWEB, obj) { READWRITE(obj.merkle, obj.hogex, obj.mweb_header); }
+};
+
 #endif // BITCOIN_MERKLEBLOCK_H

src/net_processing.cpp

@@ -1680,6 +1680,11 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c
                 } else {
                     connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock));
                 }
+            } else if (inv.IsMsgMWEBHeader()) {
+                if (pblock->GetHogEx() != nullptr && !pblock->mweb_block.IsNull()) {
+                    CMerkleBlockWithMWEB merkle_block_with_mweb(*pblock);
+                    connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::MWEBHEADER, merkle_block_with_mweb));
+                }
             }
         }
 

src/protocol.cpp

@@ -46,6 +46,7 @@ const char *CFHEADERS="cfheaders";
 const char *GETCFCHECKPT="getcfcheckpt";
 const char *CFCHECKPT="cfcheckpt";
 const char *WTXIDRELAY="wtxidrelay";
+const char *MWEBHEADER="mwebheader";
 } // namespace NetMsgType
 
 /** All known message types. Keep this in the same order as the list of
@@ -86,6 +87,7 @@ const static std::string allNetMessageTypes[] = {
     NetMsgType::GETCFCHECKPT,
     NetMsgType::CFCHECKPT,
     NetMsgType::WTXIDRELAY,
+    NetMsgType::MWEBHEADER,
 };
 const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes));
 
@@ -176,6 +178,7 @@ std::string CInv::GetCommand() const
     case MSG_BLOCK:          return cmd.append(NetMsgType::BLOCK);
     case MSG_FILTERED_BLOCK: return cmd.append(NetMsgType::MERKLEBLOCK);
     case MSG_CMPCT_BLOCK:    return cmd.append(NetMsgType::CMPCTBLOCK);
+    case MSG_MWEB_HEADER:    return cmd.append(NetMsgType::MWEBHEADER);
     default:
         throw std::out_of_range(strprintf("CInv::GetCommand(): type=%d unknown type", type));
     }

src/protocol.h

@@ -260,6 +260,13 @@ extern const char* CFCHECKPT;
  * @since protocol version 70016 as described by BIP 339.
  */
 extern const char* WTXIDRELAY;
+/**
+ * Contains a CMerkleBlockWithMWEB.
+ * Sent in response to a getdata message which requested a
+ * block using the inventory type MSG_MWEB_HEADER.
+ * @since protocol version 70017 as described by LIP-0007
+ */
+extern const char* MWEBHEADER;
 }; // namespace NetMsgType
 
 /* Get a vector of all valid message types (see above) */
@@ -427,6 +434,7 @@ enum GetDataMsg : uint32_t {
     // MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG,
     MSG_MWEB_BLOCK = MSG_WITNESS_BLOCK | MSG_MWEB_FLAG,
     MSG_MWEB_TX = MSG_WITNESS_TX | MSG_MWEB_FLAG,
+    MSG_MWEB_HEADER = 8 | MSG_MWEB_FLAG,            //!< Defined in LIP-0007
 };
 
 /** inv message data */
@@ -451,6 +459,7 @@ public:
     bool IsMsgCmpctBlk() const { return type == MSG_CMPCT_BLOCK; }
     bool IsMsgWitnessBlk() const { return type == MSG_WITNESS_BLOCK; }
     bool IsMsgMWEBBlk() const { return type == MSG_MWEB_BLOCK; }
+    bool IsMsgMWEBHeader() const { return type == MSG_MWEB_HEADER; }
 
     // Combined-message helper methods
     bool IsGenTxMsg() const
@@ -459,7 +468,7 @@ public:
     }
     bool IsGenBlkMsg() const
     {
-        return type == MSG_BLOCK || type == MSG_FILTERED_BLOCK || type == MSG_CMPCT_BLOCK || type == MSG_WITNESS_BLOCK || type == MSG_MWEB_BLOCK;
+        return type == MSG_BLOCK || type == MSG_FILTERED_BLOCK || type == MSG_CMPCT_BLOCK || type == MSG_WITNESS_BLOCK || type == MSG_MWEB_BLOCK || type == MSG_MWEB_HEADER;
     }
 
     uint32_t type;

src/version.h

@@ -9,7 +9,7 @@
  * network protocol versioning
  */
 
-static const int PROTOCOL_VERSION = 70016;
+static const int PROTOCOL_VERSION = 70017;
 
 //! initial proto version, to be increased after version/verack negotiation
 static const int INIT_PROTO_VERSION = 209;
@@ -38,6 +38,9 @@ static const int INVALID_CB_NO_BAN_VERSION = 70015;
 //! "wtxidrelay" command for wtxid-based relay starts with this version
 static const int WTXID_RELAY_VERSION = 70016;
 
+//! "mwebheader" command for light client MWEB support starts with this version
+static const int MWEB_SYNC_VERSION = 70017;
+
 // Make sure that none of the values above collide with
 // `SERIALIZE_TRANSACTION_NO_WITNESS` or `ADDRV2_FORMAT`.
 

test/functional/mweb_mining.py

@@ -8,7 +8,7 @@ from test_framework.blocktools import (create_coinbase, NORMAL_GBT_REQUEST_PARAM
 from test_framework.messages import (CBlock, MWEBBlock)
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
-from test_framework.ltc_util import create_hogex, get_mweb_header_tip, setup_mweb_chain
+from test_framework.ltc_util import create_hogex, get_mweb_header, setup_mweb_chain
 
 class MWEBMiningTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -28,7 +28,7 @@ class MWEBMiningTest(BitcoinTestFramework):
         next_height = int(gbt["height"])        
 
         # Build MWEB block
-        mweb_header = get_mweb_header_tip(node)
+        mweb_header = get_mweb_header(node)
         mweb_header.height = next_height
         mweb_header.rehash()
         mweb_block = MWEBBlock(mweb_header)
@@ -46,7 +46,7 @@ class MWEBMiningTest(BitcoinTestFramework):
         block.nBits = int(gbt["bits"], 16)
         block.nNonce = 0
         block.vtx = vtx
-        block.mweb_block = mweb_block.serialize().hex()
+        block.mweb_block = mweb_block
         block.hashMerkleRoot = block.calc_merkle_root()
 
         # Call getblocktemplate with the block proposal

test/functional/mweb_p2p.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+Test LIP-0007
+
+1. Test getdata 'mwebheader' *before* MWEB activation
+2. Test getdata 'mwebheader' *after* MWEB activation
+"""
+
+from test_framework.messages import (
+    CBlockHeader,
+    CInv,
+    Hash,
+    hash256,
+    msg_getdata,
+    MSG_MWEB_HEADER,
+)
+from test_framework.p2p import P2PInterface, p2p_lock
+from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.ltc_util import FIRST_MWEB_HEIGHT, get_hogex_tx, get_mweb_header
+from test_framework.util import assert_equal
+
+# Can be used to mimic a light client requesting MWEB data from a full node
+class MockLightClient(P2PInterface):
+    def __init__(self):
+        super().__init__()
+        self.merkle_blocks_with_mweb = {}
+        self.block_headers = {}
+
+    def request_mweb_header(self, block_hash):
+        want = msg_getdata([CInv(MSG_MWEB_HEADER, int(block_hash, 16))])
+        self.send_message(want)
+
+    def on_mwebheader(self, message):
+        self.merkle_blocks_with_mweb[message.header_hash()] = message.merkleblockwithmweb
+
+    def on_block(self, message):
+        message.block.calc_sha256()
+        self.block_headers[Hash(message.block.sha256)] = CBlockHeader(message.block)
+
+
+class MWEBP2PTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.extra_args = [['-whitelist=noban@127.0.0.1']]  # immediate tx relay
+        self.num_nodes = 1
+
+    def run_test(self):
+        node = self.nodes[0]
+        light_client = node.add_p2p_connection(MockLightClient())
+
+        self.log.info("Generate pre-MWEB blocks")
+        pre_mweb_block_hash = node.generate(FIRST_MWEB_HEIGHT - 1)[-1]
+
+        self.log.info("Request mweb_header for pre-MWEB block '{}'".format(pre_mweb_block_hash))
+        light_client.request_mweb_header(pre_mweb_block_hash)
+        
+        self.log.info("Activate MWEB")
+        node.sendtoaddress(node.getnewaddress(address_type='mweb'), 1)
+        post_mweb_block_hash = node.generate(1)[0]
+
+        self.log.info("Request mweb_header for block '{}'".format(post_mweb_block_hash))
+        light_client.request_mweb_header(post_mweb_block_hash)
+
+        self.log.info("Waiting for mweb_header")
+        light_client.wait_for_mwebheader(post_mweb_block_hash, 5)
+        light_client.wait_for_block(int(post_mweb_block_hash, 16), 5)
+
+        self.log.info("Assert results")
+
+        # Before MWEB activation, no merkle block should be returned
+        assert pre_mweb_block_hash not in light_client.merkle_blocks_with_mweb
+
+        # After MWEB activation, the requested merkle block should be returned
+        assert post_mweb_block_hash in light_client.merkle_blocks_with_mweb
+        merkle_block_with_mweb = light_client.merkle_blocks_with_mweb[post_mweb_block_hash]
+
+        # Check block header is correct
+        assert Hash.from_hex(post_mweb_block_hash) in light_client.block_headers
+        block_header = light_client.block_headers[Hash.from_hex(post_mweb_block_hash)]
+        assert_equal(block_header, merkle_block_with_mweb.merkle.header)
+
+        # Check MWEB header is correct
+        mweb_header = get_mweb_header(node, post_mweb_block_hash)
+        assert_equal(mweb_header, merkle_block_with_mweb.mweb_header)
+
+        # Check HogEx transaction is correct
+        hogex_tx = get_hogex_tx(node, post_mweb_block_hash)
+        assert_equal(hogex_tx, merkle_block_with_mweb.hogex)
+
+        # Check Merkle tree
+        merkle_tree = merkle_block_with_mweb.merkle.txn
+        assert_equal(3, merkle_tree.nTransactions)
+        assert_equal(2, len(merkle_tree.vHash))
+        
+        left_hash = Hash(merkle_tree.vHash[0])
+        right_hash = Hash(merkle_tree.vHash[1])
+        assert_equal(Hash.from_hex(hogex_tx.hash), right_hash)
+
+        right_branch_bytes = hash256(right_hash.serialize() + right_hash.serialize())
+        merkle_root_bytes = hash256(left_hash.serialize() + right_branch_bytes)
+        assert_equal(Hash.from_byte_arr(merkle_root_bytes), Hash(block_header.hashMerkleRoot))
+
+if __name__ == '__main__':
+    MWEBP2PTest().main()

test/functional/test_framework/ltc_util.py

@@ -6,17 +6,22 @@
 
 import os
 
-from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, MWEBHeader
+from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, MWEBHeader
 from test_framework.util import get_datadir_path, initialize_datadir, satoshi_round
 from test_framework.script_util import DUMMY_P2WPKH_SCRIPT, hogaddr_script
 from test_framework.test_node import TestNode
 
+FIRST_MWEB_HEIGHT = 432 # Height of the first block to contain an MWEB if using default regtest params
+
+
 """Create a txout with a given amount and scriptPubKey
 
 Mines coins as needed.
 
 confirmed - txouts created will be confirmed in the blockchain;
             unconfirmed otherwise.
+
+Returns the 'COutPoint' of the UTXO
 """
 def make_utxo(node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT):
     fee = 1*COIN
@@ -56,26 +61,59 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT):
 
     return COutPoint(int(txid, 16), 0)
 
-def setup_mweb_chain(node):
+
+"""Generates all pre-MWEB blocks, pegs 1 coin into the MWEB,
+then mines the first MWEB block which includes that pegin.
+
+mining_node - The node to use to generate blocks
+"""
+def setup_mweb_chain(mining_node):
     # Create all pre-MWEB blocks
-    node.generate(431)
+    mining_node.generate(FIRST_MWEB_HEIGHT - 1)
 
     # Pegin some coins
-    node.sendtoaddress(node.getnewaddress(address_type='mweb'), 1)
+    mining_node.sendtoaddress(mining_node.getnewaddress(address_type='mweb'), 1)
 
     # Create some blocks - activate MWEB
-    node.generate(1)
+    mining_node.generate(1)
 
-def get_hog_addr_txout(node):
-    best_block = node.getblock(node.getbestblockhash(), 2)
 
-    hogex_tx = best_block['tx'][-1] # TODO: Should validate that the tx is marked as a hogex tx
-    hog_addr = hogex_tx['vout'][0]
+"""Retrieves the HogEx transaction for the block.
 
-    return CTxOut(int(hog_addr['value'] * COIN), hog_addr['scriptPubKey'])
+node - The node to use to lookup the transaction.
+block_hash - The block to retrieve the HogEx for. If not provided, the best block hash will be used.
 
-def get_mweb_header_tip(node):
-    best_block = node.getblock(node.getbestblockhash(), 2)
+Returns the HogEx as a 'CTransaction'
+"""
+def get_hogex_tx(node, block_hash = None):
+    block_hash = block_hash or node.getbestblockhash()
+    hogex_tx_id = node.getblock(block_hash)['tx'][-1]
+    hogex_tx = FromHex(CTransaction(), node.getrawtransaction(hogex_tx_id, False, block_hash)) # TODO: Should validate that the tx is marked as a hogex tx
+    hogex_tx.rehash()
+    return hogex_tx
+    
+
+"""Retrieves the HogAddr for a block.
+
+node - The node to use to lookup the HogAddr.
+block_hash - The block to retrieve the HogAddr for. If not provided, the best block hash will be used.
+
+Returns the HogAddr as a 'CTxOut'
+"""
+def get_hog_addr_txout(node, block_hash = None):
+    return get_hogex_tx(node, block_hash).vout[0]
+    
+
+"""Retrieves the MWEB header for a block.
+
+node - The node to use to lookup the MWEB header.
+block_hash - The block to retrieve the MWEB header for. If not provided, the best block hash will be used.
+
+Returns the MWEB header as an 'MWEBHeader' or None if the block doesn't contain MWEB data.
+"""
+def get_mweb_header(node, block_hash = None):
+    block_hash = block_hash or node.getbestblockhash()
+    best_block = node.getblock(block_hash, 2)
     if not 'mweb' in best_block:
         return None
 
@@ -83,19 +121,26 @@ def get_mweb_header_tip(node):
     mweb_header.from_json(best_block['mweb'])
     return mweb_header
 
-def create_hogex(node, mweb_hash):
-    best_block = node.getblock(node.getbestblockhash(), 2)
+    
+"""Creates a HogEx transaction that commits to the provided MWEB hash.
+# TODO: In the future, this should support passing in pegins and pegouts.
 
-    hogex_tx = best_block['tx'][-1] # TODO: Should validate that the tx is marked as a hogex tx
-    hog_addr = hogex_tx['vout'][0]
+node - The node to use to lookup the latest block.
+mweb_hash - The block to retrieve the MWEB header for. If not provided, the best block hash will be used.
+
+Returns the built HogEx transaction as a 'CTransaction'
+"""
+def create_hogex(node, mweb_hash):
+    hogex_tx = get_hogex_tx(node)
 
     tx = CTransaction()
-    tx.vin = [CTxIn(COutPoint(int(hogex_tx['txid'], 16), 0))]
-    tx.vout = [CTxOut(int(hog_addr['value'] * COIN), hogaddr_script(mweb_hash))]
+    tx.vin = [CTxIn(COutPoint(hogex_tx.sha256, 0))]
+    tx.vout = [CTxOut(hogex_tx.vout[0].nValue, hogaddr_script(mweb_hash.to_byte_arr()))]
     tx.hogex = True
     tx.rehash()
     return tx
 
+
 """ Create a non-HD wallet from a temporary v15.1.0 node.
 
 Returns the path of the wallet.dat.

test/functional/test_framework/messages.py

@@ -34,7 +34,7 @@ from test_framework.siphash import siphash256
 from test_framework.util import hex_str_to_bytes, assert_equal
 
 MIN_VERSION_SUPPORTED = 60001
-MY_VERSION = 70016  # past wtxid relay
+MY_VERSION = 70017  # past wtxid relay
 MY_SUBVERSION = b"/python-p2p-tester:0.0.3/"
 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
 
@@ -71,6 +71,7 @@ MSG_TYPE_MASK = 0xffffffff >> 3
 MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG
 MSG_MWEB_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG | MSG_MWEB_FLAG
 MSG_MWEB_TX = MSG_WITNESS_TX | MSG_MWEB_FLAG
+MSG_MWEB_HEADER = 8 | MSG_MWEB_FLAG
 
 FILTER_TYPE_BASIC = 0
 
@@ -142,6 +143,52 @@ def uint256_from_compact(c):
     return v
 
 
+def deser_fixed_bytes(f, size):
+    r = []
+    for i in range(size):
+        r.append(struct.unpack("B", f.read(1))[0])
+    return r
+
+def ser_fixed_bytes(u, size):
+    rs = b""
+    for i in range(size):
+        rs += struct.pack("B", u[i])
+        #rs += struct.pack("B", u & 0xFF)
+        #u >>= 8
+    return rs
+    
+
+def deser_pubkey(f):
+    r = 0
+    for i in range(33):
+        t = struct.unpack("B", f.read(1))[0]
+        r += t << (i * 8)
+    return r
+
+def ser_pubkey(u):
+    rs = b""
+    for _ in range(33):
+        rs += struct.pack("B", u & 0xFF)
+        u >>= 8
+    return rs
+    
+
+def deser_signature(f):
+    r = 0
+    for i in range(64):
+        t = struct.unpack("B", f.read(1))[0]
+        r += t << (i * 8)
+    return r
+
+def ser_signature(u):
+    rs = b""
+    for _ in range(64):
+        rs += struct.pack("B", u & 0xFF)
+        u >>= 8
+    return rs
+        
+
+
 # deser_function_name: Allow for an alternate deserialization function on the
 # entries in the vector.
 def deser_vector(f, c, deser_function_name=None):
@@ -307,6 +354,7 @@ class CInv:
         MSG_FILTERED_BLOCK: "filtered Block",
         MSG_CMPCT_BLOCK: "CompactBlock",
         MSG_WTX: "WTX",
+        MSG_MWEB_HEADER: "MWEB Header"
     }
 
     def __init__(self, t=0, h=0):
@@ -647,6 +695,8 @@ class CTransaction:
         return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
             % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 
+    def __eq__(self, other):
+        return isinstance(other, CTransaction) and repr(self) == repr(other)
 
 class CBlockHeader:
     __slots__ = ("hash", "hashMerkleRoot", "hashPrevBlock", "nBits", "nNonce",
@@ -723,6 +773,9 @@ class CBlockHeader:
             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
                time.ctime(self.nTime), self.nBits, self.nNonce)
 
+    def __eq__(self, other):
+        return isinstance(other, CBlockHeader) and repr(self) == repr(other)
+
 BLOCK_HEADER_SIZE = len(CBlockHeader().serialize())
 assert_equal(BLOCK_HEADER_SIZE, 80)
 
@@ -1871,8 +1924,55 @@ class msg_cfcheckpt:
 
 import blake3 as BLAKE3
 
+def hex_reverse(h):
+    return "".join(reversed([h[i:i+2] for i in range(0, len(h), 2)]))
+
+class Hash:
+    __slots__ = ("val")
+
+    def __init__(self, val = 0):
+        self.val = val
+        
+    @classmethod
+    def from_hex(cls, hex_str):
+        return cls(int(hex_str, 16))
+
+    @classmethod
+    def from_rev_hex(cls, hex_str):
+        return cls(int(hex_reverse(hex_str), 16))
+        
+    @classmethod
+    def from_byte_arr(cls, b):
+        return cls(deser_uint256(BytesIO(b)))
+    
+    def to_hex(self):
+        return self.serialize().hex()
+
+    def to_byte_arr(self):
+        return hex_str_to_bytes(self.to_hex())
+
+    @classmethod
+    def deserialize(cls, f):
+        return cls(deser_uint256(f))
+
+    def serialize(self):
+        return ser_uint256(self.val)
+
+    def __hash__(self):
+        return hash(self.val)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "Hash(val=0x{:x})".format(self.val)
+
+    def __eq__(self, other):
+        return isinstance(other, Hash) and self.val == other.val
+
+
 def blake3(s):
-    return BLAKE3.blake3(s).hexdigest()
+    return Hash.from_rev_hex(BLAKE3.blake3(s).hexdigest())
 
 def ser_varint(n):
     r = b""
@@ -1904,7 +2004,7 @@ def ser_mweb_block(b):
     if b == None:
         return struct.pack("B", 0)
     else:
-        return struct.pack("B", 1) + hex_str_to_bytes(b)
+        return struct.pack("B", 1) + b.serialize()
 
 def deser_mweb_block(f):
     has_mweb = struct.unpack("B", f.read(1))[0]
@@ -1919,14 +2019,260 @@ def ser_mweb_tx(t):
     if t == None:
         return struct.pack("B", 0)
     else:
-        return struct.pack("B", 1) + hex_str_to_bytes(t)
+        return struct.pack("B", 1) + t.serialize()
 
 def deser_mweb_tx(f):
     has_mweb = struct.unpack("B", f.read(1))[0]
     if has_mweb == 1:
-        return binascii.hexlify(f.read())
+        mweb_tx = MWEBTransaction()
+        return mweb_tx.deserialize(f)
     else:
         return None
+        
+
+class MWEBInput:
+    __slots__ = ("features", "output_id", "commitment", "input_pubkey",
+                "output_pubkey", "extradata", "signature", "hash")
+
+    def __init__(self):
+        self.features = 0
+        self.output_id = Hash()
+        self.commitment = None
+        self.input_pubkey = None
+        self.output_pubkey = None
+        self.extradata = None
+        self.signature = None
+        self.hash = None
+
+    def deserialize(self, f):
+        self.features = f.read(1)
+        self.output_id = Hash.deserialize(f)
+        self.commitment = deser_pubkey(f)
+        self.output_pubkey = deser_pubkey(f)
+        if self.features & 1:
+            self.input_pubkey = deser_pubkey(f)
+        self.extradata = None
+        if self.features & 2:
+            self.extradata = deser_fixed_bytes(f, deser_compact_size(f))
+        self.signature = deser_signature(f)
+        self.rehash()
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("B", self.features)
+        r += ser_uint256(self.output_id)
+        r += ser_pubkey(self.commitment)
+        r += ser_pubkey(self.output_pubkey)
+        if self.features & 1:
+            r += ser_pubkey(self.input_pubkey)
+        if self.features & 2:
+            r += ser_compact_size(self.extradata)
+            r += ser_fixed_bytes(self.extradata, len(self.extradata))#extradata
+        r += ser_signature(self.signature)
+        return r
+    
+    def rehash(self):
+        self.hash = blake3(self.serialize())
+        return self.hash.to_hex()
+
+class MWEBOutputMessage:
+    __slots__ = ("features", "key_exchange_pubkey", "view_tag", "masked_value",
+                "masked_nonce", "extradata", "hash")
+
+    def __init__(self):
+        self.features = 0
+        self.key_exchange_pubkey = None
+        self.view_tag = 0
+        self.masked_value = None
+        self.masked_nonce = None
+        self.extradata = None
+        self.hash = None
+
+    def deserialize(self, f):
+        self.features = struct.unpack("B", f.read(1))[0]
+        if self.features & 1:
+            self.key_exchange_pubkey = deser_pubkey(f)
+            self.view_tag = struct.unpack("B", f.read(1))[0]
+            self.masked_value = struct.unpack("<q", f.read(8))[0]
+            self.masked_nonce = deser_fixed_bytes(f, 16)
+        self.extradata = None
+        if self.features & 2:
+            self.extradata = deser_fixed_bytes(f, deser_compact_size(f))
+        self.rehash()
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("B", self.features)
+        if self.features & 1:
+            r += ser_pubkey(self.key_exchange_pubkey)
+            r += struct.pack("B", self.view_tag)
+            r += struct.pack("<q", self.masked_value)
+            r += ser_fixed_bytes(self.masked_nonce, 16)
+        if self.features & 2:
+            r += ser_compact_size(self.extradata)
+            r += ser_fixed_bytes(self.extradata, len(self.extradata))
+        return r
+    
+    def rehash(self):
+        self.hash = blake3(self.serialize())
+        return self.hash.to_hex()
+
+class MWEBOutput:
+    __slots__ = ("commitment", "sender_pubkey", "receiver_pubkey", "message",
+                "proof", "signature", "hash")
+
+    def __init__(self):
+        self.commitment = None
+        self.sender_pubkey = None
+        self.receiver_pubkey = None
+        self.message = MWEBOutputMessage()
+        self.proof = None
+        self.signature = None
+        self.hash = None
+
+    def deserialize(self, f):
+        self.commitment = deser_pubkey(f)
+        self.sender_pubkey = deser_pubkey(f)
+        self.receiver_pubkey = deser_pubkey(f)
+        self.message.deserialize(f)
+        self.proof = deser_fixed_bytes(f, 675)
+        self.signature = deser_signature(f)
+        self.rehash()
+
+    def serialize(self):
+        r = b""
+        r += ser_pubkey(self.commitment)
+        r += ser_pubkey(self.sender_pubkey)
+        r += ser_pubkey(self.receiver_pubkey)
+        r += self.message.serialize()
+        r += ser_fixed_bytes(self.proof, 675)
+        r += ser_signature(self.signature)
+        return r
+    
+    def rehash(self):
+        self.hash = blake3(self.serialize())
+        return self.hash.to_hex()
+
+class MWEBKernel:
+    __slots__ = ("features", "fee", "pegin", "pegouts", "lock_height",
+                "stealth_excess", "extradata", "excess", "signature", "hash")
+
+    def __init__(self):
+        self.features = 0
+        self.fee = None
+        self.pegin = None
+        self.pegouts = None
+        self.lock_height = None
+        self.stealth_excess = None
+        self.extradata = None
+        self.excess = None
+        self.signature = None
+        self.hash = None
+
+    def deserialize(self, f):
+        self.features = struct.unpack("B", f.read(1))[0]
+        self.fee = None
+        if self.features & 1:
+            self.fee = deser_varint(f)
+        self.pegin = None
+        if self.features & 2:
+            self.pegin = deser_varint(f)
+        self.pegouts = None
+        if self.features & 4:
+            self.pegouts =  []# TODO: deser_vector(f, CPegout)
+        self.lock_height = None
+        if self.features & 8:
+            self.lock_height = deser_varint(f)
+        self.stealth_excess = None
+        if self.features & 16:
+            self.stealth_excess = Hash.deserialize(f)
+        self.extradata = None
+        if self.features & 32:
+            self.extradata = f.read(deser_compact_size(f))
+        self.excess = deser_pubkey(f)
+        self.signature = deser_signature(f)
+        self.rehash()
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("B", self.features)
+        if self.features & 1:
+            r += ser_varint(self.fee)
+        if self.features & 2:
+            r += ser_varint(self.pegin)
+        if self.features & 4:
+            r += ser_vector(self.pegouts)
+        if self.features & 8:
+            r += ser_varint(self.lock_height)
+        if self.features & 16:
+            r += self.stealth_excess.serialize()
+        if self.features & 32:
+            r += ser_compact_size(len(self.extradata))
+            for i in range(len(self.extradata)):
+                struct.pack("B", self.extradata[i])
+        r += ser_pubkey(self.excess)
+        r += ser_signature(self.signature)
+        return r
+    
+    def rehash(self):
+        self.hash = blake3(self.serialize())
+        return self.hash.to_hex()
+
+    def __repr__(self):
+        return "MWEBKernel(features=%d, excess=%s)" % (self.features, repr(self.excess))
+
+class MWEBTxBody:
+    __slots__ = ("inputs", "outputs", "kernels")
+
+    def __init__(self):
+        self.inputs = []
+        self.outputs = []
+        self.kernels = []
+
+    def deserialize(self, f):
+        self.inputs = deser_vector(f, MWEBInput)
+        self.outputs = deser_vector(f, MWEBOutput)
+        self.kernels = deser_vector(f, MWEBKernel)
+
+    def serialize(self):
+        r = b""
+        r += ser_vector(self.inputs)
+        r += ser_vector(self.outputs)
+        r += ser_vector(self.kernels)
+        return r
+
+    def __repr__(self):
+        return "MWEBTxBody(inputs=%s, outputs=%d, kernels=%s)" % (repr(self.inputs), repr(self.outputs), repr(self.kernels))
+
+
+class MWEBTransaction:
+    __slots__ = ("kernel_offset", "stealth_offset", "body", "hash")
+
+    def __init__(self):
+        self.kernel_offset = Hash()
+        self.stealth_offset = Hash()
+        self.body = MWEBTxBody()
+        self.hash = None
+
+    def deserialize(self, f):
+        self.kernel_offset = Hash.deserialize(f)
+        self.stealth_offset = Hash.deserialize(f)
+        self.body.deserialize(f)
+        self.rehash()
+
+    def serialize(self):
+        r = b""
+        r += self.kernel_offset.serialize()
+        r += self.stealth_offset.serialize()
+        r += self.body.serialize()
+        return r
+    
+    def rehash(self):
+        self.hash = blake3(self.serialize())
+        return self.hash.to_hex()
+
+    def __repr__(self):
+        return "MWEBTransaction(kernel_offset=%s, stealth_offset=%d, body=%s, hash=%s)" % (repr(self.kernel_offset), repr(self.stealth_offset), repr(self.body), repr(self.hash))
 
 class MWEBHeader:
     __slots__ = ("height", "output_root", "kernel_root", "leafset_root",
@@ -1934,33 +2280,33 @@ class MWEBHeader:
 
     def __init__(self):
         self.height = 0
-        self.output_root = 0
-        self.kernel_root = 0
-        self.leafset_root = 0
-        self.kernel_offset = 0
-        self.stealth_offset = 0
+        self.output_root = Hash()
+        self.kernel_root = Hash()
+        self.leafset_root = Hash()
+        self.kernel_offset = Hash()
+        self.stealth_offset = Hash()
         self.num_txos = 0
         self.num_kernels = 0
         self.hash = None
 
     def from_json(self, mweb_json):
         self.height = mweb_json['height']
-        self.output_root = hex_str_to_bytes(mweb_json['output_root'])
-        self.kernel_root = hex_str_to_bytes(mweb_json['kernel_root'])
-        self.leafset_root = hex_str_to_bytes(mweb_json['leaf_root'])
-        self.kernel_offset = hex_str_to_bytes(mweb_json['kernel_offset'])
-        self.stealth_offset = hex_str_to_bytes(mweb_json['stealth_offset'])
+        self.output_root = Hash.from_rev_hex(mweb_json['output_root'])
+        self.kernel_root = Hash.from_rev_hex(mweb_json['kernel_root'])
+        self.leafset_root = Hash.from_rev_hex(mweb_json['leaf_root'])
+        self.kernel_offset = Hash.from_rev_hex(mweb_json['kernel_offset'])
+        self.stealth_offset = Hash.from_rev_hex(mweb_json['stealth_offset'])
         self.num_txos = mweb_json['num_txos']
         self.num_kernels = mweb_json['num_kernels']
         self.rehash()
 
     def deserialize(self, f):
         self.height = deser_varint(f)
-        self.output_root = deser_uint256(f)
-        self.kernel_root = deser_uint256(f)
-        self.leafset_root = deser_uint256(f)
-        self.kernel_offset = deser_uint256(f)
-        self.stealth_offset = deser_uint256(f)
+        self.output_root = Hash.deserialize(f)
+        self.kernel_root = Hash.deserialize(f)
+        self.leafset_root = Hash.deserialize(f)
+        self.kernel_offset = Hash.deserialize(f)
+        self.stealth_offset = Hash.deserialize(f)
         self.num_txos = deser_varint(f)
         self.num_kernels = deser_varint(f)
         self.rehash()
@@ -1968,46 +2314,94 @@ class MWEBHeader:
     def serialize(self):
         r = b""
         r += ser_varint(self.height)
-        r += self.output_root
-        r += self.kernel_root
-        r += self.leafset_root
-        r += self.kernel_offset
-        r += self.stealth_offset
+        r += self.output_root.serialize()
+        r += self.kernel_root.serialize()
+        r += self.leafset_root.serialize()
+        r += self.kernel_offset.serialize()
+        r += self.stealth_offset.serialize()
         r += ser_varint(self.num_txos)
         r += ser_varint(self.num_kernels)
         return r
     
     def rehash(self):
         self.hash = blake3(self.serialize())
-        return self.hash
+        return self.hash.to_hex()
 
     def __repr__(self):
-        return "MWEBHeader(height=%s, hash=%s)" % (repr(self.height), repr(self.hash))
+        return ("MWEBHeader(height=%s, output_root=%s, kernel_root=%s, leafset_root=%s, kernel_offset=%s, stealth_offset=%s, num_txos=%d, num_kernels=%d, hash=%s)" %
+            (repr(self.height), repr(self.output_root), repr(self.kernel_root), repr(self.leafset_root), repr(self.kernel_offset), repr(self.stealth_offset), self.num_txos, self.num_kernels, repr(self.hash)))
+
+    def __eq__(self, other):
+        return isinstance(other, MWEBHeader) and self.hash == other.hash
 
-# TODO: Finish this class
 class MWEBBlock:
-    __slots__ = ("header", "inputs", "outputs", "kernels")
+    __slots__ = ("header", "body")
 
-    def __init__(self, header = MWEBHeader(), inputs = [], outputs = [], kernels = []):
+    def __init__(self, header = MWEBHeader()):
         self.header = copy.deepcopy(header)
-        self.inputs = inputs
-        self.outputs = outputs
-        self.kernels = kernels
+        self.body = MWEBTxBody()
 
     def deserialize(self, f):
         self.header.deserialize(f)
-        #self.inputs = deser_vector(f, MWEBInput)
-        #self.outputs = deser_vector(f, MWEBOutput)
-        #self.kernels = deser_vector(f, MWEBKernel)
+        self.body.deserialize(f)
         self.rehash()
 
     def serialize(self):
         r = b""
         r += self.header.serialize()
-        r += ser_vector(self.inputs)
-        r += ser_vector(self.outputs)
-        r += ser_vector(self.kernels)
+        r += self.body.serialize()
         return r
     
     def rehash(self):
-        return self.header.rehash()
+        return self.header.rehash()
+
+    def __repr__(self):
+        return "MWEBBlock(header=%s, body=%s)" % (repr(self.header), repr(self.body))
+
+
+class CMerkleBlockWithMWEB:
+    __slots__ = ("merkle", "hogex", "mweb_header")
+
+    def __init__(self):
+        self.merkle = CMerkleBlock()
+        self.hogex = CTransaction()
+        self.mweb_header = MWEBHeader()
+
+    def deserialize(self, f):
+        self.merkle.deserialize(f)
+        self.hogex.deserialize(f)
+        self.mweb_header.deserialize(f)
+
+    def serialize(self):
+        r = b""
+        r += self.merkle.serialize()
+        r += self.hogex.serialize_with_mweb()
+        r += self.mweb_header.serialize()
+        return r
+
+    def __repr__(self):
+        return "CMerkleBlockWithMWEB(merkle=%s, hogex=%s, mweb_header=%s)" % (repr(self.merkle), repr(self.hogex), repr(self.mweb_header))
+
+
+class msg_mwebheader:
+    __slots__ = ("merkleblockwithmweb",)
+    msgtype = b"mwebheader"
+
+    def __init__(self, merkleblockwithmweb=None):
+        if merkleblockwithmweb is None:
+            self.merkleblockwithmweb = CMerkleBlockWithMWEB()
+        else:
+            self.merkleblockwithmweb = merkleblockwithmweb
+
+    def deserialize(self, f):
+        self.merkleblockwithmweb.deserialize(f)
+
+    def serialize(self):
+        return self.merkleblockwithmweb.serialize()
+
+    def header_hash(self):
+        self.merkleblockwithmweb.merkle.header.rehash()
+        return self.merkleblockwithmweb.merkle.header.hash
+
+    def __repr__(self):
+        return "msg_mwebheader(merkleblockwithmweb=%s)" % (repr(self.merkleblockwithmweb))

test/functional/test_framework/p2p.py

@@ -54,6 +54,7 @@ from test_framework.messages import (
     msg_inv,
     msg_mempool,
     msg_merkleblock,
+    msg_mwebheader,
     msg_notfound,
     msg_ping,
     msg_pong,
@@ -102,6 +103,7 @@ MESSAGEMAP = {
     b"inv": msg_inv,
     b"mempool": msg_mempool,
     b"merkleblock": msg_merkleblock,
+    b"mwebheader": msg_mwebheader,
     b"notfound": msg_notfound,
     b"ping": msg_ping,
     b"pong": msg_pong,
@@ -390,6 +392,7 @@ class P2PInterface(P2PConnection):
     def on_getdata(self, message): pass
     def on_getheaders(self, message): pass
     def on_headers(self, message): pass
+    def on_hogexheader(self, message): pass
     def on_mempool(self, message): pass
     def on_merkleblock(self, message): pass
     def on_notfound(self, message): pass
@@ -475,6 +478,18 @@ class P2PInterface(P2PConnection):
 
         self.wait_until(test_function, timeout=timeout)
 
+    def wait_for_mwebheader(self, blockhash, timeout=60):
+        """Waits for an mwebheader message
+        
+        The hash of the block header must match the provided blockhash"""
+        def test_function():
+            last_mwebheader = self.last_message.get('mwebheader')
+            if not last_mwebheader:
+                return False
+            return last_mwebheader.merkleblockwithmweb.merkle.header.rehash() == int(blockhash, 16)
+
+        self.wait_until(test_function, timeout=timeout)
+
     def wait_for_getdata(self, hash_list, timeout=60):
         """Waits for a getdata message.
 

test/functional/test_runner.py

@@ -248,6 +248,7 @@ BASE_SCRIPTS = [
     'mweb_basic.py',
     'mweb_mining.py',
     'mweb_reorg.py',
+    'mweb_p2p.py',
     'mweb_pegout_all.py',
     'mweb_node_compatibility.py',
     'mweb_wallet_address.py',