p2p_compactblocks.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2016-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Test compact blocks (BIP 152).""" 6 import random 7 8 from test_framework.blocktools import ( 9 COINBASE_MATURITY, 10 NORMAL_GBT_REQUEST_PARAMS, 11 add_witness_commitment, 12 create_block, 13 ) 14 from test_framework.messages import ( 15 BlockTransactions, 16 BlockTransactionsRequest, 17 CBlock, 18 CBlockHeader, 19 CInv, 20 COutPoint, 21 CTransaction, 22 CTxIn, 23 CTxInWitness, 24 CTxOut, 25 from_hex, 26 HeaderAndShortIDs, 27 MSG_BLOCK, 28 MSG_CMPCT_BLOCK, 29 MSG_WITNESS_FLAG, 30 P2PHeaderAndShortIDs, 31 PrefilledTransaction, 32 calculate_shortid, 33 msg_block, 34 msg_blocktxn, 35 msg_cmpctblock, 36 msg_getblocktxn, 37 msg_getdata, 38 msg_getheaders, 39 msg_headers, 40 msg_inv, 41 msg_no_witness_block, 42 msg_no_witness_blocktxn, 43 msg_sendcmpct, 44 msg_sendheaders, 45 msg_tx, 46 ser_uint256, 47 tx_from_hex, 48 ) 49 from test_framework.p2p import ( 50 P2PInterface, 51 p2p_lock, 52 ) 53 from test_framework.script import ( 54 CScript, 55 OP_DROP, 56 OP_TRUE, 57 OP_RETURN, 58 ) 59 from test_framework.test_framework import BitcoinTestFramework 60 from test_framework.util import ( 61 assert_not_equal, 62 assert_equal, 63 softfork_active, 64 ) 65 from test_framework.wallet import MiniWallet 66 67 68 # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. 69 class TestP2PConn(P2PInterface): 70 def __init__(self): 71 super().__init__() 72 self.last_sendcmpct = [] 73 self.block_announced = False 74 # Store the hashes of blocks we've seen announced. 75 # This is for synchronizing the p2p message traffic, 76 # so we can eg wait until a particular block is announced. 77 self.announced_blockhashes = set() 78 79 def on_sendcmpct(self, message): 80 self.last_sendcmpct.append(message) 81 82 def on_cmpctblock(self, message): 83 self.block_announced = True 84 self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.hash_int) 85 86 def on_headers(self, message): 87 self.block_announced = True 88 for x in self.last_message["headers"].headers: 89 self.announced_blockhashes.add(x.hash_int) 90 91 def on_inv(self, message): 92 for x in self.last_message["inv"].inv: 93 if x.type == MSG_BLOCK: 94 self.block_announced = True 95 self.announced_blockhashes.add(x.hash) 96 97 # Requires caller to hold p2p_lock 98 def received_block_announcement(self): 99 return self.block_announced 100 101 def clear_block_announcement(self): 102 with p2p_lock: 103 self.block_announced = False 104 self.last_message.pop("inv", None) 105 self.last_message.pop("headers", None) 106 self.last_message.pop("cmpctblock", None) 107 108 def clear_getblocktxn(self): 109 with p2p_lock: 110 self.last_message.pop("getblocktxn", None) 111 112 def get_headers(self, locator, hashstop): 113 msg = msg_getheaders() 114 msg.locator.vHave = locator 115 msg.hashstop = hashstop 116 self.send_without_ping(msg) 117 118 def send_header_for_blocks(self, new_blocks): 119 headers_message = msg_headers() 120 headers_message.headers = [CBlockHeader(b) for b in new_blocks] 121 self.send_without_ping(headers_message) 122 123 def request_headers_and_sync(self, locator, hashstop=0): 124 self.clear_block_announcement() 125 self.get_headers(locator, hashstop) 126 self.wait_until(self.received_block_announcement, timeout=30) 127 self.clear_block_announcement() 128 129 # Block until a block announcement for a particular block hash is 130 # received. 131 def wait_for_block_announcement(self, block_hash, timeout=30): 132 def received_hash(): 133 return (block_hash in self.announced_blockhashes) 134 self.wait_until(received_hash, timeout=timeout) 135 136 def send_await_disconnect(self, message, timeout=30): 137 """Sends a message to the node and wait for disconnect. 138 139 This is used when we want to send a message into the node that we expect 140 will get us disconnected, eg an invalid block.""" 141 self.send_without_ping(message) 142 self.wait_for_disconnect(timeout=timeout) 143 144 class CompactBlocksTest(BitcoinTestFramework): 145 def set_test_params(self): 146 self.setup_clean_chain = True 147 self.num_nodes = 1 148 self.extra_args = [[ 149 "-acceptnonstdtxn=1", 150 ]] 151 self.utxos = [] 152 153 def build_block_on_tip(self, node): 154 block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) 155 block.solve() 156 return block 157 158 # Create 10 more anyone-can-spend utxo's for testing. 159 def make_utxos(self): 160 block = self.build_block_on_tip(self.nodes[0]) 161 self.segwit_node.send_and_ping(msg_no_witness_block(block)) 162 assert_equal(self.nodes[0].getbestblockhash(), block.hash_hex) 163 self.generate(self.wallet, COINBASE_MATURITY) 164 165 total_value = block.vtx[0].vout[0].nValue 166 out_value = total_value // 10 167 tx = CTransaction() 168 tx.vin.append(CTxIn(COutPoint(block.vtx[0].txid_int, 0), b'')) 169 for _ in range(10): 170 tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) 171 172 block2 = self.build_block_on_tip(self.nodes[0]) 173 block2.vtx.append(tx) 174 block2.hashMerkleRoot = block2.calc_merkle_root() 175 block2.solve() 176 self.segwit_node.send_and_ping(msg_no_witness_block(block2)) 177 assert_equal(self.nodes[0].getbestblockhash(), block2.hash_hex) 178 self.utxos.extend([[tx.txid_int, i, out_value] for i in range(10)]) 179 180 181 # Test "sendcmpct" (between peers preferring the same version): 182 # - No compact block announcements unless sendcmpct is sent. 183 # - If sendcmpct is sent with version = 1, the message is ignored. 184 # - If sendcmpct is sent with version > 2, the message is ignored. 185 # - If sendcmpct is sent with boolean 0, then block announcements are not 186 # made with compact blocks. 187 # - If sendcmpct is then sent with boolean 1, then new block announcements 188 # are made with compact blocks. 189 def test_sendcmpct(self, test_node): 190 node = self.nodes[0] 191 192 # Make sure we get a SENDCMPCT message from our peer 193 def received_sendcmpct(): 194 return (len(test_node.last_sendcmpct) > 0) 195 test_node.wait_until(received_sendcmpct, timeout=30) 196 with p2p_lock: 197 # Check that version 2 is received. 198 assert_equal(test_node.last_sendcmpct[0].version, 2) 199 test_node.last_sendcmpct = [] 200 201 tip = int(node.getbestblockhash(), 16) 202 203 def check_announcement_of_new_block(node, peer, predicate): 204 peer.clear_block_announcement() 205 block_hash = int(self.generate(node, 1)[0], 16) 206 peer.wait_for_block_announcement(block_hash, timeout=30) 207 assert peer.block_announced 208 209 with p2p_lock: 210 assert predicate(peer), ( 211 "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( 212 block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) 213 214 # We shouldn't get any block announcements via cmpctblock yet. 215 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 216 217 # Try one more time, this time after requesting headers. 218 test_node.request_headers_and_sync(locator=[tip]) 219 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) 220 221 # Test a few ways of using sendcmpct that should NOT 222 # result in compact block announcements. 223 # Before each test, sync the headers chain. 224 test_node.request_headers_and_sync(locator=[tip]) 225 226 # Now try a SENDCMPCT message with too-low version 227 test_node.send_and_ping(msg_sendcmpct(announce=True, version=1)) 228 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 229 230 # Headers sync before next test. 231 test_node.request_headers_and_sync(locator=[tip]) 232 233 # Now try a SENDCMPCT message with too-high version 234 test_node.send_and_ping(msg_sendcmpct(announce=True, version=3)) 235 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 236 237 # Headers sync before next test. 238 test_node.request_headers_and_sync(locator=[tip]) 239 240 # Now try a SENDCMPCT message with valid version, but announce=False 241 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 242 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 243 244 # Headers sync before next test. 245 test_node.request_headers_and_sync(locator=[tip]) 246 247 # Finally, try a SENDCMPCT message with announce=True 248 test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 249 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 250 251 # Try one more time (no headers sync should be needed!) 252 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 253 254 # Try one more time, after turning on sendheaders 255 test_node.send_and_ping(msg_sendheaders()) 256 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 257 258 # Try one more time, after sending a version=1, announce=false message. 259 test_node.send_and_ping(msg_sendcmpct(announce=False, version=1)) 260 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 261 262 # Now turn off announcements 263 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 264 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) 265 266 # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. 267 def test_invalid_cmpctblock_message(self): 268 self.generate(self.nodes[0], COINBASE_MATURITY + 1) 269 block = self.build_block_on_tip(self.nodes[0]) 270 271 cmpct_block = P2PHeaderAndShortIDs() 272 cmpct_block.header = CBlockHeader(block) 273 cmpct_block.prefilled_txn_length = 1 274 # This index will be too high 275 prefilled_txn = PrefilledTransaction(1, block.vtx[0]) 276 cmpct_block.prefilled_txn = [prefilled_txn] 277 self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) 278 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) 279 280 # Compare the generated shortids to what we expect based on BIP 152, given 281 # bitcoind's choice of nonce. 282 def test_compactblock_construction(self, test_node): 283 node = self.nodes[0] 284 # Generate a bunch of transactions. 285 self.generate(node, COINBASE_MATURITY + 1) 286 num_transactions = 25 287 288 segwit_tx_generated = False 289 for _ in range(num_transactions): 290 hex_tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['hex'] 291 tx = tx_from_hex(hex_tx) 292 if not tx.wit.is_null(): 293 segwit_tx_generated = True 294 295 assert segwit_tx_generated # check that our test is not broken 296 297 # Wait until we've seen the block announcement for the resulting tip 298 tip = int(node.getbestblockhash(), 16) 299 test_node.wait_for_block_announcement(tip) 300 301 # Make sure we will receive a fast-announce compact block 302 self.request_cb_announcements(test_node) 303 304 # Now mine a block, and look at the resulting compact block. 305 test_node.clear_block_announcement() 306 block_hash = int(self.generate(node, 1)[0], 16) 307 308 # Store the raw block in our internal format. 309 block = from_hex(CBlock(), node.getblock("%064x" % block_hash, False)) 310 311 # Wait until the block was announced (via compact blocks) 312 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 313 314 # Now fetch and check the compact block 315 header_and_shortids = None 316 with p2p_lock: 317 # Convert the on-the-wire representation to absolute indexes 318 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 319 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 320 321 # Now fetch the compact block using a normal non-announce getdata 322 test_node.clear_block_announcement() 323 inv = CInv(MSG_CMPCT_BLOCK, block_hash) 324 test_node.send_without_ping(msg_getdata([inv])) 325 326 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 327 328 # Now fetch and check the compact block 329 header_and_shortids = None 330 with p2p_lock: 331 # Convert the on-the-wire representation to absolute indexes 332 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 333 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 334 335 def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block): 336 # Check that we got the right block! 337 assert_equal(header_and_shortids.header.hash_int, block_hash) 338 339 # Make sure the prefilled_txn appears to have included the coinbase 340 assert len(header_and_shortids.prefilled_txn) >= 1 341 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) 342 343 # Check that all prefilled_txn entries match what's in the block. 344 for entry in header_and_shortids.prefilled_txn: 345 # This checks the non-witness parts of the tx agree 346 assert_equal(entry.tx.txid_hex, block.vtx[entry.index].txid_hex) 347 348 # And this checks the witness 349 assert_equal(entry.tx.wtxid_hex, block.vtx[entry.index].wtxid_hex) 350 351 # Check that the cmpctblock message announced all the transactions. 352 assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) 353 354 # And now check that all the shortids are as expected as well. 355 # Determine the siphash keys to use. 356 [k0, k1] = header_and_shortids.get_siphash_keys() 357 358 index = 0 359 while index < len(block.vtx): 360 if (len(header_and_shortids.prefilled_txn) > 0 and 361 header_and_shortids.prefilled_txn[0].index == index): 362 # Already checked prefilled transactions above 363 header_and_shortids.prefilled_txn.pop(0) 364 else: 365 tx_hash = block.vtx[index].wtxid_int 366 shortid = calculate_shortid(k0, k1, tx_hash) 367 assert_equal(shortid, header_and_shortids.shortids[0]) 368 header_and_shortids.shortids.pop(0) 369 index += 1 370 371 # Test that bitcoind requests compact blocks when we announce new blocks 372 # via header or inv, and that responding to getblocktxn causes the block 373 # to be successfully reconstructed. 374 def test_compactblock_requests(self, test_node): 375 node = self.nodes[0] 376 # Try announcing a block with an inv or header, expect a compactblock 377 # request 378 for announce in ["inv", "header"]: 379 block = self.build_block_on_tip(node) 380 381 if announce == "inv": 382 test_node.send_without_ping(msg_inv([CInv(MSG_BLOCK, block.hash_int)])) 383 test_node.wait_for_getheaders(timeout=30) 384 test_node.send_header_for_blocks([block]) 385 else: 386 test_node.send_header_for_blocks([block]) 387 test_node.wait_for_getdata([block.hash_int], timeout=30) 388 assert_equal(test_node.last_message["getdata"].inv[0].type, 4) 389 390 # Send back a compactblock message that omits the coinbase 391 comp_block = HeaderAndShortIDs() 392 comp_block.header = CBlockHeader(block) 393 comp_block.nonce = 0 394 [k0, k1] = comp_block.get_siphash_keys() 395 coinbase_hash = block.vtx[0].wtxid_int 396 comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] 397 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 398 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 399 # Expect a getblocktxn message. 400 with p2p_lock: 401 assert "getblocktxn" in test_node.last_message 402 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 403 assert_equal(absolute_indexes, [0]) # should be a coinbase request 404 405 # Send the coinbase, and verify that the tip advances. 406 msg = msg_blocktxn() 407 msg.block_transactions.blockhash = block.hash_int 408 msg.block_transactions.transactions = [block.vtx[0]] 409 test_node.send_and_ping(msg) 410 assert_equal(node.getbestblockhash(), block.hash_hex) 411 412 # Create a chain of transactions from given utxo, and add to a new block. 413 def build_block_with_transactions(self, node, utxo, num_transactions): 414 block = self.build_block_on_tip(node) 415 416 for _ in range(num_transactions): 417 tx = CTransaction() 418 tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) 419 tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) 420 utxo = [tx.txid_int, 0, tx.vout[0].nValue] 421 block.vtx.append(tx) 422 423 block.hashMerkleRoot = block.calc_merkle_root() 424 block.solve() 425 return block 426 427 # Test that we only receive getblocktxn requests for transactions that the 428 # node needs, and that responding to them causes the block to be 429 # reconstructed. 430 def test_getblocktxn_requests(self, test_node): 431 node = self.nodes[0] 432 433 def test_getblocktxn_response(compact_block, peer, expected_result): 434 msg = msg_cmpctblock(compact_block.to_p2p()) 435 peer.send_and_ping(msg) 436 with p2p_lock: 437 assert "getblocktxn" in peer.last_message 438 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() 439 assert_equal(absolute_indexes, expected_result) 440 441 def test_tip_after_message(node, peer, msg, tip): 442 peer.send_and_ping(msg) 443 assert_equal(int(node.getbestblockhash(), 16), tip) 444 445 # First try announcing compactblocks that won't reconstruct, and verify 446 # that we receive getblocktxn messages back. 447 utxo = self.utxos.pop(0) 448 449 block = self.build_block_with_transactions(node, utxo, 5) 450 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 451 comp_block = HeaderAndShortIDs() 452 comp_block.initialize_from_block(block, use_witness=True) 453 454 test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) 455 456 msg_bt = msg_no_witness_blocktxn() 457 msg_bt = msg_blocktxn() # serialize with witnesses 458 msg_bt.block_transactions = BlockTransactions(block.hash_int, block.vtx[1:]) 459 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 460 461 utxo = self.utxos.pop(0) 462 block = self.build_block_with_transactions(node, utxo, 5) 463 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 464 465 # Now try interspersing the prefilled transactions 466 comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True) 467 test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) 468 msg_bt.block_transactions = BlockTransactions(block.hash_int, block.vtx[2:5]) 469 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 470 471 # Now try giving one transaction ahead of time. 472 utxo = self.utxos.pop(0) 473 block = self.build_block_with_transactions(node, utxo, 5) 474 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 475 test_node.send_and_ping(msg_tx(block.vtx[1])) 476 assert block.vtx[1].txid_hex in node.getrawmempool() 477 478 # Prefill 4 out of the 6 transactions, and verify that only the one 479 # that was not in the mempool is requested. 480 comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True) 481 test_getblocktxn_response(comp_block, test_node, [5]) 482 483 msg_bt.block_transactions = BlockTransactions(block.hash_int, [block.vtx[5]]) 484 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 485 486 # Now provide all transactions to the node before the block is 487 # announced and verify reconstruction happens immediately. 488 utxo = self.utxos.pop(0) 489 block = self.build_block_with_transactions(node, utxo, 10) 490 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 491 for tx in block.vtx[1:]: 492 test_node.send_without_ping(msg_tx(tx)) 493 test_node.sync_with_ping() 494 # Make sure all transactions were accepted. 495 mempool = node.getrawmempool() 496 for tx in block.vtx[1:]: 497 assert tx.txid_hex in mempool 498 499 # Clear out last request. 500 with p2p_lock: 501 test_node.last_message.pop("getblocktxn", None) 502 503 # Send compact block 504 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 505 test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.hash_int) 506 with p2p_lock: 507 # Shouldn't have gotten a request for any transaction 508 assert "getblocktxn" not in test_node.last_message 509 510 # Incorrectly responding to a getblocktxn shouldn't cause the block to be 511 # permanently failed. 512 def test_incorrect_blocktxn_response(self, test_node): 513 node = self.nodes[0] 514 utxo = self.utxos.pop(0) 515 516 block = self.build_block_with_transactions(node, utxo, 10) 517 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 518 # Relay the first 5 transactions from the block in advance 519 for tx in block.vtx[1:6]: 520 test_node.send_without_ping(msg_tx(tx)) 521 test_node.sync_with_ping() 522 # Make sure all transactions were accepted. 523 mempool = node.getrawmempool() 524 for tx in block.vtx[1:6]: 525 assert tx.txid_hex in mempool 526 527 # Send compact block 528 comp_block = HeaderAndShortIDs() 529 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 530 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 531 absolute_indexes = [] 532 with p2p_lock: 533 assert "getblocktxn" in test_node.last_message 534 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 535 assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) 536 537 # Now give an incorrect response. 538 # Note that it's possible for bitcoind to be smart enough to know we're 539 # lying, since it could check to see if the shortid matches what we're 540 # sending, and eg disconnect us for misbehavior. If that behavior 541 # change was made, we could just modify this test by having a 542 # different peer provide the block further down, so that we're still 543 # verifying that the block isn't marked bad permanently. This is good 544 # enough for now. 545 msg = msg_blocktxn() 546 msg.block_transactions = BlockTransactions(block.hash_int, [block.vtx[5]] + block.vtx[7:]) 547 test_node.send_and_ping(msg) 548 549 # Tip should not have updated 550 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 551 552 # We should receive a getdata request 553 test_node.wait_for_getdata([block.hash_int], timeout=10) 554 assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \ 555 test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG 556 557 # Deliver the block 558 test_node.send_and_ping(msg_block(block)) 559 assert_equal(node.getbestblockhash(), block.hash_hex) 560 561 # Multiple blocktxn responses will cause a node to get disconnected. 562 def test_multiple_blocktxn_response(self, test_node): 563 node = self.nodes[0] 564 utxo = self.utxos[0] 565 566 block = self.build_block_with_transactions(node, utxo, 2) 567 568 # Send compact block 569 comp_block = HeaderAndShortIDs() 570 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 571 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 572 absolute_indexes = [] 573 with p2p_lock: 574 assert "getblocktxn" in test_node.last_message 575 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 576 assert_equal(absolute_indexes, [1, 2]) 577 578 # Send a blocktxn that does not succeed in reconstruction, triggering 579 # getdata fallback. 580 msg = msg_blocktxn() 581 msg.block_transactions = BlockTransactions(block.hash_int, [block.vtx[2]] + [block.vtx[1]]) 582 test_node.send_and_ping(msg) 583 584 # Tip should not have updated 585 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 586 587 # We should receive a getdata request 588 test_node.wait_for_getdata([block.hash_int], timeout=10) 589 assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \ 590 test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG 591 592 # Send the same blocktxn and assert the sender gets disconnected. 593 with node.assert_debug_log(['previous compact block reconstruction attempt failed']): 594 test_node.send_without_ping(msg) 595 test_node.wait_for_disconnect() 596 597 def test_getblocktxn_handler(self, test_node): 598 node = self.nodes[0] 599 # bitcoind will not send blocktxn responses for blocks whose height is 600 # more than 10 blocks deep. 601 MAX_GETBLOCKTXN_DEPTH = 10 602 chain_height = node.getblockcount() 603 current_height = chain_height 604 while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): 605 block_hash = node.getblockhash(current_height) 606 block = from_hex(CBlock(), node.getblock(block_hash, False)) 607 608 msg = msg_getblocktxn() 609 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) 610 num_to_request = random.randint(1, len(block.vtx)) 611 msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) 612 test_node.send_without_ping(msg) 613 test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) 614 615 with p2p_lock: 616 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) 617 all_indices = msg.block_txn_request.to_absolute() 618 for index in all_indices: 619 tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) 620 assert_equal(tx.txid_hex, block.vtx[index].txid_hex) 621 # Check that the witness matches 622 assert_equal(tx.wtxid_hex, block.vtx[index].wtxid_hex) 623 test_node.last_message.pop("blocktxn", None) 624 current_height -= 1 625 626 # Next request should send a full block response, as we're past the 627 # allowed depth for a blocktxn response. 628 block_hash = node.getblockhash(current_height) 629 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) 630 with p2p_lock: 631 test_node.last_message.pop("block", None) 632 test_node.last_message.pop("blocktxn", None) 633 test_node.send_and_ping(msg) 634 with p2p_lock: 635 assert_equal(test_node.last_message["block"].block.hash_hex, block_hash) 636 assert "blocktxn" not in test_node.last_message 637 638 # Request with out-of-bounds tx index results in disconnect 639 bad_peer = self.nodes[0].add_p2p_connection(TestP2PConn()) 640 block_hash = node.getblockhash(chain_height) 641 block = from_hex(CBlock(), node.getblock(block_hash, False)) 642 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [len(block.vtx)]) 643 with node.assert_debug_log(['getblocktxn with out-of-bounds tx indices']): 644 bad_peer.send_without_ping(msg) 645 bad_peer.wait_for_disconnect() 646 647 def test_low_work_compactblocks(self, test_node): 648 # A compactblock with insufficient work won't get its header included 649 node = self.nodes[0] 650 hashPrevBlock = int(node.getblockhash(node.getblockcount() - 150), 16) 651 block = self.build_block_on_tip(node) 652 block.hashPrevBlock = hashPrevBlock 653 block.solve() 654 655 comp_block = HeaderAndShortIDs() 656 comp_block.initialize_from_block(block) 657 with self.nodes[0].assert_debug_log(['[net] Ignoring low-work compact block from peer 0']): 658 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 659 660 tips = node.getchaintips() 661 found = False 662 for x in tips: 663 if x["hash"] == block.hash_hex: 664 found = True 665 break 666 assert not found 667 668 def test_compactblocks_not_at_tip(self, test_node): 669 node = self.nodes[0] 670 # Test that requesting old compactblocks doesn't work. 671 MAX_CMPCTBLOCK_DEPTH = 5 672 new_blocks = [] 673 for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): 674 test_node.clear_block_announcement() 675 new_blocks.append(self.generate(node, 1)[0]) 676 test_node.wait_until(test_node.received_block_announcement, timeout=30) 677 678 test_node.clear_block_announcement() 679 test_node.send_without_ping(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 680 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 681 682 test_node.clear_block_announcement() 683 self.generate(node, 1) 684 test_node.wait_until(test_node.received_block_announcement, timeout=30) 685 test_node.clear_block_announcement() 686 with p2p_lock: 687 test_node.last_message.pop("block", None) 688 test_node.send_without_ping(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 689 test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) 690 with p2p_lock: 691 assert_equal(test_node.last_message["block"].block.hash_hex, new_blocks[0]) 692 693 # Generate an old compactblock, and verify that it's not accepted. 694 cur_height = node.getblockcount() 695 hashPrevBlock = int(node.getblockhash(cur_height - 5), 16) 696 block = self.build_block_on_tip(node) 697 block.hashPrevBlock = hashPrevBlock 698 block.solve() 699 700 comp_block = HeaderAndShortIDs() 701 comp_block.initialize_from_block(block) 702 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 703 704 tips = node.getchaintips() 705 found = False 706 for x in tips: 707 if x["hash"] == block.hash_hex: 708 assert_equal(x["status"], "headers-only") 709 found = True 710 break 711 assert found 712 713 # Requesting this block via getblocktxn should silently fail 714 # (to avoid fingerprinting attacks). 715 msg = msg_getblocktxn() 716 msg.block_txn_request = BlockTransactionsRequest(block.hash_int, [0]) 717 with p2p_lock: 718 test_node.last_message.pop("blocktxn", None) 719 test_node.send_and_ping(msg) 720 with p2p_lock: 721 assert "blocktxn" not in test_node.last_message 722 723 def test_end_to_end_block_relay(self, listeners): 724 node = self.nodes[0] 725 utxo = self.utxos.pop(0) 726 727 block = self.build_block_with_transactions(node, utxo, 10) 728 729 [l.clear_block_announcement() for l in listeners] 730 731 # serialize without witness (this block has no witnesses anyway). 732 # TODO: repeat this test with witness tx's to a segwit node. 733 node.submitblock(block.serialize().hex()) 734 735 for l in listeners: 736 l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) 737 with p2p_lock: 738 for l in listeners: 739 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.hash_int, block.hash_int) 740 741 # Test that we don't get disconnected if we relay a compact block with valid header, 742 # but invalid transactions. 743 def test_invalid_tx_in_compactblock(self, test_node): 744 node = self.nodes[0] 745 assert len(self.utxos) 746 utxo = self.utxos[0] 747 748 block = self.build_block_with_transactions(node, utxo, 5) 749 block.hashMerkleRoot = block.calc_merkle_root() 750 # Drop the coinbase witness but include the witness commitment. 751 add_witness_commitment(block) 752 block.vtx[0].wit.vtxinwit = [] 753 block.solve() 754 755 # Now send the compact block with all transactions prefilled, and 756 # verify that we don't get disconnected. 757 comp_block = HeaderAndShortIDs() 758 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 759 msg = msg_cmpctblock(comp_block.to_p2p()) 760 test_node.send_and_ping(msg) 761 762 # Check that the tip didn't advance 763 assert_not_equal(node.getbestblockhash(), block.hash_hex) 764 test_node.sync_with_ping() 765 766 # Re-establish a proper witness commitment with the coinbase witness, but 767 # invalidate the last tx in the block. 768 block.vtx[4].vin[0].scriptSig = CScript([OP_RETURN]) 769 block.hashMerkleRoot = block.calc_merkle_root() 770 add_witness_commitment(block) 771 block.solve() 772 773 # This will lead to a consensus failure for which we also won't be disconnected but which 774 # will be cached. 775 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 776 msg = msg_cmpctblock(comp_block.to_p2p()) 777 test_node.send_and_ping(msg) 778 779 # The tip still didn't advance. 780 assert_not_equal(node.getbestblockhash(), block.hash_hex) 781 test_node.sync_with_ping() 782 783 # The failure above was cached. Submitting the compact block again will return a cached 784 # consensus error (the code path is different) and still not get us disconnected (nor 785 # advance the tip). 786 test_node.send_and_ping(msg) 787 assert_not_equal(node.getbestblockhash(), block.hash_hex) 788 test_node.sync_with_ping() 789 790 # Now, announcing a second block building on top of the invalid one will get us disconnected. 791 block.hashPrevBlock = block.hash_int 792 block.solve() 793 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 794 msg = msg_cmpctblock(comp_block.to_p2p()) 795 test_node.send_await_disconnect(msg) 796 797 # Helper for enabling cb announcements 798 # Send the sendcmpct request and sync headers 799 def request_cb_announcements(self, peer): 800 node = self.nodes[0] 801 tip = node.getbestblockhash() 802 peer.get_headers(locator=[int(tip, 16)], hashstop=0) 803 peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) 804 805 def test_compactblock_reconstruction_stalling_peer(self, stalling_peer, delivery_peer): 806 node = self.nodes[0] 807 assert len(self.utxos) 808 809 def announce_cmpct_block(node, peer): 810 utxo = self.utxos.pop(0) 811 block = self.build_block_with_transactions(node, utxo, 5) 812 813 cmpct_block = HeaderAndShortIDs() 814 cmpct_block.initialize_from_block(block) 815 msg = msg_cmpctblock(cmpct_block.to_p2p()) 816 peer.send_and_ping(msg) 817 with p2p_lock: 818 assert "getblocktxn" in peer.last_message 819 return block, cmpct_block 820 821 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 822 823 for tx in block.vtx[1:]: 824 delivery_peer.send_without_ping(msg_tx(tx)) 825 delivery_peer.sync_with_ping() 826 mempool = node.getrawmempool() 827 for tx in block.vtx[1:]: 828 assert tx.txid_hex in mempool 829 830 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 831 assert_equal(node.getbestblockhash(), block.hash_hex) 832 833 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 834 835 # Now test that delivering an invalid compact block won't break relay 836 837 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 838 for tx in block.vtx[1:]: 839 delivery_peer.send_without_ping(msg_tx(tx)) 840 delivery_peer.sync_with_ping() 841 842 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()] 843 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] 844 845 cmpct_block.use_witness = True 846 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 847 assert_not_equal(node.getbestblockhash(), block.hash_hex) 848 849 msg = msg_no_witness_blocktxn() 850 msg.block_transactions.blockhash = block.hash_int 851 msg.block_transactions.transactions = block.vtx[1:] 852 stalling_peer.send_and_ping(msg) 853 assert_equal(node.getbestblockhash(), block.hash_hex) 854 855 def test_highbandwidth_mode_states_via_getpeerinfo(self): 856 # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent 857 hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 858 859 # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` 860 # match the given parameters for the last peer of a given node 861 def assert_highbandwidth_states(node, hb_to, hb_from): 862 peerinfo = node.getpeerinfo()[-1] 863 assert_equal(peerinfo['bip152_hb_to'], hb_to) 864 assert_equal(peerinfo['bip152_hb_from'], hb_from) 865 866 # initially, neither node has selected the other peer as high-bandwidth yet 867 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=False) 868 869 # peer requests high-bandwidth mode by sending sendcmpct(1) 870 hb_test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 871 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=True) 872 873 # peer generates a block and sends it to node, which should 874 # select the peer as high-bandwidth (up to 3 peers according to BIP 152) 875 block = self.build_block_on_tip(self.nodes[0]) 876 hb_test_node.send_and_ping(msg_block(block)) 877 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=True) 878 879 # peer requests low-bandwidth mode by sending sendcmpct(0) 880 hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 881 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False) 882 883 def test_compactblock_reconstruction_parallel_reconstruction(self, stalling_peer, delivery_peer, inbound_peer, outbound_peer): 884 """ All p2p connections are inbound except outbound_peer. We test that ultimate parallel slot 885 can only be taken by an outbound node unless prior attempts were done by an outbound 886 """ 887 node = self.nodes[0] 888 assert len(self.utxos) 889 890 def announce_cmpct_block(node, peer, txn_count): 891 utxo = self.utxos.pop(0) 892 block = self.build_block_with_transactions(node, utxo, txn_count) 893 894 cmpct_block = HeaderAndShortIDs() 895 cmpct_block.initialize_from_block(block) 896 msg = msg_cmpctblock(cmpct_block.to_p2p()) 897 peer.send_and_ping(msg) 898 with p2p_lock: 899 assert "getblocktxn" in peer.last_message 900 return block, cmpct_block 901 902 for name, peer in [("delivery", delivery_peer), ("inbound", inbound_peer), ("outbound", outbound_peer)]: 903 self.log.info(f"Setting {name} as high bandwidth peer") 904 block, cmpct_block = announce_cmpct_block(node, peer, 1) 905 msg = msg_blocktxn() 906 msg.block_transactions.blockhash = block.hash_int 907 msg.block_transactions.transactions = block.vtx[1:] 908 peer.send_and_ping(msg) 909 assert_equal(node.getbestblockhash(), block.hash_hex) 910 peer.clear_getblocktxn() 911 912 # Test the simple parallel download case... 913 for num_missing in [1, 5, 20]: 914 915 # Remaining low-bandwidth peer is stalling_peer, who announces first 916 assert_equal([peer['bip152_hb_to'] for peer in node.getpeerinfo()], [False, True, True, True]) 917 918 block, cmpct_block = announce_cmpct_block(node, stalling_peer, num_missing) 919 920 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 921 with p2p_lock: 922 # The second peer to announce should still get a getblocktxn 923 assert "getblocktxn" in delivery_peer.last_message 924 assert_not_equal(node.getbestblockhash(), block.hash_hex) 925 926 inbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 927 with p2p_lock: 928 # The third inbound peer to announce should *not* get a getblocktxn 929 assert "getblocktxn" not in inbound_peer.last_message 930 assert_not_equal(node.getbestblockhash(), block.hash_hex) 931 932 outbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 933 with p2p_lock: 934 # The third peer to announce should get a getblocktxn if outbound 935 assert "getblocktxn" in outbound_peer.last_message 936 assert_not_equal(node.getbestblockhash(), block.hash_hex) 937 938 # Second peer completes the compact block first 939 msg = msg_blocktxn() 940 msg.block_transactions.blockhash = block.hash_int 941 msg.block_transactions.transactions = block.vtx[1:] 942 delivery_peer.send_and_ping(msg) 943 assert_equal(node.getbestblockhash(), block.hash_hex) 944 945 # Nothing bad should happen if we get a late fill from the first peer... 946 stalling_peer.send_and_ping(msg) 947 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 948 949 delivery_peer.clear_getblocktxn() 950 inbound_peer.clear_getblocktxn() 951 outbound_peer.clear_getblocktxn() 952 953 954 def run_test(self): 955 self.wallet = MiniWallet(self.nodes[0]) 956 957 # Setup the p2p connections 958 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 959 self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 960 self.onemore_inbound_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 961 self.outbound_node = self.nodes[0].add_outbound_p2p_connection(TestP2PConn(), p2p_idx=3, connection_type="outbound-full-relay") 962 963 # We will need UTXOs to construct transactions in later tests. 964 self.make_utxos() 965 966 assert softfork_active(self.nodes[0], "segwit") 967 968 self.log.info("Testing SENDCMPCT p2p message... ") 969 self.test_sendcmpct(self.segwit_node) 970 self.test_sendcmpct(self.additional_segwit_node) 971 self.test_sendcmpct(self.onemore_inbound_node) 972 self.test_sendcmpct(self.outbound_node) 973 974 self.log.info("Testing compactblock construction...") 975 self.test_compactblock_construction(self.segwit_node) 976 977 self.log.info("Testing compactblock requests (segwit node)... ") 978 self.test_compactblock_requests(self.segwit_node) 979 980 self.log.info("Testing getblocktxn requests (segwit node)...") 981 self.test_getblocktxn_requests(self.segwit_node) 982 983 self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") 984 self.test_getblocktxn_handler(self.segwit_node) 985 986 self.log.info("Testing compactblock requests/announcements not at chain tip...") 987 self.test_compactblocks_not_at_tip(self.segwit_node) 988 989 self.log.info("Testing handling of low-work compact blocks...") 990 self.test_low_work_compactblocks(self.segwit_node) 991 992 self.log.info("Testing handling of incorrect blocktxn responses...") 993 self.test_incorrect_blocktxn_response(self.segwit_node) 994 995 self.log.info("Testing reconstructing compact blocks with a stalling peer...") 996 self.test_compactblock_reconstruction_stalling_peer(self.segwit_node, self.additional_segwit_node) 997 998 self.log.info("Testing reconstructing compact blocks from multiple peers...") 999 self.test_compactblock_reconstruction_parallel_reconstruction(stalling_peer=self.segwit_node, inbound_peer=self.onemore_inbound_node, delivery_peer=self.additional_segwit_node, outbound_peer=self.outbound_node) 1000 1001 # Test that if we submitblock to node1, we'll get a compact block 1002 # announcement to all peers. 1003 # (Post-segwit activation, blocks won't propagate from node0 to node1 1004 # automatically, so don't bother testing a block announced to node0.) 1005 self.log.info("Testing end-to-end block relay...") 1006 self.request_cb_announcements(self.segwit_node) 1007 self.request_cb_announcements(self.additional_segwit_node) 1008 self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node]) 1009 1010 self.log.info("Testing handling of invalid compact blocks...") 1011 self.test_invalid_tx_in_compactblock(self.segwit_node) 1012 1013 # The previous test will lead to a disconnection. Reconnect before continuing. 1014 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 1015 1016 self.log.info("Testing handling of multiple blocktxn responses...") 1017 self.test_multiple_blocktxn_response(self.segwit_node) 1018 1019 # The previous test will lead to a disconnection. Reconnect before continuing. 1020 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 1021 1022 self.log.info("Testing invalid index in cmpctblock message...") 1023 self.test_invalid_cmpctblock_message() 1024 1025 self.log.info("Testing high-bandwidth mode states via getpeerinfo...") 1026 self.test_highbandwidth_mode_states_via_getpeerinfo() 1027 1028 1029 if __name__ == '__main__': 1030 CompactBlocksTest(__file__).main()