p2p_compactblocks.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2016-2022 The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Test compact blocks (BIP 152).""" 6 import random 7 8 from test_framework.blocktools import ( 9 COINBASE_MATURITY, 10 NORMAL_GBT_REQUEST_PARAMS, 11 add_witness_commitment, 12 create_block, 13 ) 14 from test_framework.messages import ( 15 BlockTransactions, 16 BlockTransactionsRequest, 17 CBlock, 18 CBlockHeader, 19 CInv, 20 COutPoint, 21 CTransaction, 22 CTxIn, 23 CTxInWitness, 24 CTxOut, 25 from_hex, 26 HeaderAndShortIDs, 27 MSG_BLOCK, 28 MSG_CMPCT_BLOCK, 29 MSG_WITNESS_FLAG, 30 P2PHeaderAndShortIDs, 31 PrefilledTransaction, 32 calculate_shortid, 33 msg_block, 34 msg_blocktxn, 35 msg_cmpctblock, 36 msg_getblocktxn, 37 msg_getdata, 38 msg_getheaders, 39 msg_headers, 40 msg_inv, 41 msg_no_witness_block, 42 msg_no_witness_blocktxn, 43 msg_sendcmpct, 44 msg_sendheaders, 45 msg_tx, 46 ser_uint256, 47 tx_from_hex, 48 ) 49 from test_framework.p2p import ( 50 P2PInterface, 51 p2p_lock, 52 ) 53 from test_framework.script import ( 54 CScript, 55 OP_DROP, 56 OP_TRUE, 57 ) 58 from test_framework.test_framework import BitcoinTestFramework 59 from test_framework.util import ( 60 assert_equal, 61 softfork_active, 62 ) 63 from test_framework.wallet import MiniWallet 64 65 66 # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. 67 class TestP2PConn(P2PInterface): 68 def __init__(self): 69 super().__init__() 70 self.last_sendcmpct = [] 71 self.block_announced = False 72 # Store the hashes of blocks we've seen announced. 73 # This is for synchronizing the p2p message traffic, 74 # so we can eg wait until a particular block is announced. 75 self.announced_blockhashes = set() 76 77 def on_sendcmpct(self, message): 78 self.last_sendcmpct.append(message) 79 80 def on_cmpctblock(self, message): 81 self.block_announced = True 82 self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() 83 self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) 84 85 def on_headers(self, message): 86 self.block_announced = True 87 for x in self.last_message["headers"].headers: 88 x.calc_sha256() 89 self.announced_blockhashes.add(x.sha256) 90 91 def on_inv(self, message): 92 for x in self.last_message["inv"].inv: 93 if x.type == MSG_BLOCK: 94 self.block_announced = True 95 self.announced_blockhashes.add(x.hash) 96 97 # Requires caller to hold p2p_lock 98 def received_block_announcement(self): 99 return self.block_announced 100 101 def clear_block_announcement(self): 102 with p2p_lock: 103 self.block_announced = False 104 self.last_message.pop("inv", None) 105 self.last_message.pop("headers", None) 106 self.last_message.pop("cmpctblock", None) 107 108 def clear_getblocktxn(self): 109 with p2p_lock: 110 self.last_message.pop("getblocktxn", None) 111 112 def get_headers(self, locator, hashstop): 113 msg = msg_getheaders() 114 msg.locator.vHave = locator 115 msg.hashstop = hashstop 116 self.send_message(msg) 117 118 def send_header_for_blocks(self, new_blocks): 119 headers_message = msg_headers() 120 headers_message.headers = [CBlockHeader(b) for b in new_blocks] 121 self.send_message(headers_message) 122 123 def request_headers_and_sync(self, locator, hashstop=0): 124 self.clear_block_announcement() 125 self.get_headers(locator, hashstop) 126 self.wait_until(self.received_block_announcement, timeout=30) 127 self.clear_block_announcement() 128 129 # Block until a block announcement for a particular block hash is 130 # received. 131 def wait_for_block_announcement(self, block_hash, timeout=30): 132 def received_hash(): 133 return (block_hash in self.announced_blockhashes) 134 self.wait_until(received_hash, timeout=timeout) 135 136 def send_await_disconnect(self, message, timeout=30): 137 """Sends a message to the node and wait for disconnect. 138 139 This is used when we want to send a message into the node that we expect 140 will get us disconnected, eg an invalid block.""" 141 self.send_message(message) 142 self.wait_for_disconnect(timeout) 143 144 class CompactBlocksTest(BitcoinTestFramework): 145 def set_test_params(self): 146 self.setup_clean_chain = True 147 self.num_nodes = 1 148 self.extra_args = [[ 149 "-acceptnonstdtxn=1", 150 ]] 151 self.utxos = [] 152 153 def build_block_on_tip(self, node): 154 block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) 155 block.solve() 156 return block 157 158 # Create 10 more anyone-can-spend utxo's for testing. 159 def make_utxos(self): 160 block = self.build_block_on_tip(self.nodes[0]) 161 self.segwit_node.send_and_ping(msg_no_witness_block(block)) 162 assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 163 self.generate(self.wallet, COINBASE_MATURITY) 164 165 total_value = block.vtx[0].vout[0].nValue 166 out_value = total_value // 10 167 tx = CTransaction() 168 tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) 169 for _ in range(10): 170 tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) 171 tx.rehash() 172 173 block2 = self.build_block_on_tip(self.nodes[0]) 174 block2.vtx.append(tx) 175 block2.hashMerkleRoot = block2.calc_merkle_root() 176 block2.solve() 177 self.segwit_node.send_and_ping(msg_no_witness_block(block2)) 178 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) 179 self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) 180 181 182 # Test "sendcmpct" (between peers preferring the same version): 183 # - No compact block announcements unless sendcmpct is sent. 184 # - If sendcmpct is sent with version = 1, the message is ignored. 185 # - If sendcmpct is sent with version > 2, the message is ignored. 186 # - If sendcmpct is sent with boolean 0, then block announcements are not 187 # made with compact blocks. 188 # - If sendcmpct is then sent with boolean 1, then new block announcements 189 # are made with compact blocks. 190 def test_sendcmpct(self, test_node): 191 node = self.nodes[0] 192 193 # Make sure we get a SENDCMPCT message from our peer 194 def received_sendcmpct(): 195 return (len(test_node.last_sendcmpct) > 0) 196 test_node.wait_until(received_sendcmpct, timeout=30) 197 with p2p_lock: 198 # Check that version 2 is received. 199 assert_equal(test_node.last_sendcmpct[0].version, 2) 200 test_node.last_sendcmpct = [] 201 202 tip = int(node.getbestblockhash(), 16) 203 204 def check_announcement_of_new_block(node, peer, predicate): 205 peer.clear_block_announcement() 206 block_hash = int(self.generate(node, 1)[0], 16) 207 peer.wait_for_block_announcement(block_hash, timeout=30) 208 assert peer.block_announced 209 210 with p2p_lock: 211 assert predicate(peer), ( 212 "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( 213 block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) 214 215 # We shouldn't get any block announcements via cmpctblock yet. 216 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 217 218 # Try one more time, this time after requesting headers. 219 test_node.request_headers_and_sync(locator=[tip]) 220 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) 221 222 # Test a few ways of using sendcmpct that should NOT 223 # result in compact block announcements. 224 # Before each test, sync the headers chain. 225 test_node.request_headers_and_sync(locator=[tip]) 226 227 # Now try a SENDCMPCT message with too-low version 228 test_node.send_and_ping(msg_sendcmpct(announce=True, version=1)) 229 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 230 231 # Headers sync before next test. 232 test_node.request_headers_and_sync(locator=[tip]) 233 234 # Now try a SENDCMPCT message with too-high version 235 test_node.send_and_ping(msg_sendcmpct(announce=True, version=3)) 236 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 237 238 # Headers sync before next test. 239 test_node.request_headers_and_sync(locator=[tip]) 240 241 # Now try a SENDCMPCT message with valid version, but announce=False 242 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 243 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 244 245 # Headers sync before next test. 246 test_node.request_headers_and_sync(locator=[tip]) 247 248 # Finally, try a SENDCMPCT message with announce=True 249 test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 250 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 251 252 # Try one more time (no headers sync should be needed!) 253 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 254 255 # Try one more time, after turning on sendheaders 256 test_node.send_and_ping(msg_sendheaders()) 257 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 258 259 # Try one more time, after sending a version=1, announce=false message. 260 test_node.send_and_ping(msg_sendcmpct(announce=False, version=1)) 261 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 262 263 # Now turn off announcements 264 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 265 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) 266 267 # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. 268 def test_invalid_cmpctblock_message(self): 269 self.generate(self.nodes[0], COINBASE_MATURITY + 1) 270 block = self.build_block_on_tip(self.nodes[0]) 271 272 cmpct_block = P2PHeaderAndShortIDs() 273 cmpct_block.header = CBlockHeader(block) 274 cmpct_block.prefilled_txn_length = 1 275 # This index will be too high 276 prefilled_txn = PrefilledTransaction(1, block.vtx[0]) 277 cmpct_block.prefilled_txn = [prefilled_txn] 278 self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) 279 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) 280 281 # Compare the generated shortids to what we expect based on BIP 152, given 282 # bitcoind's choice of nonce. 283 def test_compactblock_construction(self, test_node): 284 node = self.nodes[0] 285 # Generate a bunch of transactions. 286 self.generate(node, COINBASE_MATURITY + 1) 287 num_transactions = 25 288 289 segwit_tx_generated = False 290 for _ in range(num_transactions): 291 hex_tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['hex'] 292 tx = tx_from_hex(hex_tx) 293 if not tx.wit.is_null(): 294 segwit_tx_generated = True 295 296 assert segwit_tx_generated # check that our test is not broken 297 298 # Wait until we've seen the block announcement for the resulting tip 299 tip = int(node.getbestblockhash(), 16) 300 test_node.wait_for_block_announcement(tip) 301 302 # Make sure we will receive a fast-announce compact block 303 self.request_cb_announcements(test_node) 304 305 # Now mine a block, and look at the resulting compact block. 306 test_node.clear_block_announcement() 307 block_hash = int(self.generate(node, 1)[0], 16) 308 309 # Store the raw block in our internal format. 310 block = from_hex(CBlock(), node.getblock("%064x" % block_hash, False)) 311 for tx in block.vtx: 312 tx.calc_sha256() 313 block.rehash() 314 315 # Wait until the block was announced (via compact blocks) 316 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 317 318 # Now fetch and check the compact block 319 header_and_shortids = None 320 with p2p_lock: 321 # Convert the on-the-wire representation to absolute indexes 322 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 323 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 324 325 # Now fetch the compact block using a normal non-announce getdata 326 test_node.clear_block_announcement() 327 inv = CInv(MSG_CMPCT_BLOCK, block_hash) 328 test_node.send_message(msg_getdata([inv])) 329 330 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 331 332 # Now fetch and check the compact block 333 header_and_shortids = None 334 with p2p_lock: 335 # Convert the on-the-wire representation to absolute indexes 336 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 337 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 338 339 def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block): 340 # Check that we got the right block! 341 header_and_shortids.header.calc_sha256() 342 assert_equal(header_and_shortids.header.sha256, block_hash) 343 344 # Make sure the prefilled_txn appears to have included the coinbase 345 assert len(header_and_shortids.prefilled_txn) >= 1 346 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) 347 348 # Check that all prefilled_txn entries match what's in the block. 349 for entry in header_and_shortids.prefilled_txn: 350 entry.tx.calc_sha256() 351 # This checks the non-witness parts of the tx agree 352 assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) 353 354 # And this checks the witness 355 wtxid = entry.tx.calc_sha256(True) 356 assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) 357 358 # Check that the cmpctblock message announced all the transactions. 359 assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) 360 361 # And now check that all the shortids are as expected as well. 362 # Determine the siphash keys to use. 363 [k0, k1] = header_and_shortids.get_siphash_keys() 364 365 index = 0 366 while index < len(block.vtx): 367 if (len(header_and_shortids.prefilled_txn) > 0 and 368 header_and_shortids.prefilled_txn[0].index == index): 369 # Already checked prefilled transactions above 370 header_and_shortids.prefilled_txn.pop(0) 371 else: 372 tx_hash = block.vtx[index].calc_sha256(True) 373 shortid = calculate_shortid(k0, k1, tx_hash) 374 assert_equal(shortid, header_and_shortids.shortids[0]) 375 header_and_shortids.shortids.pop(0) 376 index += 1 377 378 # Test that bitcoind requests compact blocks when we announce new blocks 379 # via header or inv, and that responding to getblocktxn causes the block 380 # to be successfully reconstructed. 381 def test_compactblock_requests(self, test_node): 382 node = self.nodes[0] 383 # Try announcing a block with an inv or header, expect a compactblock 384 # request 385 for announce in ["inv", "header"]: 386 block = self.build_block_on_tip(node) 387 388 if announce == "inv": 389 test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) 390 test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) 391 test_node.send_header_for_blocks([block]) 392 else: 393 test_node.send_header_for_blocks([block]) 394 test_node.wait_for_getdata([block.sha256], timeout=30) 395 assert_equal(test_node.last_message["getdata"].inv[0].type, 4) 396 397 # Send back a compactblock message that omits the coinbase 398 comp_block = HeaderAndShortIDs() 399 comp_block.header = CBlockHeader(block) 400 comp_block.nonce = 0 401 [k0, k1] = comp_block.get_siphash_keys() 402 coinbase_hash = block.vtx[0].calc_sha256(True) 403 comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] 404 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 405 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 406 # Expect a getblocktxn message. 407 with p2p_lock: 408 assert "getblocktxn" in test_node.last_message 409 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 410 assert_equal(absolute_indexes, [0]) # should be a coinbase request 411 412 # Send the coinbase, and verify that the tip advances. 413 msg = msg_blocktxn() 414 msg.block_transactions.blockhash = block.sha256 415 msg.block_transactions.transactions = [block.vtx[0]] 416 test_node.send_and_ping(msg) 417 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 418 419 # Create a chain of transactions from given utxo, and add to a new block. 420 def build_block_with_transactions(self, node, utxo, num_transactions): 421 block = self.build_block_on_tip(node) 422 423 for _ in range(num_transactions): 424 tx = CTransaction() 425 tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) 426 tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) 427 tx.rehash() 428 utxo = [tx.sha256, 0, tx.vout[0].nValue] 429 block.vtx.append(tx) 430 431 block.hashMerkleRoot = block.calc_merkle_root() 432 block.solve() 433 return block 434 435 # Test that we only receive getblocktxn requests for transactions that the 436 # node needs, and that responding to them causes the block to be 437 # reconstructed. 438 def test_getblocktxn_requests(self, test_node): 439 node = self.nodes[0] 440 441 def test_getblocktxn_response(compact_block, peer, expected_result): 442 msg = msg_cmpctblock(compact_block.to_p2p()) 443 peer.send_and_ping(msg) 444 with p2p_lock: 445 assert "getblocktxn" in peer.last_message 446 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() 447 assert_equal(absolute_indexes, expected_result) 448 449 def test_tip_after_message(node, peer, msg, tip): 450 peer.send_and_ping(msg) 451 assert_equal(int(node.getbestblockhash(), 16), tip) 452 453 # First try announcing compactblocks that won't reconstruct, and verify 454 # that we receive getblocktxn messages back. 455 utxo = self.utxos.pop(0) 456 457 block = self.build_block_with_transactions(node, utxo, 5) 458 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 459 comp_block = HeaderAndShortIDs() 460 comp_block.initialize_from_block(block, use_witness=True) 461 462 test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) 463 464 msg_bt = msg_no_witness_blocktxn() 465 msg_bt = msg_blocktxn() # serialize with witnesses 466 msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) 467 test_tip_after_message(node, test_node, msg_bt, block.sha256) 468 469 utxo = self.utxos.pop(0) 470 block = self.build_block_with_transactions(node, utxo, 5) 471 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 472 473 # Now try interspersing the prefilled transactions 474 comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True) 475 test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) 476 msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) 477 test_tip_after_message(node, test_node, msg_bt, block.sha256) 478 479 # Now try giving one transaction ahead of time. 480 utxo = self.utxos.pop(0) 481 block = self.build_block_with_transactions(node, utxo, 5) 482 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 483 test_node.send_and_ping(msg_tx(block.vtx[1])) 484 assert block.vtx[1].hash in node.getrawmempool() 485 486 # Prefill 4 out of the 6 transactions, and verify that only the one 487 # that was not in the mempool is requested. 488 comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True) 489 test_getblocktxn_response(comp_block, test_node, [5]) 490 491 msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) 492 test_tip_after_message(node, test_node, msg_bt, block.sha256) 493 494 # Now provide all transactions to the node before the block is 495 # announced and verify reconstruction happens immediately. 496 utxo = self.utxos.pop(0) 497 block = self.build_block_with_transactions(node, utxo, 10) 498 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 499 for tx in block.vtx[1:]: 500 test_node.send_message(msg_tx(tx)) 501 test_node.sync_with_ping() 502 # Make sure all transactions were accepted. 503 mempool = node.getrawmempool() 504 for tx in block.vtx[1:]: 505 assert tx.hash in mempool 506 507 # Clear out last request. 508 with p2p_lock: 509 test_node.last_message.pop("getblocktxn", None) 510 511 # Send compact block 512 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 513 test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) 514 with p2p_lock: 515 # Shouldn't have gotten a request for any transaction 516 assert "getblocktxn" not in test_node.last_message 517 518 # Incorrectly responding to a getblocktxn shouldn't cause the block to be 519 # permanently failed. 520 def test_incorrect_blocktxn_response(self, test_node): 521 node = self.nodes[0] 522 utxo = self.utxos.pop(0) 523 524 block = self.build_block_with_transactions(node, utxo, 10) 525 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 526 # Relay the first 5 transactions from the block in advance 527 for tx in block.vtx[1:6]: 528 test_node.send_message(msg_tx(tx)) 529 test_node.sync_with_ping() 530 # Make sure all transactions were accepted. 531 mempool = node.getrawmempool() 532 for tx in block.vtx[1:6]: 533 assert tx.hash in mempool 534 535 # Send compact block 536 comp_block = HeaderAndShortIDs() 537 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 538 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 539 absolute_indexes = [] 540 with p2p_lock: 541 assert "getblocktxn" in test_node.last_message 542 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 543 assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) 544 545 # Now give an incorrect response. 546 # Note that it's possible for bitcoind to be smart enough to know we're 547 # lying, since it could check to see if the shortid matches what we're 548 # sending, and eg disconnect us for misbehavior. If that behavior 549 # change was made, we could just modify this test by having a 550 # different peer provide the block further down, so that we're still 551 # verifying that the block isn't marked bad permanently. This is good 552 # enough for now. 553 msg = msg_blocktxn() 554 msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) 555 test_node.send_and_ping(msg) 556 557 # Tip should not have updated 558 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 559 560 # We should receive a getdata request 561 test_node.wait_for_getdata([block.sha256], timeout=10) 562 assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \ 563 test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG 564 565 # Deliver the block 566 test_node.send_and_ping(msg_block(block)) 567 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 568 569 def test_getblocktxn_handler(self, test_node): 570 node = self.nodes[0] 571 # bitcoind will not send blocktxn responses for blocks whose height is 572 # more than 10 blocks deep. 573 MAX_GETBLOCKTXN_DEPTH = 10 574 chain_height = node.getblockcount() 575 current_height = chain_height 576 while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): 577 block_hash = node.getblockhash(current_height) 578 block = from_hex(CBlock(), node.getblock(block_hash, False)) 579 580 msg = msg_getblocktxn() 581 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) 582 num_to_request = random.randint(1, len(block.vtx)) 583 msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) 584 test_node.send_message(msg) 585 test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) 586 587 [tx.calc_sha256() for tx in block.vtx] 588 with p2p_lock: 589 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) 590 all_indices = msg.block_txn_request.to_absolute() 591 for index in all_indices: 592 tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) 593 tx.calc_sha256() 594 assert_equal(tx.sha256, block.vtx[index].sha256) 595 # Check that the witness matches 596 assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) 597 test_node.last_message.pop("blocktxn", None) 598 current_height -= 1 599 600 # Next request should send a full block response, as we're past the 601 # allowed depth for a blocktxn response. 602 block_hash = node.getblockhash(current_height) 603 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) 604 with p2p_lock: 605 test_node.last_message.pop("block", None) 606 test_node.last_message.pop("blocktxn", None) 607 test_node.send_and_ping(msg) 608 with p2p_lock: 609 test_node.last_message["block"].block.calc_sha256() 610 assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) 611 assert "blocktxn" not in test_node.last_message 612 613 # Request with out-of-bounds tx index results in disconnect 614 bad_peer = self.nodes[0].add_p2p_connection(TestP2PConn()) 615 block_hash = node.getblockhash(chain_height) 616 block = from_hex(CBlock(), node.getblock(block_hash, False)) 617 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [len(block.vtx)]) 618 with node.assert_debug_log(['getblocktxn with out-of-bounds tx indices']): 619 bad_peer.send_message(msg) 620 bad_peer.wait_for_disconnect() 621 622 def test_low_work_compactblocks(self, test_node): 623 # A compactblock with insufficient work won't get its header included 624 node = self.nodes[0] 625 hashPrevBlock = int(node.getblockhash(node.getblockcount() - 150), 16) 626 block = self.build_block_on_tip(node) 627 block.hashPrevBlock = hashPrevBlock 628 block.solve() 629 630 comp_block = HeaderAndShortIDs() 631 comp_block.initialize_from_block(block) 632 with self.nodes[0].assert_debug_log(['[net] Ignoring low-work compact block from peer 0']): 633 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 634 635 tips = node.getchaintips() 636 found = False 637 for x in tips: 638 if x["hash"] == block.hash: 639 found = True 640 break 641 assert not found 642 643 def test_compactblocks_not_at_tip(self, test_node): 644 node = self.nodes[0] 645 # Test that requesting old compactblocks doesn't work. 646 MAX_CMPCTBLOCK_DEPTH = 5 647 new_blocks = [] 648 for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): 649 test_node.clear_block_announcement() 650 new_blocks.append(self.generate(node, 1)[0]) 651 test_node.wait_until(test_node.received_block_announcement, timeout=30) 652 653 test_node.clear_block_announcement() 654 test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 655 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 656 657 test_node.clear_block_announcement() 658 self.generate(node, 1) 659 test_node.wait_until(test_node.received_block_announcement, timeout=30) 660 test_node.clear_block_announcement() 661 with p2p_lock: 662 test_node.last_message.pop("block", None) 663 test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 664 test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) 665 with p2p_lock: 666 test_node.last_message["block"].block.calc_sha256() 667 assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) 668 669 # Generate an old compactblock, and verify that it's not accepted. 670 cur_height = node.getblockcount() 671 hashPrevBlock = int(node.getblockhash(cur_height - 5), 16) 672 block = self.build_block_on_tip(node) 673 block.hashPrevBlock = hashPrevBlock 674 block.solve() 675 676 comp_block = HeaderAndShortIDs() 677 comp_block.initialize_from_block(block) 678 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 679 680 tips = node.getchaintips() 681 found = False 682 for x in tips: 683 if x["hash"] == block.hash: 684 assert_equal(x["status"], "headers-only") 685 found = True 686 break 687 assert found 688 689 # Requesting this block via getblocktxn should silently fail 690 # (to avoid fingerprinting attacks). 691 msg = msg_getblocktxn() 692 msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) 693 with p2p_lock: 694 test_node.last_message.pop("blocktxn", None) 695 test_node.send_and_ping(msg) 696 with p2p_lock: 697 assert "blocktxn" not in test_node.last_message 698 699 def test_end_to_end_block_relay(self, listeners): 700 node = self.nodes[0] 701 utxo = self.utxos.pop(0) 702 703 block = self.build_block_with_transactions(node, utxo, 10) 704 705 [l.clear_block_announcement() for l in listeners] 706 707 # serialize without witness (this block has no witnesses anyway). 708 # TODO: repeat this test with witness tx's to a segwit node. 709 node.submitblock(block.serialize().hex()) 710 711 for l in listeners: 712 l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) 713 with p2p_lock: 714 for l in listeners: 715 l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() 716 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) 717 718 # Test that we don't get disconnected if we relay a compact block with valid header, 719 # but invalid transactions. 720 def test_invalid_tx_in_compactblock(self, test_node): 721 node = self.nodes[0] 722 assert len(self.utxos) 723 utxo = self.utxos[0] 724 725 block = self.build_block_with_transactions(node, utxo, 5) 726 del block.vtx[3] 727 block.hashMerkleRoot = block.calc_merkle_root() 728 # Drop the coinbase witness but include the witness commitment. 729 add_witness_commitment(block) 730 block.vtx[0].wit.vtxinwit = [] 731 block.solve() 732 733 # Now send the compact block with all transactions prefilled, and 734 # verify that we don't get disconnected. 735 comp_block = HeaderAndShortIDs() 736 comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=True) 737 msg = msg_cmpctblock(comp_block.to_p2p()) 738 test_node.send_and_ping(msg) 739 740 # Check that the tip didn't advance 741 assert int(node.getbestblockhash(), 16) is not block.sha256 742 test_node.sync_with_ping() 743 744 # Helper for enabling cb announcements 745 # Send the sendcmpct request and sync headers 746 def request_cb_announcements(self, peer): 747 node = self.nodes[0] 748 tip = node.getbestblockhash() 749 peer.get_headers(locator=[int(tip, 16)], hashstop=0) 750 peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) 751 752 def test_compactblock_reconstruction_stalling_peer(self, stalling_peer, delivery_peer): 753 node = self.nodes[0] 754 assert len(self.utxos) 755 756 def announce_cmpct_block(node, peer): 757 utxo = self.utxos.pop(0) 758 block = self.build_block_with_transactions(node, utxo, 5) 759 760 cmpct_block = HeaderAndShortIDs() 761 cmpct_block.initialize_from_block(block) 762 msg = msg_cmpctblock(cmpct_block.to_p2p()) 763 peer.send_and_ping(msg) 764 with p2p_lock: 765 assert "getblocktxn" in peer.last_message 766 return block, cmpct_block 767 768 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 769 770 for tx in block.vtx[1:]: 771 delivery_peer.send_message(msg_tx(tx)) 772 delivery_peer.sync_with_ping() 773 mempool = node.getrawmempool() 774 for tx in block.vtx[1:]: 775 assert tx.hash in mempool 776 777 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 778 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 779 780 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 781 782 # Now test that delivering an invalid compact block won't break relay 783 784 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 785 for tx in block.vtx[1:]: 786 delivery_peer.send_message(msg_tx(tx)) 787 delivery_peer.sync_with_ping() 788 789 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()] 790 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] 791 792 cmpct_block.use_witness = True 793 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 794 assert int(node.getbestblockhash(), 16) != block.sha256 795 796 msg = msg_no_witness_blocktxn() 797 msg.block_transactions.blockhash = block.sha256 798 msg.block_transactions.transactions = block.vtx[1:] 799 stalling_peer.send_and_ping(msg) 800 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 801 802 def test_highbandwidth_mode_states_via_getpeerinfo(self): 803 # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent 804 hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 805 806 # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` 807 # match the given parameters for the last peer of a given node 808 def assert_highbandwidth_states(node, hb_to, hb_from): 809 peerinfo = node.getpeerinfo()[-1] 810 assert_equal(peerinfo['bip152_hb_to'], hb_to) 811 assert_equal(peerinfo['bip152_hb_from'], hb_from) 812 813 # initially, neither node has selected the other peer as high-bandwidth yet 814 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=False) 815 816 # peer requests high-bandwidth mode by sending sendcmpct(1) 817 hb_test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 818 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=True) 819 820 # peer generates a block and sends it to node, which should 821 # select the peer as high-bandwidth (up to 3 peers according to BIP 152) 822 block = self.build_block_on_tip(self.nodes[0]) 823 hb_test_node.send_and_ping(msg_block(block)) 824 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=True) 825 826 # peer requests low-bandwidth mode by sending sendcmpct(0) 827 hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 828 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False) 829 830 def test_compactblock_reconstruction_parallel_reconstruction(self, stalling_peer, delivery_peer, inbound_peer, outbound_peer): 831 """ All p2p connections are inbound except outbound_peer. We test that ultimate parallel slot 832 can only be taken by an outbound node unless prior attempts were done by an outbound 833 """ 834 node = self.nodes[0] 835 assert len(self.utxos) 836 837 def announce_cmpct_block(node, peer, txn_count): 838 utxo = self.utxos.pop(0) 839 block = self.build_block_with_transactions(node, utxo, txn_count) 840 841 cmpct_block = HeaderAndShortIDs() 842 cmpct_block.initialize_from_block(block) 843 msg = msg_cmpctblock(cmpct_block.to_p2p()) 844 peer.send_and_ping(msg) 845 with p2p_lock: 846 assert "getblocktxn" in peer.last_message 847 return block, cmpct_block 848 849 for name, peer in [("delivery", delivery_peer), ("inbound", inbound_peer), ("outbound", outbound_peer)]: 850 self.log.info(f"Setting {name} as high bandwidth peer") 851 block, cmpct_block = announce_cmpct_block(node, peer, 1) 852 msg = msg_blocktxn() 853 msg.block_transactions.blockhash = block.sha256 854 msg.block_transactions.transactions = block.vtx[1:] 855 peer.send_and_ping(msg) 856 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 857 peer.clear_getblocktxn() 858 859 # Test the simple parallel download case... 860 for num_missing in [1, 5, 20]: 861 862 # Remaining low-bandwidth peer is stalling_peer, who announces first 863 assert_equal([peer['bip152_hb_to'] for peer in node.getpeerinfo()], [False, True, True, True]) 864 865 block, cmpct_block = announce_cmpct_block(node, stalling_peer, num_missing) 866 867 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 868 with p2p_lock: 869 # The second peer to announce should still get a getblocktxn 870 assert "getblocktxn" in delivery_peer.last_message 871 assert int(node.getbestblockhash(), 16) != block.sha256 872 873 inbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 874 with p2p_lock: 875 # The third inbound peer to announce should *not* get a getblocktxn 876 assert "getblocktxn" not in inbound_peer.last_message 877 assert int(node.getbestblockhash(), 16) != block.sha256 878 879 outbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 880 with p2p_lock: 881 # The third peer to announce should get a getblocktxn if outbound 882 assert "getblocktxn" in outbound_peer.last_message 883 assert int(node.getbestblockhash(), 16) != block.sha256 884 885 # Second peer completes the compact block first 886 msg = msg_blocktxn() 887 msg.block_transactions.blockhash = block.sha256 888 msg.block_transactions.transactions = block.vtx[1:] 889 delivery_peer.send_and_ping(msg) 890 assert_equal(int(node.getbestblockhash(), 16), block.sha256) 891 892 # Nothing bad should happen if we get a late fill from the first peer... 893 stalling_peer.send_and_ping(msg) 894 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) 895 896 delivery_peer.clear_getblocktxn() 897 inbound_peer.clear_getblocktxn() 898 outbound_peer.clear_getblocktxn() 899 900 901 def run_test(self): 902 self.wallet = MiniWallet(self.nodes[0]) 903 904 # Setup the p2p connections 905 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 906 self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 907 self.onemore_inbound_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 908 self.outbound_node = self.nodes[0].add_outbound_p2p_connection(TestP2PConn(), p2p_idx=3, connection_type="outbound-full-relay") 909 910 # We will need UTXOs to construct transactions in later tests. 911 self.make_utxos() 912 913 assert softfork_active(self.nodes[0], "segwit") 914 915 self.log.info("Testing SENDCMPCT p2p message... ") 916 self.test_sendcmpct(self.segwit_node) 917 self.test_sendcmpct(self.additional_segwit_node) 918 self.test_sendcmpct(self.onemore_inbound_node) 919 self.test_sendcmpct(self.outbound_node) 920 921 self.log.info("Testing compactblock construction...") 922 self.test_compactblock_construction(self.segwit_node) 923 924 self.log.info("Testing compactblock requests (segwit node)... ") 925 self.test_compactblock_requests(self.segwit_node) 926 927 self.log.info("Testing getblocktxn requests (segwit node)...") 928 self.test_getblocktxn_requests(self.segwit_node) 929 930 self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") 931 self.test_getblocktxn_handler(self.segwit_node) 932 933 self.log.info("Testing compactblock requests/announcements not at chain tip...") 934 self.test_compactblocks_not_at_tip(self.segwit_node) 935 936 self.log.info("Testing handling of low-work compact blocks...") 937 self.test_low_work_compactblocks(self.segwit_node) 938 939 self.log.info("Testing handling of incorrect blocktxn responses...") 940 self.test_incorrect_blocktxn_response(self.segwit_node) 941 942 self.log.info("Testing reconstructing compact blocks with a stalling peer...") 943 self.test_compactblock_reconstruction_stalling_peer(self.segwit_node, self.additional_segwit_node) 944 945 self.log.info("Testing reconstructing compact blocks from multiple peers...") 946 self.test_compactblock_reconstruction_parallel_reconstruction(stalling_peer=self.segwit_node, inbound_peer=self.onemore_inbound_node, delivery_peer=self.additional_segwit_node, outbound_peer=self.outbound_node) 947 948 # Test that if we submitblock to node1, we'll get a compact block 949 # announcement to all peers. 950 # (Post-segwit activation, blocks won't propagate from node0 to node1 951 # automatically, so don't bother testing a block announced to node0.) 952 self.log.info("Testing end-to-end block relay...") 953 self.request_cb_announcements(self.segwit_node) 954 self.request_cb_announcements(self.additional_segwit_node) 955 self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node]) 956 957 self.log.info("Testing handling of invalid compact blocks...") 958 self.test_invalid_tx_in_compactblock(self.segwit_node) 959 960 self.log.info("Testing invalid index in cmpctblock message...") 961 self.test_invalid_cmpctblock_message() 962 963 self.log.info("Testing high-bandwidth mode states via getpeerinfo...") 964 self.test_highbandwidth_mode_states_via_getpeerinfo() 965 966 967 if __name__ == '__main__': 968 CompactBlocksTest().main()