p2p_compactblocks.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2016-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Test compact blocks (BIP 152).""" 6 import random 7 8 from test_framework.blocktools import ( 9 COINBASE_MATURITY, 10 NORMAL_GBT_REQUEST_PARAMS, 11 add_witness_commitment, 12 create_block, 13 ) 14 from test_framework.messages import ( 15 BlockTransactions, 16 BlockTransactionsRequest, 17 CBlock, 18 CBlockHeader, 19 CInv, 20 COutPoint, 21 CTransaction, 22 CTxIn, 23 CTxInWitness, 24 CTxOut, 25 from_hex, 26 HeaderAndShortIDs, 27 MSG_BLOCK, 28 MSG_CMPCT_BLOCK, 29 MSG_WITNESS_FLAG, 30 P2PHeaderAndShortIDs, 31 PrefilledTransaction, 32 calculate_shortid, 33 msg_block, 34 msg_blocktxn, 35 msg_cmpctblock, 36 msg_getblocktxn, 37 msg_getdata, 38 msg_getheaders, 39 msg_headers, 40 msg_inv, 41 msg_no_witness_block, 42 msg_no_witness_blocktxn, 43 msg_sendcmpct, 44 msg_sendheaders, 45 msg_tx, 46 ser_uint256, 47 tx_from_hex, 48 ) 49 from test_framework.p2p import ( 50 P2PInterface, 51 p2p_lock, 52 ) 53 from test_framework.script import ( 54 CScript, 55 OP_DROP, 56 OP_TRUE, 57 OP_RETURN, 58 ) 59 from test_framework.test_framework import BitcoinTestFramework 60 from test_framework.util import ( 61 assert_not_equal, 62 assert_equal, 63 softfork_active, 64 ) 65 from test_framework.wallet import MiniWallet 66 67 68 # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. 69 class TestP2PConn(P2PInterface): 70 def __init__(self): 71 super().__init__() 72 self.last_sendcmpct = [] 73 self.block_announced = False 74 # Store the hashes of blocks we've seen announced. 75 # This is for synchronizing the p2p message traffic, 76 # so we can eg wait until a particular block is announced. 77 self.announced_blockhashes = set() 78 79 def on_sendcmpct(self, message): 80 self.last_sendcmpct.append(message) 81 82 def on_cmpctblock(self, message): 83 self.block_announced = True 84 self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.hash_int) 85 86 def on_headers(self, message): 87 self.block_announced = True 88 for x in self.last_message["headers"].headers: 89 self.announced_blockhashes.add(x.hash_int) 90 91 def on_inv(self, message): 92 for x in self.last_message["inv"].inv: 93 if x.type == MSG_BLOCK: 94 self.block_announced = True 95 self.announced_blockhashes.add(x.hash) 96 97 # Requires caller to hold p2p_lock 98 def received_block_announcement(self): 99 return self.block_announced 100 101 def clear_block_announcement(self): 102 with p2p_lock: 103 self.block_announced = False 104 self.last_message.pop("inv", None) 105 self.last_message.pop("headers", None) 106 self.last_message.pop("cmpctblock", None) 107 108 def clear_getblocktxn(self): 109 with p2p_lock: 110 self.last_message.pop("getblocktxn", None) 111 112 def get_headers(self, locator, hashstop): 113 msg = msg_getheaders() 114 msg.locator.vHave = locator 115 msg.hashstop = hashstop 116 self.send_without_ping(msg) 117 118 def send_header_for_blocks(self, new_blocks): 119 headers_message = msg_headers() 120 headers_message.headers = [CBlockHeader(b) for b in new_blocks] 121 self.send_without_ping(headers_message) 122 123 def request_headers_and_sync(self, locator, hashstop=0): 124 self.clear_block_announcement() 125 self.get_headers(locator, hashstop) 126 self.wait_until(self.received_block_announcement, timeout=30) 127 self.clear_block_announcement() 128 129 # Block until a block announcement for a particular block hash is 130 # received. 131 def wait_for_block_announcement(self, block_hash, timeout=30): 132 def received_hash(): 133 return (block_hash in self.announced_blockhashes) 134 self.wait_until(received_hash, timeout=timeout) 135 136 def send_await_disconnect(self, message, timeout=30): 137 """Sends a message to the node and wait for disconnect. 138 139 This is used when we want to send a message into the node that we expect 140 will get us disconnected, eg an invalid block.""" 141 self.send_without_ping(message) 142 self.wait_for_disconnect(timeout=timeout) 143 144 class CompactBlocksTest(BitcoinTestFramework): 145 def set_test_params(self): 146 self.setup_clean_chain = True 147 self.num_nodes = 1 148 self.extra_args = [[ 149 "-acceptnonstdtxn=1", 150 ]] 151 self.utxos = [] 152 153 def build_block_on_tip(self, node): 154 block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) 155 block.solve() 156 return block 157 158 # Create 10 more anyone-can-spend utxo's for testing. 159 def make_utxos(self): 160 block = self.build_block_on_tip(self.nodes[0]) 161 self.segwit_node.send_and_ping(msg_no_witness_block(block)) 162 assert_equal(self.nodes[0].getbestblockhash(), block.hash_hex) 163 self.generate(self.wallet, COINBASE_MATURITY) 164 165 total_value = block.vtx[0].vout[0].nValue 166 out_value = total_value // 10 167 tx = CTransaction() 168 tx.vin.append(CTxIn(COutPoint(block.vtx[0].txid_int, 0), b'')) 169 for _ in range(10): 170 tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) 171 172 block2 = self.build_block_on_tip(self.nodes[0]) 173 block2.vtx.append(tx) 174 block2.hashMerkleRoot = block2.calc_merkle_root() 175 block2.solve() 176 self.segwit_node.send_and_ping(msg_no_witness_block(block2)) 177 assert_equal(self.nodes[0].getbestblockhash(), block2.hash_hex) 178 self.utxos.extend([[tx.txid_int, i, out_value] for i in range(10)]) 179 180 181 # Test "sendcmpct" (between peers preferring the same version): 182 # - No compact block announcements unless sendcmpct is sent. 183 # - If sendcmpct is sent with version = 1, the message is ignored. 184 # - If sendcmpct is sent with version > 2, the message is ignored. 185 # - If sendcmpct is sent with boolean 0, then block announcements are not 186 # made with compact blocks. 187 # - If sendcmpct is then sent with boolean 1, then new block announcements 188 # are made with compact blocks. 189 def test_sendcmpct(self, test_node): 190 node = self.nodes[0] 191 192 # Make sure we get a SENDCMPCT message from our peer 193 def received_sendcmpct(): 194 return (len(test_node.last_sendcmpct) > 0) 195 test_node.wait_until(received_sendcmpct, timeout=30) 196 with p2p_lock: 197 # Check that version 2 is received. 198 assert_equal(test_node.last_sendcmpct[0].version, 2) 199 test_node.last_sendcmpct = [] 200 201 tip = int(node.getbestblockhash(), 16) 202 203 def check_announcement_of_new_block(node, peer, predicate): 204 peer.clear_block_announcement() 205 block_hash = int(self.generate(node, 1)[0], 16) 206 peer.wait_for_block_announcement(block_hash, timeout=30) 207 assert peer.block_announced 208 209 with p2p_lock: 210 assert predicate(peer), ( 211 "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( 212 block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) 213 214 # We shouldn't get any block announcements via cmpctblock yet. 215 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 216 217 # Try one more time, this time after requesting headers. 218 test_node.request_headers_and_sync(locator=[tip]) 219 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) 220 221 # Test a few ways of using sendcmpct that should NOT 222 # result in compact block announcements. 223 # Before each test, sync the headers chain. 224 test_node.request_headers_and_sync(locator=[tip]) 225 226 # Now try a SENDCMPCT message with too-low version 227 test_node.send_and_ping(msg_sendcmpct(announce=True, version=1)) 228 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 229 230 # Headers sync before next test. 231 test_node.request_headers_and_sync(locator=[tip]) 232 233 # Now try a SENDCMPCT message with too-high version 234 test_node.send_and_ping(msg_sendcmpct(announce=True, version=3)) 235 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 236 237 # Headers sync before next test. 238 test_node.request_headers_and_sync(locator=[tip]) 239 240 # Now try a SENDCMPCT message with valid version, but announce=False 241 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 242 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) 243 244 # Headers sync before next test. 245 test_node.request_headers_and_sync(locator=[tip]) 246 247 # Finally, try a SENDCMPCT message with announce=True 248 test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 249 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 250 251 # Try one more time (no headers sync should be needed!) 252 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 253 254 # Try one more time, after turning on sendheaders 255 test_node.send_and_ping(msg_sendheaders()) 256 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 257 258 # Try one more time, after sending a version=1, announce=false message. 259 test_node.send_and_ping(msg_sendcmpct(announce=False, version=1)) 260 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) 261 262 # Now turn off announcements 263 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 264 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) 265 266 # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. 267 def test_invalid_cmpctblock_message(self): 268 self.generate(self.nodes[0], COINBASE_MATURITY + 1) 269 block = self.build_block_on_tip(self.nodes[0]) 270 271 cmpct_block = P2PHeaderAndShortIDs() 272 cmpct_block.header = CBlockHeader(block) 273 cmpct_block.prefilled_txn_length = 1 274 # This index will be too high 275 prefilled_txn = PrefilledTransaction(1, block.vtx[0]) 276 cmpct_block.prefilled_txn = [prefilled_txn] 277 self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) 278 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) 279 280 # Compare the generated shortids to what we expect based on BIP 152, given 281 # bitcoind's choice of nonce. 282 def test_compactblock_construction(self, test_node): 283 node = self.nodes[0] 284 # Generate a bunch of transactions. 285 self.generate(node, COINBASE_MATURITY + 1) 286 num_transactions = 25 287 288 segwit_tx_generated = False 289 for _ in range(num_transactions): 290 hex_tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['hex'] 291 tx = tx_from_hex(hex_tx) 292 if not tx.wit.is_null(): 293 segwit_tx_generated = True 294 295 assert segwit_tx_generated # check that our test is not broken 296 297 # Wait until we've seen the block announcement for the resulting tip 298 tip = int(node.getbestblockhash(), 16) 299 test_node.wait_for_block_announcement(tip) 300 301 # Make sure we will receive a fast-announce compact block 302 self.request_cb_announcements(test_node) 303 304 # Now mine a block, and look at the resulting compact block. 305 test_node.clear_block_announcement() 306 block_hash = int(self.generate(node, 1)[0], 16) 307 308 # Store the raw block in our internal format. 309 block = from_hex(CBlock(), node.getblock("%064x" % block_hash, False)) 310 311 # Wait until the block was announced (via compact blocks) 312 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 313 314 # Now fetch and check the compact block 315 header_and_shortids = None 316 with p2p_lock: 317 # Convert the on-the-wire representation to absolute indexes 318 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 319 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 320 321 # Now fetch the compact block using a normal non-announce getdata 322 test_node.clear_block_announcement() 323 inv = CInv(MSG_CMPCT_BLOCK, block_hash) 324 test_node.send_without_ping(msg_getdata([inv])) 325 326 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 327 328 # Now fetch and check the compact block 329 header_and_shortids = None 330 with p2p_lock: 331 # Convert the on-the-wire representation to absolute indexes 332 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) 333 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) 334 335 def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block): 336 # Check that we got the right block! 337 assert_equal(header_and_shortids.header.hash_int, block_hash) 338 339 # Make sure the prefilled_txn appears to have included the coinbase 340 assert len(header_and_shortids.prefilled_txn) >= 1 341 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) 342 343 # Check that all prefilled_txn entries match what's in the block. 344 for entry in header_and_shortids.prefilled_txn: 345 # This checks the non-witness parts of the tx agree 346 assert_equal(entry.tx.txid_hex, block.vtx[entry.index].txid_hex) 347 348 # And this checks the witness 349 assert_equal(entry.tx.wtxid_hex, block.vtx[entry.index].wtxid_hex) 350 351 # Check that the cmpctblock message announced all the transactions. 352 assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) 353 354 # And now check that all the shortids are as expected as well. 355 # Determine the siphash keys to use. 356 [k0, k1] = header_and_shortids.get_siphash_keys() 357 358 index = 0 359 while index < len(block.vtx): 360 if (len(header_and_shortids.prefilled_txn) > 0 and 361 header_and_shortids.prefilled_txn[0].index == index): 362 # Already checked prefilled transactions above 363 header_and_shortids.prefilled_txn.pop(0) 364 else: 365 tx_hash = block.vtx[index].wtxid_int 366 shortid = calculate_shortid(k0, k1, tx_hash) 367 assert_equal(shortid, header_and_shortids.shortids[0]) 368 header_and_shortids.shortids.pop(0) 369 index += 1 370 371 # Test that bitcoind requests compact blocks when we announce new blocks 372 # via header or inv, and that responding to getblocktxn causes the block 373 # to be successfully reconstructed. 374 def test_compactblock_requests(self, test_node): 375 node = self.nodes[0] 376 # Try announcing a block with an inv or header, expect a compactblock 377 # request 378 for announce in ["inv", "header"]: 379 block = self.build_block_on_tip(node) 380 381 if announce == "inv": 382 test_node.send_without_ping(msg_inv([CInv(MSG_BLOCK, block.hash_int)])) 383 test_node.wait_for_getheaders(timeout=30) 384 test_node.send_header_for_blocks([block]) 385 else: 386 test_node.send_header_for_blocks([block]) 387 test_node.wait_for_getdata([block.hash_int], timeout=30) 388 assert_equal(test_node.last_message["getdata"].inv[0].type, 4) 389 390 # Send back a compactblock message that omits the coinbase 391 comp_block = HeaderAndShortIDs() 392 comp_block.header = CBlockHeader(block) 393 comp_block.nonce = 0 394 [k0, k1] = comp_block.get_siphash_keys() 395 coinbase_hash = block.vtx[0].wtxid_int 396 comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] 397 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 398 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 399 # Expect a getblocktxn message. 400 with p2p_lock: 401 assert "getblocktxn" in test_node.last_message 402 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 403 assert_equal(absolute_indexes, [0]) # should be a coinbase request 404 405 # Send the coinbase, and verify that the tip advances. 406 msg = msg_blocktxn() 407 msg.block_transactions.blockhash = block.hash_int 408 msg.block_transactions.transactions = [block.vtx[0]] 409 test_node.send_and_ping(msg) 410 assert_equal(node.getbestblockhash(), block.hash_hex) 411 412 # Create a chain of transactions from given utxo, and add to a new block. 413 def build_block_with_transactions(self, node, utxo, num_transactions): 414 block = self.build_block_on_tip(node) 415 416 for _ in range(num_transactions): 417 tx = CTransaction() 418 tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) 419 tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) 420 utxo = [tx.txid_int, 0, tx.vout[0].nValue] 421 block.vtx.append(tx) 422 423 block.hashMerkleRoot = block.calc_merkle_root() 424 block.solve() 425 return block 426 427 # Test that we only receive getblocktxn requests for transactions that the 428 # node needs, and that responding to them causes the block to be 429 # reconstructed. 430 def test_getblocktxn_requests(self, test_node): 431 node = self.nodes[0] 432 433 def test_getblocktxn_response(compact_block, peer, expected_result): 434 msg = msg_cmpctblock(compact_block.to_p2p()) 435 peer.send_and_ping(msg) 436 with p2p_lock: 437 assert "getblocktxn" in peer.last_message 438 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() 439 assert_equal(absolute_indexes, expected_result) 440 441 def test_tip_after_message(node, peer, msg, tip): 442 peer.send_and_ping(msg) 443 assert_equal(int(node.getbestblockhash(), 16), tip) 444 445 # First try announcing compactblocks that won't reconstruct, and verify 446 # that we receive getblocktxn messages back. 447 utxo = self.utxos.pop(0) 448 449 block = self.build_block_with_transactions(node, utxo, 5) 450 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 451 comp_block = HeaderAndShortIDs() 452 comp_block.initialize_from_block(block, use_witness=True) 453 454 test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) 455 456 msg_bt = msg_no_witness_blocktxn() 457 msg_bt = msg_blocktxn() # serialize with witnesses 458 msg_bt.block_transactions = BlockTransactions(block.hash_int, block.vtx[1:]) 459 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 460 461 utxo = self.utxos.pop(0) 462 block = self.build_block_with_transactions(node, utxo, 5) 463 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 464 465 # Now try interspersing the prefilled transactions 466 comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True) 467 test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) 468 msg_bt.block_transactions = BlockTransactions(block.hash_int, block.vtx[2:5]) 469 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 470 471 # Now try giving one transaction ahead of time. 472 utxo = self.utxos.pop(0) 473 block = self.build_block_with_transactions(node, utxo, 5) 474 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 475 test_node.send_and_ping(msg_tx(block.vtx[1])) 476 assert block.vtx[1].txid_hex in node.getrawmempool() 477 478 # Prefill 4 out of the 6 transactions, and verify that only the one 479 # that was not in the mempool is requested. 480 comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True) 481 test_getblocktxn_response(comp_block, test_node, [5]) 482 483 msg_bt.block_transactions = BlockTransactions(block.hash_int, [block.vtx[5]]) 484 test_tip_after_message(node, test_node, msg_bt, block.hash_int) 485 486 # Now provide all transactions to the node before the block is 487 # announced and verify reconstruction happens immediately. 488 utxo = self.utxos.pop(0) 489 block = self.build_block_with_transactions(node, utxo, 10) 490 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 491 for tx in block.vtx[1:]: 492 test_node.send_without_ping(msg_tx(tx)) 493 test_node.sync_with_ping() 494 # Make sure all transactions were accepted. 495 mempool = node.getrawmempool() 496 for tx in block.vtx[1:]: 497 assert tx.txid_hex in mempool 498 499 # Clear out last request. 500 with p2p_lock: 501 test_node.last_message.pop("getblocktxn", None) 502 503 # Send compact block 504 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 505 test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.hash_int) 506 with p2p_lock: 507 # Shouldn't have gotten a request for any transaction 508 assert "getblocktxn" not in test_node.last_message 509 510 # Incorrectly responding to a getblocktxn shouldn't cause the block to be 511 # permanently failed. 512 def test_incorrect_blocktxn_response(self, test_node): 513 node = self.nodes[0] 514 utxo = self.utxos.pop(0) 515 516 block = self.build_block_with_transactions(node, utxo, 10) 517 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 518 # Relay the first 5 transactions from the block in advance 519 for tx in block.vtx[1:6]: 520 test_node.send_without_ping(msg_tx(tx)) 521 test_node.sync_with_ping() 522 # Make sure all transactions were accepted. 523 mempool = node.getrawmempool() 524 for tx in block.vtx[1:6]: 525 assert tx.txid_hex in mempool 526 527 # Send compact block 528 comp_block = HeaderAndShortIDs() 529 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 530 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 531 absolute_indexes = [] 532 with p2p_lock: 533 assert "getblocktxn" in test_node.last_message 534 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 535 assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) 536 537 # Now give an incorrect response. 538 # Note that it's possible for bitcoind to be smart enough to know we're 539 # lying, since it could check to see if the shortid matches what we're 540 # sending, and eg disconnect us for misbehavior. If that behavior 541 # change was made, we could just modify this test by having a 542 # different peer provide the block further down, so that we're still 543 # verifying that the block isn't marked bad permanently. This is good 544 # enough for now. 545 msg = msg_blocktxn() 546 msg.block_transactions = BlockTransactions(block.hash_int, [block.vtx[5]] + block.vtx[7:]) 547 test_node.send_and_ping(msg) 548 549 # Tip should not have updated 550 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 551 552 # We should receive a getdata request 553 test_node.wait_for_getdata([block.hash_int], timeout=10) 554 assert test_node.last_message["getdata"].inv[0].type in (MSG_BLOCK, MSG_BLOCK | MSG_WITNESS_FLAG) 555 556 # Deliver the block 557 test_node.send_and_ping(msg_block(block)) 558 assert_equal(node.getbestblockhash(), block.hash_hex) 559 560 # Multiple blocktxn responses will cause a node to get disconnected. 561 def test_multiple_blocktxn_response(self, test_node): 562 node = self.nodes[0] 563 utxo = self.utxos[0] 564 565 block = self.build_block_with_transactions(node, utxo, 2) 566 567 # Send compact block 568 comp_block = HeaderAndShortIDs() 569 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) 570 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 571 absolute_indexes = [] 572 with p2p_lock: 573 assert "getblocktxn" in test_node.last_message 574 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() 575 assert_equal(absolute_indexes, [1, 2]) 576 577 # Send a blocktxn that does not succeed in reconstruction, triggering 578 # getdata fallback. 579 msg = msg_blocktxn() 580 msg.block_transactions = BlockTransactions(block.hash_int, [block.vtx[2]] + [block.vtx[1]]) 581 test_node.send_and_ping(msg) 582 583 # Tip should not have updated 584 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) 585 586 # We should receive a getdata request 587 test_node.wait_for_getdata([block.hash_int], timeout=10) 588 assert test_node.last_message["getdata"].inv[0].type in (MSG_BLOCK, MSG_BLOCK | MSG_WITNESS_FLAG) 589 590 # Send the same blocktxn and assert the sender gets disconnected. 591 with node.assert_debug_log(['previous compact block reconstruction attempt failed']): 592 test_node.send_without_ping(msg) 593 test_node.wait_for_disconnect() 594 595 def test_getblocktxn_handler(self, test_node): 596 node = self.nodes[0] 597 # bitcoind will not send blocktxn responses for blocks whose height is 598 # more than 10 blocks deep. 599 MAX_GETBLOCKTXN_DEPTH = 10 600 chain_height = node.getblockcount() 601 current_height = chain_height 602 while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): 603 block_hash = node.getblockhash(current_height) 604 block = from_hex(CBlock(), node.getblock(block_hash, False)) 605 606 msg = msg_getblocktxn() 607 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) 608 num_to_request = random.randint(1, len(block.vtx)) 609 msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) 610 test_node.send_without_ping(msg) 611 test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) 612 613 with p2p_lock: 614 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) 615 all_indices = msg.block_txn_request.to_absolute() 616 for index in all_indices: 617 tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) 618 assert_equal(tx.txid_hex, block.vtx[index].txid_hex) 619 # Check that the witness matches 620 assert_equal(tx.wtxid_hex, block.vtx[index].wtxid_hex) 621 test_node.last_message.pop("blocktxn", None) 622 current_height -= 1 623 624 # Next request should send a full block response, as we're past the 625 # allowed depth for a blocktxn response. 626 block_hash = node.getblockhash(current_height) 627 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) 628 with p2p_lock: 629 test_node.last_message.pop("block", None) 630 test_node.last_message.pop("blocktxn", None) 631 test_node.send_and_ping(msg) 632 with p2p_lock: 633 assert_equal(test_node.last_message["block"].block.hash_hex, block_hash) 634 assert "blocktxn" not in test_node.last_message 635 636 # Request with out-of-bounds tx index results in disconnect 637 bad_peer = self.nodes[0].add_p2p_connection(TestP2PConn()) 638 block_hash = node.getblockhash(chain_height) 639 block = from_hex(CBlock(), node.getblock(block_hash, False)) 640 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [len(block.vtx)]) 641 with node.assert_debug_log(['getblocktxn with out-of-bounds tx indices']): 642 bad_peer.send_without_ping(msg) 643 bad_peer.wait_for_disconnect() 644 645 def test_low_work_compactblocks(self, test_node): 646 # A compactblock with insufficient work won't get its header included 647 node = self.nodes[0] 648 hashPrevBlock = int(node.getblockhash(node.getblockcount() - 150), 16) 649 block = self.build_block_on_tip(node) 650 block.hashPrevBlock = hashPrevBlock 651 block.solve() 652 653 comp_block = HeaderAndShortIDs() 654 comp_block.initialize_from_block(block) 655 with self.nodes[0].assert_debug_log(['[net] Ignoring low-work compact block from peer 0']): 656 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 657 658 tips = node.getchaintips() 659 found = False 660 for x in tips: 661 if x["hash"] == block.hash_hex: 662 found = True 663 break 664 assert not found 665 666 def test_compactblocks_not_at_tip(self, test_node): 667 node = self.nodes[0] 668 # Test that requesting old compactblocks doesn't work. 669 MAX_CMPCTBLOCK_DEPTH = 5 670 new_blocks = [] 671 for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): 672 test_node.clear_block_announcement() 673 new_blocks.append(self.generate(node, 1)[0]) 674 test_node.wait_until(test_node.received_block_announcement, timeout=30) 675 676 test_node.clear_block_announcement() 677 test_node.send_without_ping(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 678 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) 679 680 test_node.clear_block_announcement() 681 self.generate(node, 1) 682 test_node.wait_until(test_node.received_block_announcement, timeout=30) 683 test_node.clear_block_announcement() 684 with p2p_lock: 685 test_node.last_message.pop("block", None) 686 test_node.send_without_ping(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) 687 test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) 688 with p2p_lock: 689 assert_equal(test_node.last_message["block"].block.hash_hex, new_blocks[0]) 690 691 # Generate an old compactblock, and verify that it's not accepted. 692 cur_height = node.getblockcount() 693 hashPrevBlock = int(node.getblockhash(cur_height - 5), 16) 694 block = self.build_block_on_tip(node) 695 block.hashPrevBlock = hashPrevBlock 696 block.solve() 697 698 comp_block = HeaderAndShortIDs() 699 comp_block.initialize_from_block(block) 700 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) 701 702 tips = node.getchaintips() 703 found = False 704 for x in tips: 705 if x["hash"] == block.hash_hex: 706 assert_equal(x["status"], "headers-only") 707 found = True 708 break 709 assert found 710 711 # Requesting this block via getblocktxn should silently fail 712 # (to avoid fingerprinting attacks). 713 msg = msg_getblocktxn() 714 msg.block_txn_request = BlockTransactionsRequest(block.hash_int, [0]) 715 with p2p_lock: 716 test_node.last_message.pop("blocktxn", None) 717 test_node.send_and_ping(msg) 718 with p2p_lock: 719 assert "blocktxn" not in test_node.last_message 720 721 def test_end_to_end_block_relay(self, listeners): 722 node = self.nodes[0] 723 utxo = self.utxos.pop(0) 724 725 block = self.build_block_with_transactions(node, utxo, 10) 726 727 [l.clear_block_announcement() for l in listeners] 728 729 # serialize without witness (this block has no witnesses anyway). 730 # TODO: repeat this test with witness tx's to a segwit node. 731 node.submitblock(block.serialize().hex()) 732 733 for l in listeners: 734 l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) 735 with p2p_lock: 736 for l in listeners: 737 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.hash_int, block.hash_int) 738 739 # Test that we don't get disconnected if we relay a compact block with valid header, 740 # but invalid transactions. 741 def test_invalid_tx_in_compactblock(self, test_node): 742 node = self.nodes[0] 743 assert len(self.utxos) 744 utxo = self.utxos[0] 745 746 block = self.build_block_with_transactions(node, utxo, 5) 747 block.hashMerkleRoot = block.calc_merkle_root() 748 # Drop the coinbase witness but include the witness commitment. 749 add_witness_commitment(block) 750 block.vtx[0].wit.vtxinwit = [] 751 block.solve() 752 753 # Now send the compact block with all transactions prefilled, and 754 # verify that we don't get disconnected. 755 comp_block = HeaderAndShortIDs() 756 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 757 msg = msg_cmpctblock(comp_block.to_p2p()) 758 test_node.send_and_ping(msg) 759 760 # Check that the tip didn't advance 761 assert_not_equal(node.getbestblockhash(), block.hash_hex) 762 test_node.sync_with_ping() 763 764 # Re-establish a proper witness commitment with the coinbase witness, but 765 # invalidate the last tx in the block. 766 block.vtx[4].vin[0].scriptSig = CScript([OP_RETURN]) 767 block.hashMerkleRoot = block.calc_merkle_root() 768 add_witness_commitment(block) 769 block.solve() 770 771 # This will lead to a consensus failure for which we also won't be disconnected but which 772 # will be cached. 773 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 774 msg = msg_cmpctblock(comp_block.to_p2p()) 775 test_node.send_and_ping(msg) 776 777 # The tip still didn't advance. 778 assert_not_equal(node.getbestblockhash(), block.hash_hex) 779 test_node.sync_with_ping() 780 781 # The failure above was cached. Submitting the compact block again will return a cached 782 # consensus error (the code path is different) and still not get us disconnected (nor 783 # advance the tip). 784 test_node.send_and_ping(msg) 785 assert_not_equal(node.getbestblockhash(), block.hash_hex) 786 test_node.sync_with_ping() 787 788 # Now, announcing a second block building on top of the invalid one will get us disconnected. 789 block.hashPrevBlock = block.hash_int 790 block.solve() 791 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True) 792 msg = msg_cmpctblock(comp_block.to_p2p()) 793 test_node.send_await_disconnect(msg) 794 795 # Helper for enabling cb announcements 796 # Send the sendcmpct request and sync headers 797 def request_cb_announcements(self, peer): 798 node = self.nodes[0] 799 tip = node.getbestblockhash() 800 peer.get_headers(locator=[int(tip, 16)], hashstop=0) 801 peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) 802 803 def test_compactblock_reconstruction_stalling_peer(self, stalling_peer, delivery_peer): 804 node = self.nodes[0] 805 assert len(self.utxos) 806 807 def announce_cmpct_block(node, peer): 808 utxo = self.utxos.pop(0) 809 block = self.build_block_with_transactions(node, utxo, 5) 810 811 cmpct_block = HeaderAndShortIDs() 812 cmpct_block.initialize_from_block(block) 813 msg = msg_cmpctblock(cmpct_block.to_p2p()) 814 peer.send_and_ping(msg) 815 with p2p_lock: 816 assert "getblocktxn" in peer.last_message 817 return block, cmpct_block 818 819 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 820 821 for tx in block.vtx[1:]: 822 delivery_peer.send_without_ping(msg_tx(tx)) 823 delivery_peer.sync_with_ping() 824 mempool = node.getrawmempool() 825 for tx in block.vtx[1:]: 826 assert tx.txid_hex in mempool 827 828 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 829 assert_equal(node.getbestblockhash(), block.hash_hex) 830 831 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 832 833 # Now test that delivering an invalid compact block won't break relay 834 835 block, cmpct_block = announce_cmpct_block(node, stalling_peer) 836 for tx in block.vtx[1:]: 837 delivery_peer.send_without_ping(msg_tx(tx)) 838 delivery_peer.sync_with_ping() 839 840 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()] 841 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] 842 843 cmpct_block.use_witness = True 844 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 845 assert_not_equal(node.getbestblockhash(), block.hash_hex) 846 847 msg = msg_no_witness_blocktxn() 848 msg.block_transactions.blockhash = block.hash_int 849 msg.block_transactions.transactions = block.vtx[1:] 850 stalling_peer.send_and_ping(msg) 851 assert_equal(node.getbestblockhash(), block.hash_hex) 852 853 def test_highbandwidth_mode_states_via_getpeerinfo(self): 854 # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent 855 hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 856 857 # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` 858 # match the given parameters for the last peer of a given node 859 def assert_highbandwidth_states(node, hb_to, hb_from): 860 peerinfo = node.getpeerinfo()[-1] 861 assert_equal(peerinfo['bip152_hb_to'], hb_to) 862 assert_equal(peerinfo['bip152_hb_from'], hb_from) 863 864 # initially, neither node has selected the other peer as high-bandwidth yet 865 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=False) 866 867 # peer requests high-bandwidth mode by sending sendcmpct(1) 868 hb_test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) 869 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=True) 870 871 # peer generates a block and sends it to node, which should 872 # select the peer as high-bandwidth (up to 3 peers according to BIP 152) 873 block = self.build_block_on_tip(self.nodes[0]) 874 hb_test_node.send_and_ping(msg_block(block)) 875 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=True) 876 877 # peer requests low-bandwidth mode by sending sendcmpct(0) 878 hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) 879 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False) 880 881 def test_compactblock_reconstruction_parallel_reconstruction(self, stalling_peer, delivery_peer, inbound_peer, outbound_peer): 882 """ All p2p connections are inbound except outbound_peer. We test that ultimate parallel slot 883 can only be taken by an outbound node unless prior attempts were done by an outbound 884 """ 885 node = self.nodes[0] 886 assert len(self.utxos) 887 888 def announce_cmpct_block(node, peer, txn_count): 889 utxo = self.utxos.pop(0) 890 block = self.build_block_with_transactions(node, utxo, txn_count) 891 892 cmpct_block = HeaderAndShortIDs() 893 cmpct_block.initialize_from_block(block) 894 msg = msg_cmpctblock(cmpct_block.to_p2p()) 895 peer.send_and_ping(msg) 896 with p2p_lock: 897 assert "getblocktxn" in peer.last_message 898 return block, cmpct_block 899 900 for name, peer in [("delivery", delivery_peer), ("inbound", inbound_peer), ("outbound", outbound_peer)]: 901 self.log.info(f"Setting {name} as high bandwidth peer") 902 block, cmpct_block = announce_cmpct_block(node, peer, 1) 903 msg = msg_blocktxn() 904 msg.block_transactions.blockhash = block.hash_int 905 msg.block_transactions.transactions = block.vtx[1:] 906 peer.send_and_ping(msg) 907 assert_equal(node.getbestblockhash(), block.hash_hex) 908 peer.clear_getblocktxn() 909 910 # Test the simple parallel download case... 911 for num_missing in [1, 5, 20]: 912 913 # Remaining low-bandwidth peer is stalling_peer, who announces first 914 assert_equal([peer['bip152_hb_to'] for peer in node.getpeerinfo()], [False, True, True, True]) 915 916 block, cmpct_block = announce_cmpct_block(node, stalling_peer, num_missing) 917 918 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 919 with p2p_lock: 920 # The second peer to announce should still get a getblocktxn 921 assert "getblocktxn" in delivery_peer.last_message 922 assert_not_equal(node.getbestblockhash(), block.hash_hex) 923 924 inbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 925 with p2p_lock: 926 # The third inbound peer to announce should *not* get a getblocktxn 927 assert "getblocktxn" not in inbound_peer.last_message 928 assert_not_equal(node.getbestblockhash(), block.hash_hex) 929 930 outbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) 931 with p2p_lock: 932 # The third peer to announce should get a getblocktxn if outbound 933 assert "getblocktxn" in outbound_peer.last_message 934 assert_not_equal(node.getbestblockhash(), block.hash_hex) 935 936 # Second peer completes the compact block first 937 msg = msg_blocktxn() 938 msg.block_transactions.blockhash = block.hash_int 939 msg.block_transactions.transactions = block.vtx[1:] 940 delivery_peer.send_and_ping(msg) 941 assert_equal(node.getbestblockhash(), block.hash_hex) 942 943 # Nothing bad should happen if we get a late fill from the first peer... 944 stalling_peer.send_and_ping(msg) 945 self.utxos.append([block.vtx[-1].txid_int, 0, block.vtx[-1].vout[0].nValue]) 946 947 delivery_peer.clear_getblocktxn() 948 inbound_peer.clear_getblocktxn() 949 outbound_peer.clear_getblocktxn() 950 951 952 def run_test(self): 953 self.wallet = MiniWallet(self.nodes[0]) 954 955 # Setup the p2p connections 956 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 957 self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 958 self.onemore_inbound_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 959 self.outbound_node = self.nodes[0].add_outbound_p2p_connection(TestP2PConn(), p2p_idx=3, connection_type="outbound-full-relay") 960 961 # We will need UTXOs to construct transactions in later tests. 962 self.make_utxos() 963 964 assert softfork_active(self.nodes[0], "segwit") 965 966 self.log.info("Testing SENDCMPCT p2p message... ") 967 self.test_sendcmpct(self.segwit_node) 968 self.test_sendcmpct(self.additional_segwit_node) 969 self.test_sendcmpct(self.onemore_inbound_node) 970 self.test_sendcmpct(self.outbound_node) 971 972 self.log.info("Testing compactblock construction...") 973 self.test_compactblock_construction(self.segwit_node) 974 975 self.log.info("Testing compactblock requests (segwit node)... ") 976 self.test_compactblock_requests(self.segwit_node) 977 978 self.log.info("Testing getblocktxn requests (segwit node)...") 979 self.test_getblocktxn_requests(self.segwit_node) 980 981 self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") 982 self.test_getblocktxn_handler(self.segwit_node) 983 984 self.log.info("Testing compactblock requests/announcements not at chain tip...") 985 self.test_compactblocks_not_at_tip(self.segwit_node) 986 987 self.log.info("Testing handling of low-work compact blocks...") 988 self.test_low_work_compactblocks(self.segwit_node) 989 990 self.log.info("Testing handling of incorrect blocktxn responses...") 991 self.test_incorrect_blocktxn_response(self.segwit_node) 992 993 self.log.info("Testing reconstructing compact blocks with a stalling peer...") 994 self.test_compactblock_reconstruction_stalling_peer(self.segwit_node, self.additional_segwit_node) 995 996 self.log.info("Testing reconstructing compact blocks from multiple peers...") 997 self.test_compactblock_reconstruction_parallel_reconstruction(stalling_peer=self.segwit_node, inbound_peer=self.onemore_inbound_node, delivery_peer=self.additional_segwit_node, outbound_peer=self.outbound_node) 998 999 # Test that if we submitblock to node1, we'll get a compact block 1000 # announcement to all peers. 1001 # (Post-segwit activation, blocks won't propagate from node0 to node1 1002 # automatically, so don't bother testing a block announced to node0.) 1003 self.log.info("Testing end-to-end block relay...") 1004 self.request_cb_announcements(self.segwit_node) 1005 self.request_cb_announcements(self.additional_segwit_node) 1006 self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node]) 1007 1008 self.log.info("Testing handling of invalid compact blocks...") 1009 self.test_invalid_tx_in_compactblock(self.segwit_node) 1010 1011 # The previous test will lead to a disconnection. Reconnect before continuing. 1012 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 1013 1014 self.log.info("Testing handling of multiple blocktxn responses...") 1015 self.test_multiple_blocktxn_response(self.segwit_node) 1016 1017 # The previous test will lead to a disconnection. Reconnect before continuing. 1018 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) 1019 1020 self.log.info("Testing invalid index in cmpctblock message...") 1021 self.test_invalid_cmpctblock_message() 1022 1023 self.log.info("Testing high-bandwidth mode states via getpeerinfo...") 1024 self.test_highbandwidth_mode_states_via_getpeerinfo() 1025 1026 1027 if __name__ == '__main__': 1028 CompactBlocksTest(__file__).main()