p2p_orphan_handling.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2023-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 6 import time 7 8 from test_framework.blocktools import MAX_STANDARD_TX_WEIGHT 9 from test_framework.mempool_util import ( 10 create_large_orphan, 11 tx_in_orphanage, 12 ) 13 from test_framework.messages import ( 14 CInv, 15 DEFAULT_ANCESTOR_LIMIT, 16 MSG_TX, 17 MSG_WITNESS_TX, 18 MSG_WTX, 19 malleate_tx_to_invalid_witness, 20 msg_getdata, 21 msg_inv, 22 msg_notfound, 23 msg_tx, 24 tx_from_hex, 25 ) 26 from test_framework.p2p import ( 27 GETDATA_TX_INTERVAL, 28 NONPREF_PEER_TX_DELAY, 29 OVERLOADED_PEER_TX_DELAY, 30 p2p_lock, 31 P2PInterface, 32 P2PTxInvStore, 33 TXID_RELAY_DELAY, 34 ) 35 from test_framework.util import ( 36 assert_not_equal, 37 assert_equal, 38 ) 39 from test_framework.test_framework import BitcoinTestFramework 40 from test_framework.wallet import ( 41 MiniWallet, 42 MiniWalletMode, 43 ) 44 45 # Time to bump forward (using setmocktime) before waiting for the node to send getdata(tx) in response 46 # to an inv(tx), in seconds. This delay includes all possible delays + 1, so it should only be used 47 # when the value of the delay is not interesting. If we want to test that the node waits x seconds 48 # for one peer and y seconds for another, use specific values instead. 49 TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1 50 51 def cleanup(func): 52 def wrapper(self): 53 try: 54 func(self) 55 finally: 56 # Clear mempool 57 self.generate(self.nodes[0], 1) 58 self.nodes[0].disconnect_p2ps() 59 # Check that mempool and orphanage have been cleared 60 self.wait_until(lambda: len(self.nodes[0].getorphantxs()) == 0) 61 assert_equal(0, len(self.nodes[0].getrawmempool())) 62 63 self.restart_node(0, extra_args=["-persistmempool=0"]) 64 # Allow use of bumpmocktime again 65 self.nodes[0].setmocktime(int(time.time())) 66 self.wallet.rescan_utxos(include_mempool=True) 67 return wrapper 68 69 class PeerTxRelayer(P2PTxInvStore): 70 """A P2PTxInvStore that also remembers all of the getdata and tx messages it receives.""" 71 def __init__(self, wtxidrelay=True): 72 super().__init__(wtxidrelay=wtxidrelay) 73 self._tx_received = [] 74 self._getdata_received = [] 75 76 @property 77 def tx_received(self): 78 with p2p_lock: 79 return self._tx_received 80 81 @property 82 def getdata_received(self): 83 with p2p_lock: 84 return self._getdata_received 85 86 def on_tx(self, message): 87 self._tx_received.append(message) 88 89 def on_getdata(self, message): 90 self._getdata_received.append(message) 91 92 def wait_for_parent_requests(self, txids): 93 """Wait for requests for missing parents by txid with witness data (MSG_WITNESS_TX or 94 WitnessTx). Requires that the getdata message match these txids exactly; all txids must be 95 requested and no additional requests are allowed.""" 96 def test_function(): 97 last_getdata = self.last_message.get('getdata') 98 if not last_getdata: 99 return False 100 return len(last_getdata.inv) == len(txids) and all([item.type == MSG_WITNESS_TX and item.hash in txids for item in last_getdata.inv]) 101 self.wait_until(test_function, timeout=10) 102 103 def assert_no_immediate_response(self, message): 104 """Check that the node does not immediately respond to this message with any of getdata, 105 inv, tx. The node may respond later. 106 """ 107 prev_lastmessage = self.last_message 108 self.send_and_ping(message) 109 after_lastmessage = self.last_message 110 for msgtype in ["getdata", "inv", "tx"]: 111 if msgtype not in prev_lastmessage: 112 assert msgtype not in after_lastmessage 113 else: 114 assert_equal(prev_lastmessage[msgtype], after_lastmessage[msgtype]) 115 116 def assert_never_requested(self, txhash): 117 """Check that the node has never sent us a getdata for this hash (int type)""" 118 self.sync_with_ping() 119 for getdata in self.getdata_received: 120 for request in getdata.inv: 121 assert_not_equal(request.hash, txhash) 122 123 class OrphanHandlingTest(BitcoinTestFramework): 124 def set_test_params(self): 125 self.num_nodes = 1 126 self.extra_args = [[]] 127 128 def create_parent_and_child(self): 129 """Create package with 1 parent and 1 child, normal fees (no cpfp).""" 130 parent = self.wallet.create_self_transfer() 131 child = self.wallet.create_self_transfer(utxo_to_spend=parent['new_utxo']) 132 return child["tx"].wtxid_hex, child["tx"], parent["tx"] 133 134 def relay_transaction(self, peer, tx): 135 """Relay transaction using MSG_WTX""" 136 wtxid = tx.wtxid_int 137 peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=wtxid)])) 138 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP) 139 peer.wait_for_getdata([wtxid]) 140 peer.send_and_ping(msg_tx(tx)) 141 142 @cleanup 143 def test_arrival_timing_orphan(self): 144 self.log.info("Test missing parents that arrive during delay are not requested") 145 node = self.nodes[0] 146 tx_parent_arrives = self.wallet.create_self_transfer() 147 tx_parent_doesnt_arrive = self.wallet.create_self_transfer() 148 # Fake orphan spends nonexistent outputs of the two parents 149 tx_fake_orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[ 150 {"txid": tx_parent_doesnt_arrive["txid"], "vout": 10, "value": tx_parent_doesnt_arrive["new_utxo"]["value"]}, 151 {"txid": tx_parent_arrives["txid"], "vout": 10, "value": tx_parent_arrives["new_utxo"]["value"]} 152 ]) 153 154 peer_spy = node.add_p2p_connection(PeerTxRelayer()) 155 peer_normal = node.add_p2p_connection(PeerTxRelayer()) 156 # This transaction is an orphan because it is missing inputs. It is a "fake" orphan that the 157 # spy peer has crafted to learn information about tx_parent_arrives even though it isn't 158 # able to spend a real output of it, but it could also just be a normal, real child tx. 159 # The node should not immediately respond with a request for orphan parents. 160 # Also, no request should be sent later because it will be resolved by 161 # the time the request is scheduled to be sent. 162 peer_spy.assert_no_immediate_response(msg_tx(tx_fake_orphan["tx"])) 163 164 # Node receives transaction. It attempts to obfuscate the exact timing at which this 165 # transaction entered its mempool. Send unsolicited because otherwise we need to wait for 166 # request delays. 167 peer_normal.send_and_ping(msg_tx(tx_parent_arrives["tx"])) 168 assert tx_parent_arrives["txid"] in node.getrawmempool() 169 170 # Spy peer should not be able to query the node for the parent yet, since it hasn't been 171 # announced / insufficient time has elapsed. 172 parent_inv = CInv(t=MSG_WTX, h=tx_parent_arrives["tx"].wtxid_int) 173 assert_equal(len(peer_spy.get_invs()), 0) 174 peer_spy.assert_no_immediate_response(msg_getdata([parent_inv])) 175 176 # Request would be scheduled with this delay because it is not a preferred relay peer. 177 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY) 178 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16)) 179 peer_spy.assert_never_requested(int(tx_parent_doesnt_arrive["txid"], 16)) 180 # Request would be scheduled with this delay because it is by txid. 181 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY) 182 peer_spy.wait_for_parent_requests([int(tx_parent_doesnt_arrive["txid"], 16)]) 183 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16)) 184 185 @cleanup 186 def test_orphan_rejected_parents_exceptions(self): 187 node = self.nodes[0] 188 peer1 = node.add_p2p_connection(PeerTxRelayer()) 189 peer2 = node.add_p2p_connection(PeerTxRelayer()) 190 191 self.log.info("Test orphan handling when a nonsegwit parent is known to be invalid") 192 parent_overly_large_nonsegwit = self.wallet_nonsegwit.create_self_transfer(target_vsize=int(MAX_STANDARD_TX_WEIGHT / 4) + 1) 193 assert_equal(parent_overly_large_nonsegwit["txid"], parent_overly_large_nonsegwit["tx"].wtxid_hex) 194 parent_other = self.wallet_nonsegwit.create_self_transfer() 195 child_nonsegwit = self.wallet_nonsegwit.create_self_transfer_multi( 196 utxos_to_spend=[parent_other["new_utxo"], parent_overly_large_nonsegwit["new_utxo"]]) 197 198 # Relay the parent. It should be rejected (and not reconsiderable) because it violated size limitations. 199 self.relay_transaction(peer1, parent_overly_large_nonsegwit["tx"]) 200 assert parent_overly_large_nonsegwit["txid"] not in node.getrawmempool() 201 202 # Relay the child. It should not be accepted because it has missing inputs. 203 # Its parent should not be requested because its hash (txid == wtxid) has been added to the rejection filter. 204 self.relay_transaction(peer2, child_nonsegwit["tx"]) 205 assert child_nonsegwit["txid"] not in node.getrawmempool() 206 assert not tx_in_orphanage(node, child_nonsegwit["tx"]) 207 208 # No parents are requested. 209 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL) 210 peer1.assert_never_requested(int(parent_other["txid"], 16)) 211 peer2.assert_never_requested(int(parent_other["txid"], 16)) 212 peer2.assert_never_requested(int(parent_overly_large_nonsegwit["txid"], 16)) 213 214 self.log.info("Test orphan handling when a segwit parent was invalid but may be retried with another witness") 215 parent_low_fee = self.wallet.create_self_transfer(fee_rate=0) 216 child_low_fee = self.wallet.create_self_transfer(utxo_to_spend=parent_low_fee["new_utxo"]) 217 218 # Relay the low fee parent. It should not be accepted. 219 self.relay_transaction(peer1, parent_low_fee["tx"]) 220 assert parent_low_fee["txid"] not in node.getrawmempool() 221 222 # Relay the child. It should not be accepted because it has missing inputs. 223 self.relay_transaction(peer2, child_low_fee["tx"]) 224 assert child_low_fee["txid"] not in node.getrawmempool() 225 assert tx_in_orphanage(node, child_low_fee["tx"]) 226 227 # The parent should be requested because even though the txid commits to the fee, it doesn't 228 # commit to the feerate. Delayed because it's by txid and this is not a preferred relay peer. 229 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 230 peer2.wait_for_getdata([parent_low_fee["tx"].txid_int]) 231 232 self.log.info("Test orphan handling when a parent was previously downloaded with witness stripped") 233 parent_normal = self.wallet.create_self_transfer() 234 parent1_witness_stripped = tx_from_hex(parent_normal["tx"].serialize_without_witness().hex()) 235 child_invalid_witness = self.wallet.create_self_transfer(utxo_to_spend=parent_normal["new_utxo"]) 236 237 # Relay the parent with witness stripped. It should not be accepted. 238 self.relay_transaction(peer1, parent1_witness_stripped) 239 assert_equal(parent_normal["txid"], parent1_witness_stripped.txid_hex) 240 assert parent1_witness_stripped.txid_hex not in node.getrawmempool() 241 242 # Relay the child. It should not be accepted because it has missing inputs. 243 self.relay_transaction(peer2, child_invalid_witness["tx"]) 244 assert child_invalid_witness["txid"] not in node.getrawmempool() 245 assert tx_in_orphanage(node, child_invalid_witness["tx"]) 246 247 # The parent should be requested since the unstripped wtxid would differ. Delayed because 248 # it's by txid and this is not a preferred relay peer. 249 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 250 peer2.wait_for_getdata([parent_normal["tx"].txid_int]) 251 252 # parent_normal can be relayed again even though parent1_witness_stripped was rejected 253 self.relay_transaction(peer1, parent_normal["tx"]) 254 assert_equal(set(node.getrawmempool()), set([parent_normal["txid"], child_invalid_witness["txid"]])) 255 256 @cleanup 257 def test_orphan_multiple_parents(self): 258 node = self.nodes[0] 259 peer = node.add_p2p_connection(PeerTxRelayer()) 260 261 self.log.info("Test orphan parent requests with a mixture of confirmed, in-mempool and missing parents") 262 # This UTXO confirmed a long time ago. 263 utxo_conf_old = self.wallet.send_self_transfer(from_node=node)["new_utxo"] 264 txid_conf_old = utxo_conf_old["txid"] 265 self.generate(self.wallet, 10) 266 267 # Create a fake reorg to trigger BlockDisconnected, which resets the rolling bloom filter. 268 # The alternative is to mine thousands of transactions to push it out of the filter. 269 last_block = node.getbestblockhash() 270 node.invalidateblock(last_block) 271 node.preciousblock(last_block) 272 node.syncwithvalidationinterfacequeue() 273 274 # This UTXO confirmed recently. 275 utxo_conf_recent = self.wallet.send_self_transfer(from_node=node)["new_utxo"] 276 self.generate(node, 1) 277 278 # This UTXO is unconfirmed and in the mempool. 279 assert_equal(len(node.getrawmempool()), 0) 280 mempool_tx = self.wallet.send_self_transfer(from_node=node) 281 utxo_unconf_mempool = mempool_tx["new_utxo"] 282 283 # This UTXO is unconfirmed and missing. 284 missing_tx = self.wallet.create_self_transfer() 285 utxo_unconf_missing = missing_tx["new_utxo"] 286 assert missing_tx["txid"] not in node.getrawmempool() 287 288 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[utxo_conf_old, 289 utxo_conf_recent, utxo_unconf_mempool, utxo_unconf_missing]) 290 291 self.relay_transaction(peer, orphan["tx"]) 292 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 293 peer.sync_with_ping() 294 assert tx_in_orphanage(node, orphan["tx"]) 295 assert_equal(len(peer.last_message["getdata"].inv), 2) 296 peer.wait_for_parent_requests([int(txid_conf_old, 16), int(missing_tx["txid"], 16)]) 297 298 # Even though the peer would send a notfound for the "old" confirmed transaction, the node 299 # doesn't give up on the orphan. Once all of the missing parents are received, it should be 300 # submitted to mempool. 301 peer.send_without_ping(msg_notfound(vec=[CInv(MSG_WITNESS_TX, int(txid_conf_old, 16))])) 302 # Sync with ping to ensure orphans are reconsidered 303 peer.send_and_ping(msg_tx(missing_tx["tx"])) 304 assert_equal(node.getmempoolentry(orphan["txid"])["ancestorcount"], 3) 305 306 @cleanup 307 def test_orphans_overlapping_parents(self): 308 node = self.nodes[0] 309 # In the process of relaying inflight_parent_AB 310 peer_txrequest = node.add_p2p_connection(PeerTxRelayer()) 311 # Sends the orphans 312 peer_orphans = node.add_p2p_connection(PeerTxRelayer()) 313 314 confirmed_utxos = [self.wallet_nonsegwit.get_utxo() for _ in range(4)] 315 assert all([utxo["confirmations"] > 0 for utxo in confirmed_utxos]) 316 self.log.info("Test handling of multiple orphans with missing parents that are already being requested") 317 # Parent of child_A only 318 missing_parent_A = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[0]) 319 # Parents of child_A and child_B 320 missing_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[1]) 321 inflight_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[2]) 322 # Parent of child_B only 323 missing_parent_B = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[3]) 324 child_A = self.wallet_nonsegwit.create_self_transfer_multi( 325 utxos_to_spend=[missing_parent_A["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]] 326 ) 327 child_B = self.wallet_nonsegwit.create_self_transfer_multi( 328 utxos_to_spend=[missing_parent_B["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]] 329 ) 330 331 # The wtxid and txid need to be the same for the node to recognize that the missing input 332 # and in-flight request for inflight_parent_AB are the same transaction. 333 assert_equal(inflight_parent_AB["txid"], inflight_parent_AB["wtxid"]) 334 335 # Announce inflight_parent_AB and wait for getdata 336 peer_txrequest.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=inflight_parent_AB["tx"].wtxid_int)])) 337 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY) 338 peer_txrequest.wait_for_getdata([inflight_parent_AB["tx"].wtxid_int]) 339 340 self.log.info("Test that the node does not request a parent if it has an in-flight txrequest") 341 # Relay orphan child_A 342 self.relay_transaction(peer_orphans, child_A["tx"]) 343 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 344 assert tx_in_orphanage(node, child_A["tx"]) 345 # There are 3 missing parents. missing_parent_A and missing_parent_AB should be requested. 346 # But inflight_parent_AB should not, because there is already an in-flight request for it. 347 peer_orphans.wait_for_parent_requests([int(missing_parent_A["txid"], 16), int(missing_parent_AB["txid"], 16)]) 348 349 self.log.info("Test that the node does not request a parent if it has an in-flight orphan parent request") 350 # Relay orphan child_B 351 self.relay_transaction(peer_orphans, child_B["tx"]) 352 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 353 assert tx_in_orphanage(node, child_B["tx"]) 354 # Only missing_parent_B should be requested. Not inflight_parent_AB or missing_parent_AB 355 # because they are already being requested from peer_txrequest and peer_orphans respectively. 356 peer_orphans.wait_for_parent_requests([int(missing_parent_B["txid"], 16)]) 357 peer_orphans.assert_never_requested(int(inflight_parent_AB["txid"], 16)) 358 359 # But inflight_parent_AB will be requested eventually if original peer doesn't respond 360 node.bumpmocktime(GETDATA_TX_INTERVAL) 361 peer_orphans.wait_for_parent_requests([int(inflight_parent_AB["txid"], 16)]) 362 363 @cleanup 364 def test_orphan_of_orphan(self): 365 node = self.nodes[0] 366 peer = node.add_p2p_connection(PeerTxRelayer()) 367 368 self.log.info("Test handling of an orphan with a parent who is another orphan") 369 missing_grandparent = self.wallet_nonsegwit.create_self_transfer() 370 missing_parent_orphan = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=missing_grandparent["new_utxo"]) 371 missing_parent = self.wallet_nonsegwit.create_self_transfer() 372 orphan = self.wallet_nonsegwit.create_self_transfer_multi(utxos_to_spend=[missing_parent["new_utxo"], missing_parent_orphan["new_utxo"]]) 373 374 # The node should put missing_parent_orphan into the orphanage and request missing_grandparent 375 self.relay_transaction(peer, missing_parent_orphan["tx"]) 376 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 377 assert tx_in_orphanage(node, missing_parent_orphan["tx"]) 378 peer.wait_for_parent_requests([int(missing_grandparent["txid"], 16)]) 379 380 # The node should put the orphan into the orphanage and request missing_parent, skipping 381 # missing_parent_orphan because it already has it in the orphanage. 382 self.relay_transaction(peer, orphan["tx"]) 383 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 384 assert tx_in_orphanage(node, orphan["tx"]) 385 peer.wait_for_parent_requests([int(missing_parent["txid"], 16)]) 386 387 @cleanup 388 def test_orphan_inherit_rejection(self): 389 node = self.nodes[0] 390 peer1 = node.add_p2p_connection(PeerTxRelayer()) 391 peer2 = node.add_p2p_connection(PeerTxRelayer()) 392 peer3 = node.add_p2p_connection(PeerTxRelayer(wtxidrelay=False)) 393 394 self.log.info("Test that an orphan with rejected parents, along with any descendants, cannot be retried with an alternate witness") 395 parent_overly_large_nonsegwit = self.wallet_nonsegwit.create_self_transfer(target_vsize=int(MAX_STANDARD_TX_WEIGHT / 4) + 1) 396 assert_equal(parent_overly_large_nonsegwit["txid"], parent_overly_large_nonsegwit["tx"].wtxid_hex) 397 child = self.wallet.create_self_transfer(utxo_to_spend=parent_overly_large_nonsegwit["new_utxo"]) 398 grandchild = self.wallet.create_self_transfer(utxo_to_spend=child["new_utxo"]) 399 assert_not_equal(child["txid"], child["tx"].wtxid_hex) 400 assert_not_equal(grandchild["txid"], grandchild["tx"].wtxid_hex) 401 402 # Relay the parent. It should be rejected because it pays 0 fees. 403 self.relay_transaction(peer1, parent_overly_large_nonsegwit["tx"]) 404 assert parent_overly_large_nonsegwit["txid"] not in node.getrawmempool() 405 406 # Relay the child. It should be rejected for having missing parents, and this rejection is 407 # cached by txid and wtxid. 408 self.relay_transaction(peer1, child["tx"]) 409 assert_equal(0, len(node.getrawmempool())) 410 assert not tx_in_orphanage(node, child["tx"]) 411 peer1.assert_never_requested(parent_overly_large_nonsegwit["txid"]) 412 413 # Grandchild should also not be kept in orphanage because its parent has been rejected. 414 self.relay_transaction(peer2, grandchild["tx"]) 415 assert_equal(0, len(node.getrawmempool())) 416 assert not tx_in_orphanage(node, grandchild["tx"]) 417 peer2.assert_never_requested(child["txid"]) 418 peer2.assert_never_requested(child["tx"].wtxid_hex) 419 420 # The child should never be requested, even if announced again with potentially different witness. 421 # Sync with ping to ensure orphans are reconsidered 422 peer3.send_and_ping(msg_inv([CInv(t=MSG_TX, h=int(child["txid"], 16))])) 423 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP) 424 peer3.assert_never_requested(child["txid"]) 425 426 @cleanup 427 def test_same_txid_orphan(self): 428 self.log.info("Check what happens when orphan with same txid is already in orphanage") 429 node = self.nodes[0] 430 431 tx_parent = self.wallet.create_self_transfer() 432 433 # Create the real child 434 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"]) 435 436 # Create a fake version of the child 437 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_child) 438 439 bad_peer = node.add_p2p_connection(P2PInterface()) 440 honest_peer = node.add_p2p_connection(P2PInterface()) 441 442 # 1. Fake orphan is received first. It is missing an input. 443 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) 444 assert tx_in_orphanage(node, tx_orphan_bad_wit) 445 446 # 2. Node requests the missing parent by txid. 447 parent_txid_int = int(tx_parent["txid"], 16) 448 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 449 bad_peer.wait_for_getdata([parent_txid_int]) 450 451 # 3. Honest peer relays the real child, which is also missing parents and should be placed 452 # in the orphanage. 453 with node.assert_debug_log(["missingorspent"]): 454 honest_peer.send_and_ping(msg_tx(tx_child["tx"])) 455 assert tx_in_orphanage(node, tx_child["tx"]) 456 457 # Time out the previous request for the parent (node will not request the same transaction 458 # from multiple nodes at the same time) 459 node.bumpmocktime(GETDATA_TX_INTERVAL) 460 461 # 4. The parent is requested. Honest peer sends it. 462 honest_peer.wait_for_getdata([parent_txid_int]) 463 # Sync with ping to ensure orphans are reconsidered 464 honest_peer.send_and_ping(msg_tx(tx_parent["tx"])) 465 466 # 5. After parent is accepted, orphans should be reconsidered. 467 # The real child should be accepted and the fake one rejected. 468 node_mempool = node.getrawmempool() 469 assert tx_parent["txid"] in node_mempool 470 assert tx_child["txid"] in node_mempool 471 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) 472 473 @cleanup 474 def test_same_txid_orphan_of_orphan(self): 475 self.log.info("Check what happens when orphan's parent with same txid is already in orphanage") 476 node = self.nodes[0] 477 478 tx_grandparent = self.wallet.create_self_transfer() 479 480 # Create middle tx (both parent and child) which will be in orphanage. 481 tx_middle = self.wallet.create_self_transfer(utxo_to_spend=tx_grandparent["new_utxo"]) 482 483 # Create a fake version of the middle tx 484 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_middle) 485 486 # Create grandchild spending from tx_middle (and spending from tx_orphan_bad_wit since they 487 # have the same txid). 488 tx_grandchild = self.wallet.create_self_transfer(utxo_to_spend=tx_middle["new_utxo"]) 489 490 bad_peer = node.add_p2p_connection(P2PInterface()) 491 honest_peer = node.add_p2p_connection(P2PInterface()) 492 493 # 1. Fake orphan is received first. It is missing an input. 494 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) 495 assert tx_in_orphanage(node, tx_orphan_bad_wit) 496 497 # 2. Node requests missing tx_grandparent by txid. 498 grandparent_txid_int = int(tx_grandparent["txid"], 16) 499 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 500 bad_peer.wait_for_getdata([grandparent_txid_int]) 501 502 # 3. Honest peer relays the grandchild, which is missing a parent. The parent by txid already 503 # exists in orphanage, but should be re-requested because the node shouldn't assume that the 504 # witness data is the same. In this case, a same-txid-different-witness transaction exists! 505 honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"])) 506 assert tx_in_orphanage(node, tx_grandchild["tx"]) 507 middle_txid_int = int(tx_middle["txid"], 16) 508 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 509 honest_peer.wait_for_getdata([middle_txid_int]) 510 511 # 4. Honest peer relays the real child, which is also missing parents and should be placed 512 # in the orphanage. 513 honest_peer.send_and_ping(msg_tx(tx_middle["tx"])) 514 assert tx_in_orphanage(node, tx_middle["tx"]) 515 assert_equal(len(node.getrawmempool()), 0) 516 517 # 5. Honest peer sends tx_grandparent 518 honest_peer.send_and_ping(msg_tx(tx_grandparent["tx"])) 519 520 # 6. After parent is accepted, orphans should be reconsidered. 521 # The real child should be accepted and the fake one rejected. 522 node_mempool = node.getrawmempool() 523 assert tx_grandparent["txid"] in node_mempool 524 assert tx_middle["txid"] in node_mempool 525 assert tx_grandchild["txid"] in node_mempool 526 assert_equal(node.getmempoolentry(tx_middle["txid"])["wtxid"], tx_middle["wtxid"]) 527 self.wait_until(lambda: len(node.getorphantxs()) == 0) 528 529 @cleanup 530 def test_orphan_txid_inv(self): 531 self.log.info("Check node does not ignore announcement with same txid as tx in orphanage") 532 node = self.nodes[0] 533 534 tx_parent = self.wallet.create_self_transfer() 535 536 # Create the real child and fake version 537 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"]) 538 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_child) 539 540 bad_peer = node.add_p2p_connection(PeerTxRelayer()) 541 # Must not send wtxidrelay because otherwise the inv(TX) will be ignored later 542 honest_peer = node.add_p2p_connection(P2PInterface(wtxidrelay=False)) 543 544 # 1. Fake orphan is received first. It is missing an input. 545 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) 546 assert tx_in_orphanage(node, tx_orphan_bad_wit) 547 548 # 2. Node requests the missing parent by txid. 549 parent_txid_int = int(tx_parent["txid"], 16) 550 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 551 bad_peer.wait_for_getdata([parent_txid_int]) 552 553 # 3. Honest peer announces the real child, by txid (this isn't common but the node should 554 # still keep track of it). 555 child_txid_int = int(tx_child["txid"], 16) 556 honest_peer.send_and_ping(msg_inv([CInv(t=MSG_TX, h=child_txid_int)])) 557 558 # 4. The child is requested. Honest peer sends it. 559 node.bumpmocktime(TXREQUEST_TIME_SKIP) 560 honest_peer.wait_for_getdata([child_txid_int]) 561 honest_peer.send_and_ping(msg_tx(tx_child["tx"])) 562 assert tx_in_orphanage(node, tx_child["tx"]) 563 564 # 5. After first parent request times out, the node sends another one for the missing parent 565 # of the real orphan child. 566 node.bumpmocktime(GETDATA_TX_INTERVAL) 567 honest_peer.wait_for_getdata([parent_txid_int]) 568 honest_peer.send_and_ping(msg_tx(tx_parent["tx"])) 569 570 # 6. After parent is accepted, orphans should be reconsidered. 571 # The real child should be accepted and the fake one rejected. This may happen in either 572 # order since the message-processing is randomized. If tx_orphan_bad_wit is validated first, 573 # its consensus error leads to disconnection of bad_peer. If tx_child is validated first, 574 # tx_orphan_bad_wit is rejected for txn-same-nonwitness-data-in-mempool (no punishment). 575 node_mempool = node.getrawmempool() 576 assert tx_parent["txid"] in node_mempool 577 assert tx_child["txid"] in node_mempool 578 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) 579 self.wait_until(lambda: len(node.getorphantxs()) == 0) 580 581 582 @cleanup 583 def test_orphan_handling_prefer_outbound(self): 584 self.log.info("Test that the node prefers requesting from outbound peers") 585 node = self.nodes[0] 586 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child() 587 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16)) 588 589 peer_inbound = node.add_p2p_connection(PeerTxRelayer()) 590 peer_outbound = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=1) 591 592 # Inbound peer relays the transaction. 593 peer_inbound.send_and_ping(msg_inv([orphan_inv])) 594 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP) 595 peer_inbound.wait_for_getdata([int(orphan_wtxid, 16)]) 596 597 # Both peers send invs for the orphan, so the node can expect both to know its ancestors. 598 peer_outbound.send_and_ping(msg_inv([orphan_inv])) 599 600 peer_inbound.send_and_ping(msg_tx(orphan_tx)) 601 602 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are) 603 orphanage = node.getorphantxs(verbosity=2) 604 assert_equal(orphanage[0]["wtxid"], orphan_wtxid) 605 assert_equal(len(orphanage[0]["from"]), 2) 606 607 # The outbound peer should be preferred for getting orphan parents 608 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY) 609 peer_outbound.wait_for_parent_requests([parent_tx.txid_int]) 610 611 # There should be no request to the inbound peer 612 peer_inbound.assert_never_requested(parent_tx.txid_int) 613 614 self.log.info("Test that, if the preferred peer doesn't respond, the node sends another request") 615 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL) 616 peer_inbound.sync_with_ping() 617 peer_inbound.wait_for_parent_requests([parent_tx.txid_int]) 618 619 @cleanup 620 def test_maximal_package_protected(self): 621 self.log.info("Test that a node only announcing a maximally sized ancestor package is protected in orphanage") 622 self.nodes[0].setmocktime(int(time.time())) 623 node = self.nodes[0] 624 625 peer_normal = node.add_p2p_connection(P2PInterface()) 626 peer_doser = node.add_p2p_connection(P2PInterface()) 627 628 # Each of the num_peers peers creates a distinct set of orphans 629 large_orphans = [create_large_orphan() for _ in range(60)] 630 631 # Check to make sure these are orphans, within max standard size (to be accepted into the orphanage) 632 for large_orphan in large_orphans: 633 testres = node.testmempoolaccept([large_orphan.serialize().hex()]) 634 assert not testres[0]["allowed"] 635 assert_equal(testres[0]["reject-reason"], "missing-inputs") 636 637 num_individual_dosers = 20 638 self.log.info(f"Connect {num_individual_dosers} peers and send a very large orphan from each one") 639 # This test assumes that unrequested transactions are processed (skipping inv and 640 # getdata steps because they require going through request delays) 641 # Connect 20 peers and have each of them send a large orphan. 642 for large_orphan in large_orphans[:num_individual_dosers]: 643 peer_doser_individual = node.add_p2p_connection(P2PInterface()) 644 peer_doser_individual.send_and_ping(msg_tx(large_orphan)) 645 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + 1) 646 peer_doser_individual.wait_for_getdata([large_orphan.vin[0].prevout.hash]) 647 648 # Make sure that these transactions are going through the orphan handling codepaths. 649 # Subsequent rounds will not wait for getdata because the time mocking will cause the 650 # normal package request to time out. 651 self.wait_until(lambda: len(node.getorphantxs()) == num_individual_dosers) 652 653 # Now honest peer will send a maximally sized ancestor package of 24 orphans chaining 654 # off of a single missing transaction, with a total vsize 404,000Wu 655 ancestor_package = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT - 1) 656 sum_ancestor_package_vsize = sum([tx["tx"].get_vsize() for tx in ancestor_package]) 657 final_tx = self.wallet.create_self_transfer(utxo_to_spend=ancestor_package[-1]["new_utxo"], target_vsize=101000 - sum_ancestor_package_vsize) 658 ancestor_package.append(final_tx) 659 660 # Peer sends all but first tx to fill up orphange with their orphans 661 for orphan in ancestor_package[1:]: 662 peer_normal.send_and_ping(msg_tx(orphan["tx"])) 663 664 orphan_set = node.getorphantxs() 665 for orphan in ancestor_package[1:]: 666 assert orphan["txid"] in orphan_set 667 668 # Wait for ultimate parent request (the root ancestor transaction) 669 parent_txid_int = ancestor_package[0]["tx"].txid_int 670 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 671 672 self.wait_until(lambda: "getdata" in peer_normal.last_message and parent_txid_int in [inv.hash for inv in peer_normal.last_message.get("getdata").inv]) 673 674 self.log.info("Send another round of very large orphans from a DoSy peer") 675 for large_orphan in large_orphans[num_individual_dosers:]: 676 peer_doser.send_and_ping(msg_tx(large_orphan)) 677 678 self.log.info("Provide the top ancestor. The whole package should be re-evaluated after enough time.") 679 peer_normal.send_and_ping(msg_tx(ancestor_package[0]["tx"])) 680 681 # Wait until all transactions have been processed. When the last tx is accepted, it's 682 # guaranteed to have all ancestors. 683 self.wait_until(lambda: node.getmempoolentry(final_tx["txid"])["ancestorcount"] == DEFAULT_ANCESTOR_LIMIT) 684 685 @cleanup 686 def test_announcers_before_and_after(self): 687 self.log.info("Test that the node uses all peers who announced the tx prior to realizing it's an orphan") 688 node = self.nodes[0] 689 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child() 690 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16)) 691 692 # Announces before tx is sent, disconnects while node is requesting parents 693 peer_early_disconnected = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=3) 694 # Announces before tx is sent, doesn't respond to parent request 695 peer_early_unresponsive = node.add_p2p_connection(PeerTxRelayer()) 696 697 # Announces after tx is sent 698 peer_late_announcer = node.add_p2p_connection(PeerTxRelayer()) 699 700 # Both peers send invs for the orphan, so the node can expect both to know its ancestors. 701 peer_early_disconnected.send_and_ping(msg_inv([orphan_inv])) 702 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP) 703 peer_early_disconnected.wait_for_getdata([int(orphan_wtxid, 16)]) 704 peer_early_unresponsive.send_and_ping(msg_inv([orphan_inv])) 705 peer_early_disconnected.send_and_ping(msg_tx(orphan_tx)) 706 707 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are) 708 orphanage = node.getorphantxs(verbosity=2) 709 assert_equal(len(orphanage), 1) 710 assert_equal(orphanage[0]["wtxid"], orphan_wtxid) 711 assert_equal(len(orphanage[0]["from"]), 2) 712 713 # Peer disconnects before responding to request 714 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY) 715 peer_early_disconnected.wait_for_parent_requests([parent_tx.txid_int]) 716 peer_early_disconnected.peer_disconnect() 717 718 # The orphan should have 1 announcer left after the node finishes disconnecting peer_early_disconnected. 719 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1) 720 721 # The node should retry with the other peer that announced the orphan earlier. 722 # This node's request was additionally delayed because it's an inbound peer. 723 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY) 724 peer_early_unresponsive.wait_for_parent_requests([parent_tx.txid_int]) 725 726 self.log.info("Test that the node uses peers who announce the tx after realizing it's an orphan") 727 peer_late_announcer.send_and_ping(msg_inv([orphan_inv])) 728 729 # The orphan should have 2 announcers now 730 orphanage = node.getorphantxs(verbosity=2) 731 assert_equal(orphanage[0]["wtxid"], orphan_wtxid) 732 assert_equal(len(orphanage[0]["from"]), 2) 733 734 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL) 735 peer_late_announcer.wait_for_parent_requests([parent_tx.txid_int]) 736 737 @cleanup 738 def test_parents_change(self): 739 self.log.info("Test that, if a parent goes missing during orphan reso, it is requested") 740 node = self.nodes[0] 741 # Orphan will have 2 parents, 1 missing and 1 already in mempool when received. 742 # Create missing parent. 743 parent_missing = self.wallet.create_self_transfer() 744 745 # Create parent that will already be in mempool, but become missing during orphan resolution. 746 # Get 3 UTXOs for replacement-cycled parent, UTXOS A, B, C 747 coin_A = self.wallet.get_utxo(confirmed_only=True) 748 coin_B = self.wallet.get_utxo(confirmed_only=True) 749 coin_C = self.wallet.get_utxo(confirmed_only=True) 750 # parent_peekaboo_AB spends A and B. It is replaced by tx_replacer_BC (conflicting UTXO B), 751 # and then replaced by tx_replacer_C (conflicting UTXO C). This replacement cycle is used to 752 # ensure that parent_peekaboo_AB can be reintroduced without requiring package RBF. 753 FEE_INCREMENT = 2400 754 parent_peekaboo_AB = self.wallet.create_self_transfer_multi( 755 utxos_to_spend=[coin_A, coin_B], 756 num_outputs=1, 757 fee_per_output=FEE_INCREMENT 758 ) 759 tx_replacer_BC = self.wallet.create_self_transfer_multi( 760 utxos_to_spend=[coin_B, coin_C], 761 num_outputs=1, 762 fee_per_output=2*FEE_INCREMENT 763 ) 764 tx_replacer_C = self.wallet.create_self_transfer( 765 utxo_to_spend=coin_C, 766 fee_per_output=3*FEE_INCREMENT 767 ) 768 769 # parent_peekaboo_AB starts out in the mempool 770 node.sendrawtransaction(parent_peekaboo_AB["hex"]) 771 772 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[parent_peekaboo_AB["new_utxos"][0], parent_missing["new_utxo"]]) 773 orphan_wtxid = orphan["wtxid"] 774 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16)) 775 776 # peer1 sends the orphan and gets a request for the missing parent 777 peer1 = node.add_p2p_connection(PeerTxRelayer()) 778 peer1.send_and_ping(msg_inv([orphan_inv])) 779 node.bumpmocktime(TXREQUEST_TIME_SKIP) 780 peer1.wait_for_getdata([int(orphan_wtxid, 16)]) 781 peer1.send_and_ping(msg_tx(orphan["tx"])) 782 self.wait_until(lambda: node.getorphantxs(verbosity=0) == [orphan["txid"]]) 783 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) 784 peer1.wait_for_getdata([int(parent_missing["txid"], 16)]) 785 786 # Replace parent_peekaboo_AB so that is a newly missing parent. 787 # Then, replace the replacement so that it can be resubmitted. 788 node.sendrawtransaction(tx_replacer_BC["hex"]) 789 assert tx_replacer_BC["txid"] in node.getrawmempool() 790 node.sendrawtransaction(tx_replacer_C["hex"]) 791 assert tx_replacer_BC["txid"] not in node.getrawmempool() 792 assert parent_peekaboo_AB["txid"] not in node.getrawmempool() 793 assert tx_replacer_C["txid"] in node.getrawmempool() 794 795 # Second peer is an additional announcer for this orphan, but its missing parents are different from when it was 796 # previously announced. 797 peer2 = node.add_p2p_connection(PeerTxRelayer()) 798 peer2.send_and_ping(msg_inv([orphan_inv])) 799 assert_equal(len(node.getorphantxs(verbosity=2)[0]["from"]), 2) 800 801 # Disconnect peer1. peer2 should become the new candidate for orphan resolution. 802 peer1.peer_disconnect() 803 self.wait_until(lambda: node.num_test_p2p_connections() == 1) 804 peer2.sync_with_ping() # Sync with the 'net' thread which completes the disconnection fully 805 node.bumpmocktime(TXREQUEST_TIME_SKIP) 806 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1) 807 # Both parents should be requested, now that they are both missing. 808 peer2.wait_for_parent_requests([int(parent_peekaboo_AB["txid"], 16), int(parent_missing["txid"], 16)]) 809 peer2.send_and_ping(msg_tx(parent_missing["tx"])) 810 peer2.send_and_ping(msg_tx(parent_peekaboo_AB["tx"])) 811 812 final_mempool = node.getrawmempool() 813 assert parent_missing["txid"] in final_mempool 814 assert parent_peekaboo_AB["txid"] in final_mempool 815 assert orphan["txid"] in final_mempool 816 assert tx_replacer_C["txid"] in final_mempool 817 818 def run_test(self): 819 self.nodes[0].setmocktime(int(time.time())) 820 self.wallet_nonsegwit = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK) 821 self.generate(self.wallet_nonsegwit, 10) 822 self.wallet = MiniWallet(self.nodes[0]) 823 self.generate(self.wallet, 160) 824 825 self.test_arrival_timing_orphan() 826 self.test_orphan_rejected_parents_exceptions() 827 self.test_orphan_multiple_parents() 828 self.test_orphans_overlapping_parents() 829 self.test_orphan_of_orphan() 830 self.test_orphan_inherit_rejection() 831 self.test_same_txid_orphan() 832 self.test_same_txid_orphan_of_orphan() 833 self.test_orphan_txid_inv() 834 self.test_orphan_handling_prefer_outbound() 835 self.test_announcers_before_and_after() 836 self.test_parents_change() 837 self.test_maximal_package_protected() 838 839 840 if __name__ == '__main__': 841 OrphanHandlingTest(__file__).main()