rpc_packages.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2021-2022 The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """RPCs that handle raw transaction packages.""" 6 7 from decimal import Decimal 8 import random 9 10 from test_framework.blocktools import COINBASE_MATURITY 11 from test_framework.messages import ( 12 MAX_BIP125_RBF_SEQUENCE, 13 tx_from_hex, 14 ) 15 from test_framework.p2p import P2PTxInvStore 16 from test_framework.test_framework import BitcoinTestFramework 17 from test_framework.util import ( 18 assert_equal, 19 assert_fee_amount, 20 assert_raises_rpc_error, 21 ) 22 from test_framework.wallet import ( 23 DEFAULT_FEE, 24 MiniWallet, 25 ) 26 27 28 class RPCPackagesTest(BitcoinTestFramework): 29 def set_test_params(self): 30 self.num_nodes = 1 31 self.setup_clean_chain = True 32 # whitelist peers to speed up tx relay / mempool sync 33 self.noban_tx_relay = True 34 35 def assert_testres_equal(self, package_hex, testres_expected): 36 """Shuffle package_hex and assert that the testmempoolaccept result matches testres_expected. This should only 37 be used to test packages where the order does not matter. The ordering of transactions in package_hex and 38 testres_expected must match. 39 """ 40 shuffled_indeces = list(range(len(package_hex))) 41 random.shuffle(shuffled_indeces) 42 shuffled_package = [package_hex[i] for i in shuffled_indeces] 43 shuffled_testres = [testres_expected[i] for i in shuffled_indeces] 44 assert_equal(shuffled_testres, self.nodes[0].testmempoolaccept(shuffled_package)) 45 46 def run_test(self): 47 node = self.nodes[0] 48 49 # get an UTXO that requires signature to be spent 50 deterministic_address = node.get_deterministic_priv_key().address 51 blockhash = self.generatetoaddress(node, 1, deterministic_address)[0] 52 coinbase = node.getblock(blockhash=blockhash, verbosity=2)["tx"][0] 53 coin = { 54 "txid": coinbase["txid"], 55 "amount": coinbase["vout"][0]["value"], 56 "scriptPubKey": coinbase["vout"][0]["scriptPubKey"], 57 "vout": 0, 58 "height": 0 59 } 60 61 self.wallet = MiniWallet(self.nodes[0]) 62 self.generate(self.wallet, COINBASE_MATURITY + 100) # blocks generated for inputs 63 64 self.log.info("Create some transactions") 65 # Create some transactions that can be reused throughout the test. Never submit these to mempool. 66 self.independent_txns_hex = [] 67 self.independent_txns_testres = [] 68 for _ in range(3): 69 tx_hex = self.wallet.create_self_transfer(fee_rate=Decimal("0.0001"))["hex"] 70 testres = self.nodes[0].testmempoolaccept([tx_hex]) 71 assert testres[0]["allowed"] 72 self.independent_txns_hex.append(tx_hex) 73 # testmempoolaccept returns a list of length one, avoid creating a 2D list 74 self.independent_txns_testres.append(testres[0]) 75 self.independent_txns_testres_blank = [{ 76 "txid": res["txid"], "wtxid": res["wtxid"]} for res in self.independent_txns_testres] 77 78 self.test_independent(coin) 79 self.test_chain() 80 self.test_multiple_children() 81 self.test_multiple_parents() 82 self.test_conflicting() 83 self.test_rbf() 84 self.test_submitpackage() 85 self.test_maxfeerate_maxburn_submitpackage() 86 87 def test_independent(self, coin): 88 self.log.info("Test multiple independent transactions in a package") 89 node = self.nodes[0] 90 # For independent transactions, order doesn't matter. 91 self.assert_testres_equal(self.independent_txns_hex, self.independent_txns_testres) 92 93 self.log.info("Test an otherwise valid package with an extra garbage tx appended") 94 address = node.get_deterministic_priv_key().address 95 garbage_tx = node.createrawtransaction([{"txid": "00" * 32, "vout": 5}], {address: 1}) 96 tx = tx_from_hex(garbage_tx) 97 # Only the txid and wtxids are returned because validation is incomplete for the independent txns. 98 # Package validation is atomic: if the node cannot find a UTXO for any single tx in the package, 99 # it terminates immediately to avoid unnecessary, expensive signature verification. 100 package_bad = self.independent_txns_hex + [garbage_tx] 101 testres_bad = self.independent_txns_testres_blank + [{"txid": tx.rehash(), "wtxid": tx.getwtxid(), "allowed": False, "reject-reason": "missing-inputs"}] 102 self.assert_testres_equal(package_bad, testres_bad) 103 104 self.log.info("Check testmempoolaccept tells us when some transactions completed validation successfully") 105 tx_bad_sig_hex = node.createrawtransaction([{"txid": coin["txid"], "vout": 0}], 106 {address : coin["amount"] - Decimal("0.0001")}) 107 tx_bad_sig = tx_from_hex(tx_bad_sig_hex) 108 testres_bad_sig = node.testmempoolaccept(self.independent_txns_hex + [tx_bad_sig_hex]) 109 # By the time the signature for the last transaction is checked, all the other transactions 110 # have been fully validated, which is why the node returns full validation results for all 111 # transactions here but empty results in other cases. 112 assert_equal(testres_bad_sig, self.independent_txns_testres + [{ 113 "txid": tx_bad_sig.rehash(), 114 "wtxid": tx_bad_sig.getwtxid(), "allowed": False, 115 "reject-reason": "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)" 116 }]) 117 118 self.log.info("Check testmempoolaccept reports txns in packages that exceed max feerate") 119 tx_high_fee = self.wallet.create_self_transfer(fee=Decimal("0.999")) 120 testres_high_fee = node.testmempoolaccept([tx_high_fee["hex"]]) 121 assert_equal(testres_high_fee, [ 122 {"txid": tx_high_fee["txid"], "wtxid": tx_high_fee["wtxid"], "allowed": False, "reject-reason": "max-fee-exceeded"} 123 ]) 124 package_high_fee = [tx_high_fee["hex"]] + self.independent_txns_hex 125 testres_package_high_fee = node.testmempoolaccept(package_high_fee) 126 assert_equal(testres_package_high_fee, testres_high_fee + self.independent_txns_testres_blank) 127 128 def test_chain(self): 129 node = self.nodes[0] 130 131 chain = self.wallet.create_self_transfer_chain(chain_length=25) 132 chain_hex = [t["hex"] for t in chain] 133 chain_txns = [t["tx"] for t in chain] 134 135 self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") 136 assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), 137 [{"txid": tx.rehash(), "wtxid": tx.getwtxid(), "package-error": "package-not-sorted"} for tx in chain_txns[::-1]]) 138 139 self.log.info("Testmempoolaccept a chain of 25 transactions") 140 testres_multiple = node.testmempoolaccept(rawtxs=chain_hex) 141 142 testres_single = [] 143 # Test accept and then submit each one individually, which should be identical to package test accept 144 for rawtx in chain_hex: 145 testres = node.testmempoolaccept([rawtx]) 146 testres_single.append(testres[0]) 147 # Submit the transaction now so its child should have no problem validating 148 node.sendrawtransaction(rawtx) 149 assert_equal(testres_single, testres_multiple) 150 151 # Clean up by clearing the mempool 152 self.generate(node, 1) 153 154 def test_multiple_children(self): 155 node = self.nodes[0] 156 self.log.info("Testmempoolaccept a package in which a transaction has two children within the package") 157 158 parent_tx = self.wallet.create_self_transfer_multi(num_outputs=2) 159 assert node.testmempoolaccept([parent_tx["hex"]])[0]["allowed"] 160 161 # Child A 162 child_a_tx = self.wallet.create_self_transfer(utxo_to_spend=parent_tx["new_utxos"][0]) 163 assert not node.testmempoolaccept([child_a_tx["hex"]])[0]["allowed"] 164 165 # Child B 166 child_b_tx = self.wallet.create_self_transfer(utxo_to_spend=parent_tx["new_utxos"][1]) 167 assert not node.testmempoolaccept([child_b_tx["hex"]])[0]["allowed"] 168 169 self.log.info("Testmempoolaccept with entire package, should work with children in either order") 170 testres_multiple_ab = node.testmempoolaccept(rawtxs=[parent_tx["hex"], child_a_tx["hex"], child_b_tx["hex"]]) 171 testres_multiple_ba = node.testmempoolaccept(rawtxs=[parent_tx["hex"], child_b_tx["hex"], child_a_tx["hex"]]) 172 assert all([testres["allowed"] for testres in testres_multiple_ab + testres_multiple_ba]) 173 174 testres_single = [] 175 # Test accept and then submit each one individually, which should be identical to package testaccept 176 for rawtx in [parent_tx["hex"], child_a_tx["hex"], child_b_tx["hex"]]: 177 testres = node.testmempoolaccept([rawtx]) 178 testres_single.append(testres[0]) 179 # Submit the transaction now so its child should have no problem validating 180 node.sendrawtransaction(rawtx) 181 assert_equal(testres_single, testres_multiple_ab) 182 183 def test_multiple_parents(self): 184 node = self.nodes[0] 185 self.log.info("Testmempoolaccept a package in which a transaction has multiple parents within the package") 186 187 for num_parents in [2, 10, 24]: 188 # Test a package with num_parents parents and 1 child transaction. 189 parent_coins = [] 190 package_hex = [] 191 192 for _ in range(num_parents): 193 # Package accept should work with the parents in any order (as long as parents come before child) 194 parent_tx = self.wallet.create_self_transfer() 195 parent_coins.append(parent_tx["new_utxo"]) 196 package_hex.append(parent_tx["hex"]) 197 198 child_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins, fee_per_output=2000) 199 for _ in range(10): 200 random.shuffle(package_hex) 201 testres_multiple = node.testmempoolaccept(rawtxs=package_hex + [child_tx['hex']]) 202 assert all([testres["allowed"] for testres in testres_multiple]) 203 204 testres_single = [] 205 # Test accept and then submit each one individually, which should be identical to package testaccept 206 for rawtx in package_hex + [child_tx["hex"]]: 207 testres_single.append(node.testmempoolaccept([rawtx])[0]) 208 # Submit the transaction now so its child should have no problem validating 209 node.sendrawtransaction(rawtx) 210 assert_equal(testres_single, testres_multiple) 211 212 def test_conflicting(self): 213 node = self.nodes[0] 214 coin = self.wallet.get_utxo() 215 216 # tx1 and tx2 share the same inputs 217 tx1 = self.wallet.create_self_transfer(utxo_to_spend=coin, fee_rate=DEFAULT_FEE) 218 tx2 = self.wallet.create_self_transfer(utxo_to_spend=coin, fee_rate=2*DEFAULT_FEE) 219 220 # Ensure tx1 and tx2 are valid by themselves 221 assert node.testmempoolaccept([tx1["hex"]])[0]["allowed"] 222 assert node.testmempoolaccept([tx2["hex"]])[0]["allowed"] 223 224 self.log.info("Test duplicate transactions in the same package") 225 testres = node.testmempoolaccept([tx1["hex"], tx1["hex"]]) 226 assert_equal(testres, [ 227 {"txid": tx1["txid"], "wtxid": tx1["wtxid"], "package-error": "package-contains-duplicates"}, 228 {"txid": tx1["txid"], "wtxid": tx1["wtxid"], "package-error": "package-contains-duplicates"} 229 ]) 230 231 self.log.info("Test conflicting transactions in the same package") 232 testres = node.testmempoolaccept([tx1["hex"], tx2["hex"]]) 233 assert_equal(testres, [ 234 {"txid": tx1["txid"], "wtxid": tx1["wtxid"], "package-error": "conflict-in-package"}, 235 {"txid": tx2["txid"], "wtxid": tx2["wtxid"], "package-error": "conflict-in-package"} 236 ]) 237 238 def test_rbf(self): 239 node = self.nodes[0] 240 241 coin = self.wallet.get_utxo() 242 fee = Decimal("0.00125000") 243 replaceable_tx = self.wallet.create_self_transfer(utxo_to_spend=coin, sequence=MAX_BIP125_RBF_SEQUENCE, fee = fee) 244 testres_replaceable = node.testmempoolaccept([replaceable_tx["hex"]])[0] 245 assert_equal(testres_replaceable["txid"], replaceable_tx["txid"]) 246 assert_equal(testres_replaceable["wtxid"], replaceable_tx["wtxid"]) 247 assert testres_replaceable["allowed"] 248 assert_equal(testres_replaceable["vsize"], replaceable_tx["tx"].get_vsize()) 249 assert_equal(testres_replaceable["fees"]["base"], fee) 250 assert_fee_amount(fee, replaceable_tx["tx"].get_vsize(), testres_replaceable["fees"]["effective-feerate"]) 251 assert_equal(testres_replaceable["fees"]["effective-includes"], [replaceable_tx["wtxid"]]) 252 253 # Replacement transaction is identical except has double the fee 254 replacement_tx = self.wallet.create_self_transfer(utxo_to_spend=coin, sequence=MAX_BIP125_RBF_SEQUENCE, fee = 2 * fee) 255 testres_rbf_conflicting = node.testmempoolaccept([replaceable_tx["hex"], replacement_tx["hex"]]) 256 assert_equal(testres_rbf_conflicting, [ 257 {"txid": replaceable_tx["txid"], "wtxid": replaceable_tx["wtxid"], "package-error": "conflict-in-package"}, 258 {"txid": replacement_tx["txid"], "wtxid": replacement_tx["wtxid"], "package-error": "conflict-in-package"} 259 ]) 260 261 self.log.info("Test that packages cannot conflict with mempool transactions, even if a valid BIP125 RBF") 262 # This transaction is a valid BIP125 replace-by-fee 263 self.wallet.sendrawtransaction(from_node=node, tx_hex=replaceable_tx["hex"]) 264 testres_rbf_single = node.testmempoolaccept([replacement_tx["hex"]]) 265 assert testres_rbf_single[0]["allowed"] 266 testres_rbf_package = self.independent_txns_testres_blank + [{ 267 "txid": replacement_tx["txid"], "wtxid": replacement_tx["wtxid"], "allowed": False, 268 "reject-reason": "bip125-replacement-disallowed" 269 }] 270 self.assert_testres_equal(self.independent_txns_hex + [replacement_tx["hex"]], testres_rbf_package) 271 272 def assert_equal_package_results(self, node, testmempoolaccept_result, submitpackage_result): 273 """Assert that a successful submitpackage result is consistent with testmempoolaccept 274 results and getmempoolentry info. Note that the result structs are different and, due to 275 policy differences between testmempoolaccept and submitpackage (i.e. package feerate), 276 some information may be different. 277 """ 278 for testres_tx in testmempoolaccept_result: 279 # Grab this result from the submitpackage_result 280 submitres_tx = submitpackage_result["tx-results"][testres_tx["wtxid"]] 281 assert_equal(submitres_tx["txid"], testres_tx["txid"]) 282 # No "allowed" if the tx was already in the mempool 283 if "allowed" in testres_tx and testres_tx["allowed"]: 284 assert_equal(submitres_tx["vsize"], testres_tx["vsize"]) 285 assert_equal(submitres_tx["fees"]["base"], testres_tx["fees"]["base"]) 286 entry_info = node.getmempoolentry(submitres_tx["txid"]) 287 assert_equal(submitres_tx["vsize"], entry_info["vsize"]) 288 assert_equal(submitres_tx["fees"]["base"], entry_info["fees"]["base"]) 289 290 def test_submit_child_with_parents(self, num_parents, partial_submit): 291 node = self.nodes[0] 292 peer = node.add_p2p_connection(P2PTxInvStore()) 293 294 package_txns = [] 295 presubmitted_wtxids = set() 296 for _ in range(num_parents): 297 parent_tx = self.wallet.create_self_transfer(fee=DEFAULT_FEE) 298 package_txns.append(parent_tx) 299 if partial_submit and random.choice([True, False]): 300 node.sendrawtransaction(parent_tx["hex"]) 301 presubmitted_wtxids.add(parent_tx["wtxid"]) 302 child_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[tx["new_utxo"] for tx in package_txns], fee_per_output=10000) #DEFAULT_FEE 303 package_txns.append(child_tx) 304 305 testmempoolaccept_result = node.testmempoolaccept(rawtxs=[tx["hex"] for tx in package_txns]) 306 submitpackage_result = node.submitpackage(package=[tx["hex"] for tx in package_txns]) 307 308 # Check that each result is present, with the correct size and fees 309 assert_equal(submitpackage_result["package_msg"], "success") 310 for package_txn in package_txns: 311 tx = package_txn["tx"] 312 assert tx.getwtxid() in submitpackage_result["tx-results"] 313 wtxid = tx.getwtxid() 314 assert wtxid in submitpackage_result["tx-results"] 315 tx_result = submitpackage_result["tx-results"][wtxid] 316 assert_equal(tx_result["txid"], tx.rehash()) 317 assert_equal(tx_result["vsize"], tx.get_vsize()) 318 assert_equal(tx_result["fees"]["base"], DEFAULT_FEE) 319 if wtxid not in presubmitted_wtxids: 320 assert_fee_amount(DEFAULT_FEE, tx.get_vsize(), tx_result["fees"]["effective-feerate"]) 321 assert_equal(tx_result["fees"]["effective-includes"], [wtxid]) 322 323 # submitpackage result should be consistent with testmempoolaccept and getmempoolentry 324 self.assert_equal_package_results(node, testmempoolaccept_result, submitpackage_result) 325 326 # The node should announce each transaction. No guarantees for propagation. 327 peer.wait_for_broadcast([tx["tx"].getwtxid() for tx in package_txns]) 328 self.generate(node, 1) 329 330 def test_submitpackage(self): 331 node = self.nodes[0] 332 333 self.log.info("Submitpackage valid packages with 1 child and some number of parents") 334 for num_parents in [1, 2, 24]: 335 self.test_submit_child_with_parents(num_parents, False) 336 self.test_submit_child_with_parents(num_parents, True) 337 338 self.log.info("Submitpackage only allows packages of 1 child with its parents") 339 # Chain of 3 transactions has too many generations 340 legacy_pool = node.getrawmempool() 341 chain_hex = [t["hex"] for t in self.wallet.create_self_transfer_chain(chain_length=3)] 342 assert_raises_rpc_error(-25, "package topology disallowed", node.submitpackage, chain_hex) 343 assert_equal(legacy_pool, node.getrawmempool()) 344 345 # Create a transaction chain such as only the parent gets accepted (by making the child's 346 # version non-standard). Make sure the parent does get broadcast. 347 self.log.info("If a package is partially submitted, transactions included in mempool get broadcast") 348 peer = node.add_p2p_connection(P2PTxInvStore()) 349 txs = self.wallet.create_self_transfer_chain(chain_length=2) 350 bad_child = tx_from_hex(txs[1]["hex"]) 351 bad_child.nVersion = -1 352 hex_partial_acceptance = [txs[0]["hex"], bad_child.serialize().hex()] 353 res = node.submitpackage(hex_partial_acceptance) 354 assert_equal(res["package_msg"], "transaction failed") 355 first_wtxid = txs[0]["tx"].getwtxid() 356 assert "error" not in res["tx-results"][first_wtxid] 357 sec_wtxid = bad_child.getwtxid() 358 assert_equal(res["tx-results"][sec_wtxid]["error"], "version") 359 peer.wait_for_broadcast([first_wtxid]) 360 361 def test_maxfeerate_maxburn_submitpackage(self): 362 node = self.nodes[0] 363 # clear mempool 364 deterministic_address = node.get_deterministic_priv_key().address 365 self.generatetoaddress(node, 1, deterministic_address) 366 367 self.log.info("Submitpackage maxfeerate arg testing") 368 chained_txns = self.wallet.create_self_transfer_chain(chain_length=2) 369 minrate_btc_kvb = min([chained_txn["fee"] / chained_txn["tx"].get_vsize() * 1000 for chained_txn in chained_txns]) 370 chain_hex = [t["hex"] for t in chained_txns] 371 pkg_result = node.submitpackage(chain_hex, maxfeerate=minrate_btc_kvb - Decimal("0.00000001")) 372 assert_equal(pkg_result["tx-results"][chained_txns[0]["wtxid"]]["error"], "max feerate exceeded") 373 assert_equal(pkg_result["tx-results"][chained_txns[1]["wtxid"]]["error"], "bad-txns-inputs-missingorspent") 374 assert_equal(node.getrawmempool(), []) 375 376 self.log.info("Submitpackage maxburnamount arg testing") 377 tx = tx_from_hex(chain_hex[1]) 378 tx.vout[-1].scriptPubKey = b'a' * 10001 # scriptPubKey bigger than 10k IsUnspendable 379 chain_hex = [chain_hex[0], tx.serialize().hex()] 380 # burn test is run before any package evaluation; nothing makes it in and we get broader exception 381 assert_raises_rpc_error(-25, "Unspendable output exceeds maximum configured by user", node.submitpackage, chain_hex, 0, chained_txns[1]["new_utxo"]["value"] - Decimal("0.00000001")) 382 assert_equal(node.getrawmempool(), []) 383 384 # Relax the restrictions for both and send it; parent gets through as own subpackage 385 pkg_result = node.submitpackage(chain_hex, maxfeerate=minrate_btc_kvb, maxburnamount=chained_txns[1]["new_utxo"]["value"]) 386 assert "error" not in pkg_result["tx-results"][chained_txns[0]["wtxid"]] 387 assert_equal(pkg_result["tx-results"][tx.getwtxid()]["error"], "scriptpubkey") 388 assert_equal(node.getrawmempool(), [chained_txns[0]["txid"]]) 389 390 if __name__ == "__main__": 391 RPCPackagesTest().main()