feature_fee_estimation.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2014-2022 The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Test fee estimation code.""" 6 from copy import deepcopy 7 from decimal import Decimal, ROUND_DOWN 8 import os 9 import random 10 import time 11 12 from test_framework.messages import ( 13 COIN, 14 ) 15 from test_framework.test_framework import BitcoinTestFramework 16 from test_framework.util import ( 17 assert_not_equal, 18 assert_equal, 19 assert_greater_than, 20 assert_greater_than_or_equal, 21 assert_raises_rpc_error, 22 satoshi_round, 23 ) 24 from test_framework.wallet import MiniWallet 25 26 MAX_FILE_AGE = 60 27 SECONDS_PER_HOUR = 60 * 60 28 29 def small_txpuzzle_randfee( 30 wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment, batch_reqs 31 ): 32 """Create and send a transaction with a random fee using MiniWallet. 33 34 The function takes a list of confirmed outputs and unconfirmed outputs 35 and attempts to use the confirmed list first for its inputs. 36 It adds the newly created outputs to the unconfirmed list. 37 Returns (raw transaction, fee).""" 38 39 # It's best to exponentially distribute our random fees 40 # because the buckets are exponentially spaced. 41 # Exponentially distributed from 1-128 * fee_increment 42 rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) 43 # Total fee ranges from min_fee to min_fee + 127*fee_increment 44 fee = min_fee - fee_increment + satoshi_round(rand_fee, rounding=ROUND_DOWN) 45 utxos_to_spend = [] 46 total_in = Decimal("0.00000000") 47 while total_in <= (amount + fee) and len(conflist) > 0: 48 t = conflist.pop(0) 49 total_in += t["value"] 50 utxos_to_spend.append(t) 51 while total_in <= (amount + fee) and len(unconflist) > 0: 52 t = unconflist.pop(0) 53 total_in += t["value"] 54 utxos_to_spend.append(t) 55 if total_in <= amount + fee: 56 raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") 57 tx = wallet.create_self_transfer_multi( 58 utxos_to_spend=utxos_to_spend, 59 fee_per_output=0, 60 )["tx"] 61 tx.vout[0].nValue = int((total_in - amount - fee) * COIN) 62 tx.vout.append(deepcopy(tx.vout[0])) 63 tx.vout[1].nValue = int(amount * COIN) 64 txid = tx.txid_hex 65 tx_hex = tx.serialize().hex() 66 67 batch_reqs.append(from_node.sendrawtransaction.get_request(hexstring=tx_hex, maxfeerate=0)) 68 unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) 69 unconflist.append({"txid": txid, "vout": 1, "value": amount}) 70 71 return (tx.get_vsize(), fee) 72 73 74 def check_raw_estimates(node, fees_seen): 75 """Call estimaterawfee and verify that the estimates meet certain invariants.""" 76 77 delta = 1.0e-6 # account for rounding error 78 for i in range(1, 26): 79 for _, e in node.estimaterawfee(i).items(): 80 feerate = float(e["feerate"]) 81 assert_greater_than(feerate, 0) 82 83 if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): 84 raise AssertionError( 85 f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" 86 ) 87 88 89 def check_smart_estimates(node, fees_seen): 90 """Call estimatesmartfee and verify that the estimates meet certain invariants.""" 91 92 delta = 1.0e-6 # account for rounding error 93 all_smart_estimates = [node.estimatesmartfee(i) for i in range(1, 26)] 94 mempoolMinFee = node.getmempoolinfo()["mempoolminfee"] 95 minRelaytxFee = node.getmempoolinfo()["minrelaytxfee"] 96 feerate_ceiling = max(max(fees_seen), float(mempoolMinFee), float(minRelaytxFee)) 97 last_feerate = feerate_ceiling 98 for i, e in enumerate(all_smart_estimates): # estimate is for i+1 99 feerate = float(e["feerate"]) 100 assert_greater_than(feerate, 0) 101 assert_greater_than_or_equal(feerate, float(mempoolMinFee)) 102 assert_greater_than_or_equal(feerate, float(minRelaytxFee)) 103 104 if feerate + delta < min(fees_seen) or feerate - delta > feerate_ceiling: 105 raise AssertionError( 106 f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{feerate_ceiling})" 107 ) 108 if feerate - delta > last_feerate: 109 raise AssertionError( 110 f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms" 111 ) 112 last_feerate = feerate 113 114 if i == 0: 115 assert_equal(e["blocks"], 2) 116 else: 117 assert_greater_than_or_equal(i + 1, e["blocks"]) 118 119 120 def check_estimates(node, fees_seen): 121 check_raw_estimates(node, fees_seen) 122 check_smart_estimates(node, fees_seen) 123 124 125 def make_tx(wallet, utxo, feerate): 126 """Create a 1in-1out transaction with a specific input and feerate (sat/vb).""" 127 return wallet.create_self_transfer( 128 utxo_to_spend=utxo, 129 fee_rate=Decimal(feerate * 1000) / COIN, 130 ) 131 132 def check_fee_estimates_btw_modes(node, expected_conservative, expected_economical): 133 fee_est_conservative = node.estimatesmartfee(1, estimate_mode="conservative")['feerate'] 134 fee_est_economical = node.estimatesmartfee(1, estimate_mode="economical")['feerate'] 135 fee_est_default = node.estimatesmartfee(1)['feerate'] 136 assert_equal(fee_est_conservative, expected_conservative) 137 assert_equal(fee_est_economical, expected_economical) 138 assert_equal(fee_est_default, expected_economical) 139 140 141 class EstimateFeeTest(BitcoinTestFramework): 142 def set_test_params(self): 143 self.num_nodes = 3 144 # whitelist peers to speed up tx relay / mempool sync 145 self.noban_tx_relay = True 146 self.extra_args = [ 147 [], 148 ["-blockmaxweight=72000"], 149 ["-blockmaxweight=36000"], 150 ] 151 152 def setup_network(self): 153 """ 154 We'll setup the network to have 3 nodes that all mine with different parameters. 155 But first we need to use one node to create a lot of outputs 156 which we will use to generate our transactions. 157 """ 158 self.add_nodes(3, extra_args=self.extra_args) 159 # Use node0 to mine blocks for input splitting 160 # Node1 mines small blocks but that are bigger than the expected transaction rate. 161 # NOTE: the CreateNewBlock code starts counting block weight at 4,000 weight, 162 # (68k weight is room enough for 120 or so transactions) 163 # Node2 is a stingy miner, that 164 # produces too small blocks (room for only 55 or so transactions) 165 166 def transact_and_mine(self, numblocks, mining_node): 167 min_fee = Decimal("0.00001") 168 # We will now mine numblocks blocks generating on average 100 transactions between each block 169 # We shuffle our confirmed txout set before each set of transactions 170 # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible 171 # resorting to tx's that depend on the mempool when those run out 172 for _ in range(numblocks): 173 random.shuffle(self.confutxo) 174 batch_sendtx_reqs = [] 175 for _ in range(random.randrange(100 - 50, 100 + 50)): 176 from_index = random.randint(1, 2) 177 (tx_bytes, fee) = small_txpuzzle_randfee( 178 self.wallet, 179 self.nodes[from_index], 180 self.confutxo, 181 self.memutxo, 182 Decimal("0.005"), 183 min_fee, 184 min_fee, 185 batch_sendtx_reqs, 186 ) 187 tx_kbytes = tx_bytes / 1000.0 188 self.fees_per_kb.append(float(fee) / tx_kbytes) 189 for node in self.nodes: 190 node.batch(batch_sendtx_reqs) 191 self.sync_mempools(wait=0.1) 192 mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] 193 # update which txouts are confirmed 194 newmem = [] 195 for utx in self.memutxo: 196 if utx["txid"] in mined: 197 self.confutxo.append(utx) 198 else: 199 newmem.append(utx) 200 self.memutxo = newmem 201 202 def initial_split(self, node): 203 """Split two coinbase UTxOs into many small coins""" 204 self.confutxo = self.wallet.send_self_transfer_multi( 205 from_node=node, 206 utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], 207 num_outputs=2048)['new_utxos'] 208 while len(node.getrawmempool()) > 0: 209 self.generate(node, 1, sync_fun=self.no_op) 210 211 def sanity_check_estimates_range(self): 212 """Populate estimation buckets, assert estimates are in a sane range and 213 are strictly increasing as the target decreases.""" 214 self.fees_per_kb = [] 215 self.memutxo = [] 216 self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") 217 218 for _ in range(2): 219 self.log.info( 220 "Creating transactions and mining them with a block size that can't keep up" 221 ) 222 # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine 223 self.transact_and_mine(10, self.nodes[2]) 224 check_estimates(self.nodes[1], self.fees_per_kb) 225 226 self.log.info( 227 "Creating transactions and mining them at a block size that is just big enough" 228 ) 229 # Generate transactions while mining 10 more blocks, this time with node1 230 # which mines blocks with capacity just above the rate that transactions are being created 231 self.transact_and_mine(10, self.nodes[1]) 232 check_estimates(self.nodes[1], self.fees_per_kb) 233 234 # Finish by mining a normal-sized block: 235 while len(self.nodes[1].getrawmempool()) > 0: 236 self.generate(self.nodes[1], 1) 237 238 self.log.info("Final estimates after emptying mempools") 239 check_estimates(self.nodes[1], self.fees_per_kb) 240 241 def test_estimates_with_highminrelaytxfee(self): 242 high_val = 3 * self.nodes[1].estimatesmartfee(2)["feerate"] 243 self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"]) 244 check_smart_estimates(self.nodes[1], self.fees_per_kb) 245 self.restart_node(1) 246 247 def sanity_check_rbf_estimates(self, utxos): 248 """During 5 blocks, broadcast low fee transactions. Only 10% of them get 249 confirmed and the remaining ones get RBF'd with a high fee transaction at 250 the next block. 251 The block policy estimator should return the high feerate. 252 """ 253 # The broadcaster and block producer 254 node = self.nodes[0] 255 miner = self.nodes[1] 256 # In sat/vb 257 low_feerate = 1 258 high_feerate = 10 259 # Cache the utxos of which to replace the spender after it failed to get 260 # confirmed 261 utxos_to_respend = [] 262 txids_to_replace = [] 263 264 assert_greater_than_or_equal(len(utxos), 250) 265 for _ in range(5): 266 # Broadcast 45 low fee transactions that will need to be RBF'd 267 txs = [] 268 for _ in range(45): 269 u = utxos.pop(0) 270 tx = make_tx(self.wallet, u, low_feerate) 271 utxos_to_respend.append(u) 272 txids_to_replace.append(tx["txid"]) 273 txs.append(tx) 274 # Broadcast 5 low fee transaction which don't need to 275 for _ in range(5): 276 tx = make_tx(self.wallet, utxos.pop(0), low_feerate) 277 txs.append(tx) 278 batch_send_tx = [node.sendrawtransaction.get_request(tx["hex"]) for tx in txs] 279 for n in self.nodes: 280 n.batch(batch_send_tx) 281 # Mine the transactions on another node 282 self.sync_mempools(wait=0.1, nodes=[node, miner]) 283 for txid in txids_to_replace: 284 miner.prioritisetransaction(txid=txid, fee_delta=-COIN) 285 self.generate(miner, 1) 286 # RBF the low-fee transactions 287 while len(utxos_to_respend) > 0: 288 u = utxos_to_respend.pop(0) 289 tx = make_tx(self.wallet, u, high_feerate) 290 node.sendrawtransaction(tx["hex"]) 291 txs.append(tx) 292 dec_txs = [res["result"] for res in node.batch([node.decoderawtransaction.get_request(tx["hex"]) for tx in txs])] 293 self.wallet.scan_txs(dec_txs) 294 295 296 # Mine the last replacement txs 297 self.sync_mempools(wait=0.1, nodes=[node, miner]) 298 self.generate(miner, 1) 299 300 # Only 10% of the transactions were really confirmed with a low feerate, 301 # the rest needed to be RBF'd. We must return the 90% conf rate feerate. 302 high_feerate_kvb = Decimal(high_feerate) / COIN * 10 ** 3 303 est_feerate = node.estimatesmartfee(2)["feerate"] 304 assert_equal(est_feerate, high_feerate_kvb) 305 306 def test_old_fee_estimate_file(self): 307 # Get the initial fee rate while node is running 308 fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] 309 310 # Restart node to ensure fee_estimate.dat file is read 311 self.restart_node(0) 312 assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) 313 314 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 315 316 # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE 317 self.stop_node(0) 318 last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR 319 os.utime(fee_dat, (last_modified_time, last_modified_time)) 320 321 # Start node and ensure the fee_estimates.dat file was not read 322 self.start_node(0) 323 assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) 324 325 326 def test_estimate_dat_is_flushed_periodically(self): 327 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 328 os.remove(fee_dat) if os.path.exists(fee_dat) else None 329 330 # Verify that fee_estimates.dat does not exist 331 assert_equal(os.path.isfile(fee_dat), False) 332 333 # Verify if the string "Flushed fee estimates to fee_estimates.dat." is present in the debug log file. 334 # If present, it indicates that fee estimates have been successfully flushed to disk. 335 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 336 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 337 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 338 339 # Verify that fee estimates were flushed and fee_estimates.dat file is created 340 assert_equal(os.path.isfile(fee_dat), True) 341 342 # Verify that the estimates remain the same if there are no blocks in the flush interval 343 block_hash_before = self.nodes[0].getbestblockhash() 344 fee_dat_initial_content = open(fee_dat, "rb").read() 345 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 346 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 347 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 348 349 # Verify that there were no blocks in between the flush interval 350 assert_equal(block_hash_before, self.nodes[0].getbestblockhash()) 351 352 fee_dat_current_content = open(fee_dat, "rb").read() 353 assert_equal(fee_dat_current_content, fee_dat_initial_content) 354 355 # Verify that the estimates remain the same after shutdown with no blocks before shutdown 356 self.restart_node(0) 357 fee_dat_current_content = open(fee_dat, "rb").read() 358 assert_equal(fee_dat_current_content, fee_dat_initial_content) 359 360 # Verify that the estimates are not the same if new blocks were produced in the flush interval 361 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 362 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 363 self.generate(self.nodes[0], 5, sync_fun=self.no_op) 364 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 365 366 fee_dat_current_content = open(fee_dat, "rb").read() 367 assert_not_equal(fee_dat_current_content, fee_dat_initial_content) 368 369 fee_dat_initial_content = fee_dat_current_content 370 371 # Generate blocks before shutdown and verify that the fee estimates are not the same 372 self.generate(self.nodes[0], 5, sync_fun=self.no_op) 373 self.restart_node(0) 374 fee_dat_current_content = open(fee_dat, "rb").read() 375 assert_not_equal(fee_dat_current_content, fee_dat_initial_content) 376 377 378 def test_acceptstalefeeestimates_option(self): 379 # Get the initial fee rate while node is running 380 fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] 381 382 self.stop_node(0) 383 384 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 385 386 # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE 387 last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR 388 os.utime(fee_dat, (last_modified_time, last_modified_time)) 389 390 # Restart node with -acceptstalefeeestimates option to ensure fee_estimate.dat file is read 391 self.start_node(0,extra_args=["-acceptstalefeeestimates"]) 392 assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) 393 394 def clear_estimates(self): 395 self.log.info("Restarting node with fresh estimation") 396 self.stop_node(0) 397 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 398 os.remove(fee_dat) 399 self.start_node(0) 400 self.connect_nodes(0, 1) 401 self.connect_nodes(0, 2) 402 self.sync_blocks() 403 assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) 404 405 def broadcast_and_mine(self, broadcaster, miner, feerate, count): 406 """Broadcast and mine some number of transactions with a specified fee rate.""" 407 for _ in range(count): 408 self.wallet.send_self_transfer(from_node=broadcaster, fee_rate=feerate) 409 self.sync_mempools() 410 self.generate(miner, 1) 411 412 def test_estimation_modes(self): 413 low_feerate = Decimal("0.001") 414 high_feerate = Decimal("0.005") 415 tx_count = 24 416 # Broadcast and mine high fee transactions for the first 12 blocks. 417 for _ in range(12): 418 self.broadcast_and_mine(self.nodes[1], self.nodes[2], high_feerate, tx_count) 419 check_fee_estimates_btw_modes(self.nodes[0], high_feerate, high_feerate) 420 421 # We now track 12 blocks; short horizon stats will start decaying. 422 # Broadcast and mine low fee transactions for the next 4 blocks. 423 for _ in range(4): 424 self.broadcast_and_mine(self.nodes[1], self.nodes[2], low_feerate, tx_count) 425 # conservative mode will consider longer time horizons while economical mode does not 426 # Check the fee estimates for both modes after mining low fee transactions. 427 check_fee_estimates_btw_modes(self.nodes[0], high_feerate, low_feerate) 428 429 430 def run_test(self): 431 self.log.info("This test is time consuming, please be patient") 432 self.log.info("Splitting inputs so we can generate tx's") 433 434 # Split two coinbases into many small utxos 435 self.start_node(0) 436 self.wallet = MiniWallet(self.nodes[0]) 437 self.initial_split(self.nodes[0]) 438 self.log.info("Finished splitting") 439 440 # Now we can connect the other nodes, didn't want to connect them earlier 441 # so the estimates would not be affected by the splitting transactions 442 self.start_node(1) 443 self.start_node(2) 444 self.connect_nodes(1, 0) 445 self.connect_nodes(0, 2) 446 self.connect_nodes(2, 1) 447 self.sync_all() 448 449 self.log.info("Testing estimates with single transactions.") 450 self.sanity_check_estimates_range() 451 452 self.log.info("Test fee_estimates.dat is flushed periodically") 453 self.test_estimate_dat_is_flushed_periodically() 454 455 # check that estimatesmartfee feerate is greater than or equal to maximum of mempoolminfee and minrelaytxfee 456 self.log.info( 457 "Test fee rate estimation after restarting node with high minrelaytxfee" 458 ) 459 self.test_estimates_with_highminrelaytxfee() 460 461 self.log.info("Test acceptstalefeeestimates option") 462 self.test_acceptstalefeeestimates_option() 463 464 self.log.info("Test reading old fee_estimates.dat") 465 self.test_old_fee_estimate_file() 466 467 self.clear_estimates() 468 469 self.log.info("Testing estimates with RBF.") 470 self.sanity_check_rbf_estimates(self.confutxo + self.memutxo) 471 472 self.clear_estimates() 473 self.log.info("Test estimatesmartfee modes") 474 self.test_estimation_modes() 475 476 self.log.info("Testing that fee estimation is disabled in blocksonly.") 477 self.restart_node(0, ["-blocksonly"]) 478 assert_raises_rpc_error( 479 -32603, "Fee estimation disabled", self.nodes[0].estimatesmartfee, 2 480 ) 481 482 483 if __name__ == "__main__": 484 EstimateFeeTest(__file__).main()