feature_fee_estimation.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2014-2022 The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Test fee estimation code.""" 6 from copy import deepcopy 7 from decimal import Decimal 8 import os 9 import random 10 import time 11 12 from test_framework.messages import ( 13 COIN, 14 ) 15 from test_framework.test_framework import BitcoinTestFramework 16 from test_framework.util import ( 17 assert_equal, 18 assert_greater_than, 19 assert_greater_than_or_equal, 20 assert_raises_rpc_error, 21 satoshi_round, 22 ) 23 from test_framework.wallet import MiniWallet 24 25 MAX_FILE_AGE = 60 26 SECONDS_PER_HOUR = 60 * 60 27 28 def small_txpuzzle_randfee( 29 wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment, batch_reqs 30 ): 31 """Create and send a transaction with a random fee using MiniWallet. 32 33 The function takes a list of confirmed outputs and unconfirmed outputs 34 and attempts to use the confirmed list first for its inputs. 35 It adds the newly created outputs to the unconfirmed list. 36 Returns (raw transaction, fee).""" 37 38 # It's best to exponentially distribute our random fees 39 # because the buckets are exponentially spaced. 40 # Exponentially distributed from 1-128 * fee_increment 41 rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) 42 # Total fee ranges from min_fee to min_fee + 127*fee_increment 43 fee = min_fee - fee_increment + satoshi_round(rand_fee) 44 utxos_to_spend = [] 45 total_in = Decimal("0.00000000") 46 while total_in <= (amount + fee) and len(conflist) > 0: 47 t = conflist.pop(0) 48 total_in += t["value"] 49 utxos_to_spend.append(t) 50 while total_in <= (amount + fee) and len(unconflist) > 0: 51 t = unconflist.pop(0) 52 total_in += t["value"] 53 utxos_to_spend.append(t) 54 if total_in <= amount + fee: 55 raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") 56 tx = wallet.create_self_transfer_multi( 57 utxos_to_spend=utxos_to_spend, 58 fee_per_output=0, 59 )["tx"] 60 tx.vout[0].nValue = int((total_in - amount - fee) * COIN) 61 tx.vout.append(deepcopy(tx.vout[0])) 62 tx.vout[1].nValue = int(amount * COIN) 63 tx.rehash() 64 txid = tx.hash 65 tx_hex = tx.serialize().hex() 66 67 batch_reqs.append(from_node.sendrawtransaction.get_request(hexstring=tx_hex, maxfeerate=0)) 68 unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) 69 unconflist.append({"txid": txid, "vout": 1, "value": amount}) 70 71 return (tx.get_vsize(), fee) 72 73 74 def check_raw_estimates(node, fees_seen): 75 """Call estimaterawfee and verify that the estimates meet certain invariants.""" 76 77 delta = 1.0e-6 # account for rounding error 78 for i in range(1, 26): 79 for _, e in node.estimaterawfee(i).items(): 80 feerate = float(e["feerate"]) 81 assert_greater_than(feerate, 0) 82 83 if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): 84 raise AssertionError( 85 f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" 86 ) 87 88 89 def check_smart_estimates(node, fees_seen): 90 """Call estimatesmartfee and verify that the estimates meet certain invariants.""" 91 92 delta = 1.0e-6 # account for rounding error 93 last_feerate = float(max(fees_seen)) 94 all_smart_estimates = [node.estimatesmartfee(i) for i in range(1, 26)] 95 mempoolMinFee = node.getmempoolinfo()["mempoolminfee"] 96 minRelaytxFee = node.getmempoolinfo()["minrelaytxfee"] 97 for i, e in enumerate(all_smart_estimates): # estimate is for i+1 98 feerate = float(e["feerate"]) 99 assert_greater_than(feerate, 0) 100 assert_greater_than_or_equal(feerate, float(mempoolMinFee)) 101 assert_greater_than_or_equal(feerate, float(minRelaytxFee)) 102 103 if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): 104 raise AssertionError( 105 f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" 106 ) 107 if feerate - delta > last_feerate: 108 raise AssertionError( 109 f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms" 110 ) 111 last_feerate = feerate 112 113 if i == 0: 114 assert_equal(e["blocks"], 2) 115 else: 116 assert_greater_than_or_equal(i + 1, e["blocks"]) 117 118 119 def check_estimates(node, fees_seen): 120 check_raw_estimates(node, fees_seen) 121 check_smart_estimates(node, fees_seen) 122 123 124 def make_tx(wallet, utxo, feerate): 125 """Create a 1in-1out transaction with a specific input and feerate (sat/vb).""" 126 return wallet.create_self_transfer( 127 utxo_to_spend=utxo, 128 fee_rate=Decimal(feerate * 1000) / COIN, 129 ) 130 131 132 class EstimateFeeTest(BitcoinTestFramework): 133 def set_test_params(self): 134 self.num_nodes = 3 135 # whitelist peers to speed up tx relay / mempool sync 136 self.noban_tx_relay = True 137 self.extra_args = [ 138 [], 139 ["-blockmaxweight=68000"], 140 ["-blockmaxweight=32000"], 141 ] 142 143 def setup_network(self): 144 """ 145 We'll setup the network to have 3 nodes that all mine with different parameters. 146 But first we need to use one node to create a lot of outputs 147 which we will use to generate our transactions. 148 """ 149 self.add_nodes(3, extra_args=self.extra_args) 150 # Use node0 to mine blocks for input splitting 151 # Node1 mines small blocks but that are bigger than the expected transaction rate. 152 # NOTE: the CreateNewBlock code starts counting block weight at 4,000 weight, 153 # (68k weight is room enough for 120 or so transactions) 154 # Node2 is a stingy miner, that 155 # produces too small blocks (room for only 55 or so transactions) 156 157 def transact_and_mine(self, numblocks, mining_node): 158 min_fee = Decimal("0.00001") 159 # We will now mine numblocks blocks generating on average 100 transactions between each block 160 # We shuffle our confirmed txout set before each set of transactions 161 # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible 162 # resorting to tx's that depend on the mempool when those run out 163 for _ in range(numblocks): 164 random.shuffle(self.confutxo) 165 batch_sendtx_reqs = [] 166 for _ in range(random.randrange(100 - 50, 100 + 50)): 167 from_index = random.randint(1, 2) 168 (tx_bytes, fee) = small_txpuzzle_randfee( 169 self.wallet, 170 self.nodes[from_index], 171 self.confutxo, 172 self.memutxo, 173 Decimal("0.005"), 174 min_fee, 175 min_fee, 176 batch_sendtx_reqs, 177 ) 178 tx_kbytes = tx_bytes / 1000.0 179 self.fees_per_kb.append(float(fee) / tx_kbytes) 180 for node in self.nodes: 181 node.batch(batch_sendtx_reqs) 182 self.sync_mempools(wait=0.1) 183 mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] 184 # update which txouts are confirmed 185 newmem = [] 186 for utx in self.memutxo: 187 if utx["txid"] in mined: 188 self.confutxo.append(utx) 189 else: 190 newmem.append(utx) 191 self.memutxo = newmem 192 193 def initial_split(self, node): 194 """Split two coinbase UTxOs into many small coins""" 195 self.confutxo = self.wallet.send_self_transfer_multi( 196 from_node=node, 197 utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], 198 num_outputs=2048)['new_utxos'] 199 while len(node.getrawmempool()) > 0: 200 self.generate(node, 1, sync_fun=self.no_op) 201 202 def sanity_check_estimates_range(self): 203 """Populate estimation buckets, assert estimates are in a sane range and 204 are strictly increasing as the target decreases.""" 205 self.fees_per_kb = [] 206 self.memutxo = [] 207 self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") 208 209 for _ in range(2): 210 self.log.info( 211 "Creating transactions and mining them with a block size that can't keep up" 212 ) 213 # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine 214 self.transact_and_mine(10, self.nodes[2]) 215 check_estimates(self.nodes[1], self.fees_per_kb) 216 217 self.log.info( 218 "Creating transactions and mining them at a block size that is just big enough" 219 ) 220 # Generate transactions while mining 10 more blocks, this time with node1 221 # which mines blocks with capacity just above the rate that transactions are being created 222 self.transact_and_mine(10, self.nodes[1]) 223 check_estimates(self.nodes[1], self.fees_per_kb) 224 225 # Finish by mining a normal-sized block: 226 while len(self.nodes[1].getrawmempool()) > 0: 227 self.generate(self.nodes[1], 1) 228 229 self.log.info("Final estimates after emptying mempools") 230 check_estimates(self.nodes[1], self.fees_per_kb) 231 232 def test_feerate_mempoolminfee(self): 233 high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"] 234 self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"]) 235 check_estimates(self.nodes[1], self.fees_per_kb) 236 self.restart_node(1) 237 238 def sanity_check_rbf_estimates(self, utxos): 239 """During 5 blocks, broadcast low fee transactions. Only 10% of them get 240 confirmed and the remaining ones get RBF'd with a high fee transaction at 241 the next block. 242 The block policy estimator should return the high feerate. 243 """ 244 # The broadcaster and block producer 245 node = self.nodes[0] 246 miner = self.nodes[1] 247 # In sat/vb 248 low_feerate = 1 249 high_feerate = 10 250 # Cache the utxos of which to replace the spender after it failed to get 251 # confirmed 252 utxos_to_respend = [] 253 txids_to_replace = [] 254 255 assert_greater_than_or_equal(len(utxos), 250) 256 for _ in range(5): 257 # Broadcast 45 low fee transactions that will need to be RBF'd 258 txs = [] 259 for _ in range(45): 260 u = utxos.pop(0) 261 tx = make_tx(self.wallet, u, low_feerate) 262 utxos_to_respend.append(u) 263 txids_to_replace.append(tx["txid"]) 264 txs.append(tx) 265 # Broadcast 5 low fee transaction which don't need to 266 for _ in range(5): 267 tx = make_tx(self.wallet, utxos.pop(0), low_feerate) 268 txs.append(tx) 269 batch_send_tx = [node.sendrawtransaction.get_request(tx["hex"]) for tx in txs] 270 for n in self.nodes: 271 n.batch(batch_send_tx) 272 # Mine the transactions on another node 273 self.sync_mempools(wait=0.1, nodes=[node, miner]) 274 for txid in txids_to_replace: 275 miner.prioritisetransaction(txid=txid, fee_delta=-COIN) 276 self.generate(miner, 1) 277 # RBF the low-fee transactions 278 while len(utxos_to_respend) > 0: 279 u = utxos_to_respend.pop(0) 280 tx = make_tx(self.wallet, u, high_feerate) 281 node.sendrawtransaction(tx["hex"]) 282 txs.append(tx) 283 dec_txs = [res["result"] for res in node.batch([node.decoderawtransaction.get_request(tx["hex"]) for tx in txs])] 284 self.wallet.scan_txs(dec_txs) 285 286 287 # Mine the last replacement txs 288 self.sync_mempools(wait=0.1, nodes=[node, miner]) 289 self.generate(miner, 1) 290 291 # Only 10% of the transactions were really confirmed with a low feerate, 292 # the rest needed to be RBF'd. We must return the 90% conf rate feerate. 293 high_feerate_kvb = Decimal(high_feerate) / COIN * 10 ** 3 294 est_feerate = node.estimatesmartfee(2)["feerate"] 295 assert_equal(est_feerate, high_feerate_kvb) 296 297 def test_old_fee_estimate_file(self): 298 # Get the initial fee rate while node is running 299 fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] 300 301 # Restart node to ensure fee_estimate.dat file is read 302 self.restart_node(0) 303 assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) 304 305 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 306 307 # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE 308 self.stop_node(0) 309 last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR 310 os.utime(fee_dat, (last_modified_time, last_modified_time)) 311 312 # Start node and ensure the fee_estimates.dat file was not read 313 self.start_node(0) 314 assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) 315 316 317 def test_estimate_dat_is_flushed_periodically(self): 318 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 319 os.remove(fee_dat) if os.path.exists(fee_dat) else None 320 321 # Verify that fee_estimates.dat does not exist 322 assert_equal(os.path.isfile(fee_dat), False) 323 324 # Verify if the string "Flushed fee estimates to fee_estimates.dat." is present in the debug log file. 325 # If present, it indicates that fee estimates have been successfully flushed to disk. 326 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 327 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 328 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 329 330 # Verify that fee estimates were flushed and fee_estimates.dat file is created 331 assert_equal(os.path.isfile(fee_dat), True) 332 333 # Verify that the estimates remain the same if there are no blocks in the flush interval 334 block_hash_before = self.nodes[0].getbestblockhash() 335 fee_dat_initial_content = open(fee_dat, "rb").read() 336 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 337 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 338 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 339 340 # Verify that there were no blocks in between the flush interval 341 assert_equal(block_hash_before, self.nodes[0].getbestblockhash()) 342 343 fee_dat_current_content = open(fee_dat, "rb").read() 344 assert_equal(fee_dat_current_content, fee_dat_initial_content) 345 346 # Verify that the estimates remain the same after shutdown with no blocks before shutdown 347 self.restart_node(0) 348 fee_dat_current_content = open(fee_dat, "rb").read() 349 assert_equal(fee_dat_current_content, fee_dat_initial_content) 350 351 # Verify that the estimates are not the same if new blocks were produced in the flush interval 352 with self.nodes[0].assert_debug_log(expected_msgs=["Flushed fee estimates to fee_estimates.dat."], timeout=1): 353 # Mock the scheduler for an hour to flush fee estimates to fee_estimates.dat 354 self.generate(self.nodes[0], 5, sync_fun=self.no_op) 355 self.nodes[0].mockscheduler(SECONDS_PER_HOUR) 356 357 fee_dat_current_content = open(fee_dat, "rb").read() 358 assert fee_dat_current_content != fee_dat_initial_content 359 360 fee_dat_initial_content = fee_dat_current_content 361 362 # Generate blocks before shutdown and verify that the fee estimates are not the same 363 self.generate(self.nodes[0], 5, sync_fun=self.no_op) 364 self.restart_node(0) 365 fee_dat_current_content = open(fee_dat, "rb").read() 366 assert fee_dat_current_content != fee_dat_initial_content 367 368 369 def test_acceptstalefeeestimates_option(self): 370 # Get the initial fee rate while node is running 371 fee_rate = self.nodes[0].estimatesmartfee(1)["feerate"] 372 373 self.stop_node(0) 374 375 fee_dat = self.nodes[0].chain_path / "fee_estimates.dat" 376 377 # Stop the node and backdate the fee_estimates.dat file more than MAX_FILE_AGE 378 last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR 379 os.utime(fee_dat, (last_modified_time, last_modified_time)) 380 381 # Restart node with -acceptstalefeeestimates option to ensure fee_estimate.dat file is read 382 self.start_node(0,extra_args=["-acceptstalefeeestimates"]) 383 assert_equal(self.nodes[0].estimatesmartfee(1)["feerate"], fee_rate) 384 385 386 def run_test(self): 387 self.log.info("This test is time consuming, please be patient") 388 self.log.info("Splitting inputs so we can generate tx's") 389 390 # Split two coinbases into many small utxos 391 self.start_node(0) 392 self.wallet = MiniWallet(self.nodes[0]) 393 self.initial_split(self.nodes[0]) 394 self.log.info("Finished splitting") 395 396 # Now we can connect the other nodes, didn't want to connect them earlier 397 # so the estimates would not be affected by the splitting transactions 398 self.start_node(1) 399 self.start_node(2) 400 self.connect_nodes(1, 0) 401 self.connect_nodes(0, 2) 402 self.connect_nodes(2, 1) 403 self.sync_all() 404 405 self.log.info("Testing estimates with single transactions.") 406 self.sanity_check_estimates_range() 407 408 self.log.info("Test fee_estimates.dat is flushed periodically") 409 self.test_estimate_dat_is_flushed_periodically() 410 411 # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee 412 self.log.info( 413 "Test fee rate estimation after restarting node with high MempoolMinFee" 414 ) 415 self.test_feerate_mempoolminfee() 416 417 self.log.info("Test acceptstalefeeestimates option") 418 self.test_acceptstalefeeestimates_option() 419 420 self.log.info("Test reading old fee_estimates.dat") 421 self.test_old_fee_estimate_file() 422 423 self.log.info("Restarting node with fresh estimation") 424 self.stop_node(0) 425 fee_dat = os.path.join(self.nodes[0].chain_path, "fee_estimates.dat") 426 os.remove(fee_dat) 427 self.start_node(0) 428 self.connect_nodes(0, 1) 429 self.connect_nodes(0, 2) 430 431 self.log.info("Testing estimates with RBF.") 432 self.sanity_check_rbf_estimates(self.confutxo + self.memutxo) 433 434 self.log.info("Testing that fee estimation is disabled in blocksonly.") 435 self.restart_node(0, ["-blocksonly"]) 436 assert_raises_rpc_error( 437 -32603, "Fee estimation disabled", self.nodes[0].estimatesmartfee, 2 438 ) 439 440 441 if __name__ == "__main__": 442 EstimateFeeTest().main()