wallet_backwards_compatibility.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2018-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Backwards compatibility functional test 6 7 Test various backwards compatibility scenarios. Requires previous releases binaries, 8 see test/README.md. 9 10 Due to RPC changes introduced in various versions the below tests 11 won't work for older versions without some patches or workarounds. 12 13 Use only the latest patch version of each release, unless a test specifically 14 needs an older patch version. 15 """ 16 17 import json 18 import os 19 import shutil 20 21 from test_framework.blocktools import COINBASE_MATURITY 22 from test_framework.test_framework import BitcoinTestFramework 23 from test_framework.descriptors import descsum_create 24 from test_framework.messages import ser_string 25 26 from test_framework.util import ( 27 assert_equal, 28 assert_greater_than, 29 assert_raises_rpc_error, 30 ) 31 32 LAST_KEYPOOL_INDEX = 9 # Index of the last derived address with the keypool size of 10 33 34 class BackwardsCompatibilityTest(BitcoinTestFramework): 35 def set_test_params(self): 36 self.setup_clean_chain = True 37 self.num_nodes = 8 38 # Add new version after each release: 39 self.extra_args = [ 40 ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay 41 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc 42 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v25.0 43 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v24.0.1 44 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0 45 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1", f"-keypool={LAST_KEYPOOL_INDEX + 1}"], # v22.0 46 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0 47 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.20.1 48 ] 49 self.wallet_names = [self.default_wallet_name] 50 51 def skip_test_if_missing_module(self): 52 self.skip_if_no_wallet() 53 self.skip_if_no_previous_releases() 54 55 def setup_nodes(self): 56 self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ 57 None, 58 None, 59 250000, 60 240001, 61 230000, 62 220000, 63 210000, 64 200100, 65 ]) 66 67 self.start_nodes() 68 self.import_deterministic_coinbase_privkeys() 69 70 def split_version(self, node): 71 major = node.version // 10000 72 minor = (node.version % 10000) // 100 73 patch = (node.version % 100) 74 return (major, minor, patch) 75 76 def major_version_equals(self, node, major): 77 node_major, _, _ = self.split_version(node) 78 return node_major == major 79 80 def major_version_less_than(self, node, major): 81 node_major, _, _ = self.split_version(node) 82 return node_major < major 83 84 def major_version_at_least(self, node, major): 85 node_major, _, _ = self.split_version(node) 86 return node_major >= major 87 88 def test_v22_inactivehdchain_path(self): 89 self.log.info("Testing inactive hd chain bad derivation path cleanup") 90 # 0.21.x and 22.x would both produce bad derivation paths when topping up an inactive hd chain 91 # Make sure that this is being automatically cleaned up by migration 92 node_master = self.nodes[1] 93 node_v22 = self.nodes[self.num_nodes - 3] 94 wallet_name = "bad_deriv_path" 95 node_v22.createwallet(wallet_name=wallet_name, descriptors=False) 96 bad_deriv_wallet = node_v22.get_wallet_rpc(wallet_name) 97 98 # Make a dump of the wallet to get an unused address 99 dump_path = node_v22.wallets_path / f"{wallet_name}.dump" 100 bad_deriv_wallet.dumpwallet(dump_path) 101 addr = None 102 seed = None 103 with open(dump_path) as f: 104 for line in f: 105 if f"hdkeypath=m/0'/0'/{LAST_KEYPOOL_INDEX}'" in line: 106 addr = line.split(" ")[4].split("=")[1] 107 elif " hdseed=1 " in line: 108 seed = line.split(" ")[0] 109 assert addr is not None 110 assert seed is not None 111 # Rotate seed and unload 112 bad_deriv_wallet.sethdseed() 113 bad_deriv_wallet.unloadwallet() 114 # Receive at addr to trigger inactive chain topup on next load 115 self.nodes[0].sendtoaddress(addr, 1) 116 self.generate(self.nodes[0], 1, sync_fun=self.no_op) 117 self.sync_all(nodes=[self.nodes[0], node_master, node_v22]) 118 node_v22.loadwallet(wallet_name) 119 120 # Dump again to find bad hd keypath 121 bad_deriv_path = f"m/0'/0'/{LAST_KEYPOOL_INDEX}'/0'/0'/{LAST_KEYPOOL_INDEX + 1}'" 122 good_deriv_path = f"m/0h/0h/{LAST_KEYPOOL_INDEX + 1}h" 123 os.unlink(dump_path) 124 bad_deriv_wallet.dumpwallet(dump_path) 125 bad_path_addr = None 126 with open(dump_path) as f: 127 for line in f: 128 if f"hdkeypath={bad_deriv_path}" in line: 129 bad_path_addr = line.split(" ")[4].split("=")[1] 130 assert bad_path_addr is not None 131 assert_equal(bad_deriv_wallet.getaddressinfo(bad_path_addr)["hdkeypath"], bad_deriv_path) 132 133 # Verify that this bad derivation path addr is actually at m/0'/0'/10' by making a new wallet with the same seed but larger keypool 134 node_v22.createwallet(wallet_name="path_verify", descriptors=False, blank=True) 135 verify_wallet = node_v22.get_wallet_rpc("path_verify") 136 verify_wallet.sethdseed(True, seed) 137 # Bad addr is after keypool, so need to generate it by refilling 138 verify_wallet.keypoolrefill(LAST_KEYPOOL_INDEX + 2) 139 assert_equal(verify_wallet.getaddressinfo(bad_path_addr)["hdkeypath"], good_deriv_path.replace("h", "'")) 140 141 # Migrate with master 142 # Since all keymeta records are now deleted after migration, the derivation path 143 # should now be correct as it is derived on-the-fly from the inactive hd chain's descriptor 144 backup_path = node_v22.wallets_path / f"{wallet_name}.bak" 145 bad_deriv_wallet.backupwallet(backup_path) 146 wallet_dir_master = os.path.join(node_master.wallets_path, wallet_name) 147 os.makedirs(wallet_dir_master, exist_ok=True) 148 shutil.copy(backup_path, os.path.join(wallet_dir_master, "wallet.dat")) 149 node_master.migratewallet(wallet_name) 150 bad_deriv_wallet_master = node_master.get_wallet_rpc(wallet_name) 151 assert_equal(bad_deriv_wallet_master.getaddressinfo(bad_path_addr)["hdkeypath"], good_deriv_path) 152 bad_deriv_wallet_master.unloadwallet() 153 154 def check_keymeta(conn): 155 # Retrieve all records that have the "keymeta" prefix. The remaining key data varies for each record. 156 keymeta_rec = conn.execute(f"SELECT value FROM main where key >= x'{ser_string(b'keymeta').hex()}' AND key < x'{ser_string(b'keymetb').hex()}'").fetchone() 157 assert_equal(keymeta_rec, None) 158 159 wallet_db = node_master.wallets_path / wallet_name / "wallet.dat" 160 self.inspect_sqlite_db(wallet_db, check_keymeta) 161 162 def test_ignore_legacy_during_startup(self, legacy_nodes, node_master): 163 self.log.info("Test that legacy wallets are ignored during startup on v29+") 164 165 legacy_node = legacy_nodes[0] 166 wallet_name = f"legacy_up_{legacy_node.version}" 167 legacy_node.loadwallet(wallet_name) 168 legacy_wallet = legacy_node.get_wallet_rpc(wallet_name) 169 170 # Move legacy wallet to latest node 171 wallet_path = node_master.wallets_path / wallet_name 172 wallet_path.mkdir() 173 legacy_wallet.backupwallet(wallet_path / "wallet.dat") 174 legacy_wallet.unloadwallet() 175 176 # Write wallet so it is automatically loaded during init 177 settings_path = node_master.chain_path / "settings.json" 178 with settings_path.open("w") as fp: 179 json.dump({"wallet": [wallet_name]}, fp) 180 181 # Restart latest node and verify that the legacy wallet load is skipped without exiting early during init. 182 self.restart_node(node_master.index, extra_args=[]) 183 # Ensure we receive the warning message and clear the stderr pipe. 184 node_master.stderr.seek(0) 185 warning_msg = node_master.stderr.read().decode('utf-8').strip() 186 assert "The wallet appears to be a Legacy wallet, please use the wallet migration tool (migratewallet RPC or the GUI option)" in warning_msg 187 node_master.stderr.truncate(0), node_master.stderr.seek(0) # reset buffer 188 189 # Verify the node is still running (no shutdown occurred during startup) 190 node_master.getblockcount() 191 # Reset settings for any subsequent test 192 os.remove(settings_path) 193 194 def run_test(self): 195 node_miner = self.nodes[0] 196 node_master = self.nodes[1] 197 node_v21 = self.nodes[self.num_nodes - 2] 198 node_v20 = self.nodes[self.num_nodes - 1] # bdb only 199 200 legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets 201 descriptors_nodes = self.nodes[2:-1] # Nodes that support descriptor wallets 202 203 self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) 204 205 # Sanity check the test framework: 206 assert_equal(node_v20.getblockchaininfo()["blocks"], COINBASE_MATURITY + 1) 207 208 self.log.info("Test wallet backwards compatibility...") 209 # Create a number of wallets and open them in older versions: 210 211 # w1: regular wallet, created on master: update this test when default 212 # wallets can no longer be opened by older versions. 213 node_master.createwallet(wallet_name="w1") 214 wallet = node_master.get_wallet_rpc("w1") 215 info = wallet.getwalletinfo() 216 assert info['private_keys_enabled'] 217 assert info['keypoolsize'] > 0 218 # Create a confirmed transaction, receiving coins 219 address = wallet.getnewaddress() 220 node_miner.sendtoaddress(address, 10) 221 self.sync_mempools() 222 self.generate(node_miner, 1) 223 # Create a conflicting transaction using RBF 224 return_address = node_miner.getnewaddress() 225 tx1_id = node_master.sendtoaddress(return_address, 1) 226 tx2_id = node_master.bumpfee(tx1_id)["txid"] 227 # Confirm the transaction 228 self.sync_mempools() 229 self.generate(node_miner, 1) 230 # Create another conflicting transaction using RBF 231 tx3_id = node_master.sendtoaddress(return_address, 1) 232 tx4_id = node_master.bumpfee(tx3_id)["txid"] 233 self.sync_mempools() 234 # Abandon transaction, but don't confirm 235 node_master.abandontransaction(tx3_id) 236 237 # w2: wallet with private keys disabled, created on master: update this 238 # test when default wallets private keys disabled can no longer be 239 # opened by older versions. 240 node_master.createwallet(wallet_name="w2", disable_private_keys=True) 241 wallet = node_master.get_wallet_rpc("w2") 242 info = wallet.getwalletinfo() 243 assert info['private_keys_enabled'] == False 244 assert info['keypoolsize'] == 0 245 246 # w3: blank wallet, created on master: update this 247 # test when default blank wallets can no longer be opened by older versions. 248 node_master.createwallet(wallet_name="w3", blank=True) 249 wallet = node_master.get_wallet_rpc("w3") 250 info = wallet.getwalletinfo() 251 assert info['private_keys_enabled'] 252 assert info['keypoolsize'] == 0 253 254 # Unload wallets and copy to older nodes: 255 node_master_wallets_dir = node_master.wallets_path 256 node_master.unloadwallet("w1") 257 node_master.unloadwallet("w2") 258 node_master.unloadwallet("w3") 259 260 for node in legacy_nodes: 261 # Copy wallets to previous version 262 for wallet in os.listdir(node_master_wallets_dir): 263 dest = node.wallets_path / wallet 264 source = node_master_wallets_dir / wallet 265 shutil.copytree(source, dest) 266 267 self.log.info("Test that a wallet made on master can be opened on:") 268 # This test only works on the nodes that support descriptor wallets 269 # since we can no longer create legacy wallets. 270 for node in descriptors_nodes: 271 self.log.info(f"- {node.version}") 272 for wallet_name in ["w1", "w2", "w3"]: 273 if self.major_version_less_than(node, 22) and wallet_name == "w1": 274 # Descriptor wallets created after 0.21 have taproot descriptors which 0.21 does not support, tested below 275 continue 276 # Also try to reopen on master after opening on old 277 for n in [node, node_master]: 278 n.loadwallet(wallet_name) 279 wallet = n.get_wallet_rpc(wallet_name) 280 info = wallet.getwalletinfo() 281 if wallet_name == "w1": 282 assert info['private_keys_enabled'] == True 283 assert info['keypoolsize'] > 0 284 txs = wallet.listtransactions() 285 assert_equal(len(txs), 5) 286 assert_equal(txs[1]["txid"], tx1_id) 287 assert_equal(txs[2]["walletconflicts"], [tx1_id]) 288 assert_equal(txs[1]["replaced_by_txid"], tx2_id) 289 assert not txs[1]["abandoned"] 290 assert_equal(txs[1]["confirmations"], -1) 291 assert_equal(txs[2]["blockindex"], 1) 292 assert txs[3]["abandoned"] 293 assert_equal(txs[4]["walletconflicts"], [tx3_id]) 294 assert_equal(txs[3]["replaced_by_txid"], tx4_id) 295 assert not hasattr(txs[3], "blockindex") 296 elif wallet_name == "w2": 297 assert info['private_keys_enabled'] == False 298 assert info['keypoolsize'] == 0 299 else: 300 assert info['private_keys_enabled'] == True 301 assert info['keypoolsize'] == 0 302 303 # Copy back to master 304 wallet.unloadwallet() 305 if n == node: 306 shutil.rmtree(node_master.wallets_path / wallet_name) 307 shutil.copytree( 308 n.wallets_path / wallet_name, 309 node_master.wallets_path / wallet_name, 310 ) 311 312 # Check that descriptor wallets don't work on legacy only nodes 313 self.log.info("Test descriptor wallet incompatibility on v0.20") 314 # Descriptor wallets appear to be corrupted wallets to old software 315 assert self.major_version_equals(node_v20, 20) 316 for wallet_name in ["w1", "w2", "w3"]: 317 assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v20.loadwallet, wallet_name) 318 319 # w1 cannot be opened by 0.21 since it contains a taproot descriptor 320 self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors") 321 assert_raises_rpc_error(-1, "map::at", node_v21.loadwallet, "w1") 322 323 self.log.info("Test that a wallet can upgrade to and downgrade from master, from:") 324 for node in descriptors_nodes: 325 self.log.info(f"- {node.version}") 326 wallet_name = f"up_{node.version}" 327 node.createwallet(wallet_name=wallet_name, descriptors=True) 328 wallet_prev = node.get_wallet_rpc(wallet_name) 329 address = wallet_prev.getnewaddress('', "bech32") 330 addr_info = wallet_prev.getaddressinfo(address) 331 332 hdkeypath = addr_info["hdkeypath"].replace("'", "h") 333 pubkey = addr_info["pubkey"] 334 335 # Make a backup of the wallet file 336 backup_path = os.path.join(self.options.tmpdir, f"{wallet_name}.dat") 337 wallet_prev.backupwallet(backup_path) 338 339 # Remove the wallet from old node 340 wallet_prev.unloadwallet() 341 342 # Open backup with sqlite and get flags 343 def get_flags(conn): 344 flags_rec = conn.execute(f"SELECT value FROM main WHERE key = x'{ser_string(b'flags').hex()}'").fetchone() 345 return int.from_bytes(flags_rec[0], byteorder="little") 346 347 old_flags = self.inspect_sqlite_db(backup_path, get_flags) 348 349 # Restore the wallet to master 350 load_res = node_master.restorewallet(wallet_name, backup_path) 351 352 # There should be no warnings 353 assert "warnings" not in load_res 354 355 wallet = node_master.get_wallet_rpc(wallet_name) 356 info = wallet.getaddressinfo(address) 357 descriptor = f"wpkh([{info['hdmasterfingerprint']}{hdkeypath[1:]}]{pubkey})" 358 assert_equal(info["desc"], descsum_create(descriptor)) 359 360 # Make backup so the wallet can be copied back to old node 361 down_wallet_name = f"re_down_{node.version}" 362 down_backup_path = os.path.join(self.options.tmpdir, f"{down_wallet_name}.dat") 363 wallet.backupwallet(down_backup_path) 364 365 # Check that taproot descriptors can be added to 0.21 wallets 366 # This must be done after the backup is created so that 0.21 can still load 367 # the backup 368 if self.major_version_equals(node, 21): 369 assert_raises_rpc_error(-12, "No bech32m addresses available", wallet.getnewaddress, address_type="bech32m") 370 xpubs = wallet.gethdkeys(active_only=True) 371 assert_equal(len(xpubs), 1) 372 assert_equal(len(xpubs[0]["descriptors"]), 6) 373 wallet.createwalletdescriptor("bech32m") 374 xpubs = wallet.gethdkeys(active_only=True) 375 assert_equal(len(xpubs), 1) 376 assert_equal(len(xpubs[0]["descriptors"]), 8) 377 tr_descs = [desc["desc"] for desc in xpubs[0]["descriptors"] if desc["desc"].startswith("tr(")] 378 assert_equal(len(tr_descs), 2) 379 for desc in tr_descs: 380 assert info["hdmasterfingerprint"] in desc 381 wallet.getnewaddress(address_type="bech32m") 382 383 wallet.unloadwallet() 384 385 # Open the wallet with sqlite and inspect the flags and records 386 def check_upgraded_records(conn, old_flags): 387 flags_rec = conn.execute(f"SELECT value FROM main WHERE key = x'{ser_string(b'flags').hex()}'").fetchone() 388 new_flags = int.from_bytes(flags_rec[0], byteorder="little") 389 diff_flags = new_flags & ~old_flags 390 391 # Check for last hardened xpubs if the flag is newly set 392 if diff_flags & (1 << 2): 393 self.log.debug("Checking descriptor cache was upgraded") 394 # Fetch all records with the walletdescriptorlhcache prefix 395 lh_cache_recs = conn.execute(f"SELECT value FROM main where key >= x'{ser_string(b'walletdescriptorlhcache').hex()}' AND key < x'{ser_string(b'walletdescriptorlhcachf').hex()}'").fetchall() 396 assert_greater_than(len(lh_cache_recs), 0) 397 398 self.inspect_sqlite_db(down_backup_path, check_upgraded_records, old_flags) 399 400 # Check that no automatic upgrade broke downgrading the wallet 401 target_dir = node.wallets_path / down_wallet_name 402 os.makedirs(target_dir, exist_ok=True) 403 shutil.copyfile( 404 down_backup_path, 405 target_dir / "wallet.dat" 406 ) 407 node.loadwallet(down_wallet_name) 408 wallet_res = node.get_wallet_rpc(down_wallet_name) 409 info = wallet_res.getaddressinfo(address) 410 assert_equal(info, addr_info) 411 412 self.log.info("Test that a wallet from a legacy only node must be migrated, from:") 413 for node in legacy_nodes: 414 self.log.info(f"- {node.version}") 415 wallet_name = f"legacy_up_{node.version}" 416 if self.major_version_at_least(node, 21): 417 node.createwallet(wallet_name=wallet_name, descriptors=False) 418 else: 419 node.createwallet(wallet_name=wallet_name) 420 wallet_prev = node.get_wallet_rpc(wallet_name) 421 address = wallet_prev.getnewaddress('', "bech32") 422 addr_info = wallet_prev.getaddressinfo(address) 423 424 # Make a backup of the wallet file 425 backup_path = os.path.join(self.options.tmpdir, f"{wallet_name}.dat") 426 wallet_prev.backupwallet(backup_path) 427 428 # Remove the wallet from old node 429 wallet_prev.unloadwallet() 430 431 # Restore the wallet to master 432 # Legacy wallets are no longer supported. Trying to load these should result in an error 433 assert_raises_rpc_error(-18, "The wallet appears to be a Legacy wallet, please use the wallet migration tool (migratewallet RPC or the GUI option)", node_master.restorewallet, wallet_name, backup_path) 434 435 self.test_v22_inactivehdchain_path() 436 self.test_ignore_legacy_during_startup(legacy_nodes, node_master) 437 438 if __name__ == '__main__': 439 BackwardsCompatibilityTest(__file__).main()