test_runner.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2014-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Run regression test suite. 6 7 This module calls down into individual test cases via subprocess. It will 8 forward all unrecognized arguments onto the individual test scripts. 9 10 For a description of arguments recognized by test scripts, see 11 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. 12 13 """ 14 15 import argparse 16 from collections import deque 17 from concurrent import futures 18 import configparser 19 import csv 20 import datetime 21 import os 22 import pathlib 23 import platform 24 import time 25 import shutil 26 import signal 27 import subprocess 28 import sys 29 import tempfile 30 import re 31 import logging 32 from test_framework.util import ( 33 Binaries, 34 export_env_build_path, 35 get_binary_paths, 36 ) 37 38 # Minimum amount of space to run the tests. 39 MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024 40 # Additional space to run an extra job. 41 ADDITIONAL_SPACE_PER_JOB = 100 * 1024 * 1024 42 # Minimum amount of space required for --nocleanup 43 MIN_NO_CLEANUP_SPACE = 12 * 1024 * 1024 * 1024 44 45 # Formatting. Default colors to empty strings. 46 DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "") 47 try: 48 # Make sure python thinks it can write unicode to its stdout 49 "\u2713".encode("utf_8").decode(sys.stdout.encoding) 50 TICK = "✓ " 51 CROSS = "✖ " 52 CIRCLE = "○ " 53 except UnicodeDecodeError: 54 TICK = "P " 55 CROSS = "x " 56 CIRCLE = "o " 57 58 if platform.system() == 'Windows': 59 import ctypes 60 kernel32 = ctypes.windll.kernel32 # type: ignore 61 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 62 STD_OUTPUT_HANDLE = -11 63 STD_ERROR_HANDLE = -12 64 # Enable ascii color control to stdout 65 stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) 66 stdout_mode = ctypes.c_int32() 67 kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode)) 68 kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) 69 # Enable ascii color control to stderr 70 stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE) 71 stderr_mode = ctypes.c_int32() 72 kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode)) 73 kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) 74 else: 75 # primitive formatting on supported 76 # terminal via ANSI escape sequences: 77 DEFAULT = ('\033[0m', '\033[0m') 78 BOLD = ('\033[0m', '\033[1m') 79 GREEN = ('\033[0m', '\033[0;32m') 80 RED = ('\033[0m', '\033[0;31m') 81 82 TEST_EXIT_PASSED = 0 83 TEST_EXIT_SKIPPED = 77 84 85 TEST_FRAMEWORK_UNIT_TESTS = 'feature_framework_unit_tests.py' 86 87 EXTENDED_SCRIPTS = [ 88 # These tests are not run by default. 89 # Longest test should go first, to favor running tests in parallel 90 'feature_pruning.py', 91 'feature_dbcrash.py', 92 'feature_index_prune.py', 93 ] 94 95 # Special script to run each bench sanity check 96 TOOL_BENCH_SANITY_CHECK = "tool_bench_sanity_check.py" 97 98 BASE_SCRIPTS = [ 99 # Special scripts that are "expanded" later 100 TOOL_BENCH_SANITY_CHECK, 101 # Scripts that are run by default. 102 # Longest test should go first, to favor running tests in parallel 103 # vv Tests less than 5m vv 104 'feature_fee_estimation.py', 105 'feature_taproot.py', 106 'feature_block.py', 107 'mempool_ephemeral_dust.py', 108 'wallet_conflicts.py', 109 'p2p_opportunistic_1p1c.py', 110 'p2p_node_network_limited.py --v1transport', 111 'p2p_node_network_limited.py --v2transport', 112 # vv Tests less than 2m vv 113 'mining_getblocktemplate_longpoll.py', 114 'p2p_segwit.py', 115 'feature_maxuploadtarget.py', 116 'feature_assumeutxo.py', 117 'mempool_updatefromblock.py', 118 'mempool_persist.py', 119 # vv Tests less than 60s vv 120 'rpc_psbt.py', 121 'wallet_fundrawtransaction.py', 122 'wallet_bumpfee.py', 123 'wallet_v3_txs.py', 124 'wallet_backup.py', 125 'feature_segwit.py --v2transport', 126 'feature_segwit.py --v1transport', 127 'p2p_tx_download.py', 128 'wallet_avoidreuse.py', 129 'feature_abortnode.py', 130 'wallet_address_types.py', 131 'p2p_orphan_handling.py', 132 'wallet_basic.py', 133 'feature_maxtipage.py', 134 'wallet_multiwallet.py', 135 'wallet_multiwallet.py --usecli', 136 'p2p_dns_seeds.py', 137 'wallet_groups.py', 138 'p2p_blockfilters.py', 139 'feature_assumevalid.py', 140 'wallet_taproot.py', 141 'feature_bip68_sequence.py', 142 'rpc_packages.py', 143 'rpc_bind.py --ipv4', 144 'rpc_bind.py --ipv6', 145 'rpc_bind.py --nonloopback', 146 'p2p_headers_sync_with_minchainwork.py', 147 'p2p_feefilter.py', 148 'feature_csv_activation.py', 149 'p2p_sendheaders.py', 150 'feature_config_args.py', 151 'wallet_listtransactions.py', 152 'wallet_miniscript.py', 153 # vv Tests less than 30s vv 154 'p2p_invalid_messages.py', 155 'rpc_createmultisig.py', 156 'p2p_timeouts.py --v1transport', 157 'p2p_timeouts.py --v2transport', 158 'rpc_signer.py', 159 'wallet_signer.py', 160 'mempool_limit.py', 161 'rpc_txoutproof.py', 162 'rpc_orphans.py', 163 'wallet_listreceivedby.py', 164 'wallet_abandonconflict.py', 165 'wallet_anchor.py', 166 'feature_reindex.py', 167 'feature_reindex_readonly.py', 168 'wallet_labels.py', 169 'p2p_compactblocks.py', 170 'p2p_compactblocks_blocksonly.py', 171 'wallet_hd.py', 172 'wallet_blank.py', 173 'wallet_keypool_topup.py', 174 'wallet_fast_rescan.py', 175 'wallet_gethdkeys.py', 176 'wallet_createwalletdescriptor.py', 177 'interface_zmq.py', 178 'rpc_invalid_address_message.py', 179 'rpc_validateaddress.py', 180 'interface_bitcoin_cli.py', 181 'feature_bind_extra.py', 182 'mempool_resurrect.py', 183 'wallet_txn_doublespend.py --mineblock', 184 'tool_bitcoin_chainstate.py', 185 'tool_wallet.py', 186 'tool_utils.py', 187 'tool_signet_miner.py', 188 'wallet_txn_clone.py', 189 'wallet_txn_clone.py --segwit', 190 'rpc_getchaintips.py', 191 'rpc_misc.py', 192 'p2p_1p1c_network.py', 193 'interface_rest.py', 194 'mempool_spend_coinbase.py', 195 'wallet_avoid_mixing_output_types.py', 196 'mempool_reorg.py', 197 'p2p_block_sync.py --v1transport', 198 'p2p_block_sync.py --v2transport', 199 'wallet_createwallet.py --usecli', 200 'wallet_createwallet.py', 201 'wallet_reindex.py', 202 'wallet_reorgsrestore.py', 203 'interface_http.py', 204 'interface_rpc.py', 205 'interface_usdt_coinselection.py', 206 'interface_usdt_mempool.py', 207 'interface_usdt_net.py', 208 'interface_usdt_utxocache.py', 209 'interface_usdt_validation.py', 210 'rpc_users.py', 211 'rpc_whitelist.py', 212 'feature_proxy.py', 213 'wallet_signrawtransactionwithwallet.py', 214 'rpc_signrawtransactionwithkey.py', 215 'rpc_rawtransaction.py', 216 'wallet_transactiontime_rescan.py', 217 'p2p_addrv2_relay.py', 218 'p2p_compactblocks_hb.py --v1transport', 219 'p2p_compactblocks_hb.py --v2transport', 220 'p2p_disconnect_ban.py --v1transport', 221 'p2p_disconnect_ban.py --v2transport', 222 'feature_posix_fs_permissions.py', 223 'rpc_decodescript.py', 224 'rpc_blockchain.py --v1transport', 225 'rpc_blockchain.py --v2transport', 226 'mining_template_verification.py', 227 'rpc_deprecated.py', 228 'wallet_disable.py', 229 'wallet_change_address.py', 230 'p2p_addr_relay.py', 231 'p2p_getaddr_caching.py', 232 'p2p_getdata.py', 233 'p2p_addrfetch.py', 234 'rpc_net.py --v1transport', 235 'rpc_net.py --v2transport', 236 'wallet_keypool.py', 237 'wallet_descriptor.py', 238 'p2p_nobloomfilter_messages.py', 239 TEST_FRAMEWORK_UNIT_TESTS, 240 'p2p_filter.py', 241 'rpc_setban.py --v1transport', 242 'rpc_setban.py --v2transport', 243 'p2p_blocksonly.py', 244 'mining_prioritisetransaction.py', 245 'p2p_invalid_locator.py', 246 'p2p_invalid_block.py --v1transport', 247 'p2p_invalid_block.py --v2transport', 248 'p2p_invalid_tx.py --v1transport', 249 'p2p_invalid_tx.py --v2transport', 250 'p2p_v2_transport.py', 251 'p2p_v2_encrypted.py', 252 'p2p_v2_misbehaving.py', 253 'example_test.py', 254 'mempool_truc.py', 255 'wallet_multisig_descriptor_psbt.py', 256 'wallet_miniscript_decaying_multisig_descriptor_psbt.py', 257 'wallet_txn_doublespend.py', 258 'wallet_backwards_compatibility.py', 259 'wallet_txn_clone.py --mineblock', 260 'feature_notifications.py', 261 'rpc_getblockfilter.py', 262 'rpc_getblockfrompeer.py', 263 'rpc_invalidateblock.py', 264 'feature_utxo_set_hash.py', 265 'feature_rbf.py', 266 'mempool_packages.py', 267 'mempool_package_limits.py', 268 'mempool_package_rbf.py', 269 'tool_utxo_to_sqlite.py', 270 'feature_versionbits_warning.py', 271 'feature_blocksxor.py', 272 'rpc_preciousblock.py', 273 'wallet_importprunedfunds.py', 274 'p2p_leak_tx.py --v1transport', 275 'p2p_leak_tx.py --v2transport', 276 'p2p_eviction.py', 277 'p2p_outbound_eviction.py', 278 'p2p_ibd_stalling.py --v1transport', 279 'p2p_ibd_stalling.py --v2transport', 280 'p2p_net_deadlock.py --v1transport', 281 'p2p_net_deadlock.py --v2transport', 282 'wallet_signmessagewithaddress.py', 283 'rpc_signmessagewithprivkey.py', 284 'rpc_generate.py', 285 'wallet_balance.py', 286 'p2p_initial_headers_sync.py', 287 'feature_nulldummy.py', 288 'mempool_accept.py', 289 'p2p_addr_selfannouncement.py', 290 'mempool_expiry.py', 291 'wallet_importdescriptors.py', 292 'wallet_crosschain.py', 293 'mining_basic.py', 294 'mining_mainnet.py', 295 'feature_signet.py', 296 'p2p_mutated_blocks.py', 297 'rpc_named_arguments.py', 298 'feature_startupnotify.py', 299 'wallet_simulaterawtx.py', 300 'wallet_listsinceblock.py', 301 'wallet_listdescriptors.py', 302 'p2p_leak.py', 303 'wallet_encryption.py', 304 'feature_dersig.py', 305 'feature_reindex_init.py', 306 'feature_cltv.py', 307 'rpc_uptime.py', 308 'feature_discover.py', 309 'wallet_resendwallettransactions.py', 310 'wallet_fallbackfee.py', 311 'rpc_dumptxoutset.py', 312 'feature_minchainwork.py', 313 'rpc_estimatefee.py', 314 'p2p_private_broadcast.py', 315 'rpc_getblockstats.py', 316 'feature_port.py', 317 'feature_bind_port_externalip.py', 318 'wallet_create_tx.py', 319 'wallet_send.py', 320 'wallet_sendall.py', 321 'wallet_sendmany.py', 322 'wallet_spend_unconfirmed.py', 323 'wallet_rescan_unconfirmed.py', 324 'p2p_fingerprint.py', 325 'feature_uacomment.py', 326 'feature_init.py', 327 'wallet_coinbase_category.py', 328 'feature_filelock.py', 329 'feature_loadblock.py', 330 'wallet_assumeutxo.py', 331 'p2p_add_connections.py', 332 'feature_bind_port_discover.py', 333 'p2p_unrequested_blocks.py', 334 'p2p_message_capture.py', 335 'feature_includeconf.py', 336 'feature_addrman.py', 337 'feature_asmap.py', 338 'feature_chain_tiebreaks.py', 339 'feature_fastprune.py', 340 'feature_framework_miniwallet.py', 341 'mempool_unbroadcast.py', 342 'mempool_compatibility.py', 343 'mempool_accept_wtxid.py', 344 'mempool_dust.py', 345 'mempool_sigoplimit.py', 346 'rpc_deriveaddresses.py', 347 'rpc_deriveaddresses.py --usecli', 348 'p2p_ping.py', 349 'p2p_tx_privacy.py', 350 'rpc_getdescriptoractivity.py', 351 'rpc_scanblocks.py', 352 'tool_bitcoin.py', 353 'p2p_sendtxrcncl.py', 354 'rpc_scantxoutset.py', 355 'feature_torcontrol.py', 356 'feature_unsupported_utxo_db.py', 357 'mempool_cluster.py', 358 'feature_logging.py', 359 'interface_ipc.py', 360 'interface_ipc_mining.py', 361 'feature_anchors.py', 362 'mempool_datacarrier.py', 363 'feature_coinstatsindex.py', 364 'feature_coinstatsindex_compatibility.py', 365 'wallet_orphanedreward.py', 366 'wallet_musig.py', 367 'wallet_timelock.py', 368 'p2p_permissions.py', 369 'feature_blocksdir.py', 370 'wallet_startup.py', 371 'feature_remove_pruned_files_on_startup.py', 372 'p2p_i2p_ports.py', 373 'p2p_i2p_sessions.py', 374 'feature_presegwit_node_upgrade.py', 375 'feature_settings.py', 376 'rpc_getdescriptorinfo.py', 377 'rpc_gettxspendingprevout.py', 378 'rpc_help.py', 379 'feature_framework_testshell.py', 380 'tool_rpcauth.py', 381 'p2p_handshake.py', 382 'p2p_handshake.py --v2transport', 383 'interface_ipc_cli.py', 384 'feature_dirsymlinks.py', 385 'feature_help.py', 386 'feature_framework_startup_failures.py', 387 'feature_shutdown.py', 388 'wallet_migration.py', 389 'p2p_ibd_txrelay.py', 390 'p2p_seednode.py', 391 # Don't append tests at the end to avoid merge conflicts 392 # Put them in a random line within the section that fits their approximate run-time 393 ] 394 395 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests 396 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS 397 398 NON_SCRIPTS = [ 399 # These are python files that live in the functional tests directory, but are not test scripts. 400 "combine_logs.py", 401 "create_cache.py", 402 "test_runner.py", 403 ] 404 405 def main(): 406 # Parse arguments and pass through unrecognised args 407 parser = argparse.ArgumentParser(add_help=False, 408 usage='%(prog)s [test_runner.py options] [script options] [scripts]', 409 description=__doc__, 410 epilog=''' 411 Help text and arguments for individual test script:''', 412 formatter_class=argparse.RawTextHelpFormatter) 413 parser.add_argument('--ansi', action='store_true', default=sys.stdout.isatty(), help="Use ANSI colors and dots in output (enabled by default when standard output is a TTY)") 414 parser.add_argument('--combinedlogslen', '-c', type=int, default=0, metavar='n', help='On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes.') 415 parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') 416 parser.add_argument('--exclude', '-x', action='append', help='specify a script to exclude. Can be specified multiple times. The .py extension is optional.') 417 parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') 418 parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') 419 parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') 420 parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs') 421 parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") 422 parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure') 423 parser.add_argument('--filter', help='filter scripts to run by regular expression') 424 parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", 425 help="Leave bitcoinds and test.* datadir on exit or error") 426 parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') 427 428 args, unknown_args = parser.parse_known_args() 429 # Fail on self-check warnings before running the tests. 430 fail_on_warn = True 431 if not args.ansi: 432 global DEFAULT, BOLD, GREEN, RED 433 DEFAULT = ("", "") 434 BOLD = ("", "") 435 GREEN = ("", "") 436 RED = ("", "") 437 438 # args to be passed on always start with two dashes; tests are the remaining unknown args 439 tests = [arg for arg in unknown_args if arg[:2] != "--"] 440 passon_args = [arg for arg in unknown_args if arg[:2] == "--"] 441 442 # Read config generated by configure. 443 config = configparser.ConfigParser() 444 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" 445 config.read_file(open(configfile)) 446 447 passon_args.append("--configfile=%s" % configfile) 448 449 # Set up logging 450 logging_level = logging.INFO if args.quiet else logging.DEBUG 451 logging.basicConfig(format='%(message)s', level=logging_level) 452 453 # Create base test directory 454 tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) 455 456 os.makedirs(tmpdir) 457 458 logging.debug("Temporary test directory at %s" % tmpdir) 459 460 results_filepath = None 461 if args.resultsfile: 462 results_filepath = pathlib.Path(args.resultsfile) 463 # Stop early if the parent directory doesn't exist 464 assert results_filepath.parent.exists(), "Results file parent directory does not exist" 465 logging.debug("Test results will be written to " + str(results_filepath)) 466 467 enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") 468 469 if not enable_bitcoind: 470 print("No functional tests to run.") 471 print("Re-compile with the -DBUILD_DAEMON=ON build option") 472 sys.exit(1) 473 474 export_env_build_path(config) 475 476 # Build tests 477 test_list = deque() 478 if tests: 479 # Individual tests have been specified. Run specified tests that exist 480 # in the ALL_SCRIPTS list. Accept names with or without a .py extension. 481 # Specified tests can contain wildcards, but in that case the supplied 482 # paths should be coherent, e.g. the same path as that provided to call 483 # test_runner.py. Examples: 484 # `test/functional/test_runner.py test/functional/wallet*` 485 # `test/functional/test_runner.py ./test/functional/wallet*` 486 # `test_runner.py wallet*` 487 # but not: 488 # `test/functional/test_runner.py wallet*` 489 # Multiple wildcards can be passed: 490 # `test_runner.py tool* mempool*` 491 for test in tests: 492 script = test.split("/")[-1] 493 script = script + ".py" if ".py" not in script else script 494 matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)] 495 if matching_scripts: 496 test_list += matching_scripts 497 else: 498 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test)) 499 elif args.extended: 500 # Include extended tests 501 test_list += ALL_SCRIPTS 502 else: 503 # Run base tests only 504 test_list += BASE_SCRIPTS 505 506 # Remove the test cases that the user has explicitly asked to exclude. 507 # The user can specify a test case with or without the .py extension. 508 if args.exclude: 509 510 def print_warning_missing_test(test_name): 511 print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude options.".format(BOLD[1], BOLD[0], test_name)) 512 if fail_on_warn: 513 sys.exit(1) 514 515 def remove_tests(exclude_list): 516 if not exclude_list: 517 print_warning_missing_test(exclude_test) 518 for exclude_item in exclude_list: 519 print("Excluding %s" % exclude_item) 520 test_list.remove(exclude_item) 521 522 for exclude_test in args.exclude: 523 if ',' in exclude_test: 524 print("{}WARNING!{} --exclude '{}' contains a comma. Use --exclude for each test.".format(BOLD[1], BOLD[0], exclude_test)) 525 if fail_on_warn: 526 sys.exit(1) 527 528 for exclude_test in args.exclude: 529 # A space in the name indicates it has arguments such as "rpc_bind.py --ipv4" 530 if ' ' in exclude_test: 531 remove_tests([test for test in test_list if test.replace('.py', '') == exclude_test.replace('.py', '')]) 532 else: 533 # Exclude all variants of a test 534 remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]]) 535 536 if config["components"].getboolean("BUILD_BENCH") and TOOL_BENCH_SANITY_CHECK in test_list: 537 # Remove it, and expand it for each bench in the list 538 test_list.remove(TOOL_BENCH_SANITY_CHECK) 539 bench_cmd = Binaries(get_binary_paths(config), bin_dir=None).bench_argv() + ["-list"] 540 bench_list = subprocess.check_output(bench_cmd, text=True).splitlines() 541 bench_list = [f"{TOOL_BENCH_SANITY_CHECK} --bench={b}" for b in bench_list] 542 # Start with special scripts (variable, unknown runtime) 543 test_list.extendleft(reversed(bench_list)) 544 545 if args.filter: 546 test_list = deque(filter(re.compile(args.filter).search, test_list)) 547 548 if not test_list: 549 print("No valid test scripts specified. Check that your test is in one " 550 "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests") 551 sys.exit(1) 552 553 if args.help: 554 # Print help for test_runner.py, then print help of the first script (with args removed) and exit. 555 parser.print_help() 556 subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h']) 557 sys.exit(0) 558 559 # Warn if there is not enough space on tmpdir to run the tests with --nocleanup 560 if args.nocleanup: 561 if shutil.disk_usage(tmpdir).free < MIN_NO_CLEANUP_SPACE: 562 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the functional test suite with --nocleanup. " 563 f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") 564 passon_args.append("--nocleanup") 565 566 check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn) 567 check_script_prefixes() 568 569 run_tests( 570 test_list=test_list, 571 build_dir=config["environment"]["BUILDDIR"], 572 tmpdir=tmpdir, 573 jobs=args.jobs, 574 enable_coverage=args.coverage, 575 args=passon_args, 576 combined_logs_len=args.combinedlogslen, 577 failfast=args.failfast, 578 use_term_control=args.ansi, 579 results_filepath=results_filepath, 580 ) 581 582 def run_tests(*, test_list, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): 583 args = args or [] 584 585 # Some optional Python dependencies (e.g. pycapnp) may emit warnings or fail under 586 # CPython free-threaded builds when the GIL is disabled. Force it on for all 587 # functional tests so every child process inherits PYTHON_GIL=1. 588 os.environ["PYTHON_GIL"] = "1" 589 590 # Warn if bitcoind is already running 591 try: 592 # pgrep exits with code zero when one or more matching processes found 593 if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0: 594 print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) 595 except OSError: 596 # pgrep not supported 597 pass 598 599 # Warn if there is not enough space on the testing dir 600 min_space = MIN_FREE_SPACE + (jobs - 1) * ADDITIONAL_SPACE_PER_JOB 601 if shutil.disk_usage(tmpdir).free < min_space: 602 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Bitcoin functional test suite. " 603 f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.") 604 605 tests_dir = f"{build_dir}/test/functional/" 606 # This allows `test_runner.py` to work from an out-of-source build directory using a symlink, 607 # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561. 608 sys.path.append(tests_dir) 609 610 cache_tmp_dir = tempfile.TemporaryDirectory(prefix="functional_test_cache") 611 flags = [f"--cachedir={cache_tmp_dir.name}"] + args 612 613 if enable_coverage: 614 coverage = RPCCoverage() 615 flags.append(coverage.flag) 616 logging.debug("Initializing coverage directory at %s" % coverage.dir) 617 else: 618 coverage = None 619 620 if len(test_list) > 1 and jobs > 1: 621 # Populate cache 622 try: 623 subprocess.check_output([sys.executable, tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) 624 except subprocess.CalledProcessError as e: 625 sys.stdout.buffer.write(e.output) 626 raise 627 628 # Run Tests 629 job_queue = TestHandler( 630 num_tests_parallel=jobs, 631 tests_dir=tests_dir, 632 tmpdir=tmpdir, 633 test_list=test_list, 634 flags=flags, 635 use_term_control=use_term_control, 636 ) 637 start_time = time.time() 638 test_results = [] 639 640 max_len_name = len(max(test_list, key=len)) 641 test_count = len(test_list) 642 all_passed = True 643 while not job_queue.done(): 644 if failfast and not all_passed: 645 break 646 for test_result, testdir, stdout, stderr, skip_reason in job_queue.get_next(): 647 test_results.append(test_result) 648 done_str = f"{len(test_results)}/{test_count} - {BOLD[1]}{test_result.name}{BOLD[0]}" 649 if test_result.status == "Passed": 650 logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) 651 elif test_result.status == "Skipped": 652 logging.debug(f"{done_str} skipped ({skip_reason})") 653 else: 654 all_passed = False 655 print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) 656 print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') 657 print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') 658 if combined_logs_len and os.path.isdir(testdir): 659 # Print the final `combinedlogslen` lines of the combined logs 660 print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0])) 661 print('\n============') 662 print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) 663 print('============\n') 664 combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir] 665 if BOLD[0]: 666 combined_logs_args += ['--color'] 667 combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate() 668 print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) 669 670 if failfast: 671 logging.debug("Early exiting after test failure") 672 break 673 674 if "[Errno 28] No space left on device" in stdout: 675 sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n" 676 f"Test execution data left in {tmpdir}.\n" 677 f"Additional storage is needed to execute testing.") 678 679 runtime = int(time.time() - start_time) 680 print_results(test_results, max_len_name, runtime) 681 if results_filepath: 682 write_results(test_results, results_filepath, runtime) 683 684 if coverage: 685 coverage_passed = coverage.report_rpc_coverage() 686 else: 687 coverage_passed = True 688 689 # Clear up the temp directory if all subdirectories are gone 690 if not os.listdir(tmpdir): 691 os.rmdir(tmpdir) 692 693 all_passed = all_passed and coverage_passed 694 695 # Clean up dangling processes if any. This may only happen with --failfast option. 696 # Killing the process group will also terminate the current process but that is 697 # not an issue 698 if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs): 699 os.killpg(os.getpgid(0), signal.SIGKILL) 700 701 sys.exit(not all_passed) 702 703 704 def print_results(test_results, max_len_name, runtime): 705 results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] 706 707 test_results.sort(key=TestResult.sort_key) 708 all_passed = True 709 time_sum = 0 710 711 for test_result in test_results: 712 all_passed = all_passed and test_result.was_successful 713 time_sum += test_result.time 714 test_result.padding = max_len_name 715 results += str(test_result) 716 717 status = TICK + "Passed" if all_passed else CROSS + "Failed" 718 if not all_passed: 719 results += RED[1] 720 results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] 721 if not all_passed: 722 results += RED[0] 723 results += "Runtime: %s s\n" % (runtime) 724 print(results) 725 726 727 def write_results(test_results, filepath, total_runtime): 728 with open(filepath, mode="w") as results_file: 729 results_writer = csv.writer(results_file) 730 results_writer.writerow(['test', 'status', 'duration(seconds)']) 731 all_passed = True 732 for test_result in test_results: 733 all_passed = all_passed and test_result.was_successful 734 results_writer.writerow([test_result.name, test_result.status, str(test_result.time)]) 735 results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)]) 736 737 class TestHandler: 738 """ 739 Trigger the test scripts passed in via the list. 740 """ 741 def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, use_term_control): 742 assert num_tests_parallel >= 1 743 self.executor = futures.ThreadPoolExecutor(max_workers=num_tests_parallel) 744 self.num_jobs = num_tests_parallel 745 self.tests_dir = tests_dir 746 self.tmpdir = tmpdir 747 self.test_list = test_list 748 self.flags = flags 749 self.jobs = {} 750 self.use_term_control = use_term_control 751 752 def done(self): 753 return not (self.jobs or self.test_list) 754 755 def get_next(self): 756 while len(self.jobs) < self.num_jobs and self.test_list: 757 # Add tests 758 test = self.test_list.popleft() 759 portseed = len(self.test_list) 760 portseed_arg = ["--portseed={}".format(portseed)] 761 log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) 762 log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) 763 test_argv = test.split() 764 testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed) 765 tmpdir_arg = ["--tmpdir={}".format(testdir)] 766 767 def proc_wait(task): 768 task[2].wait() 769 return task 770 771 task = [ 772 test, 773 time.time(), 774 subprocess.Popen( 775 [sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg, 776 text=True, 777 stdout=log_stdout, 778 stderr=log_stderr, 779 ), 780 testdir, 781 log_stdout, 782 log_stderr, 783 ] 784 fut = self.executor.submit(proc_wait, task) 785 self.jobs[fut] = test 786 assert self.jobs # Must not be empty here 787 788 # Print remaining running jobs when all jobs have been started. 789 if not self.test_list: 790 print("Remaining jobs: [{}]".format(", ".join(sorted(self.jobs.values())))) 791 792 dot_count = 0 793 while True: 794 # Return all procs that have finished, if any. Otherwise sleep until there is one. 795 procs = futures.wait(self.jobs.keys(), timeout=.5, return_when=futures.FIRST_COMPLETED) 796 self.jobs = {fut: self.jobs[fut] for fut in procs.not_done} 797 ret = [] 798 for job in procs.done: 799 (name, start_time, proc, testdir, log_out, log_err) = job.result() 800 801 log_out.seek(0), log_err.seek(0) 802 [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)] 803 log_out.close(), log_err.close() 804 skip_reason = None 805 if proc.returncode == TEST_EXIT_PASSED and stderr == "": 806 status = "Passed" 807 elif proc.returncode == TEST_EXIT_SKIPPED: 808 status = "Skipped" 809 skip_reason = re.search(r"Test Skipped: (.*)", stdout).group(1).strip() 810 else: 811 status = "Failed" 812 813 if self.use_term_control: 814 clearline = '\r' + (' ' * dot_count) + '\r' 815 print(clearline, end='', flush=True) 816 dot_count = 0 817 ret.append((TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr, skip_reason)) 818 if ret: 819 return ret 820 if self.use_term_control: 821 print('.', end='', flush=True) 822 dot_count += 1 823 824 825 class TestResult(): 826 def __init__(self, name, status, time): 827 self.name = name 828 self.status = status 829 self.time = time 830 self.padding = 0 831 832 def sort_key(self): 833 if self.status == "Passed": 834 return 0, self.name.lower() 835 elif self.status == "Failed": 836 return 2, self.name.lower() 837 elif self.status == "Skipped": 838 return 1, self.name.lower() 839 840 def __repr__(self): 841 if self.status == "Passed": 842 color = GREEN 843 glyph = TICK 844 elif self.status == "Failed": 845 color = RED 846 glyph = CROSS 847 elif self.status == "Skipped": 848 color = DEFAULT 849 glyph = CIRCLE 850 851 return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] 852 853 @property 854 def was_successful(self): 855 return self.status != "Failed" 856 857 858 def check_script_prefixes(): 859 """Check that test scripts start with one of the allowed name prefixes.""" 860 861 good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") 862 bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] 863 864 if bad_script_names: 865 print("%sERROR:%s %d tests not meeting naming conventions:" % (BOLD[1], BOLD[0], len(bad_script_names))) 866 print(" %s" % ("\n ".join(sorted(bad_script_names)))) 867 raise AssertionError("Some tests are not following naming convention!") 868 869 870 def check_script_list(*, src_dir, fail_on_warn): 871 """Check scripts directory. 872 873 Check that all python files in this directory are categorized 874 as a test script or meta script.""" 875 script_dir = src_dir + '/test/functional/' 876 python_files = set([test_file for test_file in os.listdir(script_dir) if test_file.endswith(".py")]) 877 missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) 878 if len(missed_tests) != 0: 879 print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) 880 if fail_on_warn: 881 sys.exit(1) 882 883 884 class RPCCoverage(): 885 """ 886 Coverage reporting utilities for test_runner. 887 888 Coverage calculation works by having each test script subprocess write 889 coverage files into a particular directory. These files contain the RPC 890 commands invoked during testing, as well as a complete listing of RPC 891 commands per `bitcoin-cli help` (`rpc_interface.txt`). 892 893 After all tests complete, the commands run are combined and diff'd against 894 the complete list to calculate uncovered RPC commands. 895 896 See also: test/functional/test_framework/coverage.py 897 898 """ 899 def __init__(self): 900 self.temp_dir = tempfile.TemporaryDirectory(prefix="coverage") 901 self.dir = self.temp_dir.name 902 self.flag = '--coveragedir=%s' % self.dir 903 904 def report_rpc_coverage(self): 905 """ 906 Print out RPC commands that were unexercised by tests. 907 908 """ 909 uncovered = self._get_uncovered_rpc_commands() 910 911 if uncovered: 912 print("Uncovered RPC commands:") 913 print("".join((" - %s\n" % command) for command in sorted(uncovered))) 914 return False 915 else: 916 print("All RPC commands covered.") 917 return True 918 919 def _get_uncovered_rpc_commands(self): 920 """ 921 Return a set of currently untested RPC commands. 922 923 """ 924 # This is shared from `test/functional/test_framework/coverage.py` 925 reference_filename = 'rpc_interface.txt' 926 coverage_file_prefix = 'coverage.' 927 928 coverage_ref_filename = os.path.join(self.dir, reference_filename) 929 coverage_filenames = set() 930 all_cmds = set() 931 # Consider RPC generate covered, because it is overloaded in 932 # test_framework/test_node.py and not seen by the coverage check. 933 covered_cmds = set({'generate'}) 934 935 if not os.path.isfile(coverage_ref_filename): 936 raise RuntimeError("No coverage reference found") 937 938 with open(coverage_ref_filename, 'r') as coverage_ref_file: 939 all_cmds.update([line.strip() for line in coverage_ref_file.readlines()]) 940 941 for root, _, files in os.walk(self.dir): 942 for filename in files: 943 if filename.startswith(coverage_file_prefix): 944 coverage_filenames.add(os.path.join(root, filename)) 945 946 for filename in coverage_filenames: 947 with open(filename, 'r') as coverage_file: 948 covered_cmds.update([line.strip() for line in coverage_file.readlines()]) 949 950 return all_cmds - covered_cmds 951 952 953 if __name__ == '__main__': 954 main()