test_runner.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2019-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Run fuzz test targets. 6 """ 7 8 from concurrent.futures import ThreadPoolExecutor, as_completed 9 from pathlib import Path 10 import argparse 11 import configparser 12 import logging 13 import os 14 import random 15 import subprocess 16 import sys 17 18 19 def get_fuzz_env(*, target, source_dir): 20 symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") 21 return { 22 'FUZZ': target, 23 'UBSAN_OPTIONS': 24 f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', 25 'UBSAN_SYMBOLIZER_PATH':symbolizer, 26 "ASAN_OPTIONS": "detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", 27 'ASAN_SYMBOLIZER_PATH':symbolizer, 28 'MSAN_SYMBOLIZER_PATH':symbolizer, 29 } 30 31 32 def main(): 33 parser = argparse.ArgumentParser( 34 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 35 description='''Run the fuzz targets with all inputs from the corpus_dir once.''', 36 ) 37 parser.add_argument( 38 "-l", 39 "--loglevel", 40 dest="loglevel", 41 default="INFO", 42 help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", 43 ) 44 parser.add_argument( 45 '--valgrind', 46 action='store_true', 47 help='If true, run fuzzing binaries under the valgrind memory error detector', 48 ) 49 parser.add_argument( 50 "--empty_min_time", 51 type=int, 52 help="If set, run at least this long, if the existing fuzz inputs directory is empty.", 53 ) 54 parser.add_argument( 55 '-x', 56 '--exclude', 57 help="A comma-separated list of targets to exclude", 58 ) 59 parser.add_argument( 60 '--par', 61 '-j', 62 type=int, 63 default=4, 64 help='How many targets to merge or execute in parallel.', 65 ) 66 parser.add_argument( 67 'corpus_dir', 68 help='The corpus to run on (must contain subfolders for each fuzz target).', 69 ) 70 parser.add_argument( 71 'target', 72 nargs='*', 73 help='The target(s) to run. Default is to run all targets.', 74 ) 75 parser.add_argument( 76 '--m_dir', 77 action="append", 78 help="Merge inputs from these directories into the corpus_dir.", 79 ) 80 parser.add_argument( 81 '-g', 82 '--generate', 83 action='store_true', 84 help='Create new corpus (or extend the existing ones) by running' 85 ' the given targets for a finite number of times. Outputs them to' 86 ' the passed corpus_dir.' 87 ) 88 89 args = parser.parse_args() 90 args.corpus_dir = Path(args.corpus_dir) 91 92 # Set up logging 93 logging.basicConfig( 94 format='%(message)s', 95 level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), 96 ) 97 98 # Read config generated by configure. 99 config = configparser.ConfigParser() 100 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" 101 config.read_file(open(configfile, encoding="utf8")) 102 103 if not config["components"].getboolean("ENABLE_FUZZ_BINARY"): 104 logging.error("Must have fuzz executable built") 105 sys.exit(1) 106 107 # Build list of tests 108 test_list_all = parse_test_list( 109 fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), 110 source_dir=config['environment']['SRCDIR'], 111 ) 112 113 if not test_list_all: 114 logging.error("No fuzz targets found") 115 sys.exit(1) 116 117 logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all)))) 118 119 args.target = args.target or test_list_all # By default run all 120 test_list_error = list(set(args.target).difference(set(test_list_all))) 121 if test_list_error: 122 logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) 123 test_list_selection = list(set(test_list_all).intersection(set(args.target))) 124 if not test_list_selection: 125 logging.error("No fuzz targets selected") 126 if args.exclude: 127 for excluded_target in args.exclude.split(","): 128 if excluded_target not in test_list_selection: 129 logging.error("Target \"{}\" not found in current target list.".format(excluded_target)) 130 continue 131 test_list_selection.remove(excluded_target) 132 test_list_selection.sort() 133 134 logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) 135 136 if not args.generate: 137 test_list_missing_corpus = [] 138 for t in test_list_selection: 139 corpus_path = os.path.join(args.corpus_dir, t) 140 if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: 141 test_list_missing_corpus.append(t) 142 test_list_missing_corpus.sort() 143 if test_list_missing_corpus: 144 logging.info( 145 "Fuzzing harnesses lacking a corpus: {}".format( 146 " ".join(test_list_missing_corpus) 147 ) 148 ) 149 logging.info("Please consider adding a fuzz corpus at https://github.com/bitcoin-core/qa-assets") 150 151 try: 152 help_output = subprocess.run( 153 args=[ 154 os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), 155 '-help=1', 156 ], 157 env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), 158 timeout=20, 159 check=False, 160 stderr=subprocess.PIPE, 161 text=True, 162 ).stderr 163 using_libfuzzer = "libFuzzer" in help_output 164 if (args.generate or args.m_dir) and not using_libfuzzer: 165 logging.error("Must be built with libFuzzer") 166 sys.exit(1) 167 except subprocess.TimeoutExpired: 168 logging.error("subprocess timed out: Currently only libFuzzer is supported") 169 sys.exit(1) 170 171 with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: 172 if args.generate: 173 return generate_corpus( 174 fuzz_pool=fuzz_pool, 175 src_dir=config['environment']['SRCDIR'], 176 build_dir=config["environment"]["BUILDDIR"], 177 corpus_dir=args.corpus_dir, 178 targets=test_list_selection, 179 ) 180 181 if args.m_dir: 182 merge_inputs( 183 fuzz_pool=fuzz_pool, 184 corpus=args.corpus_dir, 185 test_list=test_list_selection, 186 src_dir=config['environment']['SRCDIR'], 187 build_dir=config["environment"]["BUILDDIR"], 188 merge_dirs=[Path(m_dir) for m_dir in args.m_dir], 189 ) 190 return 191 192 run_once( 193 fuzz_pool=fuzz_pool, 194 corpus=args.corpus_dir, 195 test_list=test_list_selection, 196 src_dir=config['environment']['SRCDIR'], 197 build_dir=config["environment"]["BUILDDIR"], 198 using_libfuzzer=using_libfuzzer, 199 use_valgrind=args.valgrind, 200 empty_min_time=args.empty_min_time, 201 ) 202 203 204 def transform_process_message_target(targets, src_dir): 205 """Add a target per process message, and also keep ("process_message", {}) to allow for 206 cross-pollination, or unlimited search""" 207 208 p2p_msg_target = "process_message" 209 if (p2p_msg_target, {}) in targets: 210 lines = subprocess.run( 211 ["git", "grep", "--function-context", "g_all_net_message_types{", src_dir / "src" / "protocol.cpp"], 212 check=True, 213 stdout=subprocess.PIPE, 214 text=True, 215 ).stdout.splitlines() 216 lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.cpp- NetMsgType::")] 217 assert len(lines) 218 targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines] 219 return targets 220 221 222 def transform_rpc_target(targets, src_dir): 223 """Add a target per RPC command, and also keep ("rpc", {}) to allow for cross-pollination, 224 or unlimited search""" 225 226 rpc_target = "rpc" 227 if (rpc_target, {}) in targets: 228 lines = subprocess.run( 229 ["git", "grep", "--function-context", "RPC_COMMANDS_SAFE_FOR_FUZZING{", src_dir / "src" / "test" / "fuzz" / "rpc.cpp"], 230 check=True, 231 stdout=subprocess.PIPE, 232 text=True, 233 ).stdout.splitlines() 234 lines = [l.split("\"", 1)[1].split("\"")[0] for l in lines if l.startswith("src/test/fuzz/rpc.cpp- \"")] 235 assert len(lines) 236 targets += [(rpc_target, {"LIMIT_TO_RPC_COMMAND": r}) for r in lines] 237 return targets 238 239 240 def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): 241 """Generates new corpus. 242 243 Run {targets} without input, and outputs the generated corpus to 244 {corpus_dir}. 245 """ 246 logging.info("Generating corpus to {}".format(corpus_dir)) 247 targets = [(t, {}) for t in targets] # expand to add dictionary for target-specific env variables 248 targets = transform_process_message_target(targets, Path(src_dir)) 249 targets = transform_rpc_target(targets, Path(src_dir)) 250 251 def job(command, t, t_env): 252 logging.debug(f"Running '{command}'") 253 logging.debug("Command '{}' output:\n'{}'\n".format( 254 command, 255 subprocess.run( 256 command, 257 env={ 258 **t_env, 259 **get_fuzz_env(target=t, source_dir=src_dir), 260 }, 261 check=True, 262 stderr=subprocess.PIPE, 263 text=True, 264 ).stderr, 265 )) 266 267 futures = [] 268 for target, t_env in targets: 269 target_corpus_dir = corpus_dir / target 270 os.makedirs(target_corpus_dir, exist_ok=True) 271 use_value_profile = int(random.random() < .3) 272 command = [ 273 os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), 274 "-rss_limit_mb=8000", 275 "-max_total_time=6000", 276 "-reload=0", 277 f"-use_value_profile={use_value_profile}", 278 target_corpus_dir, 279 ] 280 futures.append(fuzz_pool.submit(job, command, target, t_env)) 281 282 for future in as_completed(futures): 283 future.result() 284 285 286 def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dirs): 287 logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}") 288 jobs = [] 289 for t in test_list: 290 args = [ 291 os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), 292 '-rss_limit_mb=8000', 293 '-set_cover_merge=1', 294 # set_cover_merge is used instead of -merge=1 to reduce the overall 295 # size of the qa-assets git repository a bit, but more importantly, 296 # to cut the runtime to iterate over all fuzz inputs [0]. 297 # [0] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1761760866 298 '-shuffle=0', 299 '-prefer_small=1', 300 '-use_value_profile=0', 301 # use_value_profile is enabled by oss-fuzz [0], but disabled for 302 # now to avoid bloating the qa-assets git repository [1]. 303 # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 304 # [1] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1749075891 305 os.path.join(corpus, t), 306 ] + [str(m_dir / t) for m_dir in merge_dirs] 307 os.makedirs(os.path.join(corpus, t), exist_ok=True) 308 for m_dir in merge_dirs: 309 (m_dir / t).mkdir(exist_ok=True) 310 311 def job(t, args): 312 output = 'Run {} with args {}\n'.format(t, " ".join(args)) 313 output += subprocess.run( 314 args, 315 env=get_fuzz_env(target=t, source_dir=src_dir), 316 check=True, 317 stderr=subprocess.PIPE, 318 text=True, 319 ).stderr 320 logging.debug(output) 321 322 jobs.append(fuzz_pool.submit(job, t, args)) 323 324 for future in as_completed(jobs): 325 future.result() 326 327 328 def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, using_libfuzzer, use_valgrind, empty_min_time): 329 jobs = [] 330 for t in test_list: 331 corpus_path = corpus / t 332 os.makedirs(corpus_path, exist_ok=True) 333 args = [ 334 os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), 335 ] 336 empty_dir = not any(corpus_path.iterdir()) 337 if using_libfuzzer: 338 if empty_min_time and empty_dir: 339 args += [f"-max_total_time={empty_min_time}"] 340 else: 341 args += [ 342 "-runs=1", 343 corpus_path, 344 ] 345 else: 346 args += [corpus_path] 347 if use_valgrind: 348 args = ['valgrind', '--quiet', '--error-exitcode=1'] + args 349 350 def job(t, args): 351 output = 'Run {} with args {}'.format(t, args) 352 result = subprocess.run( 353 args, 354 env=get_fuzz_env(target=t, source_dir=src_dir), 355 stderr=subprocess.PIPE, 356 text=True, 357 ) 358 output += result.stderr 359 return output, result, t 360 361 jobs.append(fuzz_pool.submit(job, t, args)) 362 363 stats = [] 364 for future in as_completed(jobs): 365 output, result, target = future.result() 366 logging.debug(output) 367 if using_libfuzzer: 368 done_stat = [l for l in output.splitlines() if "DONE" in l] 369 assert len(done_stat) == 1 370 stats.append((target, done_stat[0])) 371 try: 372 result.check_returncode() 373 except subprocess.CalledProcessError as e: 374 if e.stdout: 375 logging.info(e.stdout) 376 if e.stderr: 377 logging.info(e.stderr) 378 logging.info(f"Target {result.args} failed with exit code {e.returncode}") 379 sys.exit(1) 380 381 if using_libfuzzer: 382 print("Summary:") 383 max_len = max(len(t[0]) for t in stats) 384 for t, s in sorted(stats): 385 t = t.ljust(max_len + 1) 386 print(f"{t}{s}") 387 388 389 def parse_test_list(*, fuzz_bin, source_dir): 390 test_list_all = subprocess.run( 391 fuzz_bin, 392 env={ 393 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '', 394 **get_fuzz_env(target="", source_dir=source_dir) 395 }, 396 stdout=subprocess.PIPE, 397 text=True, 398 check=True, 399 ).stdout.splitlines() 400 return test_list_all 401 402 403 if __name__ == '__main__': 404 main()