test_runner.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2019-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Run fuzz test targets. 6 """ 7 8 from concurrent.futures import ThreadPoolExecutor, as_completed 9 from pathlib import Path 10 import argparse 11 import configparser 12 import logging 13 import os 14 import random 15 import subprocess 16 import sys 17 18 19 def get_fuzz_env(*, target, source_dir): 20 symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") 21 fuzz_env = os.environ | { 22 'FUZZ': target, 23 'UBSAN_OPTIONS': 24 f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', 25 'UBSAN_SYMBOLIZER_PATH': symbolizer, 26 "ASAN_OPTIONS": "detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", 27 'ASAN_SYMBOLIZER_PATH': symbolizer, 28 'MSAN_SYMBOLIZER_PATH': symbolizer, 29 } 30 return fuzz_env 31 32 33 def main(): 34 parser = argparse.ArgumentParser( 35 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 36 description='''Run the fuzz targets with all inputs from the corpus_dir once.''', 37 ) 38 parser.add_argument( 39 "-l", 40 "--loglevel", 41 dest="loglevel", 42 default="INFO", 43 help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", 44 ) 45 parser.add_argument( 46 '--valgrind', 47 action='store_true', 48 help='If true, run fuzzing binaries under the valgrind memory error detector', 49 ) 50 parser.add_argument( 51 "--empty_min_time", 52 type=int, 53 help="If set, run at least this long, if the existing fuzz inputs directory is empty.", 54 ) 55 parser.add_argument( 56 '-x', 57 '--exclude', 58 help="A comma-separated list of targets to exclude", 59 ) 60 parser.add_argument( 61 '--par', 62 '-j', 63 type=int, 64 default=4, 65 help='How many targets to merge or execute in parallel.', 66 ) 67 parser.add_argument( 68 'corpus_dir', 69 help='The corpus to run on (must contain subfolders for each fuzz target).', 70 ) 71 parser.add_argument( 72 'target', 73 nargs='*', 74 help='The target(s) to run. Default is to run all targets.', 75 ) 76 parser.add_argument( 77 '--m_dir', 78 action="append", 79 help="Merge inputs from these directories into the corpus_dir.", 80 ) 81 parser.add_argument( 82 '-g', 83 '--generate', 84 action='store_true', 85 help='Create new corpus (or extend the existing ones) by running' 86 ' the given targets for a finite number of times. Outputs them to' 87 ' the passed corpus_dir.' 88 ) 89 90 args = parser.parse_args() 91 args.corpus_dir = Path(args.corpus_dir) 92 93 # Set up logging 94 logging.basicConfig( 95 format='%(message)s', 96 level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), 97 ) 98 99 # Read config generated by configure. 100 config = configparser.ConfigParser() 101 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" 102 config.read_file(open(configfile, encoding="utf8")) 103 104 if not config["components"].getboolean("ENABLE_FUZZ_BINARY"): 105 logging.error("Must have fuzz executable built") 106 sys.exit(1) 107 108 fuzz_bin=os.getenv("BITCOINFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'bin', 'fuzz')) 109 110 # Build list of tests 111 test_list_all = parse_test_list( 112 fuzz_bin=fuzz_bin, 113 source_dir=config['environment']['SRCDIR'], 114 ) 115 116 if not test_list_all: 117 logging.error("No fuzz targets found") 118 sys.exit(1) 119 120 logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all)))) 121 122 args.target = args.target or test_list_all # By default run all 123 test_list_error = list(set(args.target).difference(set(test_list_all))) 124 if test_list_error: 125 logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) 126 test_list_selection = list(set(test_list_all).intersection(set(args.target))) 127 if not test_list_selection: 128 logging.error("No fuzz targets selected") 129 if args.exclude: 130 for excluded_target in args.exclude.split(","): 131 if excluded_target not in test_list_selection: 132 logging.error("Target \"{}\" not found in current target list.".format(excluded_target)) 133 continue 134 test_list_selection.remove(excluded_target) 135 test_list_selection.sort() 136 137 logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) 138 139 if not args.generate: 140 test_list_missing_corpus = [] 141 for t in test_list_selection: 142 corpus_path = os.path.join(args.corpus_dir, t) 143 if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: 144 test_list_missing_corpus.append(t) 145 test_list_missing_corpus.sort() 146 if test_list_missing_corpus: 147 logging.info( 148 "Fuzzing harnesses lacking a corpus: {}".format( 149 " ".join(test_list_missing_corpus) 150 ) 151 ) 152 logging.info("Please consider adding a fuzz corpus at https://github.com/bitcoin-core/qa-assets") 153 154 try: 155 help_output = subprocess.run( 156 args=[ 157 fuzz_bin, 158 '-help=1', 159 ], 160 env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), 161 timeout=20, 162 check=False, 163 stderr=subprocess.PIPE, 164 text=True, 165 ).stderr 166 using_libfuzzer = "libFuzzer" in help_output 167 if (args.generate or args.m_dir) and not using_libfuzzer: 168 logging.error("Must be built with libFuzzer") 169 sys.exit(1) 170 except subprocess.TimeoutExpired: 171 logging.error("subprocess timed out: Currently only libFuzzer is supported") 172 sys.exit(1) 173 174 with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: 175 if args.generate: 176 return generate_corpus( 177 fuzz_pool=fuzz_pool, 178 src_dir=config['environment']['SRCDIR'], 179 fuzz_bin=fuzz_bin, 180 corpus_dir=args.corpus_dir, 181 targets=test_list_selection, 182 ) 183 184 if args.m_dir: 185 merge_inputs( 186 fuzz_pool=fuzz_pool, 187 corpus=args.corpus_dir, 188 test_list=test_list_selection, 189 src_dir=config['environment']['SRCDIR'], 190 fuzz_bin=fuzz_bin, 191 merge_dirs=[Path(m_dir) for m_dir in args.m_dir], 192 ) 193 return 194 195 run_once( 196 fuzz_pool=fuzz_pool, 197 corpus=args.corpus_dir, 198 test_list=test_list_selection, 199 src_dir=config['environment']['SRCDIR'], 200 fuzz_bin=fuzz_bin, 201 using_libfuzzer=using_libfuzzer, 202 use_valgrind=args.valgrind, 203 empty_min_time=args.empty_min_time, 204 ) 205 206 207 def transform_process_message_target(targets, src_dir): 208 """Add a target per process message, and also keep ("process_message", {}) to allow for 209 cross-pollination, or unlimited search""" 210 211 p2p_msg_target = "process_message" 212 if (p2p_msg_target, {}) in targets: 213 lines = subprocess.run( 214 ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", src_dir / "src" / "protocol.h"], 215 check=True, 216 stdout=subprocess.PIPE, 217 text=True, 218 ).stdout.splitlines() 219 lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")] 220 assert len(lines) 221 targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines] 222 return targets 223 224 225 def transform_rpc_target(targets, src_dir): 226 """Add a target per RPC command, and also keep ("rpc", {}) to allow for cross-pollination, 227 or unlimited search""" 228 229 rpc_target = "rpc" 230 if (rpc_target, {}) in targets: 231 lines = subprocess.run( 232 ["git", "grep", "--function-context", "RPC_COMMANDS_SAFE_FOR_FUZZING{", src_dir / "src" / "test" / "fuzz" / "rpc.cpp"], 233 check=True, 234 stdout=subprocess.PIPE, 235 text=True, 236 ).stdout.splitlines() 237 lines = [l.split("\"", 1)[1].split("\"")[0] for l in lines if l.startswith("src/test/fuzz/rpc.cpp- \"")] 238 assert len(lines) 239 targets += [(rpc_target, {"LIMIT_TO_RPC_COMMAND": r}) for r in lines] 240 return targets 241 242 243 def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets): 244 """Generates new corpus. 245 246 Run {targets} without input, and outputs the generated corpus to 247 {corpus_dir}. 248 """ 249 logging.info("Generating corpus to {}".format(corpus_dir)) 250 targets = [(t, {}) for t in targets] # expand to add dictionary for target-specific env variables 251 targets = transform_process_message_target(targets, Path(src_dir)) 252 targets = transform_rpc_target(targets, Path(src_dir)) 253 254 def job(command, t, t_env): 255 logging.debug(f"Running '{command}'") 256 logging.debug("Command '{}' output:\n'{}'\n".format( 257 command, 258 subprocess.run( 259 command, 260 env={ 261 **t_env, 262 **get_fuzz_env(target=t, source_dir=src_dir), 263 }, 264 check=True, 265 stderr=subprocess.PIPE, 266 text=True, 267 ).stderr, 268 )) 269 270 futures = [] 271 for target, t_env in targets: 272 target_corpus_dir = corpus_dir / target 273 os.makedirs(target_corpus_dir, exist_ok=True) 274 use_value_profile = int(random.random() < .3) 275 command = [ 276 fuzz_bin, 277 "-rss_limit_mb=8000", 278 "-max_total_time=6000", 279 "-reload=0", 280 f"-use_value_profile={use_value_profile}", 281 target_corpus_dir, 282 ] 283 futures.append(fuzz_pool.submit(job, command, target, t_env)) 284 285 for future in as_completed(futures): 286 future.result() 287 288 289 def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs): 290 logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}") 291 jobs = [] 292 for t in test_list: 293 args = [ 294 fuzz_bin, 295 '-rss_limit_mb=8000', 296 '-set_cover_merge=1', 297 # set_cover_merge is used instead of -merge=1 to reduce the overall 298 # size of the qa-assets git repository a bit, but more importantly, 299 # to cut the runtime to iterate over all fuzz inputs [0]. 300 # [0] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1761760866 301 '-shuffle=0', 302 '-prefer_small=1', 303 '-use_value_profile=0', 304 # use_value_profile is enabled by oss-fuzz [0], but disabled for 305 # now to avoid bloating the qa-assets git repository [1]. 306 # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 307 # [1] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1749075891 308 os.path.join(corpus, t), 309 ] + [str(m_dir / t) for m_dir in merge_dirs] 310 os.makedirs(os.path.join(corpus, t), exist_ok=True) 311 for m_dir in merge_dirs: 312 (m_dir / t).mkdir(exist_ok=True) 313 314 def job(t, args): 315 output = 'Run {} with args {}\n'.format(t, " ".join(args)) 316 output += subprocess.run( 317 args, 318 env=get_fuzz_env(target=t, source_dir=src_dir), 319 check=True, 320 stderr=subprocess.PIPE, 321 text=True, 322 ).stderr 323 logging.debug(output) 324 325 jobs.append(fuzz_pool.submit(job, t, args)) 326 327 for future in as_completed(jobs): 328 future.result() 329 330 331 def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time): 332 jobs = [] 333 for t in test_list: 334 corpus_path = corpus / t 335 os.makedirs(corpus_path, exist_ok=True) 336 args = [ 337 fuzz_bin, 338 ] 339 empty_dir = not any(corpus_path.iterdir()) 340 if using_libfuzzer: 341 if empty_min_time and empty_dir: 342 args += [f"-max_total_time={empty_min_time}"] 343 else: 344 args += [ 345 "-runs=1", 346 corpus_path, 347 ] 348 else: 349 args += [corpus_path] 350 if use_valgrind: 351 args = ['valgrind', '--quiet', '--error-exitcode=1'] + args 352 353 def job(t, args): 354 output = 'Run {} with args {}'.format(t, args) 355 result = subprocess.run( 356 args, 357 env=get_fuzz_env(target=t, source_dir=src_dir), 358 stderr=subprocess.PIPE, 359 text=True, 360 ) 361 output += result.stderr 362 return output, result, t 363 364 jobs.append(fuzz_pool.submit(job, t, args)) 365 366 stats = [] 367 for future in as_completed(jobs): 368 output, result, target = future.result() 369 logging.debug(output) 370 try: 371 result.check_returncode() 372 except subprocess.CalledProcessError as e: 373 if e.stdout: 374 logging.info(e.stdout) 375 if e.stderr: 376 logging.info(e.stderr) 377 logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}") 378 sys.exit(1) 379 if using_libfuzzer: 380 done_stat = [l for l in output.splitlines() if "DONE" in l] 381 assert len(done_stat) == 1 382 stats.append((target, done_stat[0])) 383 384 if using_libfuzzer: 385 print("Summary:") 386 max_len = max(len(t[0]) for t in stats) 387 for t, s in sorted(stats): 388 t = t.ljust(max_len + 1) 389 print(f"{t}{s}") 390 391 392 def parse_test_list(*, fuzz_bin, source_dir): 393 test_list_all = subprocess.run( 394 fuzz_bin, 395 env={ 396 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '', 397 **get_fuzz_env(target="", source_dir=source_dir) 398 }, 399 stdout=subprocess.PIPE, 400 text=True, 401 check=True, 402 ).stdout.splitlines() 403 return test_list_all 404 405 406 if __name__ == '__main__': 407 main()