test_runner.py
1 #!/usr/bin/env python3 2 # Copyright (c) 2019-present The Bitcoin Core developers 3 # Distributed under the MIT software license, see the accompanying 4 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 """Run fuzz test targets. 6 """ 7 8 from concurrent.futures import ThreadPoolExecutor, as_completed 9 from pathlib import Path 10 import argparse 11 import configparser 12 import logging 13 import os 14 import random 15 import subprocess 16 import sys 17 18 19 def get_fuzz_env(*, target, source_dir): 20 symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") 21 fuzz_env = os.environ | { 22 'FUZZ': target, 23 'UBSAN_OPTIONS': 24 f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', 25 'UBSAN_SYMBOLIZER_PATH': symbolizer, 26 "ASAN_OPTIONS": "detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", 27 'ASAN_SYMBOLIZER_PATH': symbolizer, 28 'MSAN_SYMBOLIZER_PATH': symbolizer, 29 } 30 return fuzz_env 31 32 33 def main(): 34 parser = argparse.ArgumentParser( 35 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 36 description='''Run the fuzz targets with all inputs from the corpus_dir once.''', 37 ) 38 parser.add_argument( 39 "-l", 40 "--loglevel", 41 dest="loglevel", 42 default="INFO", 43 help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", 44 ) 45 parser.add_argument( 46 '--valgrind', 47 action='store_true', 48 help='If true, run fuzzing binaries under the valgrind memory error detector', 49 ) 50 parser.add_argument( 51 "--empty_min_time", 52 type=int, 53 help="If set, run at least this long, if the existing fuzz inputs directory is empty.", 54 ) 55 parser.add_argument( 56 '-x', 57 '--exclude', 58 help="A comma-separated list of targets to exclude", 59 ) 60 parser.add_argument( 61 '--par', 62 '-j', 63 type=int, 64 default=4, 65 help='How many targets to merge or execute in parallel.', 66 ) 67 parser.add_argument( 68 'corpus_dir', 69 help='The corpus to run on (must contain subfolders for each fuzz target).', 70 ) 71 parser.add_argument( 72 'target', 73 nargs='*', 74 help='The target(s) to run. Default is to run all targets.', 75 ) 76 parser.add_argument( 77 '--m_dir', 78 action="append", 79 help="Merge inputs from these directories into the corpus_dir.", 80 ) 81 parser.add_argument( 82 '-g', 83 '--generate', 84 action='store_true', 85 help='Create new corpus (or extend the existing ones) by running' 86 ' the given targets for a finite number of times. Outputs them to' 87 ' the passed corpus_dir.' 88 ) 89 90 args = parser.parse_args() 91 args.corpus_dir = Path(args.corpus_dir) 92 93 # Set up logging 94 logging.basicConfig( 95 format='%(message)s', 96 level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), 97 ) 98 99 # Read config generated by configure. 100 config = configparser.ConfigParser() 101 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" 102 config.read_file(open(configfile)) 103 104 if not config["components"].getboolean("ENABLE_FUZZ_BINARY"): 105 logging.error("Must have fuzz executable built") 106 sys.exit(1) 107 108 fuzz_bin=os.getenv("BITCOINFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'bin', 'fuzz')) 109 110 # Build list of tests 111 test_list_all = parse_test_list( 112 fuzz_bin=fuzz_bin, 113 source_dir=config['environment']['SRCDIR'], 114 ) 115 116 if not test_list_all: 117 logging.error("No fuzz targets found") 118 sys.exit(1) 119 120 logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all)))) 121 122 args.target = args.target or test_list_all # By default run all 123 test_list_error = list(set(args.target).difference(set(test_list_all))) 124 if test_list_error: 125 logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) 126 test_list_selection = list(set(test_list_all).intersection(set(args.target))) 127 if not test_list_selection: 128 logging.error("No fuzz targets selected") 129 if args.exclude: 130 for excluded_target in args.exclude.split(","): 131 if excluded_target not in test_list_selection: 132 logging.error("Target \"{}\" not found in current target list.".format(excluded_target)) 133 continue 134 test_list_selection.remove(excluded_target) 135 test_list_selection.sort() 136 137 logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) 138 139 if not args.generate: 140 test_list_missing_corpus = [] 141 for t in test_list_selection: 142 corpus_path = os.path.join(args.corpus_dir, t) 143 if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: 144 test_list_missing_corpus.append(t) 145 test_list_missing_corpus.sort() 146 if test_list_missing_corpus: 147 logging.info( 148 "Fuzzing harnesses lacking a corpus: {}".format( 149 " ".join(test_list_missing_corpus) 150 ) 151 ) 152 logging.info("Please consider adding a fuzz corpus at https://github.com/bitcoin-core/qa-assets") 153 154 print("Check if using libFuzzer ... ", end='') 155 help_output = subprocess.run( 156 args=[ 157 fuzz_bin, 158 '-help=1', 159 ], 160 env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), 161 check=False, 162 stderr=subprocess.PIPE, 163 text=True, 164 ).stderr 165 using_libfuzzer = "libFuzzer" in help_output 166 print(using_libfuzzer) 167 if (args.generate or args.m_dir) and not using_libfuzzer: 168 logging.error("Must be built with libFuzzer") 169 sys.exit(1) 170 171 with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: 172 if args.generate: 173 return generate_corpus( 174 fuzz_pool=fuzz_pool, 175 src_dir=config['environment']['SRCDIR'], 176 fuzz_bin=fuzz_bin, 177 corpus_dir=args.corpus_dir, 178 targets=test_list_selection, 179 ) 180 181 if args.m_dir: 182 merge_inputs( 183 fuzz_pool=fuzz_pool, 184 corpus=args.corpus_dir, 185 test_list=test_list_selection, 186 src_dir=config['environment']['SRCDIR'], 187 fuzz_bin=fuzz_bin, 188 merge_dirs=[Path(m_dir) for m_dir in args.m_dir], 189 ) 190 return 191 192 run_once( 193 fuzz_pool=fuzz_pool, 194 corpus=args.corpus_dir, 195 test_list=test_list_selection, 196 src_dir=config['environment']['SRCDIR'], 197 fuzz_bin=fuzz_bin, 198 using_libfuzzer=using_libfuzzer, 199 use_valgrind=args.valgrind, 200 empty_min_time=args.empty_min_time, 201 ) 202 203 204 def transform_process_message_target(targets, src_dir): 205 """Add a target per process message, and also keep ("process_message", {}) to allow for 206 cross-pollination, or unlimited search""" 207 208 p2p_msg_target = "process_message" 209 if (p2p_msg_target, {}) in targets: 210 lines = subprocess.run( 211 ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", "src/protocol.h"], 212 check=True, 213 stdout=subprocess.PIPE, 214 text=True, 215 cwd=src_dir, 216 ).stdout.splitlines() 217 lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")] 218 assert len(lines) 219 targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines] 220 return targets 221 222 223 def transform_rpc_target(targets, src_dir): 224 """Add a target per RPC command, and also keep ("rpc", {}) to allow for cross-pollination, 225 or unlimited search""" 226 227 rpc_target = "rpc" 228 if (rpc_target, {}) in targets: 229 lines = subprocess.run( 230 ["git", "grep", "--function-context", "RPC_COMMANDS_SAFE_FOR_FUZZING{", "src/test/fuzz/rpc.cpp"], 231 check=True, 232 stdout=subprocess.PIPE, 233 text=True, 234 cwd=src_dir, 235 ).stdout.splitlines() 236 lines = [l.split("\"", 1)[1].split("\"")[0] for l in lines if l.startswith("src/test/fuzz/rpc.cpp- \"")] 237 assert len(lines) 238 targets += [(rpc_target, {"LIMIT_TO_RPC_COMMAND": r}) for r in lines] 239 return targets 240 241 242 def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets): 243 """Generates new corpus. 244 245 Run {targets} without input, and outputs the generated corpus to 246 {corpus_dir}. 247 """ 248 logging.info("Generating corpus to {}".format(corpus_dir)) 249 targets = [(t, {}) for t in targets] # expand to add dictionary for target-specific env variables 250 targets = transform_process_message_target(targets, Path(src_dir)) 251 targets = transform_rpc_target(targets, Path(src_dir)) 252 253 def job(command, t, t_env): 254 logging.debug(f"Running '{command}'") 255 logging.debug("Command '{}' output:\n'{}'\n".format( 256 command, 257 subprocess.run( 258 command, 259 env={ 260 **t_env, 261 **get_fuzz_env(target=t, source_dir=src_dir), 262 }, 263 check=True, 264 stderr=subprocess.PIPE, 265 text=True, 266 ).stderr, 267 )) 268 269 futures = [] 270 for target, t_env in targets: 271 target_corpus_dir = corpus_dir / target 272 os.makedirs(target_corpus_dir, exist_ok=True) 273 use_value_profile = int(random.random() < .3) 274 command = [ 275 fuzz_bin, 276 "-rss_limit_mb=8000", 277 "-max_total_time=6000", 278 "-reload=0", 279 f"-use_value_profile={use_value_profile}", 280 target_corpus_dir, 281 ] 282 futures.append(fuzz_pool.submit(job, command, target, t_env)) 283 284 for future in as_completed(futures): 285 future.result() 286 287 288 def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs): 289 logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}") 290 jobs = [] 291 for t in test_list: 292 args = [ 293 fuzz_bin, 294 '-rss_limit_mb=8000', 295 '-set_cover_merge=1', 296 # set_cover_merge is used instead of -merge=1 to reduce the overall 297 # size of the qa-assets git repository a bit, but more importantly, 298 # to cut the runtime to iterate over all fuzz inputs [0]. 299 # [0] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1761760866 300 '-shuffle=0', 301 '-prefer_small=1', 302 '-use_value_profile=0', 303 # use_value_profile is enabled by oss-fuzz [0], but disabled for 304 # now to avoid bloating the qa-assets git repository [1]. 305 # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 306 # [1] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1749075891 307 os.path.join(corpus, t), 308 ] + [str(m_dir / t) for m_dir in merge_dirs] 309 os.makedirs(os.path.join(corpus, t), exist_ok=True) 310 for m_dir in merge_dirs: 311 (m_dir / t).mkdir(exist_ok=True) 312 313 def job(t, args): 314 output = 'Run {} with args {}\n'.format(t, " ".join(args)) 315 output += subprocess.run( 316 args, 317 env=get_fuzz_env(target=t, source_dir=src_dir), 318 check=True, 319 stderr=subprocess.PIPE, 320 text=True, 321 ).stderr 322 logging.debug(output) 323 324 jobs.append(fuzz_pool.submit(job, t, args)) 325 326 for future in as_completed(jobs): 327 future.result() 328 329 330 def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time): 331 jobs = [] 332 for t in test_list: 333 corpus_path = corpus / t 334 os.makedirs(corpus_path, exist_ok=True) 335 args = [ 336 fuzz_bin, 337 ] 338 empty_dir = not any(corpus_path.iterdir()) 339 if using_libfuzzer: 340 if empty_min_time and empty_dir: 341 args += [f"-max_total_time={empty_min_time}"] 342 else: 343 args += [ 344 "-runs=1", 345 corpus_path, 346 ] 347 else: 348 args += [corpus_path] 349 if use_valgrind: 350 args = ['valgrind', '--quiet', '--error-exitcode=1'] + args 351 352 def job(t, args): 353 output = 'Run {} with args {}'.format(t, args) 354 result = subprocess.run( 355 args, 356 env=get_fuzz_env(target=t, source_dir=src_dir), 357 stderr=subprocess.PIPE, 358 text=True, 359 ) 360 output += result.stderr 361 return output, result, t 362 363 jobs.append(fuzz_pool.submit(job, t, args)) 364 365 stats = [] 366 for future in as_completed(jobs): 367 output, result, target = future.result() 368 logging.debug(output) 369 try: 370 result.check_returncode() 371 except subprocess.CalledProcessError as e: 372 if e.stdout: 373 logging.info(e.stdout) 374 if e.stderr: 375 logging.info(e.stderr) 376 logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}") 377 sys.exit(1) 378 if using_libfuzzer: 379 done_stat = [l for l in output.splitlines() if "DONE" in l] 380 assert len(done_stat) == 1 381 stats.append((target, done_stat[0])) 382 383 if using_libfuzzer: 384 print("Summary:") 385 max_len = max(len(t[0]) for t in stats) 386 for t, s in sorted(stats): 387 t = t.ljust(max_len + 1) 388 print(f"{t}{s}") 389 390 391 def parse_test_list(*, fuzz_bin, source_dir): 392 test_list_all = subprocess.run( 393 fuzz_bin, 394 env={ 395 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '', 396 **get_fuzz_env(target="", source_dir=source_dir) 397 }, 398 stdout=subprocess.PIPE, 399 text=True, 400 check=True, 401 ).stdout.splitlines() 402 return test_list_all 403 404 405 if __name__ == '__main__': 406 main()