test_runner.py
1 #!/usr/bin/env python3 2 # Copyright 2014 BitPay Inc. 3 # Copyright 2016-2017 The Bitcoin Core developers 4 # Distributed under the MIT software license, see the accompanying 5 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 6 """Test framework for bitcoin utils. 7 8 Runs automatically during `make check`. 9 10 Can also be run manually.""" 11 12 import argparse 13 import configparser 14 import difflib 15 import json 16 import logging 17 import os 18 import pprint 19 import subprocess 20 import sys 21 22 def main(): 23 config = configparser.ConfigParser() 24 config.optionxform = str 25 with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f: 26 config.read_file(f) 27 env_conf = dict(config.items('environment')) 28 29 parser = argparse.ArgumentParser(description=__doc__) 30 parser.add_argument('-v', '--verbose', action='store_true') 31 args = parser.parse_args() 32 verbose = args.verbose 33 34 if verbose: 35 level = logging.DEBUG 36 else: 37 level = logging.ERROR 38 formatter = '%(asctime)s - %(levelname)s - %(message)s' 39 # Add the format/level to the logger 40 logging.basicConfig(format=formatter, level=level) 41 42 bctester(os.path.join(env_conf["SRCDIR"], "test", "util", "data"), "bitcoin-util-test.json", env_conf) 43 44 def bctester(testDir, input_basename, buildenv): 45 """ Loads and parses the input file, runs all tests and reports results""" 46 input_filename = os.path.join(testDir, input_basename) 47 with open(input_filename, encoding="utf8") as f: 48 raw_data = f.read() 49 input_data = json.loads(raw_data) 50 51 failed_testcases = [] 52 53 for testObj in input_data: 54 try: 55 bctest(testDir, testObj, buildenv) 56 logging.info("PASSED: " + testObj["description"]) 57 except Exception: 58 logging.info("FAILED: " + testObj["description"]) 59 failed_testcases.append(testObj["description"]) 60 61 if failed_testcases: 62 error_message = "FAILED_TESTCASES:\n" 63 error_message += pprint.pformat(failed_testcases, width=400) 64 logging.error(error_message) 65 sys.exit(1) 66 else: 67 sys.exit(0) 68 69 def bctest(testDir, testObj, buildenv): 70 """Runs a single test, comparing output and RC to expected output and RC. 71 72 Raises an error if input can't be read, executable fails, or output/RC 73 are not as expected. Error is caught by bctester() and reported. 74 """ 75 # Get the exec names and arguments 76 execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"]) 77 if testObj["exec"] == "./bitcoin-util": 78 execprog = os.getenv("BITCOINUTIL", default=execprog) 79 elif testObj["exec"] == "./bitcoin-tx": 80 execprog = os.getenv("BITCOINTX", default=execprog) 81 82 execargs = testObj['args'] 83 execrun = [execprog] + execargs 84 85 # Read the input data (if there is any) 86 stdinCfg = None 87 inputData = None 88 if "input" in testObj: 89 filename = os.path.join(testDir, testObj["input"]) 90 with open(filename, encoding="utf8") as f: 91 inputData = f.read() 92 stdinCfg = subprocess.PIPE 93 94 # Read the expected output data (if there is any) 95 outputFn = None 96 outputData = None 97 outputType = None 98 if "output_cmp" in testObj: 99 outputFn = testObj['output_cmp'] 100 outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) 101 try: 102 with open(os.path.join(testDir, outputFn), encoding="utf8") as f: 103 outputData = f.read() 104 except Exception: 105 logging.error("Output file " + outputFn + " cannot be opened") 106 raise 107 if not outputData: 108 logging.error("Output data missing for " + outputFn) 109 raise Exception 110 if not outputType: 111 logging.error("Output file %s does not have a file extension" % outputFn) 112 raise Exception 113 114 # Run the test 115 proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 116 try: 117 outs = proc.communicate(input=inputData) 118 except OSError: 119 logging.error("OSError, Failed to execute " + execprog) 120 raise 121 122 if outputData: 123 data_mismatch, formatting_mismatch = False, False 124 # Parse command output and expected output 125 try: 126 a_parsed = parse_output(outs[0], outputType) 127 except Exception as e: 128 logging.error('Error parsing command output as %s: %s' % (outputType, e)) 129 raise 130 try: 131 b_parsed = parse_output(outputData, outputType) 132 except Exception as e: 133 logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) 134 raise 135 # Compare data 136 if a_parsed != b_parsed: 137 logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") 138 data_mismatch = True 139 # Compare formatting 140 if outs[0] != outputData: 141 error_message = "Output formatting mismatch for " + outputFn + ":\n" 142 error_message += "".join(difflib.context_diff(outputData.splitlines(True), 143 outs[0].splitlines(True), 144 fromfile=outputFn, 145 tofile="returned")) 146 logging.error(error_message) 147 formatting_mismatch = True 148 149 assert not data_mismatch and not formatting_mismatch 150 151 # Compare the return code to the expected return code 152 wantRC = 0 153 if "return_code" in testObj: 154 wantRC = testObj['return_code'] 155 if proc.returncode != wantRC: 156 logging.error("Return code mismatch for " + outputFn) 157 raise Exception 158 159 if "error_txt" in testObj: 160 want_error = testObj["error_txt"] 161 # Compare error text 162 # TODO: ideally, we'd compare the strings exactly and also assert 163 # That stderr is empty if no errors are expected. However, bitcoin-tx 164 # emits DISPLAY errors when running as a windows application on 165 # linux through wine. Just assert that the expected error text appears 166 # somewhere in stderr. 167 if want_error not in outs[1]: 168 logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) 169 raise Exception 170 171 def parse_output(a, fmt): 172 """Parse the output according to specified format. 173 174 Raise an error if the output can't be parsed.""" 175 if fmt == 'json': # json: compare parsed data 176 return json.loads(a) 177 elif fmt == 'hex': # hex: parse and compare binary data 178 return bytes.fromhex(a.strip()) 179 else: 180 raise NotImplementedError("Don't know how to compare %s" % fmt) 181 182 if __name__ == '__main__': 183 main()