run_test_idf_monitor.py
1 #!/usr/bin/env python 2 # 3 # Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 from __future__ import print_function 18 from __future__ import unicode_literals 19 from builtins import object 20 from io import open 21 import os 22 import sys 23 import time 24 import subprocess 25 import socket 26 import pty 27 import filecmp 28 import threading 29 import errno 30 import tempfile 31 32 test_list = ( 33 # Add new tests here. All files should be placed in IN_DIR. Columns are: 34 # Input file Filter string File with expected output Timeout 35 ('in1.txt', '', 'in1f1.txt', 60), 36 ('in1.txt', '*:V', 'in1f1.txt', 60), 37 ('in1.txt', 'hello_world', 'in1f2.txt', 60), 38 ('in1.txt', '*:N', 'in1f3.txt', 60), 39 ('in2.txt', 'boot mdf_device_handle:I mesh:E vfs:I', 'in2f1.txt', 420), 40 ('in2.txt', 'vfs', 'in2f2.txt', 420), 41 ('core1.txt', '', 'core1_out.txt', 60), 42 ) 43 44 IN_DIR = 'tests/' # tests are in this directory 45 OUT_DIR = 'outputs/' # test results are written to this directory (kept only for debugging purposes) 46 ERR_OUT = 'monitor_error_output.' 47 IDF_MONITOR_WAPPER = 'idf_monitor_wrapper.py' 48 SERIAL_ALIVE_FILE = '/tmp/serial_alive' # the existence of this file signalize that idf_monitor is ready to receive 49 50 # connection related to communicating with idf_monitor through sockets 51 HOST = 'localhost' 52 # blocking socket operations are used with timeout: 53 SOCKET_TIMEOUT = 30 54 # the test is restarted after failure (idf_monitor has to be killed): 55 RETRIES_PER_TEST = 2 56 57 58 def monitor_timeout(process): 59 if process.poll() is None: 60 # idf_monitor_wrapper is still running 61 try: 62 process.kill() 63 print('\tidf_monitor_wrapper was killed because it did not finish in time.') 64 except OSError as e: 65 if e.errno == errno.ESRCH: 66 # ignores a possible race condition which can occur when the process exits between poll() and kill() 67 pass 68 else: 69 raise 70 71 72 class TestRunner(object): 73 def __enter__(self): 74 self.serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 75 self.serversocket.bind((HOST, 0)) 76 self.port = self.serversocket.getsockname()[1] 77 self.serversocket.listen(5) 78 return self 79 80 def __exit__(self, type, value, traceback): 81 self.serversocket.shutdown(socket.SHUT_RDWR) 82 self.serversocket.close() 83 print('Socket was closed successfully') 84 85 def accept_connection(self): 86 """ returns a socket for sending the input for idf_monitor which must be closed before calling this again. """ 87 (clientsocket, address) = self.serversocket.accept() 88 # exception will be thrown here if the idf_monitor didn't connect in time 89 return clientsocket 90 91 92 def test_iteration(runner, test): 93 try: 94 # Make sure that the file doesn't exist. It will be recreated by idf_monitor_wrapper.py 95 os.remove(SERIAL_ALIVE_FILE) 96 except OSError: 97 pass 98 print('\nRunning test on {} with filter "{}" and expecting {}'.format(test[0], test[1], test[2])) 99 try: 100 with open(OUT_DIR + test[2], "w", encoding='utf-8') as o_f, \ 101 tempfile.NamedTemporaryFile(dir=OUT_DIR, prefix=ERR_OUT, mode="w", delete=False) as e_f: 102 monitor_cmd = [sys.executable, IDF_MONITOR_WAPPER, 103 '--port', 'socket://{}:{}?logging=debug'.format(HOST, runner.port), 104 '--print_filter', test[1], 105 '--serial_alive_file', SERIAL_ALIVE_FILE] 106 (master_fd, slave_fd) = pty.openpty() 107 print('\t', ' '.join(monitor_cmd), sep='') 108 print('\tstdout="{}" stderr="{}" stdin="{}"'.format(o_f.name, e_f.name, os.ttyname(slave_fd))) 109 print('\tMonitor timeout: {} seconds'.format(test[3])) 110 start = time.time() 111 # the server socket is alive so idf_monitor can start now 112 proc = subprocess.Popen(monitor_cmd, stdin=slave_fd, stdout=o_f, stderr=e_f, close_fds=True, bufsize=0) 113 # - idf_monitor's stdin needs to be connected to some pseudo-tty in docker image even when it is not 114 # used at all 115 # - setting bufsize is needed because the default value is different on Python 2 and 3 116 # - the default close_fds is also different on Python 2 and 3 117 monitor_watchdog = threading.Timer(test[3], monitor_timeout, [proc]) 118 monitor_watchdog.start() 119 client = runner.accept_connection() 120 # The connection is ready but idf_monitor cannot yet receive data (the serial reader thread is not running). 121 # This seems to happen on Ubuntu 16.04 LTS and is not related to the version of Python or pyserial. 122 # Updating to Ubuntu 18.04 LTS also helps but here, a workaround is used: A wrapper is used for IDF monitor 123 # which checks the serial reader thread and creates a file when it is running. 124 while not os.path.isfile(SERIAL_ALIVE_FILE) and proc.poll() is None: 125 print('\tSerial reader is not ready. Do a sleep...') 126 time.sleep(1) 127 # Only now can we send the inputs: 128 with open(IN_DIR + test[0], 'rb') as f: 129 print('\tSending {} to the socket'.format(f.name)) 130 for chunk in iter(lambda: f.read(1024), b''): 131 client.sendall(chunk) 132 idf_exit_sequence = b'\x1d\n' 133 print('\tSending <exit> to the socket') 134 client.sendall(idf_exit_sequence) 135 close_end_time = start + 0.75 * test[3] # time when the process is close to be killed 136 while True: 137 ret = proc.poll() 138 if ret is not None: 139 break 140 if time.time() > close_end_time: 141 # The process isn't finished yet so we are starting to send additional exit sequences because maybe 142 # the other end didn't received it. 143 print('\tSending additional <exit> to the socket') 144 client.sendall(idf_exit_sequence) 145 time.sleep(1) 146 end = time.time() 147 print('\tidf_monitor exited after {:.2f} seconds'.format(end - start)) 148 if ret < 0: 149 raise RuntimeError('idf_monitor was terminated by signal {}'.format(-ret)) 150 # idf_monitor needs to end before the socket is closed in order to exit without an exception. 151 finally: 152 if monitor_watchdog: 153 monitor_watchdog.cancel() 154 os.close(slave_fd) 155 os.close(master_fd) 156 if client: 157 client.close() 158 print('\tThe client was closed successfully') 159 f1 = IN_DIR + test[2] 160 f2 = OUT_DIR + test[2] 161 print('\tdiff {} {}'.format(f1, f2)) 162 if filecmp.cmp(f1, f2, shallow=False): 163 print('\tTest has passed') 164 else: 165 raise RuntimeError("The contents of the files are different. Please examine the artifacts.") 166 167 168 def main(): 169 gstart = time.time() 170 if not os.path.exists(OUT_DIR): 171 os.mkdir(OUT_DIR) 172 173 socket.setdefaulttimeout(SOCKET_TIMEOUT) 174 175 for test in test_list: 176 for i in range(RETRIES_PER_TEST): 177 with TestRunner() as runner: 178 # Each test (and each retry) is run with a different port (and server socket). This is done for 179 # the CI run where retry with a different socket is necessary to pass the test. According to the 180 # experiments, retry with the same port (and server socket) is not sufficient. 181 try: 182 test_iteration(runner, test) 183 # no more retries if test_iteration exited without an exception 184 break 185 except Exception as e: 186 if i < RETRIES_PER_TEST - 1: 187 print('Test has failed with exception:', e) 188 print('Another attempt will be made.') 189 else: 190 raise 191 192 gend = time.time() 193 print('Execution took {:.2f} seconds\n'.format(gend - gstart)) 194 195 196 if __name__ == "__main__": 197 main()