find_apps.py
1 #!/usr/bin/env python 2 # coding=utf-8 3 # 4 # ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps. 5 # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. 6 7 import argparse 8 import glob 9 import json 10 import logging 11 import os 12 import re 13 import sys 14 15 import typing 16 17 from find_build_apps import ( 18 BUILD_SYSTEMS, 19 BUILD_SYSTEM_CMAKE, 20 BuildSystem, 21 BuildItem, 22 setup_logging, 23 ConfigRule, 24 config_rules_from_str, 25 DEFAULT_TARGET, 26 ) 27 28 29 # Helper functions 30 31 def dict_from_sdkconfig(path): 32 """ 33 Parse the sdkconfig file at 'path', return name:value pairs as a dict 34 """ 35 regex = re.compile(r"^([^#=]+)=(.+)$") 36 result = {} 37 with open(path) as f: 38 for line in f: 39 m = regex.match(line) 40 if m: 41 val = m.group(2) 42 if val.startswith('"') and val.endswith('"'): 43 val = val[1:-1] 44 result[m.group(1)] = val 45 return result 46 47 48 # Main logic: enumerating apps and builds 49 50 51 def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, 52 build_system, config_rules, preserve_artifacts=True): 53 # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem] 54 """ 55 Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects 56 :param app_path: app directory (can be / usually will be a relative path) 57 :param work_dir: directory where the app should be copied before building. 58 May contain env. variables and placeholders. 59 :param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders. 60 :param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go 61 into stdout/stderr. 62 :param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with 63 a different CONFIG_IDF_TARGET value. 64 :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary 65 :param config_rules: mapping of sdkconfig file name patterns to configuration names 66 :param preserve_artifacts: determine if the built binary will be uploaded as artifacts. 67 :return: list of BuildItems representing build configuration of the app 68 """ 69 build_items = [] # type: typing.List[BuildItem] 70 default_config_name = "" 71 72 for rule in config_rules: 73 if not rule.file_name: 74 default_config_name = rule.config_name 75 continue 76 77 sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name)) 78 sdkconfig_paths = sorted(sdkconfig_paths) 79 for sdkconfig_path in sdkconfig_paths: 80 81 # Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument. 82 sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path) 83 target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET") 84 if target_from_config is not None and target_from_config != target_arg: 85 logging.debug("Skipping sdkconfig {} which requires target {}".format( 86 sdkconfig_path, target_from_config)) 87 continue 88 89 # Figure out the config name 90 config_name = rule.config_name or "" 91 if "*" in rule.file_name: 92 # convert glob pattern into a regex 93 regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)") 94 groups = re.match(regex_str, sdkconfig_path) 95 assert groups 96 config_name = groups.group(1) 97 98 sdkconfig_path = os.path.relpath(sdkconfig_path, app_path) 99 logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format( 100 app_path, sdkconfig_path, config_name)) 101 build_items.append( 102 BuildItem( 103 app_path, 104 work_dir, 105 build_dir, 106 build_log, 107 target_arg, 108 sdkconfig_path, 109 config_name, 110 build_system, 111 preserve_artifacts, 112 )) 113 114 if not build_items: 115 logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name)) 116 return [ 117 BuildItem( 118 app_path, 119 work_dir, 120 build_dir, 121 build_log, 122 target_arg, 123 None, 124 default_config_name, 125 build_system, 126 preserve_artifacts, 127 ) 128 ] 129 130 return build_items 131 132 133 def find_apps(build_system_class, path, recursive, exclude_list, target): 134 # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str] 135 """ 136 Find app directories in path (possibly recursively), which contain apps for the given build system, compatible 137 with the given target. 138 :param build_system_class: class derived from BuildSystem, representing the build system in use 139 :param path: path where to look for apps 140 :param recursive: whether to recursively descend into nested directories if no app is found 141 :param exclude_list: list of paths to be excluded from the recursive search 142 :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped. 143 :return: list of paths of the apps found 144 """ 145 build_system_name = build_system_class.NAME 146 logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else "")) 147 if not recursive: 148 if exclude_list: 149 logging.warn("--exclude option is ignored when used without --recursive") 150 if not build_system_class.is_app(path): 151 logging.warn("Path {} specified without --recursive flag, but no {} app found there".format( 152 path, build_system_name)) 153 return [] 154 return [path] 155 156 # The remaining part is for recursive == True 157 apps_found = [] # type: typing.List[str] 158 for root, dirs, _ in os.walk(path, topdown=True): 159 logging.debug("Entering {}".format(root)) 160 if root in exclude_list: 161 logging.debug("Skipping {} (excluded)".format(root)) 162 del dirs[:] 163 continue 164 165 if build_system_class.is_app(root): 166 logging.debug("Found {} app in {}".format(build_system_name, root)) 167 # Don't recurse into app subdirectories 168 del dirs[:] 169 170 supported_targets = build_system_class.supported_targets(root) 171 if supported_targets and target not in supported_targets: 172 logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets)) 173 continue 174 175 apps_found.append(root) 176 177 return apps_found 178 179 180 def main(): 181 parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps") 182 parser.add_argument( 183 "-v", 184 "--verbose", 185 action="count", 186 help="Increase the logging level of the script. Can be specified multiple times.", 187 ) 188 parser.add_argument( 189 "--log-file", 190 type=argparse.FileType("w"), 191 help="Write the script log to the specified file, instead of stderr", 192 ) 193 parser.add_argument( 194 "--recursive", 195 action="store_true", 196 help="Look for apps in the specified directories recursively.", 197 ) 198 parser.add_argument( 199 "--build-system", 200 choices=BUILD_SYSTEMS.keys() 201 ) 202 parser.add_argument( 203 "--work-dir", 204 help="If set, the app is first copied into the specified directory, and then built." + 205 "If not set, the work directory is the directory of the app.", 206 ) 207 parser.add_argument( 208 "--config", 209 action="append", 210 help="Adds configurations (sdkconfig file names) to build. This can either be " + 211 "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + 212 "relative to the project directory, to be used. Optional NAME can be specified, " + 213 "which can be used as a name of this configuration. FILEPATTERN is the name of " + 214 "the sdkconfig file, relative to the project directory, with at most one wildcard. " + 215 "The part captured by the wildcard is used as the name of the configuration.", 216 ) 217 parser.add_argument( 218 "--build-dir", 219 help="If set, specifies the build directory name. Can expand placeholders. Can be either a " + 220 "name relative to the work directory, or an absolute path.", 221 ) 222 parser.add_argument( 223 "--build-log", 224 help="If specified, the build log will be written to this file. Can expand placeholders.", 225 ) 226 parser.add_argument("--target", help="Build apps for given target.") 227 parser.add_argument( 228 "--format", 229 default="json", 230 choices=["json"], 231 help="Format to write the list of builds as", 232 ) 233 parser.add_argument( 234 "--exclude", 235 action="append", 236 help="Ignore specified directory (if --recursive is given). Can be used multiple times.", 237 ) 238 parser.add_argument( 239 "-o", 240 "--output", 241 type=argparse.FileType("w"), 242 help="Output the list of builds to the specified file", 243 ) 244 parser.add_argument( 245 "--app-list", 246 default=None, 247 help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. " 248 "If the file does not exist, will build all apps and upload all artifacts." 249 ) 250 parser.add_argument( 251 "-p", "--paths", 252 nargs="+", 253 help="One or more app paths." 254 ) 255 args = parser.parse_args() 256 setup_logging(args) 257 258 # Arguments Validation 259 if args.app_list: 260 conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths] 261 if any(conflict_args): 262 raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not ' 263 'be specified with "app_list"') 264 if not os.path.exists(args.app_list): 265 raise OSError("File not found {}".format(args.app_list)) 266 else: 267 # If the build target is not set explicitly, get it from the environment or use the default one (esp32) 268 if not args.target: 269 env_target = os.environ.get("IDF_TARGET") 270 if env_target: 271 logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) 272 args.target = env_target 273 else: 274 logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) 275 args.target = DEFAULT_TARGET 276 if not args.build_system: 277 logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE)) 278 args.build_system = BUILD_SYSTEM_CMAKE 279 required_args = [args.build_system, args.target, args.paths] 280 if not all(required_args): 281 raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.') 282 283 # Prepare the list of app paths, try to read from the scan_tests result. 284 # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again. 285 # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. 286 if args.app_list: 287 apps = [json.loads(line) for line in open(args.app_list)] 288 else: 289 app_dirs = [] 290 build_system_class = BUILD_SYSTEMS[args.build_system] 291 for path in args.paths: 292 app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) 293 apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs] 294 295 if not apps: 296 logging.warning("No apps found") 297 SystemExit(0) 298 299 logging.info("Found {} apps".format(len(apps))) 300 apps.sort(key=lambda x: x["app_dir"]) 301 302 # Find compatible configurations of each app, collect them as BuildItems 303 build_items = [] # type: typing.List[BuildItem] 304 config_rules = config_rules_from_str(args.config or []) 305 for app in apps: 306 build_items += find_builds_for_app( 307 app["app_dir"], 308 args.work_dir, 309 args.build_dir, 310 args.build_log, 311 args.target or app["target"], 312 args.build_system or app["build_system"], 313 config_rules, 314 app["preserve"], 315 ) 316 logging.info("Found {} builds".format(len(build_items))) 317 318 # Write out the BuildItems. Only JSON supported now (will add YAML later). 319 if args.format != "json": 320 raise NotImplementedError() 321 322 out = args.output or sys.stdout 323 out.writelines([item.to_json() + "\n" for item in build_items]) 324 325 326 if __name__ == "__main__": 327 main()