1  import copy
  2  import glob
  3  import os
  4  import os.path
  5  import re
  6  import shutil
  7  
  8  
  9  def action_extensions(base_actions, project_path=os.getcwd()):
 10      """ Describes extensions for unit tests. This function expects that actions "all" and "reconfigure" """
 11  
 12      PROJECT_NAME = "unit-test-app"
 13  
 14      # List of unit-test-app configurations.
 15      # Each file in configs/ directory defines a configuration. The format is the
 16      # same as sdkconfig file. Configuration is applied on top of sdkconfig.defaults
 17      # file from the project directory
 18      CONFIG_NAMES = os.listdir(os.path.join(project_path, "configs"))
 19  
 20      # Build (intermediate) and output (artifact) directories
 21      BUILDS_DIR = os.path.join(project_path, "builds")
 22      BINARIES_DIR = os.path.join(project_path, "output")
 23  
 24      def parse_file_to_dict(path, regex):
 25          """
 26          Parse the config file at 'path'
 27  
 28          Returns a dict of name:value.
 29          """
 30          compiled_regex = re.compile(regex)
 31          result = {}
 32          with open(path) as f:
 33              for line in f:
 34                  m = compiled_regex.match(line)
 35                  if m:
 36                      result[m.group(1)] = m.group(2)
 37          return result
 38  
 39      def parse_config(path):
 40          """
 41          Expected format with default regex is "key=value"
 42          """
 43  
 44          return parse_file_to_dict(path, r"^([^=]+)=(.+)$")
 45  
 46      def ut_apply_config(ut_apply_config_name, ctx, args):
 47          config_name = re.match(r"ut-apply-config-(.*)", ut_apply_config_name).group(1)
 48          # Make sure that define_cache_entry is list
 49          args.define_cache_entry = list(args.define_cache_entry)
 50          new_cache_values = {}
 51          sdkconfig_set = list(filter(lambda s: "SDKCONFIG=" in s, args.define_cache_entry))
 52          sdkconfig_path = os.path.join(args.project_dir, "sdkconfig")
 53  
 54          if sdkconfig_set:
 55              sdkconfig_path = sdkconfig_set[-1].split("=")[1]
 56              sdkconfig_path = os.path.abspath(sdkconfig_path)
 57  
 58          try:
 59              os.remove(sdkconfig_path)
 60          except OSError:
 61              pass
 62  
 63          if config_name in CONFIG_NAMES:
 64              # Parse the sdkconfig for components to be included/excluded and tests to be run
 65              config_path = os.path.join(project_path, "configs", config_name)
 66              config = parse_config(config_path)
 67  
 68              target = config.get("CONFIG_IDF_TARGET", "esp32").strip("'").strip('"')
 69  
 70              print("Reconfigure: config %s, target %s" % (config_name, target))
 71  
 72              # Clean up and set idf-target
 73              base_actions["actions"]["fullclean"]["callback"]("fullclean", ctx, args)
 74  
 75              new_cache_values["EXCLUDE_COMPONENTS"] = config.get("EXCLUDE_COMPONENTS", "''")
 76              new_cache_values["TEST_EXCLUDE_COMPONENTS"] = config.get("TEST_EXCLUDE_COMPONENTS", "''")
 77              new_cache_values["TEST_COMPONENTS"] = config.get("TEST_COMPONENTS", "''")
 78              new_cache_values["TESTS_ALL"] = int(new_cache_values["TEST_COMPONENTS"] == "''")
 79              new_cache_values["IDF_TARGET"] = target
 80              new_cache_values["SDKCONFIG_DEFAULTS"] = ";".join([os.path.join(project_path, "sdkconfig.defaults"), config_path])
 81  
 82              args.define_cache_entry.extend(["%s=%s" % (k, v) for k, v in new_cache_values.items()])
 83  
 84              reconfigure = base_actions["actions"]["reconfigure"]["callback"]
 85              reconfigure(None, ctx, args)
 86  
 87      # This target builds the configuration. It does not currently track dependencies,
 88      # but is good enough for CI builds if used together with clean-all-configs.
 89      # For local builds, use 'apply-config-NAME' target and then use normal 'all'
 90      # and 'flash' targets.
 91      def ut_build(ut_build_name, ctx, args):
 92          # Create a copy of the passed arguments to prevent arg modifications to accrue if
 93          # all configs are being built
 94          build_args = copy.copy(args)
 95  
 96          config_name = re.match(r"ut-build-(.*)", ut_build_name).group(1)
 97  
 98          if config_name in CONFIG_NAMES:
 99              build_args.build_dir = os.path.join(BUILDS_DIR, config_name)
100  
101              src = os.path.join(BUILDS_DIR, config_name)
102              dest = os.path.join(BINARIES_DIR, config_name)
103  
104              try:
105                  os.makedirs(dest)
106              except OSError:
107                  pass
108  
109              # Build, tweaking paths to sdkconfig and sdkconfig.defaults
110              ut_apply_config("ut-apply-config-" + config_name, ctx, build_args)
111  
112              build_target = base_actions["actions"]["all"]["callback"]
113  
114              build_target("all", ctx, build_args)
115  
116              # Copy artifacts to the output directory
117              shutil.copyfile(
118                  os.path.join(build_args.project_dir, "sdkconfig"),
119                  os.path.join(dest, "sdkconfig"),
120              )
121  
122              binaries = [PROJECT_NAME + x for x in [".elf", ".bin", ".map"]]
123  
124              for binary in binaries:
125                  shutil.copyfile(os.path.join(src, binary), os.path.join(dest, binary))
126  
127              try:
128                  os.mkdir(os.path.join(dest, "bootloader"))
129              except OSError:
130                  pass
131  
132              shutil.copyfile(
133                  os.path.join(src, "bootloader", "bootloader.bin"),
134                  os.path.join(dest, "bootloader", "bootloader.bin"),
135              )
136  
137              for partition_table in glob.glob(os.path.join(src, "partition_table", "partition-table*.bin")):
138                  try:
139                      os.mkdir(os.path.join(dest, "partition_table"))
140                  except OSError:
141                      pass
142                  shutil.copyfile(
143                      partition_table,
144                      os.path.join(dest, "partition_table", os.path.basename(partition_table)),
145                  )
146  
147              shutil.copyfile(
148                  os.path.join(src, "flasher_args.json"),
149                  os.path.join(dest, "flasher_args.json"),
150              )
151  
152              binaries = glob.glob(os.path.join(src, "*.bin"))
153              binaries = [os.path.basename(s) for s in binaries]
154  
155              for binary in binaries:
156                  shutil.copyfile(os.path.join(src, binary), os.path.join(dest, binary))
157  
158      def ut_clean(ut_clean_name, ctx, args):
159          config_name = re.match(r"ut-clean-(.*)", ut_clean_name).group(1)
160          if config_name in CONFIG_NAMES:
161              shutil.rmtree(os.path.join(BUILDS_DIR, config_name), ignore_errors=True)
162              shutil.rmtree(os.path.join(BINARIES_DIR, config_name), ignore_errors=True)
163  
164      def test_component_callback(ctx, global_args, tasks):
165          """ Convert the values passed to the -T and -E parameter to corresponding cache entry definitions TESTS_ALL and TEST_COMPONENTS """
166          test_components = global_args.test_components
167          test_exclude_components = global_args.test_exclude_components
168  
169          cache_entries = {}
170  
171          if test_components:
172              if "all" in test_components:
173                  cache_entries["TESTS_ALL"] = 1
174                  cache_entries["TEST_COMPONENTS"] = "''"
175              else:
176                  cache_entries["TESTS_ALL"] = 0
177                  cache_entries["TEST_COMPONENTS"] = " ".join(test_components)
178  
179          if test_exclude_components:
180              cache_entries["TEST_EXCLUDE_COMPONENTS"] = " ".join(test_exclude_components)
181  
182          if cache_entries:
183              global_args.define_cache_entry = list(global_args.define_cache_entry)
184              global_args.define_cache_entry.extend(["%s=%s" % (k, v) for k, v in cache_entries.items()])
185  
186      # Add global options
187      extensions = {
188          "global_options": [{
189              "names": ["-T", "--test-components"],
190              "help": "Specify the components to test.",
191              "scope": "shared",
192              "multiple": True,
193          }, {
194              "names": ["-E", "--test-exclude-components"],
195              "help": "Specify the components to exclude from testing.",
196              "scope": "shared",
197              "multiple": True,
198          }],
199          "global_action_callbacks": [test_component_callback],
200          "actions": {},
201      }
202  
203      # This generates per-config targets (clean, build, apply-config).
204      build_all_config_deps = []
205      clean_all_config_deps = []
206  
207      for config in CONFIG_NAMES:
208          config_build_action_name = "ut-build-" + config
209          config_clean_action_name = "ut-clean-" + config
210          config_apply_config_action_name = "ut-apply-config-" + config
211  
212          extensions["actions"][config_build_action_name] = {
213              "callback":
214              ut_build,
215              "help":
216              "Build unit-test-app with configuration provided in configs/NAME. " +
217              "Build directory will be builds/%s/, " % config_build_action_name +
218              "output binaries will be under output/%s/" % config_build_action_name,
219          }
220  
221          extensions["actions"][config_clean_action_name] = {
222              "callback": ut_clean,
223              "help": "Remove build and output directories for configuration %s." % config_clean_action_name,
224          }
225  
226          extensions["actions"][config_apply_config_action_name] = {
227              "callback":
228              ut_apply_config,
229              "help":
230              "Generates configuration based on configs/%s in sdkconfig file." % config_apply_config_action_name +
231              "After this, normal all/flash targets can be used. Useful for development/debugging.",
232          }
233  
234          build_all_config_deps.append(config_build_action_name)
235          clean_all_config_deps.append(config_clean_action_name)
236  
237      extensions["actions"]["ut-build-all-configs"] = {
238          "callback": ut_build,
239          "help": "Build all configurations defined in configs/ directory.",
240          "dependencies": build_all_config_deps,
241      }
242  
243      extensions["actions"]["ut-clean-all-configs"] = {
244          "callback": ut_clean,
245          "help": "Remove build and output directories for all configurations defined in configs/ directory.",
246          "dependencies": clean_all_config_deps,
247      }
248  
249      return extensions