check_kconfigs.py
1 #!/usr/bin/env python 2 # 3 # Copyright 2018 Espressif Systems (Shanghai) PTE LTD 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 from __future__ import print_function 18 from __future__ import unicode_literals 19 import os 20 import sys 21 import re 22 import argparse 23 from io import open 24 25 # regular expression for matching Kconfig files 26 RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$' 27 28 # ouput file with suggestions will get this suffix 29 OUTPUT_SUFFIX = '.new' 30 31 # ignored directories (makes sense only when run on IDF_PATH) 32 # Note: IGNORE_DIRS is a tuple in order to be able to use it directly with the startswith() built-in function which 33 # accepts tuples but no lists. 34 IGNORE_DIRS = ( 35 # Kconfigs from submodules need to be ignored: 36 os.path.join('components', 'mqtt', 'esp-mqtt'), 37 # Test Kconfigs are also ignored 38 os.path.join('tools', 'ldgen', 'test', 'data'), 39 os.path.join('tools', 'kconfig_new', 'test'), 40 ) 41 42 SPACES_PER_INDENT = 4 43 44 CONFIG_NAME_MAX_LENGTH = 40 45 46 CONFIG_NAME_MIN_PREFIX_LENGTH = 3 47 48 # The checker will not fail if it encounters this string (it can be used for temporarily resolve conflicts) 49 RE_NOERROR = re.compile(r'\s+#\s+NOERROR\s+$') 50 51 # list or rules for lines 52 LINE_ERROR_RULES = [ 53 # (regular expression for finding, error message, correction) 54 (re.compile(r'\t'), 'tabulators should be replaced by spaces', r' ' * SPACES_PER_INDENT), 55 (re.compile(r'\s+\n'), 'trailing whitespaces should be removed', r'\n'), 56 (re.compile(r'.{120}'), 'line should be shorter than 120 characters', None), 57 # "\<CR><LF>" is not recognized due to a bug in tools/kconfig/zconf.l. The bug was fixed but the rebuild of 58 # mconf-idf is not enforced and an incorrect version is supplied with all previous IDF versions. Backslashes 59 # cannot be enabled unless everybody updates mconf-idf. 60 (re.compile(r'\\\n'), 'line cannot be wrapped by backslash', None), 61 ] 62 63 64 class InputError(RuntimeError): 65 """ 66 Represents and error on the input 67 """ 68 def __init__(self, path, line_number, error_msg, suggested_line): 69 super(InputError, self).__init__('{}:{}: {}'.format(path, line_number, error_msg)) 70 self.suggested_line = suggested_line 71 72 73 class BaseChecker(object): 74 """ 75 Base class for all checker objects 76 """ 77 def __init__(self, path_in_idf): 78 self.path_in_idf = path_in_idf 79 80 def __enter__(self): 81 return self 82 83 def __exit__(self, type, value, traceback): 84 pass 85 86 87 class SourceChecker(BaseChecker): 88 # allow to source only files which will be also checked by the script 89 # Note: The rules are complex and the LineRuleChecker cannot be used 90 def process_line(self, line, line_number): 91 m = re.search(r'^\s*source(\s*)"([^"]+)"', line) 92 if m: 93 if len(m.group(1)) == 0: 94 raise InputError(self.path_in_idf, line_number, '"source" has to been followed by space', 95 line.replace('source', 'source ')) 96 path = m.group(2) 97 filename = os.path.basename(path) 98 if path in ['$COMPONENT_KCONFIGS_SOURCE_FILE', '$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']: 99 pass 100 elif not filename.startswith('Kconfig.'): 101 raise InputError(self.path_in_idf, line_number, "only filenames starting with Kconfig.* can be sourced", 102 line.replace(path, os.path.join(os.path.dirname(path), 'Kconfig.' + filename))) 103 104 105 class LineRuleChecker(BaseChecker): 106 """ 107 checks LINE_ERROR_RULES for each line 108 """ 109 def process_line(self, line, line_number): 110 suppress_errors = RE_NOERROR.search(line) is not None 111 errors = [] 112 for rule in LINE_ERROR_RULES: 113 m = rule[0].search(line) 114 if m: 115 if suppress_errors: 116 # just print but no failure 117 e = InputError(self.path_in_idf, line_number, rule[1], line) 118 print(e) 119 else: 120 errors.append(rule[1]) 121 if rule[2]: 122 line = rule[0].sub(rule[2], line) 123 if len(errors) > 0: 124 raise InputError(self.path_in_idf, line_number, "; ".join(errors), line) 125 126 127 class IndentAndNameChecker(BaseChecker): 128 """ 129 checks the indentation of each line and configuration names 130 """ 131 def __init__(self, path_in_idf, debug=False): 132 super(IndentAndNameChecker, self).__init__(path_in_idf) 133 self.debug = debug 134 self.min_prefix_length = CONFIG_NAME_MIN_PREFIX_LENGTH 135 136 # stack of the nested menuconfig items, e.g. ['mainmenu', 'menu', 'config'] 137 self.level_stack = [] 138 139 # stack common prefixes of configs 140 self.prefix_stack = [] 141 142 # if the line ends with '\' then we force the indent of the next line 143 self.force_next_indent = 0 144 145 # menu items which increase the indentation of the next line 146 self.re_increase_level = re.compile(r'''^\s* 147 ( 148 (menu(?!config)) 149 |(mainmenu) 150 |(choice) 151 |(config) 152 |(menuconfig) 153 |(help) 154 |(if) 155 |(source) 156 ) 157 ''', re.X) 158 159 # closing menu items which decrease the indentation 160 self.re_decrease_level = re.compile(r'''^\s* 161 ( 162 (endmenu) 163 |(endchoice) 164 |(endif) 165 ) 166 ''', re.X) 167 168 # matching beginning of the closing menuitems 169 self.pair_dic = {'endmenu': 'menu', 170 'endchoice': 'choice', 171 'endif': 'if', 172 } 173 174 # regex for config names 175 self.re_name = re.compile(r'''^ 176 ( 177 (?:config) 178 |(?:menuconfig) 179 |(?:choice) 180 181 )\s+ 182 (\w+) 183 ''', re.X) 184 185 # regex for new prefix stack 186 self.re_new_stack = re.compile(r'''^ 187 ( 188 (?:menu(?!config)) 189 |(?:mainmenu) 190 |(?:choice) 191 192 ) 193 ''', re.X) 194 195 def __exit__(self, type, value, traceback): 196 super(IndentAndNameChecker, self).__exit__(type, value, traceback) 197 if len(self.prefix_stack) > 0: 198 self.check_common_prefix('', 'EOF') 199 if len(self.prefix_stack) != 0: 200 if self.debug: 201 print(self.prefix_stack) 202 raise RuntimeError("Prefix stack should be empty. Perhaps a menu/choice hasn't been closed") 203 204 def del_from_level_stack(self, count): 205 """ delete count items from the end of the level_stack """ 206 if count > 0: 207 # del self.level_stack[-0:] would delete everything and we expect not to delete anything for count=0 208 del self.level_stack[-count:] 209 210 def update_level_for_inc_pattern(self, new_item): 211 if self.debug: 212 print('level+', new_item, ': ', self.level_stack, end=' -> ') 213 # "config" and "menuconfig" don't have a closing pair. So if new_item is an item which need to be indented 214 # outside the last "config" or "menuconfig" then we need to find to a parent where it belongs 215 if new_item in ['config', 'menuconfig', 'menu', 'choice', 'if', 'source']: 216 # item is not belonging to a previous "config" or "menuconfig" so need to indent to parent 217 for i, item in enumerate(reversed(self.level_stack)): 218 if item in ['menu', 'mainmenu', 'choice', 'if']: 219 # delete items ("config", "menuconfig", "help") until the appropriate parent 220 self.del_from_level_stack(i) 221 break 222 else: 223 # delete everything when configs are at top level without a parent menu, mainmenu... 224 self.del_from_level_stack(len(self.level_stack)) 225 226 self.level_stack.append(new_item) 227 if self.debug: 228 print(self.level_stack) 229 # The new indent is for the next line. Use the old one for the current line: 230 return len(self.level_stack) - 1 231 232 def update_level_for_dec_pattern(self, new_item): 233 if self.debug: 234 print('level-', new_item, ': ', self.level_stack, end=' -> ') 235 target = self.pair_dic[new_item] 236 for i, item in enumerate(reversed(self.level_stack)): 237 # find the matching beginning for the closing item in reverse-order search 238 # Note: "menuconfig", "config" and "help" don't have closing pairs and they are also on the stack. Now they 239 # will be deleted together with the "menu" or "choice" we are closing. 240 if item == target: 241 i += 1 # delete also the matching beginning 242 if self.debug: 243 print('delete ', i, end=' -> ') 244 self.del_from_level_stack(i) 245 break 246 if self.debug: 247 print(self.level_stack) 248 return len(self.level_stack) 249 250 def check_name_and_update_prefix(self, line, line_number): 251 m = self.re_name.search(line) 252 if m: 253 name = m.group(2) 254 name_length = len(name) 255 256 if name_length > CONFIG_NAME_MAX_LENGTH: 257 raise InputError(self.path_in_idf, line_number, 258 '{} is {} characters long and it should be {} at most' 259 ''.format(name, name_length, CONFIG_NAME_MAX_LENGTH), 260 line + '\n') # no suggested correction for this 261 if len(self.prefix_stack) == 0: 262 self.prefix_stack.append(name) 263 elif self.prefix_stack[-1] is None: 264 self.prefix_stack[-1] = name 265 else: 266 # this has nothing common with paths but the algorithm can be used for this also 267 self.prefix_stack[-1] = os.path.commonprefix([self.prefix_stack[-1], name]) 268 if self.debug: 269 print('prefix+', self.prefix_stack) 270 m = self.re_new_stack.search(line) 271 if m: 272 self.prefix_stack.append(None) 273 if self.debug: 274 print('prefix+', self.prefix_stack) 275 276 def check_common_prefix(self, line, line_number): 277 common_prefix = self.prefix_stack.pop() 278 if self.debug: 279 print('prefix-', self.prefix_stack) 280 if common_prefix is None: 281 return 282 common_prefix_len = len(common_prefix) 283 if common_prefix_len < self.min_prefix_length: 284 raise InputError(self.path_in_idf, line_number, 285 'The common prefix for the config names of the menu ending at this line is "{}". ' 286 'All config names in this menu should start with the same prefix of {} characters ' 287 'or more.'.format(common_prefix, self.min_prefix_length), 288 line) # no suggested correction for this 289 if len(self.prefix_stack) > 0: 290 parent_prefix = self.prefix_stack[-1] 291 if parent_prefix is None: 292 # propagate to parent level where it will influence the prefix checking with the rest which might 293 # follow later on that level 294 self.prefix_stack[-1] = common_prefix 295 else: 296 if len(self.level_stack) > 0 and self.level_stack[-1] in ['mainmenu', 'menu']: 297 # the prefix from menu is not required to propagate to the children 298 return 299 if not common_prefix.startswith(parent_prefix): 300 raise InputError(self.path_in_idf, line_number, 301 'Common prefix "{}" should start with {}' 302 ''.format(common_prefix, parent_prefix), 303 line) # no suggested correction for this 304 305 def process_line(self, line, line_number): 306 stripped_line = line.strip() 307 if len(stripped_line) == 0: 308 self.force_next_indent = 0 309 return 310 current_level = len(self.level_stack) 311 m = re.search(r'\S', line) # indent found as the first non-space character 312 if m: 313 current_indent = m.start() 314 else: 315 current_indent = 0 316 317 if current_level > 0 and self.level_stack[-1] == 'help': 318 if current_indent >= current_level * SPACES_PER_INDENT: 319 # this line belongs to 'help' 320 self.force_next_indent = 0 321 return 322 323 if self.force_next_indent > 0: 324 if current_indent != self.force_next_indent: 325 raise InputError(self.path_in_idf, line_number, 326 'Indentation consists of {} spaces instead of {}'.format(current_indent, 327 self.force_next_indent), 328 (' ' * self.force_next_indent) + line.lstrip()) 329 else: 330 if not stripped_line.endswith('\\'): 331 self.force_next_indent = 0 332 return 333 334 elif stripped_line.endswith('\\') and stripped_line.startswith(('config', 'menuconfig', 'choice')): 335 raise InputError(self.path_in_idf, line_number, 336 'Line-wrap with backslash is not supported here', 337 line) # no suggestion for this 338 339 self.check_name_and_update_prefix(stripped_line, line_number) 340 341 m = self.re_increase_level.search(line) 342 if m: 343 current_level = self.update_level_for_inc_pattern(m.group(1)) 344 else: 345 m = self.re_decrease_level.search(line) 346 if m: 347 new_item = m.group(1) 348 current_level = self.update_level_for_dec_pattern(new_item) 349 if new_item not in ['endif']: 350 # endif doesn't require to check the prefix because the items inside if/endif belong to the 351 # same prefix level 352 self.check_common_prefix(line, line_number) 353 354 expected_indent = current_level * SPACES_PER_INDENT 355 356 if stripped_line.endswith('\\'): 357 self.force_next_indent = expected_indent + SPACES_PER_INDENT 358 else: 359 self.force_next_indent = 0 360 361 if current_indent != expected_indent: 362 raise InputError(self.path_in_idf, line_number, 363 'Indentation consists of {} spaces instead of {}'.format(current_indent, expected_indent), 364 (' ' * expected_indent) + line.lstrip()) 365 366 367 def valid_directory(path): 368 if not os.path.isdir(path): 369 raise argparse.ArgumentTypeError("{} is not a valid directory!".format(path)) 370 return path 371 372 373 def main(): 374 default_path = os.getenv('IDF_PATH', None) 375 376 parser = argparse.ArgumentParser(description='Kconfig style checker') 377 parser.add_argument('--verbose', '-v', help='Print more information (useful for debugging)', 378 action='store_true', default=False) 379 parser.add_argument('--directory', '-d', help='Path to directory where Kconfigs should be recursively checked ' 380 '(for example $IDF_PATH)', 381 type=valid_directory, 382 required=default_path is None, 383 default=default_path) 384 args = parser.parse_args() 385 386 success_couter = 0 387 ignore_counter = 0 388 failure = False 389 390 # IGNORE_DIRS makes sense when the required directory is IDF_PATH 391 check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path) 392 393 for root, dirnames, filenames in os.walk(args.directory): 394 for filename in filenames: 395 full_path = os.path.join(root, filename) 396 path_in_idf = os.path.relpath(full_path, args.directory) 397 if re.search(RE_KCONFIG, filename): 398 if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS): 399 print('{}: Ignored'.format(path_in_idf)) 400 ignore_counter += 1 401 continue 402 suggestions_full_path = full_path + OUTPUT_SUFFIX 403 with open(full_path, 'r', encoding='utf-8') as f, \ 404 open(suggestions_full_path, 'w', encoding='utf-8', newline='\n') as f_o, \ 405 LineRuleChecker(path_in_idf) as line_checker, \ 406 SourceChecker(path_in_idf) as source_checker, \ 407 IndentAndNameChecker(path_in_idf, debug=args.verbose) as indent_and_name_checker: 408 try: 409 for line_number, line in enumerate(f, start=1): 410 try: 411 for checker in [line_checker, indent_and_name_checker, source_checker]: 412 checker.process_line(line, line_number) 413 # The line is correct therefore we echo it to the output file 414 f_o.write(line) 415 except InputError as e: 416 print(e) 417 failure = True 418 f_o.write(e.suggested_line) 419 except UnicodeDecodeError: 420 raise ValueError("The encoding of {} is not Unicode.".format(path_in_idf)) 421 422 if failure: 423 print('{} has been saved with suggestions for resolving the issues. Please note that the ' 424 'suggestions can be wrong and you might need to re-run the checker several times ' 425 'for solving all issues'.format(path_in_idf + OUTPUT_SUFFIX)) 426 print('Please fix the errors and run {} for checking the correctness of ' 427 'Kconfigs.'.format(os.path.relpath(os.path.abspath(__file__), args.directory))) 428 sys.exit(1) 429 else: 430 success_couter += 1 431 print('{}: OK'.format(path_in_idf)) 432 try: 433 os.remove(suggestions_full_path) 434 except Exception: 435 # not a serious error is when the file cannot be deleted 436 print('{} cannot be deleted!'.format(suggestions_full_path)) 437 elif re.search(RE_KCONFIG, filename, re.IGNORECASE): 438 # On Windows Kconfig files are working with different cases! 439 raise ValueError('Incorrect filename of {}. The case should be "Kconfig"!'.format(path_in_idf)) 440 441 if ignore_counter > 0: 442 print('{} files have been ignored.'.format(ignore_counter)) 443 444 if success_couter > 0: 445 print('{} files have been successfully checked.'.format(success_couter)) 446 447 448 if __name__ == "__main__": 449 main()