checkLinks.py
1 #!/usr/bin/python3 2 # 3 # Copyright (c) 2015-2019 The Khronos Group Inc. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # checkLinks.py - validate link/reference API constructs in files 18 # 19 # Usage: checkLinks.py [options] files > logfile 20 # 21 # Options: 22 # -follow attempt to follow include:: directives. This script isn't # an 23 # Asciidoctor processor, so only literal relative paths can # be followed. 24 # -info print some internal diagnostics. 25 # -paramcheck attempt to validate param: names against the surrounding 26 # context (the current structure/function being validated, for example). 27 # This generates many false positives, so is not enabled by default. 28 # -fatal unvalidatable links cause immediate error exit from the script. 29 # Otherwise, errors are accumulated and summarized at the end. 30 # 31 # Depends on vkapi.py, which is a Python representation of relevant parts 32 # of the Vulkan API. Only works when vkapi.py is generated for the full 33 # API, e.g. 'makeAllExts checklinks'; otherwise many false-flagged errors 34 # will occur. 35 36 import copy, os, pdb, re, string, sys 37 from vkapi import * 38 39 global curFile, curLine, sectionDepth 40 global errCount, warnCount, emittedPrefix, printInfo 41 42 curFile = '???' 43 curLine = -1 44 sectionDepth = 0 45 emittedPrefix = {} 46 printInfo = False 47 48 # Called before printing a warning or error. Only prints once prior 49 # to output for a given file. 50 def emitPrefix(): 51 global curFile, curLine, emittedPrefix 52 if (curFile not in emittedPrefix.keys()): 53 emittedPrefix[curFile] = None 54 print('Checking file:', curFile) 55 print('-------------------------------') 56 57 def info(*args, **kwargs): 58 global curFile, curLine, printInfo 59 if (printInfo): 60 61 emitPrefix() 62 print('INFO: %s line %d:' % (curFile, curLine), 63 ' '.join([str(arg) for arg in args])) 64 65 # Print a validation warning found in a file 66 def warning(*args, **kwargs): 67 global curFile, curLine, warnCount 68 69 warnCount = warnCount + 1 70 emitPrefix() 71 print('WARNING: %s line %d:' % (curFile, curLine), 72 ' '.join([str(arg) for arg in args])) 73 74 # Print a validation error found in a file 75 def error(*args, **kwargs): 76 global curFile, curLine, errCount 77 78 errCount = errCount + 1 79 emitPrefix() 80 print('ERROR: %s line %d:' % (curFile, curLine), 81 ' '.join([str(arg) for arg in args])) 82 83 # See if a tag value exists in the specified dictionary and 84 # suggest it as an alternative if so. 85 def checkTag(tag, value, dict, dictName, tagName): 86 if (value in dict.keys()): 87 warning(value, 'exists in the API but not as a', 88 tag + ': .', 'Try using the', tagName + ': tag.') 89 90 # Report an error due to an asciidoc tag which doesn't match 91 # a corresponding API entity. 92 def foundError(errType, tag, value, fatal): 93 global curFile, curLine 94 error('no such', errType, tag + ':' + value) 95 # Try some heuristics to detect likely problems such as missing vk 96 # prefixes or the wrong tag. 97 98 # Look in all the dictionaries in vkapi.py to see if the tag 99 # is just wrong but the API entity actually exists. 100 checkTag(tag, value, flags, 'flags', 'tlink/tname') 101 checkTag(tag, value, enums, 'enums', 'elink') 102 checkTag(tag, value, structs, 'structs', 'slink/sname') 103 checkTag(tag, value, handles, 'handles', 'slink/sname') 104 checkTag(tag, value, defines, 'defines', 'slink/sname') 105 checkTag(tag, value, consts, 'consts', 'ename') 106 checkTag(tag, value, protos, 'protos', 'flink/fname') 107 checkTag(tag, value, funcpointers, 'funcpointers', 'tlink/tname') 108 109 # Look for missing vk prefixes (quirky since it's case-dependent) 110 # NOT DONE YET 111 112 if fatal: 113 print('ERROR: %s line %d:' % (curFile, curLine), 114 ' '.join(['no such', errType, tag + ':' + value]), file=sys.stderr) 115 sys.exit(1) 116 117 # Look for param in the list of all parameters of the specified functions 118 # Returns True if found, False otherwise 119 def findParam(param, funclist): 120 for f in funclist: 121 if (param in protos[f]): 122 info('parameter:', param, 'found in function:', f) 123 return True 124 return False 125 126 # Initialize tracking state for checking links/includes 127 def initChecks(): 128 global curFile, curLine, curFuncs, curStruct, accumFunc, sectionDepth 129 global errCount, warnCount 130 global incPat, linkPat, pathPat, sectionPat 131 132 # Matches asciidoc single-line section tags 133 sectionPat = re.compile('^(=+) ') 134 135 # Matches any asciidoc include:: directive 136 pathPat = re.compile('^include::([\w./_]+)\[\]') 137 138 # Matches asciidoc include:: directives used in spec/ref pages (and also 139 # others such as validity). This is specific to the layout of the api/ 140 # includes and allows any path precding 'api/' followed by the category 141 # (protos, structs, enums, etc.) followed by the name of the proto, 142 # struct, etc. file. 143 incPat = re.compile('^.*api/(\w+)/(\w+)\.txt') 144 145 # Lists of current /protos/ (functions) and /structs/ includes. There 146 # can be several protos contiguously for different forms of a command 147 curFuncs = [] 148 curStruct = None 149 150 # Tag if we should accumulate funcs or start a new list. Any intervening 151 # pname: tags or struct includes will restart the list. 152 accumFunc = False 153 154 # Matches all link names in the current spec/man pages. Assumes these 155 # macro names are not trailing subsets of other macros. Used to 156 # precede the regexp with [^A-Za-z], but this didn't catch macros 157 # at start of line. 158 linkPat = re.compile('([efpst](name|link)):(\w*)') 159 160 # Total error/warning counters 161 errCount = 0 162 warnCount = 0 163 164 # Validate asciidoc internal links in specified file. 165 # infile - filename to validate 166 # follow - if True, recursively follow include:: directives 167 # paramCheck - if True, try to verify pname: refers to valid 168 # parameter/member names. This generates many false flags currently 169 # included - if True, function was called recursively 170 # fatalExit - if True, validation errors cause an error exit immediately 171 # Links checked are: 172 # fname:vkBlah - Vulkan command name (generates internal link) 173 # flink:vkBlah - Vulkan command name 174 # sname:VkBlah - Vulkan struct name (generates internal link) 175 # slink:VkBlah - Vulkan struct name 176 # elink:VkEnumName - Vulkan enumeration ('enum') type name (generates internal link) 177 # ename:VK_BLAH - Vulkan enumerant token name 178 # pname:name - parameter name to a command or a struct member 179 # tlink:name - Other Vulkan type name (generates internal link) 180 # tname:name - Other Vulkan type name 181 def checkLinks(infile, follow = False, paramCheck = True, included = False, fatalExit = False): 182 global curFile, curLine, curFuncs, curStruct, accumFunc, sectionDepth 183 global errCount, warnCount 184 global incPat, linkPat, pathPat, sectionPat 185 186 # Global state which gets saved and restored by this function 187 oldCurFile = curFile 188 oldCurLine = curLine 189 curFile = infile 190 curLine = 0 191 192 # N.b. dirname() returns an empty string for a path with no directories, 193 # unlike the shell dirname(1). 194 if (not os.path.exists(curFile)): 195 error('No such file', curFile, '- skipping check') 196 # Restore global state before exiting the function 197 curFile = oldCurFile 198 curLine = oldCurLine 199 return 200 201 inPath = os.path.dirname(curFile) 202 fp = open(curFile, 'r', encoding='utf-8') 203 204 for line in fp: 205 curLine = curLine + 1 206 207 # Track changes up and down section headers, and forget 208 # the current functions/structure when popping up a level 209 match = sectionPat.search(line) 210 if (match): 211 info('Match sectionPat for line:', line) 212 depth = len(match.group(1)) 213 if (depth < sectionDepth): 214 info('Resetting current function/structure for section:', line) 215 curFuncs = [] 216 curStruct = None 217 sectionDepth = depth 218 219 match = pathPat.search(line) 220 if (match): 221 incpath = match.group(1) 222 info('Match pathPat for line:', line) 223 info(' incpath =', incpath) 224 # An include:: directive. First check if it looks like a 225 # function or struct include file, and modify the corresponding 226 # current function or struct state accordingly. 227 match = incPat.search(incpath) 228 if (match): 229 info('Match incPat for line:', line) 230 # For prototypes, if it is preceded by 231 # another include:: directive with no intervening link: tags, 232 # add to the current function list. Otherwise start a new list. 233 # There is only one current structure. 234 category = match.group(1) 235 tag = match.group(2) 236 # @ Validate tag! 237 # @ Arguably, any intervening text should shift to accumFuncs = False, 238 # e.g. only back-to-back includes separated by blank lines would be 239 # accumulated. 240 if (category == 'protos'): 241 if (tag in protos.keys()): 242 if (accumFunc): 243 curFuncs.append(tag) 244 else: 245 curFuncs = [ tag ] 246 # Restart accumulating functions 247 accumFunc = True 248 info('curFuncs =', curFuncs, 'accumFunc =', accumFunc) 249 else: 250 error('include of nonexistent function', tag) 251 elif (category == 'structs'): 252 if (tag in structs.keys()): 253 curStruct = tag 254 # Any /structs/ include means to stop accumulating /protos/ 255 accumFunc = False 256 info('curStruct =', curStruct) 257 else: 258 error('include of nonexistent struct', tag) 259 if (follow): 260 # Actually process the included file now, recursively 261 newpath = os.path.normpath(os.path.join(inPath, incpath)) 262 info(curFile, ': including file:', newpath) 263 checkLinks(newpath, follow, paramCheck, included = True, fatalExit = fatalExit) 264 265 matches = linkPat.findall(line) 266 for match in matches: 267 # Start actual validation work. Depending on what the 268 # asciidoc tag name is, look up the value in the corresponding 269 # dictionary. 270 tag = match[0] 271 value = match[2] 272 if (tag == 'fname' or tag == 'flink'): 273 if (value not in protos.keys()): 274 foundError('function', tag, value, False) 275 elif (tag == 'sname' or tag == 'slink'): 276 if (value not in structs.keys() and 277 value not in handles.keys()): 278 foundError('aggregate/scalar/handle/define type', tag, value, False) 279 elif (tag == 'ename'): 280 if (value not in consts.keys() and value not in defines.keys()): 281 foundError('enumerant/constant', tag, value, False) 282 elif (tag == 'elink'): 283 if (value not in enums.keys() and value not in flags.keys()): 284 foundError('enum/bitflag type', tag, value, fatalExit) 285 # tname and tlink are the same except if the errors are treated as fatal 286 # They can be recombined once both are error-clean 287 elif (tag == 'tname'): 288 if (value not in funcpointers.keys() and value not in flags.keys()): 289 foundError('function pointer/other type', tag, value, fatalExit) 290 elif (tag == 'tlink'): 291 if (value not in funcpointers.keys() and value not in flags.keys()): 292 foundError('function pointer/other type', tag, value, False) 293 elif (tag == 'pname'): 294 # Any pname: tag means to stop accumulating /protos/ 295 accumFunc = False 296 # See if this parameter is in the current proto(s) and struct 297 foundParam = False 298 if (curStruct and value in structs[curStruct]): 299 info('parameter', value, 'found in struct', curStruct) 300 elif (curFuncs and findParam(value, curFuncs)): 301 True 302 else: 303 if paramCheck: 304 warning('parameter', value, 'not found. curStruct =', 305 curStruct, 'curFuncs =', curFuncs) 306 else: 307 # This is a logic error 308 error('unknown tag', tag + ':' + value) 309 fp.close() 310 311 if (errCount > 0 or warnCount > 0): 312 if (not included): 313 print('Errors found:', errCount, 'Warnings found:', warnCount) 314 print('') 315 316 if (included): 317 info('----- returning from:', infile, 'to parent file', '-----') 318 319 # Don't generate any output for files without errors 320 # else: 321 # print(curFile + ': No errors found') 322 323 # Restore global state before exiting the function 324 curFile = oldCurFile 325 curLine = oldCurLine 326 327 if __name__ == '__main__': 328 follow = False 329 paramCheck = False 330 included = False 331 fatalExit = False 332 333 totalErrCount = 0 334 totalWarnCount = 0 335 336 if (len(sys.argv) > 1): 337 for file in sys.argv[1:]: 338 if (file == '-follow'): 339 follow = True 340 elif (file == '-info'): 341 printInfo = True 342 elif file == '-paramcheck': 343 paramCheck = True 344 elif (file == '-fatal'): 345 fatalExit = True 346 else: 347 initChecks() 348 checkLinks(file, 349 follow, 350 paramCheck = paramCheck, 351 included = included, 352 fatalExit = fatalExit) 353 totalErrCount = totalErrCount + errCount 354 totalWarnCount = totalWarnCount + warnCount 355 else: 356 print('Need arguments: [-follow] [-info] [-paramcheck] [-fatal] infile [infile...]', file=sys.stderr) 357 358 if (totalErrCount > 0 or totalWarnCount > 0): 359 if (not included): 360 print('TOTAL Errors found:', totalErrCount, 'Warnings found:', 361 totalWarnCount) 362 if totalErrCount > 0: 363 sys.exit(1)