entity_db.py
1 """Provides EntityDatabase, a class that keeps track of spec-defined entities and associated macros.""" 2 3 # Copyright (c) 2018-2019 Collabora, Ltd. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 # Author(s): Ryan Pavlik <ryan.pavlik@collabora.com> 18 19 from abc import ABC, abstractmethod 20 21 from .shared import (CATEGORIES_WITH_VALIDITY, EXTENSION_CATEGORY, 22 NON_EXISTENT_MACROS, EntityData) 23 24 25 def _entityToDict(data): 26 return { 27 'macro': data.macro, 28 'filename': data.filename, 29 'category': data.category, 30 'directory': data.directory 31 } 32 33 34 class EntityDatabase(ABC): 35 """Parsed and processed information from the registry XML. 36 37 Must be subclasses for each specific API. 38 """ 39 40 ### 41 # Methods that must be implemented in subclasses. 42 ### 43 @abstractmethod 44 def makeRegistry(self): 45 """Return a Registry object that has already had loadFile() and parseTree() called. 46 47 Called only once during construction. 48 """ 49 raise NotImplementedError 50 51 @abstractmethod 52 def getNamePrefix(self): 53 """Return the (two-letter) prefix of all entity names for this API. 54 55 Called only once during construction. 56 """ 57 raise NotImplementedError 58 59 @abstractmethod 60 def getPlatformRequires(self): 61 """Return the 'requires' string associated with external/platform definitions. 62 63 This is the string found in the requires attribute of the XML for entities that 64 are externally defined in a platform include file, like the question marks in: 65 66 <type requires="???" name="int8_t"/> 67 68 In Vulkan, this is 'vk_platform'. 69 70 Called only once during construction. 71 """ 72 raise NotImplementedError 73 74 ### 75 # Methods that it is optional to **override** 76 ### 77 def getSystemTypes(self): 78 """Return an enumerable of strings that name system types. 79 80 System types use the macro `code`, and they do not generate API/validity includes. 81 82 Called only once during construction. 83 """ 84 return [] 85 86 def getGeneratedDirs(self): 87 """Return a sequence of strings that are the subdirectories of generates API includes. 88 89 Called only once during construction. 90 """ 91 return ['basetypes', 92 'defines', 93 'enums', 94 'flags', 95 'funcpointers', 96 'handles', 97 'protos', 98 'structs'] 99 100 def populateMacros(self): 101 """Perform API-specific calls, if any, to self.addMacro() and self.addMacros(). 102 103 It is recommended to implement/override this and call 104 self.addMacros(..., ..., [..., "flags"]), 105 since the base implementation, in _basicPopulateMacros(), 106 does not add any macros as pertaining to the category "flags". 107 108 Called only once during construction. 109 """ 110 pass 111 112 def populateEntities(self): 113 """Perform API-specific calls, if any, to self.addEntity().""" 114 pass 115 116 def getEntitiesWithoutValidity(self): 117 """Return an enumerable of entity names that do not generate validity includes.""" 118 return [self.mixed_case_name_prefix + 119 x for x in ['BaseInStructure', 'BaseOutStructure']] 120 121 def getExclusionSet(self): 122 """Return a set of "support=" attribute strings that should not be included in the database. 123 124 Called only during construction.""" 125 return set(('disabled',)) 126 127 ### 128 # Methods that it is optional to **extend** 129 ### 130 def handleType(self, name, info, requires): 131 """Add entities, if appropriate, for an item in registry.typedict. 132 133 Called at construction for every name, info in registry.typedict.items() 134 not immediately skipped, 135 to perform the correct associated addEntity() call, if applicable. 136 The contents of the requires attribute, if any, is passed in requires. 137 138 May be extended by API-specific code to handle some cases preferentially, 139 then calling the super implementation to handle the rest. 140 """ 141 if requires == self.platform_requires: 142 # Ah, no, don't skip this, it's just in the platform header file. 143 # TODO are these code or basetype? 144 self.addEntity(name, 'code', elem=info.elem, generates=False) 145 return 146 147 protect = info.elem.get('protect') 148 if protect: 149 self.addEntity(protect, 'dlink', 150 category='configdefines', generates=False) 151 152 alias = info.elem.get('alias') 153 if alias: 154 self.addAlias(name, alias) 155 156 cat = info.elem.get('category') 157 if cat == 'struct': 158 self.addEntity(name, 'slink', elem=info.elem) 159 160 elif cat == 'union': 161 # TODO: is this right? 162 self.addEntity(name, 'slink', elem=info.elem) 163 164 elif cat == 'enum': 165 self.addEntity( 166 name, 'elink', elem=info.elem) 167 168 elif cat == 'handle': 169 self.addEntity(name, 'slink', elem=info.elem, 170 category='handles') 171 172 elif cat == 'bitmask': 173 self.addEntity( 174 name, 'tlink', elem=info.elem, category='flags') 175 176 elif cat == 'basetype': 177 self.addEntity(name, 'basetype', 178 elem=info.elem) 179 180 elif cat == 'define': 181 self.addEntity(name, 'dlink', elem=info.elem) 182 183 elif cat == 'funcpointer': 184 self.addEntity(name, 'tlink', elem=info.elem) 185 186 elif cat == 'include': 187 # skip 188 return 189 190 elif cat is None: 191 self.addEntity(name, 'code', elem=info.elem, generates=False) 192 193 else: 194 raise RuntimeError('unrecognized category {}'.format(cat)) 195 196 def handleCommand(self, name, info): 197 """Add entities, if appropriate, for an item in registry.cmddict. 198 199 Called at construction for every name, info in registry.cmddict.items(). 200 Calls self.addEntity() accordingly. 201 """ 202 self.addEntity(name, 'flink', elem=info.elem, 203 category='commands', directory='protos') 204 205 def handleExtension(self, name, info): 206 """Add entities, if appropriate, for an item in registry.extdict. 207 208 Called at construction for every name, info in registry.extdict.items(). 209 Calls self.addEntity() accordingly. 210 """ 211 if info.supported in self._supportExclusionSet: 212 # Don't populate with disabled extensions. 213 return 214 215 # Only get the protect strings and name from extensions 216 217 self.addEntity(name, None, category=EXTENSION_CATEGORY, 218 generates=False) 219 protect = info.elem.get('protect') 220 if protect: 221 self.addEntity(protect, 'dlink', 222 category='configdefines', generates=False) 223 224 def handleEnumValue(self, name, info): 225 """Add entities, if appropriate, for an item in registry.enumdict. 226 227 Called at construction for every name, info in registry.enumdict.items(). 228 Calls self.addEntity() accordingly. 229 """ 230 self.addEntity(name, 'ename', elem=info.elem, 231 category='enumvalues', generates=False) 232 233 ### 234 # END of methods intended to be implemented, overridden, or extended in child classes! 235 ### 236 237 ### 238 # Accessors 239 ### 240 def findMacroAndEntity(self, macro, entity): 241 """Look up EntityData by macro and entity pair. 242 243 Does **not** resolve aliases.""" 244 return self._byMacroAndEntity.get((macro, entity)) 245 246 def findEntity(self, entity): 247 """Look up EntityData by entity name (case-sensitive). 248 249 If it fails, it will try resolving aliases. 250 """ 251 result = self._byEntity.get(entity) 252 if result: 253 return result 254 255 alias_set = self._aliasSetsByEntity.get(entity) 256 if alias_set: 257 for alias in alias_set: 258 if alias in self._byEntity: 259 return self.findEntity(alias) 260 261 assert(not "Alias without main entry!") 262 263 return None 264 265 def findEntityCaseInsensitive(self, entity): 266 """Look up EntityData by entity name (case-insensitive). 267 268 Does **not** resolve aliases.""" 269 return self._byLowercaseEntity.get(entity.lower()) 270 271 def getMemberElems(self, commandOrStruct): 272 """Given a command or struct name, retrieve the ETree elements for each member/param. 273 274 Returns None if the entity is not found or doesn't have members/params. 275 """ 276 data = self.findEntity(commandOrStruct) 277 278 if not data: 279 return None 280 if data.macro == 'slink': 281 tag = 'member' 282 else: 283 tag = 'param' 284 return data.elem.findall('.//{}'.format(tag)) 285 286 def getMemberNames(self, commandOrStruct): 287 """Given a command or struct name, retrieve the names of each member/param. 288 289 Returns an empty list if the entity is not found or doesn't have members/params. 290 """ 291 members = self.getMemberElems(commandOrStruct) 292 if not members: 293 return [] 294 ret = [] 295 for member in members: 296 name_tag = member.find('name') 297 if name_tag: 298 ret.append(name_tag.text) 299 return ret 300 301 def getEntityJson(self): 302 """Dump the internal entity dictionary to JSON for debugging.""" 303 import json 304 d = {entity: _entityToDict(data) 305 for entity, data in self._byEntity.items()} 306 return json.dumps(d, sort_keys=True, indent=4) 307 308 def entityHasValidity(self, entity): 309 """Estimate if we expect to see a validity include for an entity name. 310 311 Returns None if the entity name is not known, 312 otherwise a boolean: True if a validity include is expected. 313 314 Related to ValidityGenerator.isStructAlwaysValid. 315 """ 316 data = self.findEntity(entity) 317 if not data: 318 return None 319 320 if entity in self.entities_without_validity: 321 return False 322 323 if data.category == 'protos': 324 # All protos have validity 325 return True 326 327 if data.category not in CATEGORIES_WITH_VALIDITY: 328 return False 329 330 # Handle structs here. 331 members = self.getMemberElems(entity) 332 if not members: 333 return None 334 for member in members: 335 336 if member.find('name').text in ['next', 'type']: 337 return True 338 339 if member.find('type').text in ['void', 'char']: 340 return True 341 342 if member.get('noautovalidity'): 343 # Not generating validity for this member, skip it 344 continue 345 346 if member.get('len'): 347 # Array 348 return True 349 350 typetail = member.find('type').tail 351 if typetail and '*' in typetail: 352 # Pointer 353 return True 354 355 if member.get('category') in [ 356 'handle', 'enum', 'bitmask'] == 'type': 357 return True 358 359 if member.get('category') in ['struct', 'union'] and self.entityHasValidity( 360 member.find('type').text): 361 # struct or union member - recurse 362 return True 363 364 # Got this far - no validity needed 365 return False 366 367 def entityGenerates(self, entity_name): 368 """Return True if the named entity generates include file(s).""" 369 return entity_name in self._generating_entities 370 371 @property 372 def generating_entities(self): 373 """Return a sequence of all generating entity names.""" 374 return self._generating_entities.keys() 375 376 def likelyRecognizedEntity(self, entity_name): 377 """Guess (based on name prefix alone) if an entity is likely to be recognized.""" 378 return entity_name.lower().startswith(self.name_prefix) 379 380 def isLinkedMacro(self, macro): 381 """Identify if a macro is considered a "linked" macro.""" 382 return macro in self._linkedMacros 383 384 def isValidMacro(self, macro): 385 """Identify if a macro is known and valid.""" 386 if macro not in self._categoriesByMacro: 387 return False 388 389 return macro not in NON_EXISTENT_MACROS 390 391 def getCategoriesForMacro(self, macro): 392 """Identify the categories associated with a (known, valid) macro.""" 393 if macro in self._categoriesByMacro: 394 return self._categoriesByMacro[macro] 395 return None 396 397 def areAliases(self, first_entity_name, second_entity_name): 398 """Return true if the two entity names are equivalent (aliases of each other).""" 399 alias_set = self._aliasSetsByEntity.get(first_entity_name) 400 if not alias_set: 401 # If this assert fails, we have goofed in addAlias 402 assert(second_entity_name not in self._aliasSetsByEntity) 403 404 return False 405 406 return second_entity_name in alias_set 407 408 @property 409 def macros(self): 410 """Return the collection of all known entity-related markup macros.""" 411 return self._categoriesByMacro.keys() 412 413 ### 414 # Methods only used during initial setup/population of this data structure 415 ### 416 def addMacro(self, macro, categories, link=False): 417 """Add a single markup macro to the collection of categories by macro. 418 419 Also adds the macro to the set of linked macros if link=True. 420 421 If a macro has already been supplied to a call, later calls for that macro have no effect. 422 """ 423 if macro in self._categoriesByMacro: 424 return 425 self._categoriesByMacro[macro] = categories 426 if link: 427 self._linkedMacros.add(macro) 428 429 def addMacros(self, letter, macroTypes, categories): 430 """Add markup macros associated with a leading letter to the collection of categories by macro. 431 432 Also, those macros created using 'link' in macroTypes will also be added to the set of linked macros. 433 434 Basically automates a number of calls to addMacro(). 435 """ 436 for macroType in macroTypes: 437 macro = letter + macroType 438 self.addMacro(macro, categories, link=(macroType == 'link')) 439 440 def addAlias(self, entityName, aliasName): 441 """Record that entityName is an alias for aliasName.""" 442 # See if we already have something with this as the alias. 443 alias_set = self._aliasSetsByEntity.get(aliasName) 444 other_alias_set = self._aliasSetsByEntity.get(entityName) 445 if alias_set and other_alias_set: 446 # If this fails, we need to merge sets and update. 447 assert(alias_set is other_alias_set) 448 449 if not alias_set: 450 # Try looking by the other name. 451 alias_set = other_alias_set 452 453 if not alias_set: 454 # Nope, this is a new set. 455 alias_set = set() 456 self._aliasSets.append(alias_set) 457 458 # Add both names to the set 459 alias_set.add(entityName) 460 alias_set.add(aliasName) 461 462 # Associate the set with each name 463 self._aliasSetsByEntity[aliasName] = alias_set 464 self._aliasSetsByEntity[entityName] = alias_set 465 466 def addEntity(self, entityName, macro, category=None, elem=None, 467 generates=None, directory=None, filename=None): 468 """Add an entity (command, structure type, enum, enum value, etc) in the database. 469 470 If an entityName has already been supplied to a call, later calls for that entityName have no effect. 471 472 Arguments: 473 entityName -- the name of the entity. 474 macro -- the macro (without the trailing colon) that should be used to refer to this entity. 475 476 Optional keyword arguments: 477 category -- If not manually specified, looked up based on the macro. 478 elem -- The ETree element associated with the entity in the registry XML. 479 generates -- Indicates whether this entity generates api and validity include files. 480 Default depends on directory (or if not specified, category). 481 directory -- The directory that include files (under api/ and validity/) are generated in. 482 If not specified (and generates is True), the default is the same as the category, 483 which is almost always correct. 484 filename -- The relative filename (under api/ or validity/) where includes are generated for this. 485 This only matters if generates is True (default). If not specified and generates is True, 486 one will be generated based on directory and entityName. 487 """ 488 # Probably dealt with in handleType(), but just in case it wasn't. 489 if elem is not None: 490 alias = elem.get('alias') 491 if alias: 492 self.addAlias(entityName, alias) 493 494 if entityName in self._byEntity: 495 # skip if already recorded. 496 return 497 498 # Look up category based on the macro, if category isn't specified. 499 if category is None: 500 category = self._categoriesByMacro.get(macro)[0] 501 502 if generates is None: 503 potential_dir = directory or category 504 generates = potential_dir in self._generated_dirs 505 506 # If directory isn't specified and this entity generates, 507 # the directory is the same as the category. 508 if directory is None and generates: 509 directory = category 510 511 # Don't generate a filename if this entity doesn't generate includes. 512 if filename is None and generates: 513 filename = '{}/{}.txt'.format(directory, entityName) 514 515 data = EntityData( 516 entity=entityName, 517 macro=macro, 518 elem=elem, 519 filename=filename, 520 category=category, 521 directory=directory 522 ) 523 if entityName.lower() not in self._byLowercaseEntity: 524 self._byLowercaseEntity[entityName.lower()] = [] 525 526 self._byEntity[entityName] = data 527 self._byLowercaseEntity[entityName.lower()].append(data) 528 self._byMacroAndEntity[(macro, entityName)] = data 529 if generates and filename is not None: 530 self._generating_entities[entityName] = data 531 532 def __init__(self): 533 """Constructor: Do not extend or override. 534 535 Changing the behavior of other parts of this logic should be done by 536 implementing, extending, or overriding (as documented): 537 538 - Implement makeRegistry() 539 - Implement getNamePrefix() 540 - Implement getPlatformRequires() 541 - Override getSystemTypes() 542 - Override populateMacros() 543 - Override populateEntities() 544 - Extend handleType() 545 - Extend handleCommand() 546 - Extend handleExtension() 547 - Extend handleEnumValue() 548 """ 549 # Internal data that we don't want consumers of the class touching for fear of 550 # breaking invariants 551 self._byEntity = {} 552 self._byLowercaseEntity = {} 553 self._byMacroAndEntity = {} 554 self._categoriesByMacro = {} 555 self._linkedMacros = set() 556 self._aliasSetsByEntity = {} 557 self._aliasSets = [] 558 559 # Retrieve from subclass, if overridden, then store locally. 560 self._supportExclusionSet = set(self.getExclusionSet()) 561 562 # Entities that get a generated/api/category/entity.txt file. 563 self._generating_entities = {} 564 565 # Name prefix members 566 self.name_prefix = self.getNamePrefix().lower() 567 self.mixed_case_name_prefix = self.name_prefix[:1].upper( 568 ) + self.name_prefix[1:] 569 # Regex string for the name prefix that is case-insensitive. 570 self.case_insensitive_name_prefix_pattern = ''.join( 571 ('[{}{}]'.format(c.upper(), c) for c in self.name_prefix)) 572 573 registry = self.makeRegistry() 574 self.platform_requires = self.getPlatformRequires() 575 576 self._generated_dirs = set(self.getGeneratedDirs()) 577 578 # Note: Default impl requires self.mixed_case_name_prefix 579 self.entities_without_validity = set(self.getEntitiesWithoutValidity()) 580 581 # TODO: Where should flags actually go? Not mentioned in the style guide. 582 # TODO: What about flag wildcards? There are a few such uses... 583 584 # Abstract method: subclass must implement to define macros for flags 585 self.populateMacros() 586 587 # Now, do default macro population 588 self._basicPopulateMacros() 589 590 # Abstract method: subclass must implement to add any "not from the registry" (and not system type) 591 # entities 592 self.populateEntities() 593 594 # Now, do default entity population 595 self._basicPopulateEntities(registry) 596 597 ### 598 # Methods only used internally during initial setup/population of this data structure 599 ### 600 601 def _basicPopulateMacros(self): 602 """Contains calls to self.addMacro() and self.addMacros(). 603 604 If you need to change any of these, do so in your override of populateMacros(), 605 which will be called first. 606 """ 607 self.addMacro('basetype', ['basetypes']) 608 self.addMacro('code', ['code']) 609 self.addMacros('f', ['link', 'name', 'text'], ['protos']) 610 self.addMacros('s', ['link', 'name', 'text'], ['structs', 'handles']) 611 self.addMacros('e', ['link', 'name', 'text'], ['enums']) 612 self.addMacros('p', ['name', 'text'], ['parameter', 'member']) 613 self.addMacros('t', ['link', 'name'], ['funcpointers']) 614 self.addMacros('d', ['link', 'name'], ['defines', 'configdefines']) 615 616 for macro in NON_EXISTENT_MACROS: 617 # Still search for them 618 self.addMacro(macro, None) 619 620 def _basicPopulateEntities(self, registry): 621 """Contains typical calls to self.addEntity(). 622 623 If you need to change any of these, do so in your override of populateEntities(), 624 which will be called first. 625 """ 626 system_types = set(self.getSystemTypes()) 627 for t in system_types: 628 self.addEntity(t, 'code', generates=False) 629 630 for name, info in registry.typedict.items(): 631 if name in system_types: 632 # We already added these. 633 continue 634 635 requires = info.elem.get('requires') 636 637 if requires and not requires.lower().startswith(self.name_prefix): 638 # This is an externally-defined type, will skip it. 639 continue 640 641 # OK, we might actually add an entity here 642 self.handleType(name=name, info=info, requires=requires) 643 644 for name, info in registry.enumdict.items(): 645 self.handleEnumValue(name, info) 646 647 for name, info in registry.cmddict.items(): 648 self.handleCommand(name, info) 649 650 for name, info in registry.extdict.items(): 651 self.handleExtension(name, info)