xstc.py
1 #!/usr/bin/env python 2 3 # 4 # This is the MS subset of the W3C test suite for XML Schemas. 5 # This file is generated from the MS W3c test suite description file. 6 # 7 8 import sys, os 9 import exceptions, optparse 10 import libxml2 11 12 opa = optparse.OptionParser() 13 14 opa.add_option("-b", "--base", action="store", type="string", dest="baseDir", 15 default="", 16 help="""The base directory; i.e. the parent folder of the 17 "nisttest", "suntest" and "msxsdtest" directories.""") 18 19 opa.add_option("-o", "--out", action="store", type="string", dest="logFile", 20 default="test.log", 21 help="The filepath of the log file to be created") 22 23 opa.add_option("--log", action="store_true", dest="enableLog", 24 default=False, 25 help="Create the log file") 26 27 opa.add_option("--no-test-out", action="store_true", dest="disableTestStdOut", 28 default=False, 29 help="Don't output test results") 30 31 opa.add_option("-s", "--silent", action="store_true", dest="silent", default=False, 32 help="Disables display of all tests") 33 34 opa.add_option("-v", "--verbose", action="store_true", dest="verbose", 35 default=False, 36 help="Displays all tests (only if --silent is not set)") 37 38 opa.add_option("-x", "--max", type="int", dest="maxTestCount", 39 default="-1", 40 help="The maximum number of tests to be run") 41 42 opa.add_option("-t", "--test", type="string", dest="singleTest", 43 default=None, 44 help="Runs the specified test only") 45 46 opa.add_option("--tsw", "--test-starts-with", type="string", dest="testStartsWith", 47 default=None, 48 help="Runs the specified test(s), starting with the given string") 49 50 opa.add_option("--rieo", "--report-internal-errors-only", action="store_true", 51 dest="reportInternalErrOnly", default=False, 52 help="Display erroneous tests of type 'internal' only") 53 54 opa.add_option("--rueo", "--report-unimplemented-errors-only", action="store_true", 55 dest="reportUnimplErrOnly", default=False, 56 help="Display erroneous tests of type 'unimplemented' only") 57 58 opa.add_option("--rmleo", "--report-mem-leak-errors-only", action="store_true", 59 dest="reportMemLeakErrOnly", default=False, 60 help="Display erroneous tests of type 'memory leak' only") 61 62 opa.add_option("-c", "--combines", type="string", dest="combines", 63 default=None, 64 help="Combines to be run (all if omitted)") 65 66 opa.add_option("--csw", "--csw", type="string", dest="combineStartsWith", 67 default=None, 68 help="Combines to be run (all if omitted)") 69 70 opa.add_option("--rc", "--report-combines", action="store_true", 71 dest="reportCombines", default=False, 72 help="Display combine reports") 73 74 opa.add_option("--rec", "--report-err-combines", action="store_true", 75 dest="reportErrCombines", default=False, 76 help="Display erroneous combine reports only") 77 78 opa.add_option("--debug", action="store_true", 79 dest="debugEnabled", default=False, 80 help="Displays debug messages") 81 82 opa.add_option("--info", action="store_true", 83 dest="info", default=False, 84 help="Displays info on the suite only. Does not run any test.") 85 opa.add_option("--sax", action="store_true", 86 dest="validationSAX", default=False, 87 help="Use SAX2-driven validation.") 88 opa.add_option("--tn", action="store_true", 89 dest="displayTestName", default=False, 90 help="Display the test name in every case.") 91 92 (options, args) = opa.parse_args() 93 94 if options.combines is not None: 95 options.combines = options.combines.split() 96 97 ################################################ 98 # The vars below are not intended to be changed. 99 # 100 101 msgSchemaNotValidButShould = "The schema should be valid." 102 msgSchemaValidButShouldNot = "The schema should be invalid." 103 msgInstanceNotValidButShould = "The instance should be valid." 104 msgInstanceValidButShouldNot = "The instance should be invalid." 105 vendorNIST = "NIST" 106 vendorNIST_2 = "NIST-2" 107 vendorSUN = "SUN" 108 vendorMS = "MS" 109 110 ################### 111 # Helper functions. 112 # 113 vendor = None 114 115 def handleError(test, msg): 116 global options 117 if not options.silent: 118 test.addLibLog("'%s' LIB: %s" % (test.name, msg)) 119 if msg.find("Unimplemented") > -1: 120 test.failUnimplemented() 121 elif msg.find("Internal") > -1: 122 test.failInternal() 123 124 125 def fixFileNames(fileName): 126 if (fileName is None) or (fileName == ""): 127 return "" 128 dirs = fileName.split("/") 129 if dirs[1] != "Tests": 130 fileName = os.path.join(".", "Tests") 131 for dir in dirs[1:]: 132 fileName = os.path.join(fileName, dir) 133 return fileName 134 135 class XSTCTestGroup: 136 def __init__(self, name, schemaFileName, descr): 137 global vendor, vendorNIST_2 138 self.name = name 139 self.descr = descr 140 self.mainSchema = True 141 self.schemaFileName = fixFileNames(schemaFileName) 142 self.schemaParsed = False 143 self.schemaTried = False 144 145 def setSchema(self, schemaFileName, parsed): 146 if not self.mainSchema: 147 return 148 self.mainSchema = False 149 self.schemaParsed = parsed 150 self.schemaTried = True 151 152 class XSTCTestCase: 153 154 # <!-- groupName, Name, Accepted, File, Val, Descr 155 def __init__(self, isSchema, groupName, name, accepted, file, val, descr): 156 global options 157 # 158 # Constructor. 159 # 160 self.testRunner = None 161 self.isSchema = isSchema 162 self.groupName = groupName 163 self.name = name 164 self.accepted = accepted 165 self.fileName = fixFileNames(file) 166 self.val = val 167 self.descr = descr 168 self.failed = False 169 self.combineName = None 170 171 self.log = [] 172 self.libLog = [] 173 self.initialMemUsed = 0 174 self.memLeak = 0 175 self.excepted = False 176 self.bad = False 177 self.unimplemented = False 178 self.internalErr = False 179 self.noSchemaErr = False 180 self.failed = False 181 # 182 # Init the log. 183 # 184 if not options.silent: 185 if self.descr is not None: 186 self.log.append("'%s' descr: %s\n" % (self.name, self.descr)) 187 self.log.append("'%s' exp validity: %d\n" % (self.name, self.val)) 188 189 def initTest(self, runner): 190 global vendorNIST, vendorSUN, vendorMS, vendorNIST_2, options, vendor 191 # 192 # Get the test-group. 193 # 194 self.runner = runner 195 self.group = runner.getGroup(self.groupName) 196 if vendor == vendorMS or vendor == vendorSUN: 197 # 198 # Use the last given directory for the combine name. 199 # 200 dirs = self.fileName.split("/") 201 self.combineName = dirs[len(dirs) -2] 202 elif vendor == vendorNIST: 203 # 204 # NIST files are named in the following form: 205 # "NISTSchema-short-pattern-1.xsd" 206 # 207 tokens = self.name.split("-") 208 self.combineName = tokens[1] 209 elif vendor == vendorNIST_2: 210 # 211 # Group-names have the form: "atomic-normalizedString-length-1" 212 # 213 tokens = self.groupName.split("-") 214 self.combineName = "%s-%s" % (tokens[0], tokens[1]) 215 else: 216 self.combineName = "unkown" 217 raise Exception("Could not compute the combine name of a test.") 218 if (not options.silent) and (self.group.descr is not None): 219 self.log.append("'%s' group-descr: %s\n" % (self.name, self.group.descr)) 220 221 222 def addLibLog(self, msg): 223 """This one is intended to be used by the error handler 224 function""" 225 global options 226 if not options.silent: 227 self.libLog.append(msg) 228 229 def fail(self, msg): 230 global options 231 self.failed = True 232 if not options.silent: 233 self.log.append("'%s' ( FAILED: %s\n" % (self.name, msg)) 234 235 def failNoSchema(self): 236 global options 237 self.failed = True 238 self.noSchemaErr = True 239 if not options.silent: 240 self.log.append("'%s' X NO-SCHEMA\n" % (self.name)) 241 242 def failInternal(self): 243 global options 244 self.failed = True 245 self.internalErr = True 246 if not options.silent: 247 self.log.append("'%s' * INTERNAL\n" % self.name) 248 249 def failUnimplemented(self): 250 global options 251 self.failed = True 252 self.unimplemented = True 253 if not options.silent: 254 self.log.append("'%s' ? UNIMPLEMENTED\n" % self.name) 255 256 def failCritical(self, msg): 257 global options 258 self.failed = True 259 self.bad = True 260 if not options.silent: 261 self.log.append("'%s' ! BAD: %s\n" % (self.name, msg)) 262 263 def failExcept(self, e): 264 global options 265 self.failed = True 266 self.excepted = True 267 if not options.silent: 268 self.log.append("'%s' # EXCEPTION: %s\n" % (self.name, e.__str__())) 269 270 def setUp(self): 271 # 272 # Set up Libxml2. 273 # 274 self.initialMemUsed = libxml2.debugMemory(1) 275 libxml2.initParser() 276 libxml2.lineNumbersDefault(1) 277 libxml2.registerErrorHandler(handleError, self) 278 279 def tearDown(self): 280 libxml2.schemaCleanupTypes() 281 libxml2.cleanupParser() 282 self.memLeak = libxml2.debugMemory(1) - self.initialMemUsed 283 284 def isIOError(self, file, docType): 285 err = None 286 try: 287 err = libxml2.lastError() 288 except: 289 # Suppress exceptions. 290 pass 291 if (err is None): 292 return False 293 if err.domain() == libxml2.XML_FROM_IO: 294 self.failCritical("failed to access the %s resource '%s'\n" % (docType, file)) 295 296 def debugMsg(self, msg): 297 global options 298 if options.debugEnabled: 299 sys.stdout.write("'%s' DEBUG: %s\n" % (self.name, msg)) 300 301 def finalize(self): 302 global options 303 """Adds additional info to the log.""" 304 # 305 # Add libxml2 messages. 306 # 307 if not options.silent: 308 self.log.extend(self.libLog) 309 # 310 # Add memory leaks. 311 # 312 if self.memLeak != 0: 313 self.log.append("%s + memory leak: %d bytes\n" % (self.name, self.memLeak)) 314 315 def run(self): 316 """Runs a test.""" 317 global options 318 319 ##filePath = os.path.join(options.baseDir, self.fileName) 320 # filePath = "%s/%s/%s/%s" % (options.baseDir, self.test_Folder, self.schema_Folder, self.schema_File) 321 if options.displayTestName: 322 sys.stdout.write("'%s'\n" % self.name) 323 try: 324 self.validate() 325 except (Exception, libxml2.parserError, libxml2.treeError), e: 326 self.failExcept(e) 327 328 def parseSchema(fileName): 329 schema = None 330 ctxt = libxml2.schemaNewParserCtxt(fileName) 331 try: 332 try: 333 schema = ctxt.schemaParse() 334 except: 335 pass 336 finally: 337 del ctxt 338 return schema 339 340 341 class XSTCSchemaTest(XSTCTestCase): 342 343 def __init__(self, groupName, name, accepted, file, val, descr): 344 XSTCTestCase.__init__(self, 1, groupName, name, accepted, file, val, descr) 345 346 def validate(self): 347 global msgSchemaNotValidButShould, msgSchemaValidButShouldNot 348 schema = None 349 filePath = self.fileName 350 # os.path.join(options.baseDir, self.fileName) 351 valid = 0 352 try: 353 # 354 # Parse the schema. 355 # 356 self.debugMsg("loading schema: %s" % filePath) 357 schema = parseSchema(filePath) 358 self.debugMsg("after loading schema") 359 if schema is None: 360 self.debugMsg("schema is None") 361 self.debugMsg("checking for IO errors...") 362 if self.isIOError(file, "schema"): 363 return 364 self.debugMsg("checking schema result") 365 if (schema is None and self.val) or (schema is not None and self.val == 0): 366 self.debugMsg("schema result is BAD") 367 if (schema == None): 368 self.fail(msgSchemaNotValidButShould) 369 else: 370 self.fail(msgSchemaValidButShouldNot) 371 else: 372 self.debugMsg("schema result is OK") 373 finally: 374 self.group.setSchema(self.fileName, schema is not None) 375 del schema 376 377 class XSTCInstanceTest(XSTCTestCase): 378 379 def __init__(self, groupName, name, accepted, file, val, descr): 380 XSTCTestCase.__init__(self, 0, groupName, name, accepted, file, val, descr) 381 382 def validate(self): 383 instance = None 384 schema = None 385 filePath = self.fileName 386 # os.path.join(options.baseDir, self.fileName) 387 388 if not self.group.schemaParsed and self.group.schemaTried: 389 self.failNoSchema() 390 return 391 392 self.debugMsg("loading instance: %s" % filePath) 393 parserCtxt = libxml2.newParserCtxt() 394 if (parserCtxt is None): 395 # TODO: Is this one necessary, or will an exception 396 # be already raised? 397 raise Exception("Could not create the instance parser context.") 398 if not options.validationSAX: 399 try: 400 try: 401 instance = parserCtxt.ctxtReadFile(filePath, None, libxml2.XML_PARSE_NOWARNING) 402 except: 403 # Suppress exceptions. 404 pass 405 finally: 406 del parserCtxt 407 self.debugMsg("after loading instance") 408 if instance is None: 409 self.debugMsg("instance is None") 410 self.failCritical("Failed to parse the instance for unknown reasons.") 411 return 412 try: 413 # 414 # Validate the instance. 415 # 416 self.debugMsg("loading schema: %s" % self.group.schemaFileName) 417 schema = parseSchema(self.group.schemaFileName) 418 try: 419 validationCtxt = schema.schemaNewValidCtxt() 420 #validationCtxt = libxml2.schemaNewValidCtxt(None) 421 if (validationCtxt is None): 422 self.failCritical("Could not create the validation context.") 423 return 424 try: 425 self.debugMsg("validating instance") 426 if options.validationSAX: 427 instance_Err = validationCtxt.schemaValidateFile(filePath, 0) 428 else: 429 instance_Err = validationCtxt.schemaValidateDoc(instance) 430 self.debugMsg("after instance validation") 431 self.debugMsg("instance-err: %d" % instance_Err) 432 if (instance_Err != 0 and self.val == 1) or (instance_Err == 0 and self.val == 0): 433 self.debugMsg("instance result is BAD") 434 if (instance_Err != 0): 435 self.fail(msgInstanceNotValidButShould) 436 else: 437 self.fail(msgInstanceValidButShouldNot) 438 439 else: 440 self.debugMsg("instance result is OK") 441 finally: 442 del validationCtxt 443 finally: 444 del schema 445 finally: 446 if instance is not None: 447 instance.freeDoc() 448 449 450 #################### 451 # Test runner class. 452 # 453 454 class XSTCTestRunner: 455 456 CNT_TOTAL = 0 457 CNT_RAN = 1 458 CNT_SUCCEEDED = 2 459 CNT_FAILED = 3 460 CNT_UNIMPLEMENTED = 4 461 CNT_INTERNAL = 5 462 CNT_BAD = 6 463 CNT_EXCEPTED = 7 464 CNT_MEMLEAK = 8 465 CNT_NOSCHEMA = 9 466 CNT_NOTACCEPTED = 10 467 CNT_SCHEMA_TEST = 11 468 469 def __init__(self): 470 self.logFile = None 471 self.counters = self.createCounters() 472 self.testList = [] 473 self.combinesRan = {} 474 self.groups = {} 475 self.curGroup = None 476 477 def createCounters(self): 478 counters = {self.CNT_TOTAL:0, self.CNT_RAN:0, self.CNT_SUCCEEDED:0, 479 self.CNT_FAILED:0, self.CNT_UNIMPLEMENTED:0, self.CNT_INTERNAL:0, self.CNT_BAD:0, 480 self.CNT_EXCEPTED:0, self.CNT_MEMLEAK:0, self.CNT_NOSCHEMA:0, self.CNT_NOTACCEPTED:0, 481 self.CNT_SCHEMA_TEST:0} 482 483 return counters 484 485 def addTest(self, test): 486 self.testList.append(test) 487 test.initTest(self) 488 489 def getGroup(self, groupName): 490 return self.groups[groupName] 491 492 def addGroup(self, group): 493 self.groups[group.name] = group 494 495 def updateCounters(self, test, counters): 496 if test.memLeak != 0: 497 counters[self.CNT_MEMLEAK] += 1 498 if not test.failed: 499 counters[self.CNT_SUCCEEDED] +=1 500 if test.failed: 501 counters[self.CNT_FAILED] += 1 502 if test.bad: 503 counters[self.CNT_BAD] += 1 504 if test.unimplemented: 505 counters[self.CNT_UNIMPLEMENTED] += 1 506 if test.internalErr: 507 counters[self.CNT_INTERNAL] += 1 508 if test.noSchemaErr: 509 counters[self.CNT_NOSCHEMA] += 1 510 if test.excepted: 511 counters[self.CNT_EXCEPTED] += 1 512 if not test.accepted: 513 counters[self.CNT_NOTACCEPTED] += 1 514 if test.isSchema: 515 counters[self.CNT_SCHEMA_TEST] += 1 516 return counters 517 518 def displayResults(self, out, all, combName, counters): 519 out.write("\n") 520 if all: 521 if options.combines is not None: 522 out.write("combine(s): %s\n" % str(options.combines)) 523 elif combName is not None: 524 out.write("combine : %s\n" % combName) 525 out.write(" total : %d\n" % counters[self.CNT_TOTAL]) 526 if all or options.combines is not None: 527 out.write(" ran : %d\n" % counters[self.CNT_RAN]) 528 out.write(" (schemata) : %d\n" % counters[self.CNT_SCHEMA_TEST]) 529 # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED]) 530 out.write(" not accepted : %d\n" % counters[self.CNT_NOTACCEPTED]) 531 if counters[self.CNT_FAILED] > 0: 532 out.write(" failed : %d\n" % counters[self.CNT_FAILED]) 533 out.write(" -> internal : %d\n" % counters[self.CNT_INTERNAL]) 534 out.write(" -> unimpl. : %d\n" % counters[self.CNT_UNIMPLEMENTED]) 535 out.write(" -> skip-invalid-schema : %d\n" % counters[self.CNT_NOSCHEMA]) 536 out.write(" -> bad : %d\n" % counters[self.CNT_BAD]) 537 out.write(" -> exceptions : %d\n" % counters[self.CNT_EXCEPTED]) 538 out.write(" memory leaks : %d\n" % counters[self.CNT_MEMLEAK]) 539 540 def displayShortResults(self, out, all, combName, counters): 541 out.write("Ran %d of %d tests (%d schemata):" % (counters[self.CNT_RAN], 542 counters[self.CNT_TOTAL], counters[self.CNT_SCHEMA_TEST])) 543 # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED]) 544 if counters[self.CNT_NOTACCEPTED] > 0: 545 out.write(" %d not accepted" % (counters[self.CNT_NOTACCEPTED])) 546 if counters[self.CNT_FAILED] > 0 or counters[self.CNT_MEMLEAK] > 0: 547 if counters[self.CNT_FAILED] > 0: 548 out.write(" %d failed" % (counters[self.CNT_FAILED])) 549 out.write(" (") 550 if counters[self.CNT_INTERNAL] > 0: 551 out.write(" %d internal" % (counters[self.CNT_INTERNAL])) 552 if counters[self.CNT_UNIMPLEMENTED] > 0: 553 out.write(" %d unimplemented" % (counters[self.CNT_UNIMPLEMENTED])) 554 if counters[self.CNT_NOSCHEMA] > 0: 555 out.write(" %d skip-invalid-schema" % (counters[self.CNT_NOSCHEMA])) 556 if counters[self.CNT_BAD] > 0: 557 out.write(" %d bad" % (counters[self.CNT_BAD])) 558 if counters[self.CNT_EXCEPTED] > 0: 559 out.write(" %d exception" % (counters[self.CNT_EXCEPTED])) 560 out.write(" )") 561 if counters[self.CNT_MEMLEAK] > 0: 562 out.write(" %d leaks" % (counters[self.CNT_MEMLEAK])) 563 out.write("\n") 564 else: 565 out.write(" all passed\n") 566 567 def reportCombine(self, combName): 568 global options 569 570 counters = self.createCounters() 571 # 572 # Compute evaluation counters. 573 # 574 for test in self.combinesRan[combName]: 575 counters[self.CNT_TOTAL] += 1 576 counters[self.CNT_RAN] += 1 577 counters = self.updateCounters(test, counters) 578 if options.reportErrCombines and (counters[self.CNT_FAILED] == 0) and (counters[self.CNT_MEMLEAK] == 0): 579 pass 580 else: 581 if options.enableLog: 582 self.displayResults(self.logFile, False, combName, counters) 583 self.displayResults(sys.stdout, False, combName, counters) 584 585 def displayTestLog(self, test): 586 sys.stdout.writelines(test.log) 587 sys.stdout.write("~~~~~~~~~~\n") 588 589 def reportTest(self, test): 590 global options 591 592 error = test.failed or test.memLeak != 0 593 # 594 # Only erroneous tests will be written to the log, 595 # except @verbose is switched on. 596 # 597 if options.enableLog and (options.verbose or error): 598 self.logFile.writelines(test.log) 599 self.logFile.write("~~~~~~~~~~\n") 600 # 601 # if not @silent, only erroneous tests will be 602 # written to stdout, except @verbose is switched on. 603 # 604 if not options.silent: 605 if options.reportInternalErrOnly and test.internalErr: 606 self.displayTestLog(test) 607 if options.reportMemLeakErrOnly and test.memLeak != 0: 608 self.displayTestLog(test) 609 if options.reportUnimplErrOnly and test.unimplemented: 610 self.displayTestLog(test) 611 if (options.verbose or error) and (not options.reportInternalErrOnly) and (not options.reportMemLeakErrOnly) and (not options.reportUnimplErrOnly): 612 self.displayTestLog(test) 613 614 615 def addToCombines(self, test): 616 found = False 617 if self.combinesRan.has_key(test.combineName): 618 self.combinesRan[test.combineName].append(test) 619 else: 620 self.combinesRan[test.combineName] = [test] 621 622 def run(self): 623 624 global options 625 626 if options.info: 627 for test in self.testList: 628 self.addToCombines(test) 629 sys.stdout.write("Combines: %d\n" % len(self.combinesRan)) 630 sys.stdout.write("%s\n" % self.combinesRan.keys()) 631 return 632 633 if options.enableLog: 634 self.logFile = open(options.logFile, "w") 635 try: 636 for test in self.testList: 637 self.counters[self.CNT_TOTAL] += 1 638 # 639 # Filter tests. 640 # 641 if options.singleTest is not None and options.singleTest != "": 642 if (test.name != options.singleTest): 643 continue 644 elif options.combines is not None: 645 if not options.combines.__contains__(test.combineName): 646 continue 647 elif options.testStartsWith is not None: 648 if not test.name.startswith(options.testStartsWith): 649 continue 650 elif options.combineStartsWith is not None: 651 if not test.combineName.startswith(options.combineStartsWith): 652 continue 653 654 if options.maxTestCount != -1 and self.counters[self.CNT_RAN] >= options.maxTestCount: 655 break 656 self.counters[self.CNT_RAN] += 1 657 # 658 # Run the thing, dammit. 659 # 660 try: 661 test.setUp() 662 try: 663 test.run() 664 finally: 665 test.tearDown() 666 finally: 667 # 668 # Evaluate. 669 # 670 test.finalize() 671 self.reportTest(test) 672 if options.reportCombines or options.reportErrCombines: 673 self.addToCombines(test) 674 self.counters = self.updateCounters(test, self.counters) 675 finally: 676 if options.reportCombines or options.reportErrCombines: 677 # 678 # Build a report for every single combine. 679 # 680 # TODO: How to sort a dict? 681 # 682 self.combinesRan.keys().sort(None) 683 for key in self.combinesRan.keys(): 684 self.reportCombine(key) 685 686 # 687 # Display the final report. 688 # 689 if options.silent: 690 self.displayShortResults(sys.stdout, True, None, self.counters) 691 else: 692 sys.stdout.write("===========================\n") 693 self.displayResults(sys.stdout, True, None, self.counters)