check-xsddata-test-suite.py
1 #!/usr/bin/python 2 import sys 3 import time 4 import os 5 import string 6 import StringIO 7 sys.path.insert(0, "python") 8 import libxml2 9 10 # Memory debug specific 11 libxml2.debugMemory(1) 12 debug = 0 13 verbose = 0 14 quiet = 1 15 16 # 17 # the testsuite description 18 # 19 CONF=os.path.join(os.path.dirname(__file__), "test/xsdtest/xsdtestsuite.xml") 20 LOG="check-xsddata-test-suite.log" 21 22 log = open(LOG, "w") 23 nb_schemas_tests = 0 24 nb_schemas_success = 0 25 nb_schemas_failed = 0 26 nb_instances_tests = 0 27 nb_instances_success = 0 28 nb_instances_failed = 0 29 30 libxml2.lineNumbersDefault(1) 31 # 32 # Error and warnng callbacks 33 # 34 def callback(ctx, str): 35 global log 36 log.write("%s%s" % (ctx, str)) 37 38 libxml2.registerErrorHandler(callback, "") 39 40 # 41 # Resolver callback 42 # 43 resources = {} 44 def resolver(URL, ID, ctxt): 45 global resources 46 47 if resources.has_key(URL): 48 return(StringIO.StringIO(resources[URL])) 49 log.write("Resolver failure: asked %s\n" % (URL)) 50 log.write("resources: %s\n" % (resources)) 51 return None 52 53 # 54 # handle a valid instance 55 # 56 def handle_valid(node, schema): 57 global log 58 global nb_instances_success 59 global nb_instances_failed 60 61 instance = node.prop("dtd") 62 if instance == None: 63 instance = "" 64 child = node.children 65 while child != None: 66 if child.type != 'text': 67 instance = instance + child.serialize() 68 child = child.next 69 70 mem = libxml2.debugMemory(1); 71 try: 72 doc = libxml2.parseDoc(instance) 73 except: 74 doc = None 75 76 if doc == None: 77 log.write("\nFailed to parse correct instance:\n-----\n") 78 log.write(instance) 79 log.write("\n-----\n") 80 nb_instances_failed = nb_instances_failed + 1 81 return 82 83 if debug: 84 print "instance line %d" % (node.lineNo()) 85 86 try: 87 ctxt = schema.relaxNGNewValidCtxt() 88 ret = doc.relaxNGValidateDoc(ctxt) 89 del ctxt 90 except: 91 ret = -1 92 93 doc.freeDoc() 94 if mem != libxml2.debugMemory(1): 95 print "validating instance %d line %d leaks" % ( 96 nb_instances_tests, node.lineNo()) 97 98 if ret != 0: 99 log.write("\nFailed to validate correct instance:\n-----\n") 100 log.write(instance) 101 log.write("\n-----\n") 102 nb_instances_failed = nb_instances_failed + 1 103 else: 104 nb_instances_success = nb_instances_success + 1 105 106 # 107 # handle an invalid instance 108 # 109 def handle_invalid(node, schema): 110 global log 111 global nb_instances_success 112 global nb_instances_failed 113 114 instance = node.prop("dtd") 115 if instance == None: 116 instance = "" 117 child = node.children 118 while child != None: 119 if child.type != 'text': 120 instance = instance + child.serialize() 121 child = child.next 122 123 # mem = libxml2.debugMemory(1); 124 125 try: 126 doc = libxml2.parseDoc(instance) 127 except: 128 doc = None 129 130 if doc == None: 131 log.write("\nStrange: failed to parse incorrect instance:\n-----\n") 132 log.write(instance) 133 log.write("\n-----\n") 134 return 135 136 if debug: 137 print "instance line %d" % (node.lineNo()) 138 139 try: 140 ctxt = schema.relaxNGNewValidCtxt() 141 ret = doc.relaxNGValidateDoc(ctxt) 142 del ctxt 143 144 except: 145 ret = -1 146 147 doc.freeDoc() 148 # if mem != libxml2.debugMemory(1): 149 # print "validating instance %d line %d leaks" % ( 150 # nb_instances_tests, node.lineNo()) 151 152 if ret == 0: 153 log.write("\nFailed to detect validation problem in instance:\n-----\n") 154 log.write(instance) 155 log.write("\n-----\n") 156 nb_instances_failed = nb_instances_failed + 1 157 else: 158 nb_instances_success = nb_instances_success + 1 159 160 # 161 # handle an incorrect test 162 # 163 def handle_correct(node): 164 global log 165 global nb_schemas_success 166 global nb_schemas_failed 167 168 schema = "" 169 child = node.children 170 while child != None: 171 if child.type != 'text': 172 schema = schema + child.serialize() 173 child = child.next 174 175 try: 176 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 177 rngs = rngp.relaxNGParse() 178 except: 179 rngs = None 180 if rngs == None: 181 log.write("\nFailed to compile correct schema:\n-----\n") 182 log.write(schema) 183 log.write("\n-----\n") 184 nb_schemas_failed = nb_schemas_failed + 1 185 else: 186 nb_schemas_success = nb_schemas_success + 1 187 return rngs 188 189 def handle_incorrect(node): 190 global log 191 global nb_schemas_success 192 global nb_schemas_failed 193 194 schema = "" 195 child = node.children 196 while child != None: 197 if child.type != 'text': 198 schema = schema + child.serialize() 199 child = child.next 200 201 try: 202 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 203 rngs = rngp.relaxNGParse() 204 except: 205 rngs = None 206 if rngs != None: 207 log.write("\nFailed to detect schema error in:\n-----\n") 208 log.write(schema) 209 log.write("\n-----\n") 210 nb_schemas_failed = nb_schemas_failed + 1 211 else: 212 # log.write("\nSuccess detecting schema error in:\n-----\n") 213 # log.write(schema) 214 # log.write("\n-----\n") 215 nb_schemas_success = nb_schemas_success + 1 216 return None 217 218 # 219 # resource handling: keep a dictionary of URL->string mappings 220 # 221 def handle_resource(node, dir): 222 global resources 223 224 try: 225 name = node.prop('name') 226 except: 227 name = None 228 229 if name == None or name == '': 230 log.write("resource has no name") 231 return; 232 233 if dir != None: 234 # name = libxml2.buildURI(name, dir) 235 name = dir + '/' + name 236 237 res = "" 238 child = node.children 239 while child != None: 240 if child.type != 'text': 241 res = res + child.serialize() 242 child = child.next 243 resources[name] = res 244 245 # 246 # dir handling: pseudo directory resources 247 # 248 def handle_dir(node, dir): 249 try: 250 name = node.prop('name') 251 except: 252 name = None 253 254 if name == None or name == '': 255 log.write("resource has no name") 256 return; 257 258 if dir != None: 259 # name = libxml2.buildURI(name, dir) 260 name = dir + '/' + name 261 262 dirs = node.xpathEval('dir') 263 for dir in dirs: 264 handle_dir(dir, name) 265 res = node.xpathEval('resource') 266 for r in res: 267 handle_resource(r, name) 268 269 # 270 # handle a testCase element 271 # 272 def handle_testCase(node): 273 global nb_schemas_tests 274 global nb_instances_tests 275 global resources 276 277 sections = node.xpathEval('string(section)') 278 log.write("\n ======== test %d line %d section %s ==========\n" % ( 279 280 nb_schemas_tests, node.lineNo(), sections)) 281 resources = {} 282 if debug: 283 print "test %d line %d" % (nb_schemas_tests, node.lineNo()) 284 285 dirs = node.xpathEval('dir') 286 for dir in dirs: 287 handle_dir(dir, None) 288 res = node.xpathEval('resource') 289 for r in res: 290 handle_resource(r, None) 291 292 tsts = node.xpathEval('incorrect') 293 if tsts != []: 294 if len(tsts) != 1: 295 print "warning test line %d has more than one <incorrect> example" %(node.lineNo()) 296 schema = handle_incorrect(tsts[0]) 297 else: 298 tsts = node.xpathEval('correct') 299 if tsts != []: 300 if len(tsts) != 1: 301 print "warning test line %d has more than one <correct> example"% (node.lineNo()) 302 schema = handle_correct(tsts[0]) 303 else: 304 print "warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo()) 305 306 nb_schemas_tests = nb_schemas_tests + 1; 307 308 valids = node.xpathEval('valid') 309 invalids = node.xpathEval('invalid') 310 nb_instances_tests = nb_instances_tests + len(valids) + len(invalids) 311 if schema != None: 312 for valid in valids: 313 handle_valid(valid, schema) 314 for invalid in invalids: 315 handle_invalid(invalid, schema) 316 317 318 # 319 # handle a testSuite element 320 # 321 def handle_testSuite(node, level = 0): 322 global nb_schemas_tests, nb_schemas_success, nb_schemas_failed 323 global nb_instances_tests, nb_instances_success, nb_instances_failed 324 if verbose and level >= 0: 325 old_schemas_tests = nb_schemas_tests 326 old_schemas_success = nb_schemas_success 327 old_schemas_failed = nb_schemas_failed 328 old_instances_tests = nb_instances_tests 329 old_instances_success = nb_instances_success 330 old_instances_failed = nb_instances_failed 331 332 docs = node.xpathEval('documentation') 333 authors = node.xpathEval('author') 334 if docs != []: 335 msg = "" 336 for doc in docs: 337 msg = msg + doc.content + " " 338 if authors != []: 339 msg = msg + "written by " 340 for author in authors: 341 msg = msg + author.content + " " 342 if quiet == 0: 343 print msg 344 sections = node.xpathEval('section') 345 if verbose and sections != [] and level <= 0: 346 msg = "" 347 for section in sections: 348 msg = msg + section.content + " " 349 if quiet == 0: 350 print "Tests for section %s" % (msg) 351 for test in node.xpathEval('testCase'): 352 handle_testCase(test) 353 for test in node.xpathEval('testSuite'): 354 handle_testSuite(test, level + 1) 355 356 357 if verbose and level >= 0 : 358 if sections != []: 359 msg = "" 360 for section in sections: 361 msg = msg + section.content + " " 362 print "Result of tests for section %s" % (msg) 363 elif docs != []: 364 msg = "" 365 for doc in docs: 366 msg = msg + doc.content + " " 367 print "Result of tests for %s" % (msg) 368 369 if nb_schemas_tests != old_schemas_tests: 370 print "found %d test schemas: %d success %d failures" % ( 371 nb_schemas_tests - old_schemas_tests, 372 nb_schemas_success - old_schemas_success, 373 nb_schemas_failed - old_schemas_failed) 374 if nb_instances_tests != old_instances_tests: 375 print "found %d test instances: %d success %d failures" % ( 376 nb_instances_tests - old_instances_tests, 377 nb_instances_success - old_instances_success, 378 nb_instances_failed - old_instances_failed) 379 # 380 # Parse the conf file 381 # 382 libxml2.substituteEntitiesDefault(1); 383 testsuite = libxml2.parseFile(CONF) 384 385 # 386 # Error and warnng callbacks 387 # 388 def callback(ctx, str): 389 global log 390 log.write("%s%s" % (ctx, str)) 391 392 libxml2.registerErrorHandler(callback, "") 393 394 libxml2.setEntityLoader(resolver) 395 root = testsuite.getRootElement() 396 if root.name != 'testSuite': 397 print "%s doesn't start with a testSuite element, aborting" % (CONF) 398 sys.exit(1) 399 if quiet == 0: 400 print "Running Relax NG testsuite" 401 handle_testSuite(root) 402 403 if quiet == 0 or nb_schemas_failed != 0: 404 print "\nTOTAL:\nfound %d test schemas: %d success %d failures" % ( 405 nb_schemas_tests, nb_schemas_success, nb_schemas_failed) 406 if quiet == 0 or nb_instances_failed != 0: 407 print "found %d test instances: %d success %d failures" % ( 408 nb_instances_tests, nb_instances_success, nb_instances_failed) 409 410 testsuite.freeDoc() 411 412 # Memory debug specific 413 libxml2.relaxNGCleanupTypes() 414 libxml2.cleanupParser() 415 if libxml2.debugMemory(1) == 0: 416 if quiet == 0: 417 print "OK" 418 else: 419 print "Memory leak %d bytes" % (libxml2.debugMemory(1)) 420 libxml2.dumpMemory()