check-relaxng-test-suite2.py
1 #!/usr/bin/python 2 import sys 3 import time 4 import os 5 import string 6 import StringIO 7 sys.path.insert(0, "python") 8 import libxml2 9 10 # Memory debug specific 11 libxml2.debugMemory(1) 12 debug = 0 13 quiet = 1 14 15 # 16 # the testsuite description 17 # 18 CONF=os.path.join(os.path.dirname(__file__), "test/relaxng/testsuite.xml") 19 LOG="check-relaxng-test-suite2.log" 20 21 log = open(LOG, "w") 22 nb_schemas_tests = 0 23 nb_schemas_success = 0 24 nb_schemas_failed = 0 25 nb_instances_tests = 0 26 nb_instances_success = 0 27 nb_instances_failed = 0 28 29 libxml2.lineNumbersDefault(1) 30 # 31 # Resolver callback 32 # 33 resources = {} 34 def resolver(URL, ID, ctxt): 35 global resources 36 37 if resources.has_key(URL): 38 return(StringIO.StringIO(resources[URL])) 39 log.write("Resolver failure: asked %s\n" % (URL)) 40 log.write("resources: %s\n" % (resources)) 41 return None 42 43 # 44 # Load the previous results 45 # 46 #results = {} 47 #previous = {} 48 # 49 #try: 50 # res = libxml2.parseFile(RES) 51 #except: 52 # log.write("Could not parse %s" % (RES)) 53 54 # 55 # handle a valid instance 56 # 57 def handle_valid(node, schema): 58 global log 59 global nb_instances_success 60 global nb_instances_failed 61 62 instance = node.prop("dtd") 63 if instance == None: 64 instance = "" 65 child = node.children 66 while child != None: 67 if child.type != 'text': 68 instance = instance + child.serialize() 69 child = child.next 70 71 # mem = libxml2.debugMemory(1); 72 try: 73 doc = libxml2.parseDoc(instance) 74 except: 75 doc = None 76 77 if doc == None: 78 log.write("\nFailed to parse correct instance:\n-----\n") 79 log.write(instance) 80 log.write("\n-----\n") 81 nb_instances_failed = nb_instances_failed + 1 82 return 83 84 if debug: 85 print "instance line %d" % (node.lineNo()) 86 87 try: 88 ctxt = schema.relaxNGNewValidCtxt() 89 ret = doc.relaxNGValidateDoc(ctxt) 90 del ctxt 91 except: 92 ret = -1 93 94 doc.freeDoc() 95 # if mem != libxml2.debugMemory(1): 96 # print "validating instance %d line %d leaks" % ( 97 # nb_instances_tests, node.lineNo()) 98 99 if ret != 0: 100 log.write("\nFailed to validate correct instance:\n-----\n") 101 log.write(instance) 102 log.write("\n-----\n") 103 nb_instances_failed = nb_instances_failed + 1 104 else: 105 nb_instances_success = nb_instances_success + 1 106 107 # 108 # handle an invalid instance 109 # 110 def handle_invalid(node, schema): 111 global log 112 global nb_instances_success 113 global nb_instances_failed 114 115 instance = node.prop("dtd") 116 if instance == None: 117 instance = "" 118 child = node.children 119 while child != None: 120 if child.type != 'text': 121 instance = instance + child.serialize() 122 child = child.next 123 124 # mem = libxml2.debugMemory(1); 125 126 try: 127 doc = libxml2.parseDoc(instance) 128 except: 129 doc = None 130 131 if doc == None: 132 log.write("\nStrange: failed to parse incorrect instance:\n-----\n") 133 log.write(instance) 134 log.write("\n-----\n") 135 return 136 137 if debug: 138 print "instance line %d" % (node.lineNo()) 139 140 try: 141 ctxt = schema.relaxNGNewValidCtxt() 142 ret = doc.relaxNGValidateDoc(ctxt) 143 del ctxt 144 145 except: 146 ret = -1 147 148 doc.freeDoc() 149 # mem2 = libxml2.debugMemory(1) 150 # if mem != mem2: 151 # print "validating instance %d line %d leaks %d bytes" % ( 152 # nb_instances_tests, node.lineNo(), mem2 - mem) 153 154 if ret == 0: 155 log.write("\nFailed to detect validation problem in instance:\n-----\n") 156 log.write(instance) 157 log.write("\n-----\n") 158 nb_instances_failed = nb_instances_failed + 1 159 else: 160 nb_instances_success = nb_instances_success + 1 161 162 # 163 # handle an incorrect test 164 # 165 def handle_correct(node): 166 global log 167 global nb_schemas_success 168 global nb_schemas_failed 169 170 schema = "" 171 child = node.children 172 while child != None: 173 if child.type != 'text': 174 schema = schema + child.serialize() 175 child = child.next 176 177 try: 178 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 179 rngs = rngp.relaxNGParse() 180 except: 181 rngs = None 182 if rngs == None: 183 log.write("\nFailed to compile correct schema:\n-----\n") 184 log.write(schema) 185 log.write("\n-----\n") 186 nb_schemas_failed = nb_schemas_failed + 1 187 else: 188 nb_schemas_success = nb_schemas_success + 1 189 return rngs 190 191 def handle_incorrect(node): 192 global log 193 global nb_schemas_success 194 global nb_schemas_failed 195 196 schema = "" 197 child = node.children 198 while child != None: 199 if child.type != 'text': 200 schema = schema + child.serialize() 201 child = child.next 202 203 try: 204 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 205 rngs = rngp.relaxNGParse() 206 except: 207 rngs = None 208 if rngs != None: 209 log.write("\nFailed to detect schema error in:\n-----\n") 210 log.write(schema) 211 log.write("\n-----\n") 212 nb_schemas_failed = nb_schemas_failed + 1 213 else: 214 # log.write("\nSuccess detecting schema error in:\n-----\n") 215 # log.write(schema) 216 # log.write("\n-----\n") 217 nb_schemas_success = nb_schemas_success + 1 218 return None 219 220 # 221 # resource handling: keep a dictionary of URL->string mappings 222 # 223 def handle_resource(node, dir): 224 global resources 225 226 try: 227 name = node.prop('name') 228 except: 229 name = None 230 231 if name == None or name == '': 232 log.write("resource has no name") 233 return; 234 235 if dir != None: 236 # name = libxml2.buildURI(name, dir) 237 name = dir + '/' + name 238 239 res = "" 240 child = node.children 241 while child != None: 242 if child.type != 'text': 243 res = res + child.serialize() 244 child = child.next 245 resources[name] = res 246 247 # 248 # dir handling: pseudo directory resources 249 # 250 def handle_dir(node, dir): 251 try: 252 name = node.prop('name') 253 except: 254 name = None 255 256 if name == None or name == '': 257 log.write("resource has no name") 258 return; 259 260 if dir != None: 261 # name = libxml2.buildURI(name, dir) 262 name = dir + '/' + name 263 264 dirs = node.xpathEval('dir') 265 for dir in dirs: 266 handle_dir(dir, name) 267 res = node.xpathEval('resource') 268 for r in res: 269 handle_resource(r, name) 270 271 # 272 # handle a testCase element 273 # 274 def handle_testCase(node): 275 global nb_schemas_tests 276 global nb_instances_tests 277 global resources 278 279 sections = node.xpathEval('string(section)') 280 log.write("\n ======== test %d line %d section %s ==========\n" % ( 281 282 nb_schemas_tests, node.lineNo(), sections)) 283 resources = {} 284 if debug: 285 print "test %d line %d" % (nb_schemas_tests, node.lineNo()) 286 287 dirs = node.xpathEval('dir') 288 for dir in dirs: 289 handle_dir(dir, None) 290 res = node.xpathEval('resource') 291 for r in res: 292 handle_resource(r, None) 293 294 tsts = node.xpathEval('incorrect') 295 if tsts != []: 296 if len(tsts) != 1: 297 print "warning test line %d has more than one <incorrect> example" %(node.lineNo()) 298 schema = handle_incorrect(tsts[0]) 299 else: 300 tsts = node.xpathEval('correct') 301 if tsts != []: 302 if len(tsts) != 1: 303 print "warning test line %d has more than one <correct> example"% (node.lineNo()) 304 schema = handle_correct(tsts[0]) 305 else: 306 print "warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo()) 307 308 nb_schemas_tests = nb_schemas_tests + 1; 309 310 valids = node.xpathEval('valid') 311 invalids = node.xpathEval('invalid') 312 nb_instances_tests = nb_instances_tests + len(valids) + len(invalids) 313 if schema != None: 314 for valid in valids: 315 handle_valid(valid, schema) 316 for invalid in invalids: 317 handle_invalid(invalid, schema) 318 319 320 # 321 # handle a testSuite element 322 # 323 def handle_testSuite(node, level = 0): 324 global nb_schemas_tests, nb_schemas_success, nb_schemas_failed 325 global nb_instances_tests, nb_instances_success, nb_instances_failed 326 if level >= 1: 327 old_schemas_tests = nb_schemas_tests 328 old_schemas_success = nb_schemas_success 329 old_schemas_failed = nb_schemas_failed 330 old_instances_tests = nb_instances_tests 331 old_instances_success = nb_instances_success 332 old_instances_failed = nb_instances_failed 333 334 docs = node.xpathEval('documentation') 335 authors = node.xpathEval('author') 336 if docs != []: 337 msg = "" 338 for doc in docs: 339 msg = msg + doc.content + " " 340 if authors != []: 341 msg = msg + "written by " 342 for author in authors: 343 msg = msg + author.content + " " 344 if quiet == 0: 345 print msg 346 sections = node.xpathEval('section') 347 if sections != [] and level <= 0: 348 msg = "" 349 for section in sections: 350 msg = msg + section.content + " " 351 if quiet == 0: 352 print "Tests for section %s" % (msg) 353 for test in node.xpathEval('testCase'): 354 handle_testCase(test) 355 for test in node.xpathEval('testSuite'): 356 handle_testSuite(test, level + 1) 357 358 359 if level >= 1 and sections != []: 360 msg = "" 361 for section in sections: 362 msg = msg + section.content + " " 363 print "Result of tests for section %s" % (msg) 364 if nb_schemas_tests != old_schemas_tests: 365 print "found %d test schemas: %d success %d failures" % ( 366 nb_schemas_tests - old_schemas_tests, 367 nb_schemas_success - old_schemas_success, 368 nb_schemas_failed - old_schemas_failed) 369 if nb_instances_tests != old_instances_tests: 370 print "found %d test instances: %d success %d failures" % ( 371 nb_instances_tests - old_instances_tests, 372 nb_instances_success - old_instances_success, 373 nb_instances_failed - old_instances_failed) 374 # 375 # Parse the conf file 376 # 377 libxml2.substituteEntitiesDefault(1); 378 testsuite = libxml2.parseFile(CONF) 379 380 # 381 # Error and warnng callbacks 382 # 383 def callback(ctx, str): 384 global log 385 log.write("%s%s" % (ctx, str)) 386 387 libxml2.registerErrorHandler(callback, "") 388 389 libxml2.setEntityLoader(resolver) 390 root = testsuite.getRootElement() 391 if root.name != 'testSuite': 392 print "%s doesn't start with a testSuite element, aborting" % (CONF) 393 sys.exit(1) 394 if quiet == 0: 395 print "Running Relax NG testsuite" 396 handle_testSuite(root) 397 398 if quiet == 0: 399 print "\nTOTAL:\n" 400 if quiet == 0 or nb_schemas_failed != 0: 401 print "found %d test schemas: %d success %d failures" % ( 402 nb_schemas_tests, nb_schemas_success, nb_schemas_failed) 403 if quiet == 0 or nb_instances_failed != 0: 404 print "found %d test instances: %d success %d failures" % ( 405 nb_instances_tests, nb_instances_success, nb_instances_failed) 406 407 408 testsuite.freeDoc() 409 410 # Memory debug specific 411 libxml2.relaxNGCleanupTypes() 412 libxml2.cleanupParser() 413 if libxml2.debugMemory(1) == 0: 414 if quiet == 0: 415 print "OK" 416 else: 417 print "Memory leak %d bytes" % (libxml2.debugMemory(1)) 418 libxml2.dumpMemory()