check-relaxng-test-suite.py
1 #!/usr/bin/python 2 import sys 3 import time 4 import os 5 import string 6 import StringIO 7 sys.path.insert(0, "python") 8 import libxml2 9 10 # Memory debug specific 11 libxml2.debugMemory(1) 12 debug = 0 13 verbose = 0 14 quiet = 1 15 16 # 17 # the testsuite description 18 # 19 CONF=os.path.join(os.path.dirname(__file__), "test/relaxng/OASIS/spectest.xml") 20 LOG="check-relaxng-test-suite.log" 21 RES="relaxng-test-results.xml" 22 23 log = open(LOG, "w") 24 nb_schemas_tests = 0 25 nb_schemas_success = 0 26 nb_schemas_failed = 0 27 nb_instances_tests = 0 28 nb_instances_success = 0 29 nb_instances_failed = 0 30 31 libxml2.lineNumbersDefault(1) 32 # 33 # Error and warnng callbacks 34 # 35 def callback(ctx, str): 36 global log 37 log.write("%s%s" % (ctx, str)) 38 39 libxml2.registerErrorHandler(callback, "") 40 41 # 42 # Resolver callback 43 # 44 resources = {} 45 def resolver(URL, ID, ctxt): 46 global resources 47 48 if string.find(URL, '#') != -1: 49 URL = URL[0:string.find(URL, '#')] 50 if resources.has_key(URL): 51 return(StringIO.StringIO(resources[URL])) 52 log.write("Resolver failure: asked %s\n" % (URL)) 53 log.write("resources: %s\n" % (resources)) 54 return None 55 56 # 57 # Load the previous results 58 # 59 #results = {} 60 #previous = {} 61 # 62 #try: 63 # res = libxml2.parseFile(RES) 64 #except: 65 # log.write("Could not parse %s" % (RES)) 66 67 # 68 # handle a valid instance 69 # 70 def handle_valid(node, schema): 71 global log 72 global nb_instances_success 73 global nb_instances_failed 74 75 instance = "" 76 child = node.children 77 while child != None: 78 if child.type != 'text': 79 instance = instance + child.serialize() 80 child = child.next 81 82 try: 83 doc = libxml2.parseDoc(instance) 84 except: 85 doc = None 86 87 if doc == None: 88 log.write("\nFailed to parse correct instance:\n-----\n") 89 log.write(instance) 90 log.write("\n-----\n") 91 nb_instances_failed = nb_instances_failed + 1 92 return 93 94 try: 95 ctxt = schema.relaxNGNewValidCtxt() 96 ret = doc.relaxNGValidateDoc(ctxt) 97 except: 98 ret = -1 99 if ret != 0: 100 log.write("\nFailed to validate correct instance:\n-----\n") 101 log.write(instance) 102 log.write("\n-----\n") 103 nb_instances_failed = nb_instances_failed + 1 104 else: 105 nb_instances_success = nb_instances_success + 1 106 doc.freeDoc() 107 108 # 109 # handle an invalid instance 110 # 111 def handle_invalid(node, schema): 112 global log 113 global nb_instances_success 114 global nb_instances_failed 115 116 instance = "" 117 child = node.children 118 while child != None: 119 if child.type != 'text': 120 instance = instance + child.serialize() 121 child = child.next 122 123 try: 124 doc = libxml2.parseDoc(instance) 125 except: 126 doc = None 127 128 if doc == None: 129 log.write("\nStrange: failed to parse incorrect instance:\n-----\n") 130 log.write(instance) 131 log.write("\n-----\n") 132 return 133 134 try: 135 ctxt = schema.relaxNGNewValidCtxt() 136 ret = doc.relaxNGValidateDoc(ctxt) 137 except: 138 ret = -1 139 if ret == 0: 140 log.write("\nFailed to detect validation problem in instance:\n-----\n") 141 log.write(instance) 142 log.write("\n-----\n") 143 nb_instances_failed = nb_instances_failed + 1 144 else: 145 nb_instances_success = nb_instances_success + 1 146 doc.freeDoc() 147 148 # 149 # handle an incorrect test 150 # 151 def handle_correct(node): 152 global log 153 global nb_schemas_success 154 global nb_schemas_failed 155 156 schema = "" 157 child = node.children 158 while child != None: 159 if child.type != 'text': 160 schema = schema + child.serialize() 161 child = child.next 162 163 try: 164 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 165 rngs = rngp.relaxNGParse() 166 except: 167 rngs = None 168 if rngs == None: 169 log.write("\nFailed to compile correct schema:\n-----\n") 170 log.write(schema) 171 log.write("\n-----\n") 172 nb_schemas_failed = nb_schemas_failed + 1 173 else: 174 nb_schemas_success = nb_schemas_success + 1 175 return rngs 176 177 def handle_incorrect(node): 178 global log 179 global nb_schemas_success 180 global nb_schemas_failed 181 182 schema = "" 183 child = node.children 184 while child != None: 185 if child.type != 'text': 186 schema = schema + child.serialize() 187 child = child.next 188 189 try: 190 rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema)) 191 rngs = rngp.relaxNGParse() 192 except: 193 rngs = None 194 if rngs != None: 195 log.write("\nFailed to detect schema error in:\n-----\n") 196 log.write(schema) 197 log.write("\n-----\n") 198 nb_schemas_failed = nb_schemas_failed + 1 199 else: 200 # log.write("\nSuccess detecting schema error in:\n-----\n") 201 # log.write(schema) 202 # log.write("\n-----\n") 203 nb_schemas_success = nb_schemas_success + 1 204 return None 205 206 # 207 # resource handling: keep a dictionary of URL->string mappings 208 # 209 def handle_resource(node, dir): 210 global resources 211 212 try: 213 name = node.prop('name') 214 except: 215 name = None 216 217 if name == None or name == '': 218 log.write("resource has no name") 219 return; 220 221 if dir != None: 222 # name = libxml2.buildURI(name, dir) 223 name = dir + '/' + name 224 225 res = "" 226 child = node.children 227 while child != None: 228 if child.type != 'text': 229 res = res + child.serialize() 230 child = child.next 231 resources[name] = res 232 233 # 234 # dir handling: pseudo directory resources 235 # 236 def handle_dir(node, dir): 237 try: 238 name = node.prop('name') 239 except: 240 name = None 241 242 if name == None or name == '': 243 log.write("resource has no name") 244 return; 245 246 if dir != None: 247 # name = libxml2.buildURI(name, dir) 248 name = dir + '/' + name 249 250 dirs = node.xpathEval('dir') 251 for dir in dirs: 252 handle_dir(dir, name) 253 res = node.xpathEval('resource') 254 for r in res: 255 handle_resource(r, name) 256 257 # 258 # handle a testCase element 259 # 260 def handle_testCase(node): 261 global nb_schemas_tests 262 global nb_instances_tests 263 global resources 264 265 sections = node.xpathEval('string(section)') 266 log.write("\n ======== test %d line %d section %s ==========\n" % ( 267 268 nb_schemas_tests, node.lineNo(), sections)) 269 resources = {} 270 if debug: 271 print "test %d line %d" % (nb_schemas_tests, node.lineNo()) 272 273 dirs = node.xpathEval('dir') 274 for dir in dirs: 275 handle_dir(dir, None) 276 res = node.xpathEval('resource') 277 for r in res: 278 handle_resource(r, None) 279 280 tsts = node.xpathEval('incorrect') 281 if tsts != []: 282 if len(tsts) != 1: 283 print "warning test line %d has more than one <incorrect> example" %(node.lineNo()) 284 schema = handle_incorrect(tsts[0]) 285 else: 286 tsts = node.xpathEval('correct') 287 if tsts != []: 288 if len(tsts) != 1: 289 print "warning test line %d has more than one <correct> example"% (node.lineNo()) 290 schema = handle_correct(tsts[0]) 291 else: 292 print "warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo()) 293 294 nb_schemas_tests = nb_schemas_tests + 1; 295 296 valids = node.xpathEval('valid') 297 invalids = node.xpathEval('invalid') 298 nb_instances_tests = nb_instances_tests + len(valids) + len(invalids) 299 if schema != None: 300 for valid in valids: 301 handle_valid(valid, schema) 302 for invalid in invalids: 303 handle_invalid(invalid, schema) 304 305 306 # 307 # handle a testSuite element 308 # 309 def handle_testSuite(node, level = 0): 310 global nb_schemas_tests, nb_schemas_success, nb_schemas_failed 311 global nb_instances_tests, nb_instances_success, nb_instances_failed 312 global quiet 313 if level >= 1: 314 old_schemas_tests = nb_schemas_tests 315 old_schemas_success = nb_schemas_success 316 old_schemas_failed = nb_schemas_failed 317 old_instances_tests = nb_instances_tests 318 old_instances_success = nb_instances_success 319 old_instances_failed = nb_instances_failed 320 321 docs = node.xpathEval('documentation') 322 authors = node.xpathEval('author') 323 if docs != []: 324 msg = "" 325 for doc in docs: 326 msg = msg + doc.content + " " 327 if authors != []: 328 msg = msg + "written by " 329 for author in authors: 330 msg = msg + author.content + " " 331 if quiet == 0: 332 print msg 333 sections = node.xpathEval('section') 334 if sections != [] and level <= 0: 335 msg = "" 336 for section in sections: 337 msg = msg + section.content + " " 338 if quiet == 0: 339 print "Tests for section %s" % (msg) 340 for test in node.xpathEval('testCase'): 341 handle_testCase(test) 342 for test in node.xpathEval('testSuite'): 343 handle_testSuite(test, level + 1) 344 345 346 if verbose and level >= 1 and sections != []: 347 msg = "" 348 for section in sections: 349 msg = msg + section.content + " " 350 print "Result of tests for section %s" % (msg) 351 if nb_schemas_tests != old_schemas_tests: 352 print "found %d test schemas: %d success %d failures" % ( 353 nb_schemas_tests - old_schemas_tests, 354 nb_schemas_success - old_schemas_success, 355 nb_schemas_failed - old_schemas_failed) 356 if nb_instances_tests != old_instances_tests: 357 print "found %d test instances: %d success %d failures" % ( 358 nb_instances_tests - old_instances_tests, 359 nb_instances_success - old_instances_success, 360 nb_instances_failed - old_instances_failed) 361 # 362 # Parse the conf file 363 # 364 libxml2.substituteEntitiesDefault(1); 365 testsuite = libxml2.parseFile(CONF) 366 libxml2.setEntityLoader(resolver) 367 root = testsuite.getRootElement() 368 if root.name != 'testSuite': 369 print "%s doesn't start with a testSuite element, aborting" % (CONF) 370 sys.exit(1) 371 if quiet == 0: 372 print "Running Relax NG testsuite" 373 handle_testSuite(root) 374 375 if quiet == 0: 376 print "\nTOTAL:\n" 377 if quiet == 0 or nb_schemas_failed != 0: 378 print "found %d test schemas: %d success %d failures" % ( 379 nb_schemas_tests, nb_schemas_success, nb_schemas_failed) 380 if quiet == 0 or nb_instances_failed != 0: 381 print "found %d test instances: %d success %d failures" % ( 382 nb_instances_tests, nb_instances_success, nb_instances_failed) 383 384 testsuite.freeDoc() 385 386 # Memory debug specific 387 libxml2.relaxNGCleanupTypes() 388 libxml2.cleanupParser() 389 if libxml2.debugMemory(1) == 0: 390 if quiet == 0: 391 print "OK" 392 else: 393 print "Memory leak %d bytes" % (libxml2.debugMemory(1)) 394 libxml2.dumpMemory()