umsgpack.py
1 # u-msgpack-python v2.4.1 - v at sergeev.io 2 # https://github.com/vsergeev/u-msgpack-python 3 # 4 # u-msgpack-python is a lightweight MessagePack serializer and deserializer 5 # module, compatible with both Python 2 and 3, as well CPython and PyPy 6 # implementations of Python. u-msgpack-python is fully compliant with the 7 # latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In 8 # particular, it supports the new binary, UTF-8 string, and application ext 9 # types. 10 # 11 # MIT License 12 # 13 # Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev 14 # 15 # Permission is hereby granted, free of charge, to any person obtaining a copy 16 # of this software and associated documentation files (the "Software"), to deal 17 # in the Software without restriction, including without limitation the rights 18 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 19 # copies of the Software, and to permit persons to whom the Software is 20 # furnished to do so, subject to the following conditions: 21 # 22 # The above copyright notice and this permission notice shall be included in 23 # all copies or substantial portions of the Software. 24 # 25 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 30 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 31 # THE SOFTWARE. 32 # 33 """ 34 src/fallback/umsgpack/umsgpack.py 35 ================================= 36 37 u-msgpack-python v2.4.1 - v at sergeev.io 38 https://github.com/vsergeev/u-msgpack-python 39 40 u-msgpack-python is a lightweight MessagePack serializer and deserializer 41 module, compatible with both Python 2 and 3, as well CPython and PyPy 42 implementations of Python. u-msgpack-python is fully compliant with the 43 latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In 44 particular, it supports the new binary, UTF-8 string, and application ext 45 types. 46 47 License: MIT 48 """ 49 # pylint: disable=too-many-lines,too-many-branches,too-many-statements,global-statement,too-many-return-statements 50 # pylint: disable=unused-argument 51 52 import collections 53 import io 54 import struct 55 import sys 56 57 __version__ = "2.4.1" 58 "Module version string" 59 60 version = (2, 4, 1) 61 "Module version tuple" 62 63 64 ############################################################################## 65 # Ext Class 66 ############################################################################## 67 68 # Extension type for application-defined types and data 69 class Ext: # pylint: disable=old-style-class 70 """ 71 The Ext class facilitates creating a serializable extension object to store 72 an application-defined type and data byte array. 73 """ 74 75 def __init__(self, type, data): 76 """ 77 Construct a new Ext object. 78 79 Args: 80 type: application-defined type integer from 0 to 127 81 data: application-defined data byte array 82 83 Raises: 84 TypeError: 85 Specified ext type is outside of 0 to 127 range. 86 87 Example: 88 >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") 89 >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) 90 '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' 91 >>> bar = umsgpack.unpackb(_) 92 >>> print(bar["special stuff"]) 93 Ext Object (Type: 0x05, Data: 01 02 03) 94 >>> 95 """ 96 # pylint:disable=redefined-builtin 97 98 # Application ext type should be 0 <= type <= 127 99 if not isinstance(type, int) or not (type >= 0 and type <= 127): 100 raise TypeError("ext type out of range") 101 # Check data is type bytes 102 elif sys.version_info[0] == 3 and not isinstance(data, bytes): 103 raise TypeError("ext data is not type \'bytes\'") 104 elif sys.version_info[0] == 2 and not isinstance(data, str): 105 raise TypeError("ext data is not type \'str\'") 106 self.type = type 107 self.data = data 108 109 def __eq__(self, other): 110 """ 111 Compare this Ext object with another for equality. 112 """ 113 return (isinstance(other, self.__class__) and 114 self.type == other.type and 115 self.data == other.data) 116 117 def __ne__(self, other): 118 """ 119 Compare this Ext object with another for inequality. 120 """ 121 return not self.__eq__(other) 122 123 def __str__(self): 124 """ 125 String representation of this Ext object. 126 """ 127 s = "Ext Object (Type: 0x%02x, Data: " % self.type 128 s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) 129 for i in range(min(len(self.data), 8))]) 130 if len(self.data) > 8: 131 s += " ..." 132 s += ")" 133 return s 134 135 def __hash__(self): 136 """ 137 Provide a hash of this Ext object. 138 """ 139 return hash((self.type, self.data)) 140 141 142 class InvalidString(bytes): 143 """Subclass of bytes to hold invalid UTF-8 strings.""" 144 pass 145 146 ############################################################################## 147 # Exceptions 148 ############################################################################## 149 150 151 # Base Exception classes 152 class PackException(Exception): 153 "Base class for exceptions encountered during packing." 154 pass 155 156 157 class UnpackException(Exception): 158 "Base class for exceptions encountered during unpacking." 159 pass 160 161 162 # Packing error 163 class UnsupportedTypeException(PackException): 164 "Object type not supported for packing." 165 pass 166 167 168 # Unpacking error 169 class InsufficientDataException(UnpackException): 170 "Insufficient data to unpack the serialized object." 171 pass 172 173 174 class InvalidStringException(UnpackException): 175 "Invalid UTF-8 string encountered during unpacking." 176 pass 177 178 179 class ReservedCodeException(UnpackException): 180 "Reserved code encountered during unpacking." 181 pass 182 183 184 class UnhashableKeyException(UnpackException): 185 """ 186 Unhashable key encountered during map unpacking. 187 The serialized map cannot be deserialized into a Python dictionary. 188 """ 189 pass 190 191 192 class DuplicateKeyException(UnpackException): 193 "Duplicate key encountered during map unpacking." 194 pass 195 196 197 # Backwards compatibility 198 KeyNotPrimitiveException = UnhashableKeyException 199 KeyDuplicateException = DuplicateKeyException 200 201 ############################################################################# 202 # Exported Functions and Glob 203 ############################################################################# 204 205 # Exported functions and variables, set up in __init() 206 pack = None 207 packb = None 208 unpack = None 209 unpackb = None 210 dump = None 211 dumps = None 212 load = None 213 loads = None 214 215 compatibility = False 216 """ 217 Compatibility mode boolean. 218 219 When compatibility mode is enabled, u-msgpack-python will serialize both 220 unicode strings and bytes into the old "raw" msgpack type, and deserialize the 221 "raw" msgpack type into bytes. This provides backwards compatibility with the 222 old MessagePack specification. 223 224 Example: 225 >>> umsgpack.compatibility = True 226 >>> 227 >>> umsgpack.packb([u"some string", b"some bytes"]) 228 b'\x92\xabsome string\xaasome bytes' 229 >>> umsgpack.unpackb(_) 230 [b'some string', b'some bytes'] 231 >>> 232 """ 233 234 ############################################################################## 235 # Packing 236 ############################################################################## 237 238 # You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the 239 # code below. This is to allow for seamless Python 2 and 3 compatibility, as 240 # chr(obj) has a str return type instead of bytes in Python 3, and 241 # struct.pack(...) has the right return type in both versions. 242 243 244 def _pack_integer(obj, fp, options): 245 if obj < 0: 246 if obj >= -32: 247 fp.write(struct.pack("b", obj)) 248 elif obj >= -2**(8 - 1): 249 fp.write(b"\xd0" + struct.pack("b", obj)) 250 elif obj >= -2**(16 - 1): 251 fp.write(b"\xd1" + struct.pack(">h", obj)) 252 elif obj >= -2**(32 - 1): 253 fp.write(b"\xd2" + struct.pack(">i", obj)) 254 elif obj >= -2**(64 - 1): 255 fp.write(b"\xd3" + struct.pack(">q", obj)) 256 else: 257 raise UnsupportedTypeException("huge signed int") 258 else: 259 if obj <= 127: 260 fp.write(struct.pack("B", obj)) 261 elif obj <= 2**8 - 1: 262 fp.write(b"\xcc" + struct.pack("B", obj)) 263 elif obj <= 2**16 - 1: 264 fp.write(b"\xcd" + struct.pack(">H", obj)) 265 elif obj <= 2**32 - 1: 266 fp.write(b"\xce" + struct.pack(">I", obj)) 267 elif obj <= 2**64 - 1: 268 fp.write(b"\xcf" + struct.pack(">Q", obj)) 269 else: 270 raise UnsupportedTypeException("huge unsigned int") 271 272 273 def _pack_nil(obj, fp, options): 274 fp.write(b"\xc0") 275 276 277 def _pack_boolean(obj, fp, options): 278 fp.write(b"\xc3" if obj else b"\xc2") 279 280 281 def _pack_float(obj, fp, options): 282 float_precision = options.get('force_float_precision', _float_precision) 283 284 if float_precision == "double": 285 fp.write(b"\xcb" + struct.pack(">d", obj)) 286 elif float_precision == "single": 287 fp.write(b"\xca" + struct.pack(">f", obj)) 288 else: 289 raise ValueError("invalid float precision") 290 291 292 def _pack_string(obj, fp, options): 293 obj = obj.encode('utf-8') 294 if len(obj) <= 31: 295 fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) 296 elif len(obj) <= 2**8 - 1: 297 fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) 298 elif len(obj) <= 2**16 - 1: 299 fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) 300 elif len(obj) <= 2**32 - 1: 301 fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) 302 else: 303 raise UnsupportedTypeException("huge string") 304 305 306 def _pack_binary(obj, fp, options): 307 if len(obj) <= 2**8 - 1: 308 fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) 309 elif len(obj) <= 2**16 - 1: 310 fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) 311 elif len(obj) <= 2**32 - 1: 312 fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) 313 else: 314 raise UnsupportedTypeException("huge binary string") 315 316 317 def _pack_oldspec_raw(obj, fp, options): 318 if len(obj) <= 31: 319 fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) 320 elif len(obj) <= 2**16 - 1: 321 fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) 322 elif len(obj) <= 2**32 - 1: 323 fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) 324 else: 325 raise UnsupportedTypeException("huge raw string") 326 327 328 def _pack_ext(obj, fp, options): 329 if len(obj.data) == 1: 330 fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) 331 elif len(obj.data) == 2: 332 fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) 333 elif len(obj.data) == 4: 334 fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) 335 elif len(obj.data) == 8: 336 fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) 337 elif len(obj.data) == 16: 338 fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) 339 elif len(obj.data) <= 2**8 - 1: 340 fp.write(b"\xc7" + 341 struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) 342 elif len(obj.data) <= 2**16 - 1: 343 fp.write(b"\xc8" + 344 struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) 345 elif len(obj.data) <= 2**32 - 1: 346 fp.write(b"\xc9" + 347 struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) 348 else: 349 raise UnsupportedTypeException("huge ext data") 350 351 352 def _pack_array(obj, fp, options): 353 if len(obj) <= 15: 354 fp.write(struct.pack("B", 0x90 | len(obj))) 355 elif len(obj) <= 2**16 - 1: 356 fp.write(b"\xdc" + struct.pack(">H", len(obj))) 357 elif len(obj) <= 2**32 - 1: 358 fp.write(b"\xdd" + struct.pack(">I", len(obj))) 359 else: 360 raise UnsupportedTypeException("huge array") 361 362 for e in obj: 363 pack(e, fp, **options) 364 365 366 def _pack_map(obj, fp, options): 367 if len(obj) <= 15: 368 fp.write(struct.pack("B", 0x80 | len(obj))) 369 elif len(obj) <= 2**16 - 1: 370 fp.write(b"\xde" + struct.pack(">H", len(obj))) 371 elif len(obj) <= 2**32 - 1: 372 fp.write(b"\xdf" + struct.pack(">I", len(obj))) 373 else: 374 raise UnsupportedTypeException("huge array") 375 376 for k, v in list(obj.items()): 377 pack(k, fp, **options) 378 pack(v, fp, **options) 379 380 ######################################## 381 382 383 # Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type 384 def _pack2(obj, fp, **options): 385 """ 386 Serialize a Python object into MessagePack bytes. 387 388 Args: 389 obj: a Python object 390 fp: a .write()-supporting file-like object 391 392 Kwargs: 393 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type 394 to a callable that packs an instance of the type 395 into an Ext object 396 force_float_precision (str): "single" to force packing floats as 397 IEEE-754 single-precision floats, 398 "double" to force packing floats as 399 IEEE-754 double-precision floats. 400 401 Returns: 402 None. 403 404 Raises: 405 UnsupportedType(PackException): 406 Object type not supported for packing. 407 408 Example: 409 >>> f = open('test.bin', 'wb') 410 >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) 411 >>> 412 """ 413 global compatibility 414 415 ext_handlers = options.get("ext_handlers") 416 417 if obj is None: 418 _pack_nil(obj, fp, options) 419 elif ext_handlers and obj.__class__ in ext_handlers: 420 _pack_ext(ext_handlers[obj.__class__](obj), fp, options) 421 elif isinstance(obj, bool): 422 _pack_boolean(obj, fp, options) 423 elif isinstance(obj, int): 424 _pack_integer(obj, fp, options) 425 elif isinstance(obj, float): 426 _pack_float(obj, fp, options) 427 elif compatibility and isinstance(obj, str): 428 _pack_oldspec_raw(bytes(obj), fp, options) 429 elif compatibility and isinstance(obj, bytes): 430 _pack_oldspec_raw(obj, fp, options) 431 elif isinstance(obj, str): 432 _pack_string(obj, fp, options) 433 elif isinstance(obj, str): 434 _pack_binary(obj, fp, options) 435 elif isinstance(obj, (list, tuple)): 436 _pack_array(obj, fp, options) 437 elif isinstance(obj, dict): 438 _pack_map(obj, fp, options) 439 elif isinstance(obj, Ext): 440 _pack_ext(obj, fp, options) 441 elif ext_handlers: 442 # Linear search for superclass 443 t = next((t for t in list(ext_handlers.keys()) if isinstance(obj, t)), None) 444 if t: 445 _pack_ext(ext_handlers[t](obj), fp, options) 446 else: 447 raise UnsupportedTypeException( 448 "unsupported type: %s" % str(type(obj))) 449 else: 450 raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) 451 452 453 # Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type 454 def _pack3(obj, fp, **options): 455 """ 456 Serialize a Python object into MessagePack bytes. 457 458 Args: 459 obj: a Python object 460 fp: a .write()-supporting file-like object 461 462 Kwargs: 463 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type 464 to a callable that packs an instance of the type 465 into an Ext object 466 force_float_precision (str): "single" to force packing floats as 467 IEEE-754 single-precision floats, 468 "double" to force packing floats as 469 IEEE-754 double-precision floats. 470 471 Returns: 472 None. 473 474 Raises: 475 UnsupportedType(PackException): 476 Object type not supported for packing. 477 478 Example: 479 >>> f = open('test.bin', 'wb') 480 >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) 481 >>> 482 """ 483 global compatibility 484 485 ext_handlers = options.get("ext_handlers") 486 487 if obj is None: 488 _pack_nil(obj, fp, options) 489 elif ext_handlers and obj.__class__ in ext_handlers: 490 _pack_ext(ext_handlers[obj.__class__](obj), fp, options) 491 elif isinstance(obj, bool): 492 _pack_boolean(obj, fp, options) 493 elif isinstance(obj, int): 494 _pack_integer(obj, fp, options) 495 elif isinstance(obj, float): 496 _pack_float(obj, fp, options) 497 elif compatibility and isinstance(obj, str): 498 _pack_oldspec_raw(obj.encode('utf-8'), fp, options) 499 elif compatibility and isinstance(obj, bytes): 500 _pack_oldspec_raw(obj, fp, options) 501 elif isinstance(obj, str): 502 _pack_string(obj, fp, options) 503 elif isinstance(obj, bytes): 504 _pack_binary(obj, fp, options) 505 elif isinstance(obj, (list, tuple)): 506 _pack_array(obj, fp, options) 507 elif isinstance(obj, dict): 508 _pack_map(obj, fp, options) 509 elif isinstance(obj, Ext): 510 _pack_ext(obj, fp, options) 511 elif ext_handlers: 512 # Linear search for superclass 513 t = next((t for t in list(ext_handlers.keys()) if isinstance(obj, t)), None) 514 if t: 515 _pack_ext(ext_handlers[t](obj), fp, options) 516 else: 517 raise UnsupportedTypeException( 518 "unsupported type: %s" % str(type(obj))) 519 else: 520 raise UnsupportedTypeException( 521 "unsupported type: %s" % str(type(obj))) 522 523 524 def _packb2(obj, **options): 525 """ 526 Serialize a Python object into MessagePack bytes. 527 528 Args: 529 obj: a Python object 530 531 Kwargs: 532 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type 533 to a callable that packs an instance of the type 534 into an Ext object 535 force_float_precision (str): "single" to force packing floats as 536 IEEE-754 single-precision floats, 537 "double" to force packing floats as 538 IEEE-754 double-precision floats. 539 540 Returns: 541 A 'str' containing serialized MessagePack bytes. 542 543 Raises: 544 UnsupportedType(PackException): 545 Object type not supported for packing. 546 547 Example: 548 >>> umsgpack.packb({u"compact": True, u"schema": 0}) 549 '\x82\xa7compact\xc3\xa6schema\x00' 550 >>> 551 """ 552 fp = io.BytesIO() 553 _pack2(obj, fp, **options) 554 return fp.getvalue() 555 556 557 def _packb3(obj, **options): 558 """ 559 Serialize a Python object into MessagePack bytes. 560 561 Args: 562 obj: a Python object 563 564 Kwargs: 565 ext_handlers (dict): dictionary of Ext handlers, mapping a custom type 566 to a callable that packs an instance of the type 567 into an Ext object 568 force_float_precision (str): "single" to force packing floats as 569 IEEE-754 single-precision floats, 570 "double" to force packing floats as 571 IEEE-754 double-precision floats. 572 573 Returns: 574 A 'bytes' containing serialized MessagePack bytes. 575 576 Raises: 577 UnsupportedType(PackException): 578 Object type not supported for packing. 579 580 Example: 581 >>> umsgpack.packb({u"compact": True, u"schema": 0}) 582 b'\x82\xa7compact\xc3\xa6schema\x00' 583 >>> 584 """ 585 fp = io.BytesIO() 586 _pack3(obj, fp, **options) 587 return fp.getvalue() 588 589 ############################################################################# 590 # Unpacking 591 ############################################################################# 592 593 594 def _read_except(fp, n): 595 data = fp.read(n) 596 if len(data) < n: 597 raise InsufficientDataException() 598 return data 599 600 601 def _unpack_integer(code, fp, options): 602 if (ord(code) & 0xe0) == 0xe0: 603 return struct.unpack("b", code)[0] 604 elif code == b'\xd0': 605 return struct.unpack("b", _read_except(fp, 1))[0] 606 elif code == b'\xd1': 607 return struct.unpack(">h", _read_except(fp, 2))[0] 608 elif code == b'\xd2': 609 return struct.unpack(">i", _read_except(fp, 4))[0] 610 elif code == b'\xd3': 611 return struct.unpack(">q", _read_except(fp, 8))[0] 612 elif (ord(code) & 0x80) == 0x00: 613 return struct.unpack("B", code)[0] 614 elif code == b'\xcc': 615 return struct.unpack("B", _read_except(fp, 1))[0] 616 elif code == b'\xcd': 617 return struct.unpack(">H", _read_except(fp, 2))[0] 618 elif code == b'\xce': 619 return struct.unpack(">I", _read_except(fp, 4))[0] 620 elif code == b'\xcf': 621 return struct.unpack(">Q", _read_except(fp, 8))[0] 622 raise Exception("logic error, not int: 0x%02x" % ord(code)) 623 624 625 def _unpack_reserved(code, fp, options): 626 if code == b'\xc1': 627 raise ReservedCodeException( 628 "encountered reserved code: 0x%02x" % ord(code)) 629 raise Exception( 630 "logic error, not reserved code: 0x%02x" % ord(code)) 631 632 633 def _unpack_nil(code, fp, options): 634 if code == b'\xc0': 635 return None 636 raise Exception("logic error, not nil: 0x%02x" % ord(code)) 637 638 639 def _unpack_boolean(code, fp, options): 640 if code == b'\xc2': 641 return False 642 elif code == b'\xc3': 643 return True 644 raise Exception("logic error, not boolean: 0x%02x" % ord(code)) 645 646 647 def _unpack_float(code, fp, options): 648 if code == b'\xca': 649 return struct.unpack(">f", _read_except(fp, 4))[0] 650 elif code == b'\xcb': 651 return struct.unpack(">d", _read_except(fp, 8))[0] 652 raise Exception("logic error, not float: 0x%02x" % ord(code)) 653 654 655 def _unpack_string(code, fp, options): 656 if (ord(code) & 0xe0) == 0xa0: 657 length = ord(code) & ~0xe0 658 elif code == b'\xd9': 659 length = struct.unpack("B", _read_except(fp, 1))[0] 660 elif code == b'\xda': 661 length = struct.unpack(">H", _read_except(fp, 2))[0] 662 elif code == b'\xdb': 663 length = struct.unpack(">I", _read_except(fp, 4))[0] 664 else: 665 raise Exception("logic error, not string: 0x%02x" % ord(code)) 666 667 # Always return raw bytes in compatibility mode 668 global compatibility 669 if compatibility: 670 return _read_except(fp, length) 671 672 data = _read_except(fp, length) 673 try: 674 return bytes.decode(data, 'utf-8') 675 except UnicodeDecodeError: 676 if options.get("allow_invalid_utf8"): 677 return InvalidString(data) 678 raise InvalidStringException("unpacked string is invalid utf-8") 679 680 681 def _unpack_binary(code, fp, options): 682 if code == b'\xc4': 683 length = struct.unpack("B", _read_except(fp, 1))[0] 684 elif code == b'\xc5': 685 length = struct.unpack(">H", _read_except(fp, 2))[0] 686 elif code == b'\xc6': 687 length = struct.unpack(">I", _read_except(fp, 4))[0] 688 else: 689 raise Exception("logic error, not binary: 0x%02x" % ord(code)) 690 691 return _read_except(fp, length) 692 693 694 def _unpack_ext(code, fp, options): 695 if code == b'\xd4': 696 length = 1 697 elif code == b'\xd5': 698 length = 2 699 elif code == b'\xd6': 700 length = 4 701 elif code == b'\xd7': 702 length = 8 703 elif code == b'\xd8': 704 length = 16 705 elif code == b'\xc7': 706 length = struct.unpack("B", _read_except(fp, 1))[0] 707 elif code == b'\xc8': 708 length = struct.unpack(">H", _read_except(fp, 2))[0] 709 elif code == b'\xc9': 710 length = struct.unpack(">I", _read_except(fp, 4))[0] 711 else: 712 raise Exception("logic error, not ext: 0x%02x" % ord(code)) 713 714 ext = Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) 715 716 # Unpack with ext handler, if we have one 717 ext_handlers = options.get("ext_handlers") 718 if ext_handlers and ext.type in ext_handlers: 719 ext = ext_handlers[ext.type](ext) 720 721 return ext 722 723 724 def _unpack_array(code, fp, options): 725 if (ord(code) & 0xf0) == 0x90: 726 length = (ord(code) & ~0xf0) 727 elif code == b'\xdc': 728 length = struct.unpack(">H", _read_except(fp, 2))[0] 729 elif code == b'\xdd': 730 length = struct.unpack(">I", _read_except(fp, 4))[0] 731 else: 732 raise Exception("logic error, not array: 0x%02x" % ord(code)) 733 734 return [_unpack(fp, options) for _ in range(length)] 735 736 737 def _deep_list_to_tuple(obj): 738 if isinstance(obj, list): 739 return tuple([_deep_list_to_tuple(e) for e in obj]) 740 return obj 741 742 743 def _unpack_map(code, fp, options): 744 if (ord(code) & 0xf0) == 0x80: 745 length = (ord(code) & ~0xf0) 746 elif code == b'\xde': 747 length = struct.unpack(">H", _read_except(fp, 2))[0] 748 elif code == b'\xdf': 749 length = struct.unpack(">I", _read_except(fp, 4))[0] 750 else: 751 raise Exception("logic error, not map: 0x%02x" % ord(code)) 752 753 d = {} if not options.get('use_ordered_dict') \ 754 else collections.OrderedDict() 755 for _ in range(length): 756 # Unpack key 757 k = _unpack(fp, options) 758 759 if isinstance(k, list): 760 # Attempt to convert list into a hashable tuple 761 k = _deep_list_to_tuple(k) 762 elif not isinstance(k, collections.Hashable): 763 raise UnhashableKeyException( 764 "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) 765 elif k in d: 766 raise DuplicateKeyException( 767 "encountered duplicate key: %s, %s" % (str(k), str(type(k)))) 768 769 # Unpack value 770 v = _unpack(fp, options) 771 772 try: 773 d[k] = v 774 except TypeError: 775 raise UnhashableKeyException( 776 "encountered unhashable key: %s" % str(k)) 777 return d 778 779 780 def _unpack(fp, options): 781 code = _read_except(fp, 1) 782 return _unpack_dispatch_table[code](code, fp, options) 783 784 ######################################## 785 786 787 def _unpack2(fp, **options): 788 """ 789 Deserialize MessagePack bytes into a Python object. 790 791 Args: 792 fp: a .read()-supporting file-like object 793 794 Kwargs: 795 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext 796 type to a callable that unpacks an instance of 797 Ext into an object 798 use_ordered_dict (bool): unpack maps into OrderedDict, instead of 799 unordered dict (default False) 800 allow_invalid_utf8 (bool): unpack invalid strings into instances of 801 InvalidString, for access to the bytes 802 (default False) 803 804 Returns: 805 A Python object. 806 807 Raises: 808 InsufficientDataException(UnpackException): 809 Insufficient data to unpack the serialized object. 810 InvalidStringException(UnpackException): 811 Invalid UTF-8 string encountered during unpacking. 812 ReservedCodeException(UnpackException): 813 Reserved code encountered during unpacking. 814 UnhashableKeyException(UnpackException): 815 Unhashable key encountered during map unpacking. 816 The serialized map cannot be deserialized into a Python dictionary. 817 DuplicateKeyException(UnpackException): 818 Duplicate key encountered during map unpacking. 819 820 Example: 821 >>> f = open('test.bin', 'rb') 822 >>> umsgpack.unpackb(f) 823 {u'compact': True, u'schema': 0} 824 >>> 825 """ 826 return _unpack(fp, options) 827 828 829 def _unpack3(fp, **options): 830 """ 831 Deserialize MessagePack bytes into a Python object. 832 833 Args: 834 fp: a .read()-supporting file-like object 835 836 Kwargs: 837 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext 838 type to a callable that unpacks an instance of 839 Ext into an object 840 use_ordered_dict (bool): unpack maps into OrderedDict, instead of 841 unordered dict (default False) 842 allow_invalid_utf8 (bool): unpack invalid strings into instances of 843 InvalidString, for access to the bytes 844 (default False) 845 846 Returns: 847 A Python object. 848 849 Raises: 850 InsufficientDataException(UnpackException): 851 Insufficient data to unpack the serialized object. 852 InvalidStringException(UnpackException): 853 Invalid UTF-8 string encountered during unpacking. 854 ReservedCodeException(UnpackException): 855 Reserved code encountered during unpacking. 856 UnhashableKeyException(UnpackException): 857 Unhashable key encountered during map unpacking. 858 The serialized map cannot be deserialized into a Python dictionary. 859 DuplicateKeyException(UnpackException): 860 Duplicate key encountered during map unpacking. 861 862 Example: 863 >>> f = open('test.bin', 'rb') 864 >>> umsgpack.unpackb(f) 865 {'compact': True, 'schema': 0} 866 >>> 867 """ 868 return _unpack(fp, options) 869 870 871 # For Python 2, expects a str object 872 def _unpackb2(s, **options): 873 """ 874 Deserialize MessagePack bytes into a Python object. 875 876 Args: 877 s: a 'str' or 'bytearray' containing serialized MessagePack bytes 878 879 Kwargs: 880 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext 881 type to a callable that unpacks an instance of 882 Ext into an object 883 use_ordered_dict (bool): unpack maps into OrderedDict, instead of 884 unordered dict (default False) 885 allow_invalid_utf8 (bool): unpack invalid strings into instances of 886 InvalidString, for access to the bytes 887 (default False) 888 889 Returns: 890 A Python object. 891 892 Raises: 893 TypeError: 894 Packed data type is neither 'str' nor 'bytearray'. 895 InsufficientDataException(UnpackException): 896 Insufficient data to unpack the serialized object. 897 InvalidStringException(UnpackException): 898 Invalid UTF-8 string encountered during unpacking. 899 ReservedCodeException(UnpackException): 900 Reserved code encountered during unpacking. 901 UnhashableKeyException(UnpackException): 902 Unhashable key encountered during map unpacking. 903 The serialized map cannot be deserialized into a Python dictionary. 904 DuplicateKeyException(UnpackException): 905 Duplicate key encountered during map unpacking. 906 907 Example: 908 >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') 909 {u'compact': True, u'schema': 0} 910 >>> 911 """ 912 if not isinstance(s, (str, bytearray)): 913 raise TypeError("packed data must be type 'str' or 'bytearray'") 914 return _unpack(io.BytesIO(s), options) 915 916 917 # For Python 3, expects a bytes object 918 def _unpackb3(s, **options): 919 """ 920 Deserialize MessagePack bytes into a Python object. 921 922 Args: 923 s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes 924 925 Kwargs: 926 ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext 927 type to a callable that unpacks an instance of 928 Ext into an object 929 use_ordered_dict (bool): unpack maps into OrderedDict, instead of 930 unordered dict (default False) 931 allow_invalid_utf8 (bool): unpack invalid strings into instances of 932 InvalidString, for access to the bytes 933 (default False) 934 935 Returns: 936 A Python object. 937 938 Raises: 939 TypeError: 940 Packed data type is neither 'bytes' nor 'bytearray'. 941 InsufficientDataException(UnpackException): 942 Insufficient data to unpack the serialized object. 943 InvalidStringException(UnpackException): 944 Invalid UTF-8 string encountered during unpacking. 945 ReservedCodeException(UnpackException): 946 Reserved code encountered during unpacking. 947 UnhashableKeyException(UnpackException): 948 Unhashable key encountered during map unpacking. 949 The serialized map cannot be deserialized into a Python dictionary. 950 DuplicateKeyException(UnpackException): 951 Duplicate key encountered during map unpacking. 952 953 Example: 954 >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') 955 {'compact': True, 'schema': 0} 956 >>> 957 """ 958 if not isinstance(s, (bytes, bytearray)): 959 raise TypeError("packed data must be type 'bytes' or 'bytearray'") 960 return _unpack(io.BytesIO(s), options) 961 962 ############################################################################# 963 # Module Initialization 964 ############################################################################# 965 966 967 def __init(): 968 # pylint: disable=global-variable-undefined 969 970 global pack 971 global packb 972 global unpack 973 global unpackb 974 global dump 975 global dumps 976 global load 977 global loads 978 global compatibility 979 global _float_precision 980 global _unpack_dispatch_table 981 global xrange 982 983 # Compatibility mode for handling strings/bytes with the old specification 984 compatibility = False 985 986 # Auto-detect system float precision 987 if sys.float_info.mant_dig == 53: 988 _float_precision = "double" 989 else: 990 _float_precision = "single" 991 992 # Map packb and unpackb to the appropriate version 993 if sys.version_info[0] == 3: 994 pack = _pack3 995 packb = _packb3 996 dump = _pack3 997 dumps = _packb3 998 unpack = _unpack3 999 unpackb = _unpackb3 1000 load = _unpack3 1001 loads = _unpackb3 1002 xrange = range # pylint: disable=redefined-builtin 1003 else: 1004 pack = _pack2 1005 packb = _packb2 1006 dump = _pack2 1007 dumps = _packb2 1008 unpack = _unpack2 1009 unpackb = _unpackb2 1010 load = _unpack2 1011 loads = _unpackb2 1012 1013 # Build a dispatch table for fast lookup of unpacking function 1014 1015 _unpack_dispatch_table = {} 1016 # Fix uint 1017 for code in range(0, 0x7f + 1): 1018 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer 1019 # Fix map 1020 for code in range(0x80, 0x8f + 1): 1021 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map 1022 # Fix array 1023 for code in range(0x90, 0x9f + 1): 1024 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array 1025 # Fix str 1026 for code in range(0xa0, 0xbf + 1): 1027 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string 1028 # Nil 1029 _unpack_dispatch_table[b'\xc0'] = _unpack_nil 1030 # Reserved 1031 _unpack_dispatch_table[b'\xc1'] = _unpack_reserved 1032 # Boolean 1033 _unpack_dispatch_table[b'\xc2'] = _unpack_boolean 1034 _unpack_dispatch_table[b'\xc3'] = _unpack_boolean 1035 # Bin 1036 for code in range(0xc4, 0xc6 + 1): 1037 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary 1038 # Ext 1039 for code in range(0xc7, 0xc9 + 1): 1040 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext 1041 # Float 1042 _unpack_dispatch_table[b'\xca'] = _unpack_float 1043 _unpack_dispatch_table[b'\xcb'] = _unpack_float 1044 # Uint 1045 for code in range(0xcc, 0xcf + 1): 1046 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer 1047 # Int 1048 for code in range(0xd0, 0xd3 + 1): 1049 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer 1050 # Fixext 1051 for code in range(0xd4, 0xd8 + 1): 1052 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext 1053 # String 1054 for code in range(0xd9, 0xdb + 1): 1055 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string 1056 # Array 1057 _unpack_dispatch_table[b'\xdc'] = _unpack_array 1058 _unpack_dispatch_table[b'\xdd'] = _unpack_array 1059 # Map 1060 _unpack_dispatch_table[b'\xde'] = _unpack_map 1061 _unpack_dispatch_table[b'\xdf'] = _unpack_map 1062 # Negative fixint 1063 for code in range(0xe0, 0xff + 1): 1064 _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer 1065 1066 1067 __init()