123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977 |
- """Fallback pure Python implementation of msgpack"""
- import sys
- import struct
- import warnings
- if sys.version_info[0] == 3:
- PY3 = True
- int_types = int
- Unicode = str
- xrange = range
- def dict_iteritems(d):
- return d.items()
- else:
- PY3 = False
- int_types = (int, long)
- Unicode = unicode
- def dict_iteritems(d):
- return d.iteritems()
- if hasattr(sys, 'pypy_version_info'):
- # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
- # StringBuilder is fastest.
- from __pypy__ import newlist_hint
- try:
- from __pypy__.builders import BytesBuilder as StringBuilder
- except ImportError:
- from __pypy__.builders import StringBuilder
- USING_STRINGBUILDER = True
- class StringIO(object):
- def __init__(self, s=b''):
- if s:
- self.builder = StringBuilder(len(s))
- self.builder.append(s)
- else:
- self.builder = StringBuilder()
- def write(self, s):
- if isinstance(s, memoryview):
- s = s.tobytes()
- elif isinstance(s, bytearray):
- s = bytes(s)
- self.builder.append(s)
- def getvalue(self):
- return self.builder.build()
- else:
- USING_STRINGBUILDER = False
- from io import BytesIO as StringIO
- newlist_hint = lambda size: []
- from pip._vendor.msgpack.exceptions import (
- BufferFull,
- OutOfData,
- UnpackValueError,
- PackValueError,
- PackOverflowError,
- ExtraData)
- from pip._vendor.msgpack import ExtType
- EX_SKIP = 0
- EX_CONSTRUCT = 1
- EX_READ_ARRAY_HEADER = 2
- EX_READ_MAP_HEADER = 3
- TYPE_IMMEDIATE = 0
- TYPE_ARRAY = 1
- TYPE_MAP = 2
- TYPE_RAW = 3
- TYPE_BIN = 4
- TYPE_EXT = 5
- DEFAULT_RECURSE_LIMIT = 511
- def _check_type_strict(obj, t, type=type, tuple=tuple):
- if type(t) is tuple:
- return type(obj) in t
- else:
- return type(obj) is t
- def _get_data_from_buffer(obj):
- try:
- view = memoryview(obj)
- except TypeError:
- # try to use legacy buffer protocol if 2.7, otherwise re-raise
- if not PY3:
- view = memoryview(buffer(obj))
- warnings.warn("using old buffer interface to unpack %s; "
- "this leads to unpacking errors if slicing is used and "
- "will be removed in a future version" % type(obj),
- RuntimeWarning)
- else:
- raise
- if view.itemsize != 1:
- raise ValueError("cannot unpack from multi-byte object")
- return view
- def unpack(stream, **kwargs):
- warnings.warn(
- "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
- PendingDeprecationWarning)
- data = stream.read()
- return unpackb(data, **kwargs)
- def unpackb(packed, **kwargs):
- """
- Unpack an object from `packed`.
- Raises `ExtraData` when `packed` contains extra bytes.
- See :class:`Unpacker` for options.
- """
- unpacker = Unpacker(None, **kwargs)
- unpacker.feed(packed)
- try:
- ret = unpacker._unpack()
- except OutOfData:
- raise UnpackValueError("Data is not enough.")
- if unpacker._got_extradata():
- raise ExtraData(ret, unpacker._get_extradata())
- return ret
- class Unpacker(object):
- """Streaming unpacker.
- arguments:
- :param file_like:
- File-like object having `.read(n)` method.
- If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
- :param int read_size:
- Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
- :param bool use_list:
- If true, unpack msgpack array to Python list.
- Otherwise, unpack to Python tuple. (default: True)
- :param bool raw:
- If true, unpack msgpack raw to Python bytes (default).
- Otherwise, unpack to Python str (or unicode on Python 2) by decoding
- with UTF-8 encoding (recommended).
- Currently, the default is true, but it will be changed to false in
- near future. So you must specify it explicitly for keeping backward
- compatibility.
- *encoding* option which is deprecated overrides this option.
- :param callable object_hook:
- When specified, it should be callable.
- Unpacker calls it with a dict argument after unpacking msgpack map.
- (See also simplejson)
- :param callable object_pairs_hook:
- When specified, it should be callable.
- Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
- (See also simplejson)
- :param str encoding:
- Encoding used for decoding msgpack raw.
- If it is None (default), msgpack raw is deserialized to Python bytes.
- :param str unicode_errors:
- (deprecated) Used for decoding msgpack raw with *encoding*.
- (default: `'strict'`)
- :param int max_buffer_size:
- Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
- Raises `BufferFull` exception when it is insufficient.
- You should set this parameter when unpacking data from untrusted source.
- :param int max_str_len:
- Limits max length of str. (default: 2**31-1)
- :param int max_bin_len:
- Limits max length of bin. (default: 2**31-1)
- :param int max_array_len:
- Limits max length of array. (default: 2**31-1)
- :param int max_map_len:
- Limits max length of map. (default: 2**31-1)
- example of streaming deserialize from file-like object::
- unpacker = Unpacker(file_like, raw=False)
- for o in unpacker:
- process(o)
- example of streaming deserialize from socket::
- unpacker = Unpacker(raw=False)
- while True:
- buf = sock.recv(1024**2)
- if not buf:
- break
- unpacker.feed(buf)
- for o in unpacker:
- process(o)
- """
- def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
- object_hook=None, object_pairs_hook=None, list_hook=None,
- encoding=None, unicode_errors=None, max_buffer_size=0,
- ext_hook=ExtType,
- max_str_len=2147483647, # 2**32-1
- max_bin_len=2147483647,
- max_array_len=2147483647,
- max_map_len=2147483647,
- max_ext_len=2147483647):
- if encoding is not None:
- warnings.warn(
- "encoding is deprecated, Use raw=False instead.",
- PendingDeprecationWarning)
- if unicode_errors is None:
- unicode_errors = 'strict'
- if file_like is None:
- self._feeding = True
- else:
- if not callable(file_like.read):
- raise TypeError("`file_like.read` must be callable")
- self.file_like = file_like
- self._feeding = False
- #: array of bytes fed.
- self._buffer = bytearray()
- # Some very old pythons don't support `struct.unpack_from()` with a
- # `bytearray`. So we wrap it in a `buffer()` there.
- if sys.version_info < (2, 7, 6):
- self._buffer_view = buffer(self._buffer)
- else:
- self._buffer_view = self._buffer
- #: Which position we currently reads
- self._buff_i = 0
- # When Unpacker is used as an iterable, between the calls to next(),
- # the buffer is not "consumed" completely, for efficiency sake.
- # Instead, it is done sloppily. To make sure we raise BufferFull at
- # the correct moments, we have to keep track of how sloppy we were.
- # Furthermore, when the buffer is incomplete (that is: in the case
- # we raise an OutOfData) we need to rollback the buffer to the correct
- # state, which _buf_checkpoint records.
- self._buf_checkpoint = 0
- self._max_buffer_size = max_buffer_size or 2**31-1
- if read_size > self._max_buffer_size:
- raise ValueError("read_size must be smaller than max_buffer_size")
- self._read_size = read_size or min(self._max_buffer_size, 16*1024)
- self._raw = bool(raw)
- self._encoding = encoding
- self._unicode_errors = unicode_errors
- self._use_list = use_list
- self._list_hook = list_hook
- self._object_hook = object_hook
- self._object_pairs_hook = object_pairs_hook
- self._ext_hook = ext_hook
- self._max_str_len = max_str_len
- self._max_bin_len = max_bin_len
- self._max_array_len = max_array_len
- self._max_map_len = max_map_len
- self._max_ext_len = max_ext_len
- self._stream_offset = 0
- if list_hook is not None and not callable(list_hook):
- raise TypeError('`list_hook` is not callable')
- if object_hook is not None and not callable(object_hook):
- raise TypeError('`object_hook` is not callable')
- if object_pairs_hook is not None and not callable(object_pairs_hook):
- raise TypeError('`object_pairs_hook` is not callable')
- if object_hook is not None and object_pairs_hook is not None:
- raise TypeError("object_pairs_hook and object_hook are mutually "
- "exclusive")
- if not callable(ext_hook):
- raise TypeError("`ext_hook` is not callable")
- def feed(self, next_bytes):
- assert self._feeding
- view = _get_data_from_buffer(next_bytes)
- if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
- raise BufferFull
- # Strip buffer before checkpoint before reading file.
- if self._buf_checkpoint > 0:
- del self._buffer[:self._buf_checkpoint]
- self._buff_i -= self._buf_checkpoint
- self._buf_checkpoint = 0
- self._buffer += view
- def _consume(self):
- """ Gets rid of the used parts of the buffer. """
- self._stream_offset += self._buff_i - self._buf_checkpoint
- self._buf_checkpoint = self._buff_i
- def _got_extradata(self):
- return self._buff_i < len(self._buffer)
- def _get_extradata(self):
- return self._buffer[self._buff_i:]
- def read_bytes(self, n):
- return self._read(n)
- def _read(self, n):
- # (int) -> bytearray
- self._reserve(n)
- i = self._buff_i
- self._buff_i = i+n
- return self._buffer[i:i+n]
- def _reserve(self, n):
- remain_bytes = len(self._buffer) - self._buff_i - n
- # Fast path: buffer has n bytes already
- if remain_bytes >= 0:
- return
- if self._feeding:
- self._buff_i = self._buf_checkpoint
- raise OutOfData
- # Strip buffer before checkpoint before reading file.
- if self._buf_checkpoint > 0:
- del self._buffer[:self._buf_checkpoint]
- self._buff_i -= self._buf_checkpoint
- self._buf_checkpoint = 0
- # Read from file
- remain_bytes = -remain_bytes
- while remain_bytes > 0:
- to_read_bytes = max(self._read_size, remain_bytes)
- read_data = self.file_like.read(to_read_bytes)
- if not read_data:
- break
- assert isinstance(read_data, bytes)
- self._buffer += read_data
- remain_bytes -= len(read_data)
- if len(self._buffer) < n + self._buff_i:
- self._buff_i = 0 # rollback
- raise OutOfData
- def _read_header(self, execute=EX_CONSTRUCT):
- typ = TYPE_IMMEDIATE
- n = 0
- obj = None
- self._reserve(1)
- b = self._buffer[self._buff_i]
- self._buff_i += 1
- if b & 0b10000000 == 0:
- obj = b
- elif b & 0b11100000 == 0b11100000:
- obj = -1 - (b ^ 0xff)
- elif b & 0b11100000 == 0b10100000:
- n = b & 0b00011111
- typ = TYPE_RAW
- if n > self._max_str_len:
- raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
- obj = self._read(n)
- elif b & 0b11110000 == 0b10010000:
- n = b & 0b00001111
- typ = TYPE_ARRAY
- if n > self._max_array_len:
- raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
- elif b & 0b11110000 == 0b10000000:
- n = b & 0b00001111
- typ = TYPE_MAP
- if n > self._max_map_len:
- raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
- elif b == 0xc0:
- obj = None
- elif b == 0xc2:
- obj = False
- elif b == 0xc3:
- obj = True
- elif b == 0xc4:
- typ = TYPE_BIN
- self._reserve(1)
- n = self._buffer[self._buff_i]
- self._buff_i += 1
- if n > self._max_bin_len:
- raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
- obj = self._read(n)
- elif b == 0xc5:
- typ = TYPE_BIN
- self._reserve(2)
- n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
- self._buff_i += 2
- if n > self._max_bin_len:
- raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
- obj = self._read(n)
- elif b == 0xc6:
- typ = TYPE_BIN
- self._reserve(4)
- n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
- self._buff_i += 4
- if n > self._max_bin_len:
- raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
- obj = self._read(n)
- elif b == 0xc7: # ext 8
- typ = TYPE_EXT
- self._reserve(2)
- L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i)
- self._buff_i += 2
- if L > self._max_ext_len:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
- obj = self._read(L)
- elif b == 0xc8: # ext 16
- typ = TYPE_EXT
- self._reserve(3)
- L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i)
- self._buff_i += 3
- if L > self._max_ext_len:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
- obj = self._read(L)
- elif b == 0xc9: # ext 32
- typ = TYPE_EXT
- self._reserve(5)
- L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i)
- self._buff_i += 5
- if L > self._max_ext_len:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
- obj = self._read(L)
- elif b == 0xca:
- self._reserve(4)
- obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0]
- self._buff_i += 4
- elif b == 0xcb:
- self._reserve(8)
- obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0]
- self._buff_i += 8
- elif b == 0xcc:
- self._reserve(1)
- obj = self._buffer[self._buff_i]
- self._buff_i += 1
- elif b == 0xcd:
- self._reserve(2)
- obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
- self._buff_i += 2
- elif b == 0xce:
- self._reserve(4)
- obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
- self._buff_i += 4
- elif b == 0xcf:
- self._reserve(8)
- obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0]
- self._buff_i += 8
- elif b == 0xd0:
- self._reserve(1)
- obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0]
- self._buff_i += 1
- elif b == 0xd1:
- self._reserve(2)
- obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0]
- self._buff_i += 2
- elif b == 0xd2:
- self._reserve(4)
- obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0]
- self._buff_i += 4
- elif b == 0xd3:
- self._reserve(8)
- obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0]
- self._buff_i += 8
- elif b == 0xd4: # fixext 1
- typ = TYPE_EXT
- if self._max_ext_len < 1:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
- self._reserve(2)
- n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i)
- self._buff_i += 2
- elif b == 0xd5: # fixext 2
- typ = TYPE_EXT
- if self._max_ext_len < 2:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
- self._reserve(3)
- n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i)
- self._buff_i += 3
- elif b == 0xd6: # fixext 4
- typ = TYPE_EXT
- if self._max_ext_len < 4:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
- self._reserve(5)
- n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i)
- self._buff_i += 5
- elif b == 0xd7: # fixext 8
- typ = TYPE_EXT
- if self._max_ext_len < 8:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
- self._reserve(9)
- n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i)
- self._buff_i += 9
- elif b == 0xd8: # fixext 16
- typ = TYPE_EXT
- if self._max_ext_len < 16:
- raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
- self._reserve(17)
- n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i)
- self._buff_i += 17
- elif b == 0xd9:
- typ = TYPE_RAW
- self._reserve(1)
- n = self._buffer[self._buff_i]
- self._buff_i += 1
- if n > self._max_str_len:
- raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
- obj = self._read(n)
- elif b == 0xda:
- typ = TYPE_RAW
- self._reserve(2)
- n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
- self._buff_i += 2
- if n > self._max_str_len:
- raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
- obj = self._read(n)
- elif b == 0xdb:
- typ = TYPE_RAW
- self._reserve(4)
- n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
- self._buff_i += 4
- if n > self._max_str_len:
- raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
- obj = self._read(n)
- elif b == 0xdc:
- typ = TYPE_ARRAY
- self._reserve(2)
- n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
- self._buff_i += 2
- if n > self._max_array_len:
- raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
- elif b == 0xdd:
- typ = TYPE_ARRAY
- self._reserve(4)
- n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
- self._buff_i += 4
- if n > self._max_array_len:
- raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
- elif b == 0xde:
- self._reserve(2)
- n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
- self._buff_i += 2
- if n > self._max_map_len:
- raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
- typ = TYPE_MAP
- elif b == 0xdf:
- self._reserve(4)
- n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
- self._buff_i += 4
- if n > self._max_map_len:
- raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
- typ = TYPE_MAP
- else:
- raise UnpackValueError("Unknown header: 0x%x" % b)
- return typ, n, obj
- def _unpack(self, execute=EX_CONSTRUCT):
- typ, n, obj = self._read_header(execute)
- if execute == EX_READ_ARRAY_HEADER:
- if typ != TYPE_ARRAY:
- raise UnpackValueError("Expected array")
- return n
- if execute == EX_READ_MAP_HEADER:
- if typ != TYPE_MAP:
- raise UnpackValueError("Expected map")
- return n
- # TODO should we eliminate the recursion?
- if typ == TYPE_ARRAY:
- if execute == EX_SKIP:
- for i in xrange(n):
- # TODO check whether we need to call `list_hook`
- self._unpack(EX_SKIP)
- return
- ret = newlist_hint(n)
- for i in xrange(n):
- ret.append(self._unpack(EX_CONSTRUCT))
- if self._list_hook is not None:
- ret = self._list_hook(ret)
- # TODO is the interaction between `list_hook` and `use_list` ok?
- return ret if self._use_list else tuple(ret)
- if typ == TYPE_MAP:
- if execute == EX_SKIP:
- for i in xrange(n):
- # TODO check whether we need to call hooks
- self._unpack(EX_SKIP)
- self._unpack(EX_SKIP)
- return
- if self._object_pairs_hook is not None:
- ret = self._object_pairs_hook(
- (self._unpack(EX_CONSTRUCT),
- self._unpack(EX_CONSTRUCT))
- for _ in xrange(n))
- else:
- ret = {}
- for _ in xrange(n):
- key = self._unpack(EX_CONSTRUCT)
- ret[key] = self._unpack(EX_CONSTRUCT)
- if self._object_hook is not None:
- ret = self._object_hook(ret)
- return ret
- if execute == EX_SKIP:
- return
- if typ == TYPE_RAW:
- if self._encoding is not None:
- obj = obj.decode(self._encoding, self._unicode_errors)
- elif self._raw:
- obj = bytes(obj)
- else:
- obj = obj.decode('utf_8')
- return obj
- if typ == TYPE_EXT:
- return self._ext_hook(n, bytes(obj))
- if typ == TYPE_BIN:
- return bytes(obj)
- assert typ == TYPE_IMMEDIATE
- return obj
- def __iter__(self):
- return self
- def __next__(self):
- try:
- ret = self._unpack(EX_CONSTRUCT)
- self._consume()
- return ret
- except OutOfData:
- self._consume()
- raise StopIteration
- next = __next__
- def skip(self, write_bytes=None):
- self._unpack(EX_SKIP)
- if write_bytes is not None:
- warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
- write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
- self._consume()
- def unpack(self, write_bytes=None):
- ret = self._unpack(EX_CONSTRUCT)
- if write_bytes is not None:
- warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
- write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
- self._consume()
- return ret
- def read_array_header(self, write_bytes=None):
- ret = self._unpack(EX_READ_ARRAY_HEADER)
- if write_bytes is not None:
- warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
- write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
- self._consume()
- return ret
- def read_map_header(self, write_bytes=None):
- ret = self._unpack(EX_READ_MAP_HEADER)
- if write_bytes is not None:
- warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
- write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
- self._consume()
- return ret
- def tell(self):
- return self._stream_offset
- class Packer(object):
- """
- MessagePack Packer
- usage:
- packer = Packer()
- astream.write(packer.pack(a))
- astream.write(packer.pack(b))
- Packer's constructor has some keyword arguments:
- :param callable default:
- Convert user type to builtin type that Packer supports.
- See also simplejson's document.
- :param bool use_single_float:
- Use single precision float type for float. (default: False)
- :param bool autoreset:
- Reset buffer after each pack and return its content as `bytes`. (default: True).
- If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
- :param bool use_bin_type:
- Use bin type introduced in msgpack spec 2.0 for bytes.
- It also enables str8 type for unicode.
- :param bool strict_types:
- If set to true, types will be checked to be exact. Derived classes
- from serializeable types will not be serialized and will be
- treated as unsupported type and forwarded to default.
- Additionally tuples will not be serialized as lists.
- This is useful when trying to implement accurate serialization
- for python types.
- :param str encoding:
- (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
- :param str unicode_errors:
- Error handler for encoding unicode. (default: 'strict')
- """
- def __init__(self, default=None, encoding=None, unicode_errors=None,
- use_single_float=False, autoreset=True, use_bin_type=False,
- strict_types=False):
- if encoding is None:
- encoding = 'utf_8'
- else:
- warnings.warn(
- "encoding is deprecated, Use raw=False instead.",
- PendingDeprecationWarning)
- if unicode_errors is None:
- unicode_errors = 'strict'
- self._strict_types = strict_types
- self._use_float = use_single_float
- self._autoreset = autoreset
- self._use_bin_type = use_bin_type
- self._encoding = encoding
- self._unicode_errors = unicode_errors
- self._buffer = StringIO()
- if default is not None:
- if not callable(default):
- raise TypeError("default must be callable")
- self._default = default
- def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
- check=isinstance, check_type_strict=_check_type_strict):
- default_used = False
- if self._strict_types:
- check = check_type_strict
- list_types = list
- else:
- list_types = (list, tuple)
- while True:
- if nest_limit < 0:
- raise PackValueError("recursion limit exceeded")
- if obj is None:
- return self._buffer.write(b"\xc0")
- if check(obj, bool):
- if obj:
- return self._buffer.write(b"\xc3")
- return self._buffer.write(b"\xc2")
- if check(obj, int_types):
- if 0 <= obj < 0x80:
- return self._buffer.write(struct.pack("B", obj))
- if -0x20 <= obj < 0:
- return self._buffer.write(struct.pack("b", obj))
- if 0x80 <= obj <= 0xff:
- return self._buffer.write(struct.pack("BB", 0xcc, obj))
- if -0x80 <= obj < 0:
- return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
- if 0xff < obj <= 0xffff:
- return self._buffer.write(struct.pack(">BH", 0xcd, obj))
- if -0x8000 <= obj < -0x80:
- return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
- if 0xffff < obj <= 0xffffffff:
- return self._buffer.write(struct.pack(">BI", 0xce, obj))
- if -0x80000000 <= obj < -0x8000:
- return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
- if 0xffffffff < obj <= 0xffffffffffffffff:
- return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
- if -0x8000000000000000 <= obj < -0x80000000:
- return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
- if not default_used and self._default is not None:
- obj = self._default(obj)
- default_used = True
- continue
- raise PackOverflowError("Integer value out of range")
- if check(obj, (bytes, bytearray)):
- n = len(obj)
- if n >= 2**32:
- raise PackValueError("%s is too large" % type(obj).__name__)
- self._pack_bin_header(n)
- return self._buffer.write(obj)
- if check(obj, Unicode):
- if self._encoding is None:
- raise TypeError(
- "Can't encode unicode string: "
- "no encoding is specified")
- obj = obj.encode(self._encoding, self._unicode_errors)
- n = len(obj)
- if n >= 2**32:
- raise PackValueError("String is too large")
- self._pack_raw_header(n)
- return self._buffer.write(obj)
- if check(obj, memoryview):
- n = len(obj) * obj.itemsize
- if n >= 2**32:
- raise PackValueError("Memoryview is too large")
- self._pack_bin_header(n)
- return self._buffer.write(obj)
- if check(obj, float):
- if self._use_float:
- return self._buffer.write(struct.pack(">Bf", 0xca, obj))
- return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
- if check(obj, ExtType):
- code = obj.code
- data = obj.data
- assert isinstance(code, int)
- assert isinstance(data, bytes)
- L = len(data)
- if L == 1:
- self._buffer.write(b'\xd4')
- elif L == 2:
- self._buffer.write(b'\xd5')
- elif L == 4:
- self._buffer.write(b'\xd6')
- elif L == 8:
- self._buffer.write(b'\xd7')
- elif L == 16:
- self._buffer.write(b'\xd8')
- elif L <= 0xff:
- self._buffer.write(struct.pack(">BB", 0xc7, L))
- elif L <= 0xffff:
- self._buffer.write(struct.pack(">BH", 0xc8, L))
- else:
- self._buffer.write(struct.pack(">BI", 0xc9, L))
- self._buffer.write(struct.pack("b", code))
- self._buffer.write(data)
- return
- if check(obj, list_types):
- n = len(obj)
- self._pack_array_header(n)
- for i in xrange(n):
- self._pack(obj[i], nest_limit - 1)
- return
- if check(obj, dict):
- return self._pack_map_pairs(len(obj), dict_iteritems(obj),
- nest_limit - 1)
- if not default_used and self._default is not None:
- obj = self._default(obj)
- default_used = 1
- continue
- raise TypeError("Cannot serialize %r" % (obj, ))
- def pack(self, obj):
- try:
- self._pack(obj)
- except:
- self._buffer = StringIO() # force reset
- raise
- ret = self._buffer.getvalue()
- if self._autoreset:
- self._buffer = StringIO()
- elif USING_STRINGBUILDER:
- self._buffer = StringIO(ret)
- return ret
- def pack_map_pairs(self, pairs):
- self._pack_map_pairs(len(pairs), pairs)
- ret = self._buffer.getvalue()
- if self._autoreset:
- self._buffer = StringIO()
- elif USING_STRINGBUILDER:
- self._buffer = StringIO(ret)
- return ret
- def pack_array_header(self, n):
- if n >= 2**32:
- raise PackValueError
- self._pack_array_header(n)
- ret = self._buffer.getvalue()
- if self._autoreset:
- self._buffer = StringIO()
- elif USING_STRINGBUILDER:
- self._buffer = StringIO(ret)
- return ret
- def pack_map_header(self, n):
- if n >= 2**32:
- raise PackValueError
- self._pack_map_header(n)
- ret = self._buffer.getvalue()
- if self._autoreset:
- self._buffer = StringIO()
- elif USING_STRINGBUILDER:
- self._buffer = StringIO(ret)
- return ret
- def pack_ext_type(self, typecode, data):
- if not isinstance(typecode, int):
- raise TypeError("typecode must have int type.")
- if not 0 <= typecode <= 127:
- raise ValueError("typecode should be 0-127")
- if not isinstance(data, bytes):
- raise TypeError("data must have bytes type")
- L = len(data)
- if L > 0xffffffff:
- raise PackValueError("Too large data")
- if L == 1:
- self._buffer.write(b'\xd4')
- elif L == 2:
- self._buffer.write(b'\xd5')
- elif L == 4:
- self._buffer.write(b'\xd6')
- elif L == 8:
- self._buffer.write(b'\xd7')
- elif L == 16:
- self._buffer.write(b'\xd8')
- elif L <= 0xff:
- self._buffer.write(b'\xc7' + struct.pack('B', L))
- elif L <= 0xffff:
- self._buffer.write(b'\xc8' + struct.pack('>H', L))
- else:
- self._buffer.write(b'\xc9' + struct.pack('>I', L))
- self._buffer.write(struct.pack('B', typecode))
- self._buffer.write(data)
- def _pack_array_header(self, n):
- if n <= 0x0f:
- return self._buffer.write(struct.pack('B', 0x90 + n))
- if n <= 0xffff:
- return self._buffer.write(struct.pack(">BH", 0xdc, n))
- if n <= 0xffffffff:
- return self._buffer.write(struct.pack(">BI", 0xdd, n))
- raise PackValueError("Array is too large")
- def _pack_map_header(self, n):
- if n <= 0x0f:
- return self._buffer.write(struct.pack('B', 0x80 + n))
- if n <= 0xffff:
- return self._buffer.write(struct.pack(">BH", 0xde, n))
- if n <= 0xffffffff:
- return self._buffer.write(struct.pack(">BI", 0xdf, n))
- raise PackValueError("Dict is too large")
- def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
- self._pack_map_header(n)
- for (k, v) in pairs:
- self._pack(k, nest_limit - 1)
- self._pack(v, nest_limit - 1)
- def _pack_raw_header(self, n):
- if n <= 0x1f:
- self._buffer.write(struct.pack('B', 0xa0 + n))
- elif self._use_bin_type and n <= 0xff:
- self._buffer.write(struct.pack('>BB', 0xd9, n))
- elif n <= 0xffff:
- self._buffer.write(struct.pack(">BH", 0xda, n))
- elif n <= 0xffffffff:
- self._buffer.write(struct.pack(">BI", 0xdb, n))
- else:
- raise PackValueError('Raw is too large')
- def _pack_bin_header(self, n):
- if not self._use_bin_type:
- return self._pack_raw_header(n)
- elif n <= 0xff:
- return self._buffer.write(struct.pack('>BB', 0xc4, n))
- elif n <= 0xffff:
- return self._buffer.write(struct.pack(">BH", 0xc5, n))
- elif n <= 0xffffffff:
- return self._buffer.write(struct.pack(">BI", 0xc6, n))
- else:
- raise PackValueError('Bin is too large')
- def bytes(self):
- return self._buffer.getvalue()
- def reset(self):
- self._buffer = StringIO()
|