fallback.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977
  1. """Fallback pure Python implementation of msgpack"""
  2. import sys
  3. import struct
  4. import warnings
  5. if sys.version_info[0] == 3:
  6. PY3 = True
  7. int_types = int
  8. Unicode = str
  9. xrange = range
  10. def dict_iteritems(d):
  11. return d.items()
  12. else:
  13. PY3 = False
  14. int_types = (int, long)
  15. Unicode = unicode
  16. def dict_iteritems(d):
  17. return d.iteritems()
  18. if hasattr(sys, 'pypy_version_info'):
  19. # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
  20. # StringBuilder is fastest.
  21. from __pypy__ import newlist_hint
  22. try:
  23. from __pypy__.builders import BytesBuilder as StringBuilder
  24. except ImportError:
  25. from __pypy__.builders import StringBuilder
  26. USING_STRINGBUILDER = True
  27. class StringIO(object):
  28. def __init__(self, s=b''):
  29. if s:
  30. self.builder = StringBuilder(len(s))
  31. self.builder.append(s)
  32. else:
  33. self.builder = StringBuilder()
  34. def write(self, s):
  35. if isinstance(s, memoryview):
  36. s = s.tobytes()
  37. elif isinstance(s, bytearray):
  38. s = bytes(s)
  39. self.builder.append(s)
  40. def getvalue(self):
  41. return self.builder.build()
  42. else:
  43. USING_STRINGBUILDER = False
  44. from io import BytesIO as StringIO
  45. newlist_hint = lambda size: []
  46. from pip._vendor.msgpack.exceptions import (
  47. BufferFull,
  48. OutOfData,
  49. UnpackValueError,
  50. PackValueError,
  51. PackOverflowError,
  52. ExtraData)
  53. from pip._vendor.msgpack import ExtType
  54. EX_SKIP = 0
  55. EX_CONSTRUCT = 1
  56. EX_READ_ARRAY_HEADER = 2
  57. EX_READ_MAP_HEADER = 3
  58. TYPE_IMMEDIATE = 0
  59. TYPE_ARRAY = 1
  60. TYPE_MAP = 2
  61. TYPE_RAW = 3
  62. TYPE_BIN = 4
  63. TYPE_EXT = 5
  64. DEFAULT_RECURSE_LIMIT = 511
  65. def _check_type_strict(obj, t, type=type, tuple=tuple):
  66. if type(t) is tuple:
  67. return type(obj) in t
  68. else:
  69. return type(obj) is t
  70. def _get_data_from_buffer(obj):
  71. try:
  72. view = memoryview(obj)
  73. except TypeError:
  74. # try to use legacy buffer protocol if 2.7, otherwise re-raise
  75. if not PY3:
  76. view = memoryview(buffer(obj))
  77. warnings.warn("using old buffer interface to unpack %s; "
  78. "this leads to unpacking errors if slicing is used and "
  79. "will be removed in a future version" % type(obj),
  80. RuntimeWarning)
  81. else:
  82. raise
  83. if view.itemsize != 1:
  84. raise ValueError("cannot unpack from multi-byte object")
  85. return view
  86. def unpack(stream, **kwargs):
  87. warnings.warn(
  88. "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
  89. PendingDeprecationWarning)
  90. data = stream.read()
  91. return unpackb(data, **kwargs)
  92. def unpackb(packed, **kwargs):
  93. """
  94. Unpack an object from `packed`.
  95. Raises `ExtraData` when `packed` contains extra bytes.
  96. See :class:`Unpacker` for options.
  97. """
  98. unpacker = Unpacker(None, **kwargs)
  99. unpacker.feed(packed)
  100. try:
  101. ret = unpacker._unpack()
  102. except OutOfData:
  103. raise UnpackValueError("Data is not enough.")
  104. if unpacker._got_extradata():
  105. raise ExtraData(ret, unpacker._get_extradata())
  106. return ret
  107. class Unpacker(object):
  108. """Streaming unpacker.
  109. arguments:
  110. :param file_like:
  111. File-like object having `.read(n)` method.
  112. If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
  113. :param int read_size:
  114. Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
  115. :param bool use_list:
  116. If true, unpack msgpack array to Python list.
  117. Otherwise, unpack to Python tuple. (default: True)
  118. :param bool raw:
  119. If true, unpack msgpack raw to Python bytes (default).
  120. Otherwise, unpack to Python str (or unicode on Python 2) by decoding
  121. with UTF-8 encoding (recommended).
  122. Currently, the default is true, but it will be changed to false in
  123. near future. So you must specify it explicitly for keeping backward
  124. compatibility.
  125. *encoding* option which is deprecated overrides this option.
  126. :param callable object_hook:
  127. When specified, it should be callable.
  128. Unpacker calls it with a dict argument after unpacking msgpack map.
  129. (See also simplejson)
  130. :param callable object_pairs_hook:
  131. When specified, it should be callable.
  132. Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
  133. (See also simplejson)
  134. :param str encoding:
  135. Encoding used for decoding msgpack raw.
  136. If it is None (default), msgpack raw is deserialized to Python bytes.
  137. :param str unicode_errors:
  138. (deprecated) Used for decoding msgpack raw with *encoding*.
  139. (default: `'strict'`)
  140. :param int max_buffer_size:
  141. Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
  142. Raises `BufferFull` exception when it is insufficient.
  143. You should set this parameter when unpacking data from untrusted source.
  144. :param int max_str_len:
  145. Limits max length of str. (default: 2**31-1)
  146. :param int max_bin_len:
  147. Limits max length of bin. (default: 2**31-1)
  148. :param int max_array_len:
  149. Limits max length of array. (default: 2**31-1)
  150. :param int max_map_len:
  151. Limits max length of map. (default: 2**31-1)
  152. example of streaming deserialize from file-like object::
  153. unpacker = Unpacker(file_like, raw=False)
  154. for o in unpacker:
  155. process(o)
  156. example of streaming deserialize from socket::
  157. unpacker = Unpacker(raw=False)
  158. while True:
  159. buf = sock.recv(1024**2)
  160. if not buf:
  161. break
  162. unpacker.feed(buf)
  163. for o in unpacker:
  164. process(o)
  165. """
  166. def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
  167. object_hook=None, object_pairs_hook=None, list_hook=None,
  168. encoding=None, unicode_errors=None, max_buffer_size=0,
  169. ext_hook=ExtType,
  170. max_str_len=2147483647, # 2**32-1
  171. max_bin_len=2147483647,
  172. max_array_len=2147483647,
  173. max_map_len=2147483647,
  174. max_ext_len=2147483647):
  175. if encoding is not None:
  176. warnings.warn(
  177. "encoding is deprecated, Use raw=False instead.",
  178. PendingDeprecationWarning)
  179. if unicode_errors is None:
  180. unicode_errors = 'strict'
  181. if file_like is None:
  182. self._feeding = True
  183. else:
  184. if not callable(file_like.read):
  185. raise TypeError("`file_like.read` must be callable")
  186. self.file_like = file_like
  187. self._feeding = False
  188. #: array of bytes fed.
  189. self._buffer = bytearray()
  190. # Some very old pythons don't support `struct.unpack_from()` with a
  191. # `bytearray`. So we wrap it in a `buffer()` there.
  192. if sys.version_info < (2, 7, 6):
  193. self._buffer_view = buffer(self._buffer)
  194. else:
  195. self._buffer_view = self._buffer
  196. #: Which position we currently reads
  197. self._buff_i = 0
  198. # When Unpacker is used as an iterable, between the calls to next(),
  199. # the buffer is not "consumed" completely, for efficiency sake.
  200. # Instead, it is done sloppily. To make sure we raise BufferFull at
  201. # the correct moments, we have to keep track of how sloppy we were.
  202. # Furthermore, when the buffer is incomplete (that is: in the case
  203. # we raise an OutOfData) we need to rollback the buffer to the correct
  204. # state, which _buf_checkpoint records.
  205. self._buf_checkpoint = 0
  206. self._max_buffer_size = max_buffer_size or 2**31-1
  207. if read_size > self._max_buffer_size:
  208. raise ValueError("read_size must be smaller than max_buffer_size")
  209. self._read_size = read_size or min(self._max_buffer_size, 16*1024)
  210. self._raw = bool(raw)
  211. self._encoding = encoding
  212. self._unicode_errors = unicode_errors
  213. self._use_list = use_list
  214. self._list_hook = list_hook
  215. self._object_hook = object_hook
  216. self._object_pairs_hook = object_pairs_hook
  217. self._ext_hook = ext_hook
  218. self._max_str_len = max_str_len
  219. self._max_bin_len = max_bin_len
  220. self._max_array_len = max_array_len
  221. self._max_map_len = max_map_len
  222. self._max_ext_len = max_ext_len
  223. self._stream_offset = 0
  224. if list_hook is not None and not callable(list_hook):
  225. raise TypeError('`list_hook` is not callable')
  226. if object_hook is not None and not callable(object_hook):
  227. raise TypeError('`object_hook` is not callable')
  228. if object_pairs_hook is not None and not callable(object_pairs_hook):
  229. raise TypeError('`object_pairs_hook` is not callable')
  230. if object_hook is not None and object_pairs_hook is not None:
  231. raise TypeError("object_pairs_hook and object_hook are mutually "
  232. "exclusive")
  233. if not callable(ext_hook):
  234. raise TypeError("`ext_hook` is not callable")
  235. def feed(self, next_bytes):
  236. assert self._feeding
  237. view = _get_data_from_buffer(next_bytes)
  238. if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
  239. raise BufferFull
  240. # Strip buffer before checkpoint before reading file.
  241. if self._buf_checkpoint > 0:
  242. del self._buffer[:self._buf_checkpoint]
  243. self._buff_i -= self._buf_checkpoint
  244. self._buf_checkpoint = 0
  245. self._buffer += view
  246. def _consume(self):
  247. """ Gets rid of the used parts of the buffer. """
  248. self._stream_offset += self._buff_i - self._buf_checkpoint
  249. self._buf_checkpoint = self._buff_i
  250. def _got_extradata(self):
  251. return self._buff_i < len(self._buffer)
  252. def _get_extradata(self):
  253. return self._buffer[self._buff_i:]
  254. def read_bytes(self, n):
  255. return self._read(n)
  256. def _read(self, n):
  257. # (int) -> bytearray
  258. self._reserve(n)
  259. i = self._buff_i
  260. self._buff_i = i+n
  261. return self._buffer[i:i+n]
  262. def _reserve(self, n):
  263. remain_bytes = len(self._buffer) - self._buff_i - n
  264. # Fast path: buffer has n bytes already
  265. if remain_bytes >= 0:
  266. return
  267. if self._feeding:
  268. self._buff_i = self._buf_checkpoint
  269. raise OutOfData
  270. # Strip buffer before checkpoint before reading file.
  271. if self._buf_checkpoint > 0:
  272. del self._buffer[:self._buf_checkpoint]
  273. self._buff_i -= self._buf_checkpoint
  274. self._buf_checkpoint = 0
  275. # Read from file
  276. remain_bytes = -remain_bytes
  277. while remain_bytes > 0:
  278. to_read_bytes = max(self._read_size, remain_bytes)
  279. read_data = self.file_like.read(to_read_bytes)
  280. if not read_data:
  281. break
  282. assert isinstance(read_data, bytes)
  283. self._buffer += read_data
  284. remain_bytes -= len(read_data)
  285. if len(self._buffer) < n + self._buff_i:
  286. self._buff_i = 0 # rollback
  287. raise OutOfData
  288. def _read_header(self, execute=EX_CONSTRUCT):
  289. typ = TYPE_IMMEDIATE
  290. n = 0
  291. obj = None
  292. self._reserve(1)
  293. b = self._buffer[self._buff_i]
  294. self._buff_i += 1
  295. if b & 0b10000000 == 0:
  296. obj = b
  297. elif b & 0b11100000 == 0b11100000:
  298. obj = -1 - (b ^ 0xff)
  299. elif b & 0b11100000 == 0b10100000:
  300. n = b & 0b00011111
  301. typ = TYPE_RAW
  302. if n > self._max_str_len:
  303. raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
  304. obj = self._read(n)
  305. elif b & 0b11110000 == 0b10010000:
  306. n = b & 0b00001111
  307. typ = TYPE_ARRAY
  308. if n > self._max_array_len:
  309. raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
  310. elif b & 0b11110000 == 0b10000000:
  311. n = b & 0b00001111
  312. typ = TYPE_MAP
  313. if n > self._max_map_len:
  314. raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
  315. elif b == 0xc0:
  316. obj = None
  317. elif b == 0xc2:
  318. obj = False
  319. elif b == 0xc3:
  320. obj = True
  321. elif b == 0xc4:
  322. typ = TYPE_BIN
  323. self._reserve(1)
  324. n = self._buffer[self._buff_i]
  325. self._buff_i += 1
  326. if n > self._max_bin_len:
  327. raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
  328. obj = self._read(n)
  329. elif b == 0xc5:
  330. typ = TYPE_BIN
  331. self._reserve(2)
  332. n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
  333. self._buff_i += 2
  334. if n > self._max_bin_len:
  335. raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
  336. obj = self._read(n)
  337. elif b == 0xc6:
  338. typ = TYPE_BIN
  339. self._reserve(4)
  340. n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
  341. self._buff_i += 4
  342. if n > self._max_bin_len:
  343. raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
  344. obj = self._read(n)
  345. elif b == 0xc7: # ext 8
  346. typ = TYPE_EXT
  347. self._reserve(2)
  348. L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i)
  349. self._buff_i += 2
  350. if L > self._max_ext_len:
  351. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
  352. obj = self._read(L)
  353. elif b == 0xc8: # ext 16
  354. typ = TYPE_EXT
  355. self._reserve(3)
  356. L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i)
  357. self._buff_i += 3
  358. if L > self._max_ext_len:
  359. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
  360. obj = self._read(L)
  361. elif b == 0xc9: # ext 32
  362. typ = TYPE_EXT
  363. self._reserve(5)
  364. L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i)
  365. self._buff_i += 5
  366. if L > self._max_ext_len:
  367. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
  368. obj = self._read(L)
  369. elif b == 0xca:
  370. self._reserve(4)
  371. obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0]
  372. self._buff_i += 4
  373. elif b == 0xcb:
  374. self._reserve(8)
  375. obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0]
  376. self._buff_i += 8
  377. elif b == 0xcc:
  378. self._reserve(1)
  379. obj = self._buffer[self._buff_i]
  380. self._buff_i += 1
  381. elif b == 0xcd:
  382. self._reserve(2)
  383. obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
  384. self._buff_i += 2
  385. elif b == 0xce:
  386. self._reserve(4)
  387. obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
  388. self._buff_i += 4
  389. elif b == 0xcf:
  390. self._reserve(8)
  391. obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0]
  392. self._buff_i += 8
  393. elif b == 0xd0:
  394. self._reserve(1)
  395. obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0]
  396. self._buff_i += 1
  397. elif b == 0xd1:
  398. self._reserve(2)
  399. obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0]
  400. self._buff_i += 2
  401. elif b == 0xd2:
  402. self._reserve(4)
  403. obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0]
  404. self._buff_i += 4
  405. elif b == 0xd3:
  406. self._reserve(8)
  407. obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0]
  408. self._buff_i += 8
  409. elif b == 0xd4: # fixext 1
  410. typ = TYPE_EXT
  411. if self._max_ext_len < 1:
  412. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
  413. self._reserve(2)
  414. n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i)
  415. self._buff_i += 2
  416. elif b == 0xd5: # fixext 2
  417. typ = TYPE_EXT
  418. if self._max_ext_len < 2:
  419. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
  420. self._reserve(3)
  421. n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i)
  422. self._buff_i += 3
  423. elif b == 0xd6: # fixext 4
  424. typ = TYPE_EXT
  425. if self._max_ext_len < 4:
  426. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
  427. self._reserve(5)
  428. n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i)
  429. self._buff_i += 5
  430. elif b == 0xd7: # fixext 8
  431. typ = TYPE_EXT
  432. if self._max_ext_len < 8:
  433. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
  434. self._reserve(9)
  435. n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i)
  436. self._buff_i += 9
  437. elif b == 0xd8: # fixext 16
  438. typ = TYPE_EXT
  439. if self._max_ext_len < 16:
  440. raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
  441. self._reserve(17)
  442. n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i)
  443. self._buff_i += 17
  444. elif b == 0xd9:
  445. typ = TYPE_RAW
  446. self._reserve(1)
  447. n = self._buffer[self._buff_i]
  448. self._buff_i += 1
  449. if n > self._max_str_len:
  450. raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
  451. obj = self._read(n)
  452. elif b == 0xda:
  453. typ = TYPE_RAW
  454. self._reserve(2)
  455. n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
  456. self._buff_i += 2
  457. if n > self._max_str_len:
  458. raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
  459. obj = self._read(n)
  460. elif b == 0xdb:
  461. typ = TYPE_RAW
  462. self._reserve(4)
  463. n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
  464. self._buff_i += 4
  465. if n > self._max_str_len:
  466. raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
  467. obj = self._read(n)
  468. elif b == 0xdc:
  469. typ = TYPE_ARRAY
  470. self._reserve(2)
  471. n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
  472. self._buff_i += 2
  473. if n > self._max_array_len:
  474. raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
  475. elif b == 0xdd:
  476. typ = TYPE_ARRAY
  477. self._reserve(4)
  478. n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
  479. self._buff_i += 4
  480. if n > self._max_array_len:
  481. raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
  482. elif b == 0xde:
  483. self._reserve(2)
  484. n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
  485. self._buff_i += 2
  486. if n > self._max_map_len:
  487. raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
  488. typ = TYPE_MAP
  489. elif b == 0xdf:
  490. self._reserve(4)
  491. n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
  492. self._buff_i += 4
  493. if n > self._max_map_len:
  494. raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
  495. typ = TYPE_MAP
  496. else:
  497. raise UnpackValueError("Unknown header: 0x%x" % b)
  498. return typ, n, obj
  499. def _unpack(self, execute=EX_CONSTRUCT):
  500. typ, n, obj = self._read_header(execute)
  501. if execute == EX_READ_ARRAY_HEADER:
  502. if typ != TYPE_ARRAY:
  503. raise UnpackValueError("Expected array")
  504. return n
  505. if execute == EX_READ_MAP_HEADER:
  506. if typ != TYPE_MAP:
  507. raise UnpackValueError("Expected map")
  508. return n
  509. # TODO should we eliminate the recursion?
  510. if typ == TYPE_ARRAY:
  511. if execute == EX_SKIP:
  512. for i in xrange(n):
  513. # TODO check whether we need to call `list_hook`
  514. self._unpack(EX_SKIP)
  515. return
  516. ret = newlist_hint(n)
  517. for i in xrange(n):
  518. ret.append(self._unpack(EX_CONSTRUCT))
  519. if self._list_hook is not None:
  520. ret = self._list_hook(ret)
  521. # TODO is the interaction between `list_hook` and `use_list` ok?
  522. return ret if self._use_list else tuple(ret)
  523. if typ == TYPE_MAP:
  524. if execute == EX_SKIP:
  525. for i in xrange(n):
  526. # TODO check whether we need to call hooks
  527. self._unpack(EX_SKIP)
  528. self._unpack(EX_SKIP)
  529. return
  530. if self._object_pairs_hook is not None:
  531. ret = self._object_pairs_hook(
  532. (self._unpack(EX_CONSTRUCT),
  533. self._unpack(EX_CONSTRUCT))
  534. for _ in xrange(n))
  535. else:
  536. ret = {}
  537. for _ in xrange(n):
  538. key = self._unpack(EX_CONSTRUCT)
  539. ret[key] = self._unpack(EX_CONSTRUCT)
  540. if self._object_hook is not None:
  541. ret = self._object_hook(ret)
  542. return ret
  543. if execute == EX_SKIP:
  544. return
  545. if typ == TYPE_RAW:
  546. if self._encoding is not None:
  547. obj = obj.decode(self._encoding, self._unicode_errors)
  548. elif self._raw:
  549. obj = bytes(obj)
  550. else:
  551. obj = obj.decode('utf_8')
  552. return obj
  553. if typ == TYPE_EXT:
  554. return self._ext_hook(n, bytes(obj))
  555. if typ == TYPE_BIN:
  556. return bytes(obj)
  557. assert typ == TYPE_IMMEDIATE
  558. return obj
  559. def __iter__(self):
  560. return self
  561. def __next__(self):
  562. try:
  563. ret = self._unpack(EX_CONSTRUCT)
  564. self._consume()
  565. return ret
  566. except OutOfData:
  567. self._consume()
  568. raise StopIteration
  569. next = __next__
  570. def skip(self, write_bytes=None):
  571. self._unpack(EX_SKIP)
  572. if write_bytes is not None:
  573. warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
  574. write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
  575. self._consume()
  576. def unpack(self, write_bytes=None):
  577. ret = self._unpack(EX_CONSTRUCT)
  578. if write_bytes is not None:
  579. warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
  580. write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
  581. self._consume()
  582. return ret
  583. def read_array_header(self, write_bytes=None):
  584. ret = self._unpack(EX_READ_ARRAY_HEADER)
  585. if write_bytes is not None:
  586. warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
  587. write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
  588. self._consume()
  589. return ret
  590. def read_map_header(self, write_bytes=None):
  591. ret = self._unpack(EX_READ_MAP_HEADER)
  592. if write_bytes is not None:
  593. warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning)
  594. write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
  595. self._consume()
  596. return ret
  597. def tell(self):
  598. return self._stream_offset
  599. class Packer(object):
  600. """
  601. MessagePack Packer
  602. usage:
  603. packer = Packer()
  604. astream.write(packer.pack(a))
  605. astream.write(packer.pack(b))
  606. Packer's constructor has some keyword arguments:
  607. :param callable default:
  608. Convert user type to builtin type that Packer supports.
  609. See also simplejson's document.
  610. :param bool use_single_float:
  611. Use single precision float type for float. (default: False)
  612. :param bool autoreset:
  613. Reset buffer after each pack and return its content as `bytes`. (default: True).
  614. If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
  615. :param bool use_bin_type:
  616. Use bin type introduced in msgpack spec 2.0 for bytes.
  617. It also enables str8 type for unicode.
  618. :param bool strict_types:
  619. If set to true, types will be checked to be exact. Derived classes
  620. from serializeable types will not be serialized and will be
  621. treated as unsupported type and forwarded to default.
  622. Additionally tuples will not be serialized as lists.
  623. This is useful when trying to implement accurate serialization
  624. for python types.
  625. :param str encoding:
  626. (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
  627. :param str unicode_errors:
  628. Error handler for encoding unicode. (default: 'strict')
  629. """
  630. def __init__(self, default=None, encoding=None, unicode_errors=None,
  631. use_single_float=False, autoreset=True, use_bin_type=False,
  632. strict_types=False):
  633. if encoding is None:
  634. encoding = 'utf_8'
  635. else:
  636. warnings.warn(
  637. "encoding is deprecated, Use raw=False instead.",
  638. PendingDeprecationWarning)
  639. if unicode_errors is None:
  640. unicode_errors = 'strict'
  641. self._strict_types = strict_types
  642. self._use_float = use_single_float
  643. self._autoreset = autoreset
  644. self._use_bin_type = use_bin_type
  645. self._encoding = encoding
  646. self._unicode_errors = unicode_errors
  647. self._buffer = StringIO()
  648. if default is not None:
  649. if not callable(default):
  650. raise TypeError("default must be callable")
  651. self._default = default
  652. def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
  653. check=isinstance, check_type_strict=_check_type_strict):
  654. default_used = False
  655. if self._strict_types:
  656. check = check_type_strict
  657. list_types = list
  658. else:
  659. list_types = (list, tuple)
  660. while True:
  661. if nest_limit < 0:
  662. raise PackValueError("recursion limit exceeded")
  663. if obj is None:
  664. return self._buffer.write(b"\xc0")
  665. if check(obj, bool):
  666. if obj:
  667. return self._buffer.write(b"\xc3")
  668. return self._buffer.write(b"\xc2")
  669. if check(obj, int_types):
  670. if 0 <= obj < 0x80:
  671. return self._buffer.write(struct.pack("B", obj))
  672. if -0x20 <= obj < 0:
  673. return self._buffer.write(struct.pack("b", obj))
  674. if 0x80 <= obj <= 0xff:
  675. return self._buffer.write(struct.pack("BB", 0xcc, obj))
  676. if -0x80 <= obj < 0:
  677. return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
  678. if 0xff < obj <= 0xffff:
  679. return self._buffer.write(struct.pack(">BH", 0xcd, obj))
  680. if -0x8000 <= obj < -0x80:
  681. return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
  682. if 0xffff < obj <= 0xffffffff:
  683. return self._buffer.write(struct.pack(">BI", 0xce, obj))
  684. if -0x80000000 <= obj < -0x8000:
  685. return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
  686. if 0xffffffff < obj <= 0xffffffffffffffff:
  687. return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
  688. if -0x8000000000000000 <= obj < -0x80000000:
  689. return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
  690. if not default_used and self._default is not None:
  691. obj = self._default(obj)
  692. default_used = True
  693. continue
  694. raise PackOverflowError("Integer value out of range")
  695. if check(obj, (bytes, bytearray)):
  696. n = len(obj)
  697. if n >= 2**32:
  698. raise PackValueError("%s is too large" % type(obj).__name__)
  699. self._pack_bin_header(n)
  700. return self._buffer.write(obj)
  701. if check(obj, Unicode):
  702. if self._encoding is None:
  703. raise TypeError(
  704. "Can't encode unicode string: "
  705. "no encoding is specified")
  706. obj = obj.encode(self._encoding, self._unicode_errors)
  707. n = len(obj)
  708. if n >= 2**32:
  709. raise PackValueError("String is too large")
  710. self._pack_raw_header(n)
  711. return self._buffer.write(obj)
  712. if check(obj, memoryview):
  713. n = len(obj) * obj.itemsize
  714. if n >= 2**32:
  715. raise PackValueError("Memoryview is too large")
  716. self._pack_bin_header(n)
  717. return self._buffer.write(obj)
  718. if check(obj, float):
  719. if self._use_float:
  720. return self._buffer.write(struct.pack(">Bf", 0xca, obj))
  721. return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
  722. if check(obj, ExtType):
  723. code = obj.code
  724. data = obj.data
  725. assert isinstance(code, int)
  726. assert isinstance(data, bytes)
  727. L = len(data)
  728. if L == 1:
  729. self._buffer.write(b'\xd4')
  730. elif L == 2:
  731. self._buffer.write(b'\xd5')
  732. elif L == 4:
  733. self._buffer.write(b'\xd6')
  734. elif L == 8:
  735. self._buffer.write(b'\xd7')
  736. elif L == 16:
  737. self._buffer.write(b'\xd8')
  738. elif L <= 0xff:
  739. self._buffer.write(struct.pack(">BB", 0xc7, L))
  740. elif L <= 0xffff:
  741. self._buffer.write(struct.pack(">BH", 0xc8, L))
  742. else:
  743. self._buffer.write(struct.pack(">BI", 0xc9, L))
  744. self._buffer.write(struct.pack("b", code))
  745. self._buffer.write(data)
  746. return
  747. if check(obj, list_types):
  748. n = len(obj)
  749. self._pack_array_header(n)
  750. for i in xrange(n):
  751. self._pack(obj[i], nest_limit - 1)
  752. return
  753. if check(obj, dict):
  754. return self._pack_map_pairs(len(obj), dict_iteritems(obj),
  755. nest_limit - 1)
  756. if not default_used and self._default is not None:
  757. obj = self._default(obj)
  758. default_used = 1
  759. continue
  760. raise TypeError("Cannot serialize %r" % (obj, ))
  761. def pack(self, obj):
  762. try:
  763. self._pack(obj)
  764. except:
  765. self._buffer = StringIO() # force reset
  766. raise
  767. ret = self._buffer.getvalue()
  768. if self._autoreset:
  769. self._buffer = StringIO()
  770. elif USING_STRINGBUILDER:
  771. self._buffer = StringIO(ret)
  772. return ret
  773. def pack_map_pairs(self, pairs):
  774. self._pack_map_pairs(len(pairs), pairs)
  775. ret = self._buffer.getvalue()
  776. if self._autoreset:
  777. self._buffer = StringIO()
  778. elif USING_STRINGBUILDER:
  779. self._buffer = StringIO(ret)
  780. return ret
  781. def pack_array_header(self, n):
  782. if n >= 2**32:
  783. raise PackValueError
  784. self._pack_array_header(n)
  785. ret = self._buffer.getvalue()
  786. if self._autoreset:
  787. self._buffer = StringIO()
  788. elif USING_STRINGBUILDER:
  789. self._buffer = StringIO(ret)
  790. return ret
  791. def pack_map_header(self, n):
  792. if n >= 2**32:
  793. raise PackValueError
  794. self._pack_map_header(n)
  795. ret = self._buffer.getvalue()
  796. if self._autoreset:
  797. self._buffer = StringIO()
  798. elif USING_STRINGBUILDER:
  799. self._buffer = StringIO(ret)
  800. return ret
  801. def pack_ext_type(self, typecode, data):
  802. if not isinstance(typecode, int):
  803. raise TypeError("typecode must have int type.")
  804. if not 0 <= typecode <= 127:
  805. raise ValueError("typecode should be 0-127")
  806. if not isinstance(data, bytes):
  807. raise TypeError("data must have bytes type")
  808. L = len(data)
  809. if L > 0xffffffff:
  810. raise PackValueError("Too large data")
  811. if L == 1:
  812. self._buffer.write(b'\xd4')
  813. elif L == 2:
  814. self._buffer.write(b'\xd5')
  815. elif L == 4:
  816. self._buffer.write(b'\xd6')
  817. elif L == 8:
  818. self._buffer.write(b'\xd7')
  819. elif L == 16:
  820. self._buffer.write(b'\xd8')
  821. elif L <= 0xff:
  822. self._buffer.write(b'\xc7' + struct.pack('B', L))
  823. elif L <= 0xffff:
  824. self._buffer.write(b'\xc8' + struct.pack('>H', L))
  825. else:
  826. self._buffer.write(b'\xc9' + struct.pack('>I', L))
  827. self._buffer.write(struct.pack('B', typecode))
  828. self._buffer.write(data)
  829. def _pack_array_header(self, n):
  830. if n <= 0x0f:
  831. return self._buffer.write(struct.pack('B', 0x90 + n))
  832. if n <= 0xffff:
  833. return self._buffer.write(struct.pack(">BH", 0xdc, n))
  834. if n <= 0xffffffff:
  835. return self._buffer.write(struct.pack(">BI", 0xdd, n))
  836. raise PackValueError("Array is too large")
  837. def _pack_map_header(self, n):
  838. if n <= 0x0f:
  839. return self._buffer.write(struct.pack('B', 0x80 + n))
  840. if n <= 0xffff:
  841. return self._buffer.write(struct.pack(">BH", 0xde, n))
  842. if n <= 0xffffffff:
  843. return self._buffer.write(struct.pack(">BI", 0xdf, n))
  844. raise PackValueError("Dict is too large")
  845. def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
  846. self._pack_map_header(n)
  847. for (k, v) in pairs:
  848. self._pack(k, nest_limit - 1)
  849. self._pack(v, nest_limit - 1)
  850. def _pack_raw_header(self, n):
  851. if n <= 0x1f:
  852. self._buffer.write(struct.pack('B', 0xa0 + n))
  853. elif self._use_bin_type and n <= 0xff:
  854. self._buffer.write(struct.pack('>BB', 0xd9, n))
  855. elif n <= 0xffff:
  856. self._buffer.write(struct.pack(">BH", 0xda, n))
  857. elif n <= 0xffffffff:
  858. self._buffer.write(struct.pack(">BI", 0xdb, n))
  859. else:
  860. raise PackValueError('Raw is too large')
  861. def _pack_bin_header(self, n):
  862. if not self._use_bin_type:
  863. return self._pack_raw_header(n)
  864. elif n <= 0xff:
  865. return self._buffer.write(struct.pack('>BB', 0xc4, n))
  866. elif n <= 0xffff:
  867. return self._buffer.write(struct.pack(">BH", 0xc5, n))
  868. elif n <= 0xffffffff:
  869. return self._buffer.write(struct.pack(">BI", 0xc6, n))
  870. else:
  871. raise PackValueError('Bin is too large')
  872. def bytes(self):
  873. return self._buffer.getvalue()
  874. def reset(self):
  875. self._buffer = StringIO()