From 6494e91846691d68b9c26093ff9f4523943f6f82 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jul 2018 19:54:44 +0900 Subject: [PATCH 01/12] travis: Run test on Python 3.4 (#307) Python 3.4 is not supported officially. But keep running test for a while, to know when msgpack-python stop working on Python 3.4 actually. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7aac6648..823c8b14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ cache: pip python: - "2.7" + - "3.4" - "3.5" - "3.6" - "3.7-dev" From 88a8904de03da7d43ca168d53452f992ba1b9b22 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Fri, 28 Sep 2018 22:22:38 +0000 Subject: [PATCH 02/12] Rolls back legacy python and jython fixes in favor of implicit type casting. The current patches did not work under jython-2.7.1 where implicit casting of buffer or memoryview doesn't work. It may also be the jython is a little pickier about string casting non string bytes due to the underlying strong typing of java. See issues #303 & #304. Rolls back / replaces: aa41e2fef714da8c02e2cf830d408619d2cf902f 5f684aed82d0d08079b9aa74e1d41cc2a367833d b10cf78f54a5daab866b19c32e45e207d838f52b --- msgpack/fallback.py | 60 +++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c0e5fd66..72a3707a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -231,6 +231,14 @@ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, raise TypeError("`file_like.read` must be callable") self.file_like = file_like self._feeding = False + # Some old pythons don't support `self.unpack_from()` `bytearray`. + if sys.version_info < (2, 7, 6): + self._unpack_from = staticmethod( + lambda f,b,o=0: struct.unpack_from(f, b[:o+2].tobytes(), o) + ) + else: + self._unpack_from = staticmethod(struct.unpack_from) + #: array of bytes fed. self._buffer = bytearray() @@ -289,7 +297,7 @@ def feed(self, next_bytes): self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 - self._buffer += view + self._buffer += bytearray(view) def _consume(self): """ Gets rid of the used parts of the buffer. """ @@ -388,7 +396,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = struct.unpack_from(">H", self._buffer, self._buff_i)[0] + n = self.unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -396,7 +404,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + n = self.unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -404,7 +412,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = struct.unpack_from('Bb', self._buffer, self._buff_i) + L, n = self.unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -412,7 +420,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i) + L, n = self.unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -420,18 +428,18 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i) + L, n = self.unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -439,66 +447,66 @@ def _read_header(self, execute=EX_CONSTRUCT): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = struct.unpack_from("b", self._buffer, self._buff_i)[0] + obj = self.unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0] + obj = self.unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i) + n, obj = self.unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i) + n, obj = self.unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i) + n, obj = self.unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i) + n, obj = self.unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i) + n, obj = self.unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -511,7 +519,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = self.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -519,7 +527,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = self.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -527,27 +535,27 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = self.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = self.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = self.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = self.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 7b0e41c2c3d550e934755ac919d673c9cc407e39 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Fri, 28 Sep 2018 22:41:40 +0000 Subject: [PATCH 03/12] Reverts jython and legacy python / _buffer_view changes. --- msgpack/fallback.py | 63 ++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 20ad4c90..c0e5fd66 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,13 +100,6 @@ def _get_data_from_buffer(obj): return view -# Jython's memoryview support is incomplete -# See https://github.com/msgpack/msgpack-python/issues/303 -_is_jython = sys.platform.startswith('java') -if _is_jython: - _get_data_from_buffer = bytes - - def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", @@ -241,12 +234,6 @@ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, #: array of bytes fed. self._buffer = bytearray() - # Some very old pythons don't support `struct.unpack_from()` with a - # `bytearray`. So we wrap it in a `buffer()` there. - if sys.version_info < (2, 7, 6): - self._buffer_view = buffer(self._buffer) - else: - self._buffer_view = self._buffer #: Which position we currently reads self._buff_i = 0 @@ -401,7 +388,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + n = struct.unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -409,7 +396,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + n = struct.unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -417,7 +404,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i) + L, n = struct.unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -425,7 +412,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i) + L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -433,18 +420,18 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i) + L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -452,66 +439,66 @@ def _read_header(self, execute=EX_CONSTRUCT): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0] + obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i) + n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i) + n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i) + n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i) + n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i) + n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -524,7 +511,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -532,7 +519,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -540,27 +527,27 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 2baaf7b87b4ef4f5ebd15991b1fcf04577c33660 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Fri, 28 Sep 2018 23:26:23 +0000 Subject: [PATCH 04/12] Fix original patch. --- msgpack/fallback.py | 52 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 72a3707a..1d087f9d 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -231,7 +231,7 @@ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, raise TypeError("`file_like.read` must be callable") self.file_like = file_like self._feeding = False - # Some old pythons don't support `self.unpack_from()` `bytearray`. + # Some old pythons don't support `self._unpack_from()` `bytearray`. if sys.version_info < (2, 7, 6): self._unpack_from = staticmethod( lambda f,b,o=0: struct.unpack_from(f, b[:o+2].tobytes(), o) @@ -396,7 +396,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = self.unpack_from(">H", self._buffer, self._buff_i)[0] + n = self._unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -404,7 +404,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = self.unpack_from(">I", self._buffer, self._buff_i)[0] + n = self._unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -412,7 +412,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = self.unpack_from('Bb', self._buffer, self._buff_i) + L, n = self._unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -420,7 +420,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = self.unpack_from('>Hb', self._buffer, self._buff_i) + L, n = self._unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -428,18 +428,18 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = self.unpack_from('>Ib', self._buffer, self._buff_i) + L, n = self._unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = self.unpack_from(">f", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = self.unpack_from(">d", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -447,66 +447,66 @@ def _read_header(self, execute=EX_CONSTRUCT): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = self.unpack_from(">H", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = self.unpack_from(">I", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = self.unpack_from(">Q", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = self.unpack_from("b", self._buffer, self._buff_i)[0] + obj = self._unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = self.unpack_from(">h", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = self.unpack_from(">i", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = self.unpack_from(">q", self._buffer, self._buff_i)[0] + obj = self._unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = self.unpack_from("b1s", self._buffer, self._buff_i) + n, obj = self._unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = self.unpack_from("b2s", self._buffer, self._buff_i) + n, obj = self._unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = self.unpack_from("b4s", self._buffer, self._buff_i) + n, obj = self._unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = self.unpack_from("b8s", self._buffer, self._buff_i) + n, obj = self._unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = self.unpack_from("b16s", self._buffer, self._buff_i) + n, obj = self._unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -519,7 +519,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = self.unpack_from(">H", self._buffer, self._buff_i) + n, = self._unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -527,7 +527,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = self.unpack_from(">I", self._buffer, self._buff_i) + n, = self._unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -535,27 +535,27 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = self.unpack_from(">H", self._buffer, self._buff_i) + n, = self._unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = self.unpack_from(">I", self._buffer, self._buff_i) + n, = self._unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = self.unpack_from(">H", self._buffer, self._buff_i) + n, = self._unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = self.unpack_from(">I", self._buffer, self._buff_i) + n, = self._unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 394b339b73911177b15a14857f02a037ae592808 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Sat, 29 Sep 2018 00:30:29 +0000 Subject: [PATCH 05/12] Another shot to pass py3 tests. --- msgpack/fallback.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1d087f9d..177a748c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -205,7 +205,12 @@ class Unpacker(object): for o in unpacker: process(o) """ - + # Some old pythons don't support `self._unpack_from()` `bytearray`. + _unpack_from = staticmethod( + struct.unpack_from if sys.version_info >= (2, 7, 6) + else lambda f,b,o=0: struct.unpack_from(f, b[:o+2].tobytes(),o) + ) + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, @@ -231,14 +236,6 @@ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, raise TypeError("`file_like.read` must be callable") self.file_like = file_like self._feeding = False - # Some old pythons don't support `self._unpack_from()` `bytearray`. - if sys.version_info < (2, 7, 6): - self._unpack_from = staticmethod( - lambda f,b,o=0: struct.unpack_from(f, b[:o+2].tobytes(), o) - ) - else: - self._unpack_from = staticmethod(struct.unpack_from) - #: array of bytes fed. self._buffer = bytearray() From ea0065ffd4f4742c0ff9cbc6abfd49b233f9920d Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 14:29:06 +0000 Subject: [PATCH 06/12] move _unpack_from to global function. --- msgpack/fallback.py | 62 +++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 177a748c..61ba411a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -126,6 +126,13 @@ def unpackb(packed, **kwargs): return ret +if sys.version < (2, 7, 6): + def _unpack_from(f, b, o=0): + return struct.unpack_from(f, b[:o+2].tobytes(), o) +else: + _unpack_from = struct.unpack_from + + class Unpacker(object): """Streaming unpacker. @@ -205,11 +212,6 @@ class Unpacker(object): for o in unpacker: process(o) """ - # Some old pythons don't support `self._unpack_from()` `bytearray`. - _unpack_from = staticmethod( - struct.unpack_from if sys.version_info >= (2, 7, 6) - else lambda f,b,o=0: struct.unpack_from(f, b[:o+2].tobytes(),o) - ) def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, @@ -393,7 +395,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = self._unpack_from(">H", self._buffer, self._buff_i)[0] + n = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -401,7 +403,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = self._unpack_from(">I", self._buffer, self._buff_i)[0] + n = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -409,7 +411,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = self._unpack_from('Bb', self._buffer, self._buff_i) + L, n = _unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -417,7 +419,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = self._unpack_from('>Hb', self._buffer, self._buff_i) + L, n = _unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -425,18 +427,18 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = self._unpack_from('>Ib', self._buffer, self._buff_i) + L, n = _unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = self._unpack_from(">f", self._buffer, self._buff_i)[0] + obj = _unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = self._unpack_from(">d", self._buffer, self._buff_i)[0] + obj = _unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -444,66 +446,66 @@ def _read_header(self, execute=EX_CONSTRUCT): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = self._unpack_from(">H", self._buffer, self._buff_i)[0] + obj = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = self._unpack_from(">I", self._buffer, self._buff_i)[0] + obj = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = self._unpack_from(">Q", self._buffer, self._buff_i)[0] + obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = self._unpack_from("b", self._buffer, self._buff_i)[0] + obj = _unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = self._unpack_from(">h", self._buffer, self._buff_i)[0] + obj = _unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = self._unpack_from(">i", self._buffer, self._buff_i)[0] + obj = _unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = self._unpack_from(">q", self._buffer, self._buff_i)[0] + obj = _unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = self._unpack_from("b1s", self._buffer, self._buff_i) + n, obj = _unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = self._unpack_from("b2s", self._buffer, self._buff_i) + n, obj = _unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = self._unpack_from("b4s", self._buffer, self._buff_i) + n, obj = _unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = self._unpack_from("b8s", self._buffer, self._buff_i) + n, obj = _unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = self._unpack_from("b16s", self._buffer, self._buff_i) + n, obj = _unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -516,7 +518,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = self._unpack_from(">H", self._buffer, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -524,7 +526,7 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = self._unpack_from(">I", self._buffer, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -532,27 +534,27 @@ def _read_header(self, execute=EX_CONSTRUCT): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = self._unpack_from(">H", self._buffer, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = self._unpack_from(">I", self._buffer, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = self._unpack_from(">H", self._buffer, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = self._unpack_from(">I", self._buffer, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 3b60399d2419921ce0d0a32d61c4fad4c83b9ab7 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 14:34:04 +0000 Subject: [PATCH 07/12] fix failed test. --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 61ba411a..63f0f7c2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -128,7 +128,7 @@ def unpackb(packed, **kwargs): if sys.version < (2, 7, 6): def _unpack_from(f, b, o=0): - return struct.unpack_from(f, b[:o+2].tobytes(), o) + return struct.unpack_from(f, bytes(b[:o+2]), o) else: _unpack_from = struct.unpack_from From b86a1b8e235659e4ec6412465ceac27e47a708d2 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 14:40:30 +0000 Subject: [PATCH 08/12] fix failed test. --- msgpack/fallback.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 63f0f7c2..8b35aef4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -128,7 +128,8 @@ def unpackb(packed, **kwargs): if sys.version < (2, 7, 6): def _unpack_from(f, b, o=0): - return struct.unpack_from(f, bytes(b[:o+2]), o) + """Explicit typcast for legacy struct.unpack_from""" + return struct.unpack_from(f, bytes(b), o) else: _unpack_from = struct.unpack_from From 88be776dc5c519c6a14de41e5cc5a8b7adf440bc Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 14:43:56 +0000 Subject: [PATCH 09/12] need coffee, thank you travis --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8b35aef4..2c7fc70a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -126,7 +126,7 @@ def unpackb(packed, **kwargs): return ret -if sys.version < (2, 7, 6): +if sys.version_info < (2, 7, 6): def _unpack_from(f, b, o=0): """Explicit typcast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) From 9d4909a987be6da5e29b41bed3455a9584cb6042 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 15:52:25 +0000 Subject: [PATCH 10/12] Another approach to typecasting fix for jython. For some reason += throws TypeError, but .extend triggers implicit typecasting. --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 2c7fc70a..b38c5c7d 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -297,7 +297,7 @@ def feed(self, next_bytes): self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 - self._buffer += bytearray(view) + self._buffer.extend(view) def _consume(self): """ Gets rid of the used parts of the buffer. """ From 1fabfcda692f9979e99caed6f4ef95da5cd66508 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 16:02:11 +0000 Subject: [PATCH 11/12] Add a comment so the non-obvious work around doesn't get housecleaned. --- msgpack/fallback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b38c5c7d..52b2a86a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -297,6 +297,7 @@ def feed(self, next_bytes): self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 + # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython self._buffer.extend(view) def _consume(self): From 9422e072653597517199163389384f32849a5278 Mon Sep 17 00:00:00 2001 From: Ray Ferguson Date: Mon, 1 Oct 2018 16:06:53 +0000 Subject: [PATCH 12/12] Remove sloppy whitespace --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 52b2a86a..0b60ba2a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -213,7 +213,7 @@ class Unpacker(object): for o in unpacker: process(o) """ - + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0,