From 9d45926a596028e39ec59dd909a56eb5e9e8fee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 2 Aug 2022 06:19:56 +0200 Subject: [PATCH 01/63] Usef `__BYTE_ORDER__` instead of `__BYTE_ORDER` (#513) __BYTE_ORDER__ is common predefined macro available on at least gcc and clang. __BYTE_ORDER is macro defined in platform specific headers. --- msgpack/sysdep.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ed9c1bc0..ae28f0c5 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,14 +61,12 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif -#else -#include /* __BYTE_ORDER */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define __BIG_ENDIAN__ #elif _WIN32 #define __LITTLE_ENDIAN__ From edca770071fc702e0b4c33f87fb0fa3682b486b4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 8 Aug 2022 15:08:40 +0900 Subject: [PATCH 02/63] Fix build error caused by ntohs, ntohl (#514) --- msgpack/sysdep.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ae28f0c5..70673004 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,6 +61,8 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif +#else /* _WIN32 */ +#include /* ntohs, ntohl */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) @@ -93,7 +95,7 @@ typedef unsigned int _msgpack_atomic_counter_t; #ifdef _WIN32 # if defined(ntohl) # define _msgpack_be32(x) ntohl(x) -# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) +# elif defined(_byteswap_ulong) || defined(_MSC_VER) # define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) # else # define _msgpack_be32(x) \ @@ -106,7 +108,7 @@ typedef unsigned int _msgpack_atomic_counter_t; # define _msgpack_be32(x) ntohl(x) #endif -#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) +#if defined(_byteswap_uint64) || defined(_MSC_VER) # define _msgpack_be64(x) (_byteswap_uint64(x)) #elif defined(bswap_64) # define _msgpack_be64(x) bswap_64(x) From 44a80603838ea480e66c9235036ff742f4013200 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 9 Sep 2022 09:16:12 +0200 Subject: [PATCH 03/63] Add python 3.11 wheels (#517) --- .github/workflows/wheel.yml | 4 ++-- setup.cfg | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d73898c2..770b565a 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] + os: [ubuntu-22.04, windows-2022, macos-10.15] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.6.0 + uses: pypa/cibuildwheel@v2.9.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/setup.cfg b/setup.cfg index 1cb6ce36..e3782842 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From c3995669f1f821596714240c2cd07943810f8658 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 08:08:58 +0000 Subject: [PATCH 04/63] Remove unused code --- msgpack/pack.h | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 4f3ce1d9..1e849acc 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -71,7 +71,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ static inline int msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) { -#if PY_MAJOR_VERSION >= 3 assert(PyUnicode_Check(o)); Py_ssize_t len; @@ -87,31 +86,6 @@ msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) if (ret) return ret; return msgpack_pack_raw_body(pk, buf, len); -#else - PyObject *bytes; - Py_ssize_t len; - int ret; - - // py2 - bytes = PyUnicode_AsUTF8String(o); - if (bytes == NULL) - return -1; - - len = PyString_GET_SIZE(bytes); - if (len > limit) { - Py_DECREF(bytes); - return -2; - } - - ret = msgpack_pack_raw(pk, len); - if (ret) { - Py_DECREF(bytes); - return -1; - } - ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len); - Py_DECREF(bytes); - return ret; -#endif } #ifdef __cplusplus From b82d0b62f187552b8108602d7b0451ac362a29cc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:13:44 +0900 Subject: [PATCH 05/63] fallback: Fix packing multidim memoryview (#527) Fix #526 --- msgpack/fallback.py | 2 +- test/test_memoryview.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f560c7b5..e8cebc1b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -814,7 +814,7 @@ def _pack( self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): - n = len(obj) * obj.itemsize + n = obj.nbytes if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 84941db8..a0939a69 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -96,3 +96,11 @@ def test_bin32_from_byte(): def test_bin32_from_float(): _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) + + +def test_multidim_memoryview(): + # See https://github.com/msgpack/msgpack-python/issues/526 + view = memoryview(b"\00" * 6) + data = view.cast(view.format, (3, 2)) + packed = packb(data) + assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' From 10082295536098d90681da5d7199ca384e8b8ff8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:47:15 +0900 Subject: [PATCH 06/63] Release v1.0.5rc1 (#528) --- .github/workflows/black.yaml | 4 ++-- .github/workflows/test.yml | 6 +++--- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 0a0a737d..1e28b7b5 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.x' architecture: 'x64' - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Black Code Formatter run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d01d74cb..5e41167f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ jobs: test: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] - py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] + os: [ubuntu-22.04, windows-2022, macos-10.15] + py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} cache: "pip" diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 770b565a..b2879fe5 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.9.0 + uses: pypa/cibuildwheel@v2.12.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index a11c8144..1b7ae2c3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.5rc1 +======== + +Release Date: 2023-01-18 + +* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) +* Add Python 3.11 wheels (#517) +* fallback: Fix packing multidimensional memoryview (#527) + 1.0.4 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 50710218..501b291b 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 4) -__version__ = "1.0.4" +version = (1, 0, 5, 'rc', 1) +__version__ = "1.0.5rc1" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From e3ef909c47e5a245fc9d711e9d974a5f9df99303 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 13:07:24 +0000 Subject: [PATCH 07/63] Action: Use setup-python@v4 --- .github/workflows/wheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index b2879fe5..23050088 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -22,10 +22,10 @@ jobs: with: platforms: arm64 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 + - name: Set up Python 3.x + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.x" cache: "pip" - name: Prepare From dcb775031c0b1d575b90e822e81e845ebfda4a2e Mon Sep 17 00:00:00 2001 From: Anthon van der Neut Date: Sun, 5 Mar 2023 15:45:38 +0100 Subject: [PATCH 08/63] minor type in exception message (#533) interger -> integer --- msgpack/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 25544c55..23e0d6b4 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -56,7 +56,7 @@ def __init__(self, seconds, nanoseconds=0): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ if not isinstance(seconds, int_types): - raise TypeError("seconds must be an interger") + raise TypeError("seconds must be an integer") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): From aa9ce3e2bbc1d3d0476396892c46e704292455ab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jun 2022 14:37:21 +0900 Subject: [PATCH 09/63] Action: Run publish on tag creation. --- .github/workflows/wheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 23050088..5f103a37 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -49,8 +49,8 @@ jobs: name: Wheels path: wheelhouse - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + - name: Publish Wheels to PyPI + if: github.event_name == 'create' && github.event.ref_type == 'tag' uses: pypa/gh-action-pypi-publish@release/v1 with: packages_dir: wheelhouse From 4c55f809fe2231130cf99b20538b26b92f1bea31 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:43:28 +0900 Subject: [PATCH 10/63] Release v1.0.5 (#534) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 1b7ae2c3..53547996 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.5rc1 -======== +1.0.5 +===== -Release Date: 2023-01-18 +Release Date: 2023-03-08 * Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) * Add Python 3.11 wheels (#517) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 501b291b..1300b866 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import sys -version = (1, 0, 5, 'rc', 1) -__version__ = "1.0.5rc1" +version = (1, 0, 5) +__version__ = "1.0.5" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From 35b2d246cfdb19484caa5789512bf71ee378caec Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:47:52 +0900 Subject: [PATCH 11/63] Action: Update wheel workflow --- .github/workflows/wheel.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5f103a37..6cf2fe94 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -48,10 +48,3 @@ jobs: with: name: Wheels path: wheelhouse - - - name: Publish Wheels to PyPI - if: github.event_name == 'create' && github.event.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages_dir: wheelhouse - password: ${{ secrets.PYPI_API_TOKEN }} From 0516c2c2a97ef48a2becf30bc8b2365ca16199f1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 01:22:38 +0900 Subject: [PATCH 12/63] Action: Update test workflow --- .github/workflows/test.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e41167f..88781025 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: test: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} @@ -41,9 +41,3 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test - - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} From 802cbc9495ed059b62b46f057cf3d71f756e2480 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 1 Apr 2023 00:02:25 +0900 Subject: [PATCH 13/63] Add security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..75f0c541 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. \ No newline at end of file From 45f848695c855966d8a46656d8de1a2734d934ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 8 Apr 2023 02:18:25 -0300 Subject: [PATCH 14/63] fix: build status badge (#538) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb816485..7f7c4234 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MessagePack for Python -[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python) +[![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) ## What's this From feec06206c7f8dd8efeac6177badbf5b256b36e4 Mon Sep 17 00:00:00 2001 From: sblondon Date: Sun, 21 May 2023 09:26:39 +0200 Subject: [PATCH 15/63] Drop python2 support (#519) The PR removes python2 references and cases. Close #518 Co-authored-by: Inada Naoki --- README.md | 4 +-- docs/conf.py | 20 ++++++------ msgpack/__init__.py | 2 +- msgpack/_packer.pyx | 1 - msgpack/_unpacker.pyx | 2 +- msgpack/ext.py | 24 +++----------- msgpack/fallback.py | 70 ++++++++++++----------------------------- setup.py | 3 +- test/test_buffer.py | 1 - test/test_case.py | 2 +- test/test_except.py | 2 +- test/test_extension.py | 15 ++------- test/test_memoryview.py | 5 --- test/test_pack.py | 6 ---- test/test_timestamp.py | 27 +++++++--------- test/test_unpack.py | 4 +-- tox.ini | 13 -------- 17 files changed, 58 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index 7f7c4234..61f99e1f 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ and `raw=True` options. ```pycon >>> import msgpack ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=False), raw=True) [b'spam', b'eggs'] ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=True), raw=False) [b'spam', 'eggs'] ``` diff --git a/docs/conf.py b/docs/conf.py index 6b432be0..91ce77f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,8 +40,8 @@ master_doc = "index" # General information about the project. -project = u"msgpack" -copyright = u"Inada Naoki" +project = "msgpack" +copyright = "Inada Naoki" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -181,7 +181,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"), + ("index", "msgpack.tex", "msgpack Documentation", "Author", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -209,7 +209,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)] +man_pages = [("index", "msgpack", "msgpack Documentation", ["Author"], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -224,8 +224,8 @@ ( "index", "msgpack", - u"msgpack Documentation", - u"Author", + "msgpack Documentation", + "Author", "msgpack", "One line description of project.", "Miscellaneous", @@ -245,10 +245,10 @@ # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u"msgpack" -epub_author = u"Author" -epub_publisher = u"Author" -epub_copyright = u"2013, Author" +epub_title = "msgpack" +epub_author = "Author" +epub_publisher = "Author" +epub_copyright = "2013, Author" # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1300b866..6c10dc23 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -10,7 +10,7 @@ __version__ = "1.0.5" -if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: +if os.environ.get("MSGPACK_PUREPYTHON"): from .fallback import Packer, unpackb, Unpacker else: try: diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 396da0c2..074b39fc 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -98,7 +98,6 @@ cdef class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 8b06661e..d5dc5ea5 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -236,7 +236,7 @@ cdef class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. diff --git a/msgpack/ext.py b/msgpack/ext.py index 23e0d6b4..07f96a58 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -5,19 +5,6 @@ import struct -PY2 = sys.version_info[0] == 2 - -if PY2: - int_types = (int, long) - _utc = None -else: - int_types = int - try: - _utc = datetime.timezone.utc - except AttributeError: - _utc = datetime.timezone(datetime.timedelta(0)) - - class ExtType(namedtuple("ExtType", "code data")): """ExtType represents ext type in msgpack.""" @@ -55,9 +42,9 @@ def __init__(self, seconds, nanoseconds=0): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ - if not isinstance(seconds, int_types): + if not isinstance(seconds, int): raise TypeError("seconds must be an integer") - if not isinstance(nanoseconds, int_types): + if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): raise ValueError( @@ -174,11 +161,10 @@ def to_unix_nano(self): def to_datetime(self): """Get the timestamp as a UTC datetime. - Python 2 is not supported. - :rtype: datetime. """ - return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta( + utc = datetime.timezone.utc + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( seconds=self.to_unix() ) @@ -186,8 +172,6 @@ def to_datetime(self): def from_datetime(dt): """Create a Timestamp from datetime with tzinfo. - Python 2 is not supported. - :rtype: Timestamp """ return Timestamp.from_unix(dt.timestamp()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e8cebc1b..618c3622 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,22 +4,6 @@ import struct -PY2 = sys.version_info[0] == 2 -if PY2: - int_types = (int, long) - - def dict_iteritems(d): - return d.iteritems() - -else: - int_types = int - unicode = str - xrange = range - - def dict_iteritems(d): - return d.items() - - if sys.version_info < (3, 5): # Ugly hack... RecursionError = RuntimeError @@ -134,15 +118,6 @@ def unpackb(packed, **kwargs): return ret -if sys.version_info < (2, 7, 6): - - def _unpack_from(f, b, o=0): - """Explicit type cast for legacy struct.unpack_from""" - return struct.unpack_from(f, bytes(b), o) - -else: - _unpack_from = struct.unpack_from - _NO_FORMAT_USED = "" _MSGPACK_HEADERS = { 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), @@ -202,7 +177,7 @@ class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. @@ -477,7 +452,7 @@ def _read_header(self): size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + n = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: n = self._buffer[self._buff_i] self._buff_i += size @@ -487,7 +462,7 @@ def _read_header(self): elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - L, n = _unpack_from(fmt, self._buffer, self._buff_i) + L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -496,7 +471,7 @@ def _read_header(self): size, fmt = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + obj = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: obj = self._buffer[self._buff_i] self._buff_i += size @@ -507,13 +482,13 @@ def _read_header(self): "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) ) self._reserve(size + 1) - n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 elif 0xD9 <= b <= 0xDB: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) else: n = self._buffer[self._buff_i] self._buff_i += size @@ -523,7 +498,7 @@ def _read_header(self): elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: raise ValueError( @@ -532,7 +507,7 @@ def _read_header(self): elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) @@ -554,12 +529,12 @@ def _unpack(self, execute=EX_CONSTRUCT): # TODO should we eliminate the recursion? if typ == TYPE_ARRAY: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call `list_hook` self._unpack(EX_SKIP) return ret = newlist_hint(n) - for i in xrange(n): + for i in range(n): ret.append(self._unpack(EX_CONSTRUCT)) if self._list_hook is not None: ret = self._list_hook(ret) @@ -567,7 +542,7 @@ def _unpack(self, execute=EX_CONSTRUCT): return ret if self._use_list else tuple(ret) if typ == TYPE_MAP: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call hooks self._unpack(EX_SKIP) self._unpack(EX_SKIP) @@ -575,17 +550,17 @@ def _unpack(self, execute=EX_CONSTRUCT): if self._object_pairs_hook is not None: ret = self._object_pairs_hook( (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in xrange(n) + for _ in range(n) ) else: ret = {} - for _ in xrange(n): + for _ in range(n): key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (unicode, bytes): + if self._strict_map_key and type(key) not in (str, bytes): raise ValueError( "%s is not allowed for map key" % str(type(key)) ) - if not PY2 and type(key) is str: + if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: @@ -698,7 +673,6 @@ class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') @@ -743,8 +717,6 @@ def __init__( self._autoreset = autoreset self._use_bin_type = use_bin_type self._buffer = StringIO() - if PY2 and datetime: - raise ValueError("datetime is not supported in Python 2") self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" if default is not None: @@ -774,7 +746,7 @@ def _pack( if obj: return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc2") - if check(obj, int_types): + if check(obj, int): if 0 <= obj < 0x80: return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: @@ -806,7 +778,7 @@ def _pack( raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) - if check(obj, unicode): + if check(obj, str): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) if n >= 2**32: @@ -855,13 +827,11 @@ def _pack( if check(obj, list_types): n = len(obj) self._pack_array_header(n) - for i in xrange(n): + for i in range(n): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): - return self._pack_map_pairs( - len(obj), dict_iteritems(obj), nest_limit - 1 - ) + return self._pack_map_pairs(len(obj), obj.items(), nest_limit - 1) if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None: obj = Timestamp.from_datetime(obj) @@ -1004,7 +974,7 @@ def reset(self): def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER or PY2: + if USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/setup.py b/setup.py index 9630cda0..15ba7741 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ PYPY = hasattr(sys, "pypy_version_info") -PY2 = sys.version_info[0] == 2 class NoCython(Exception): @@ -79,7 +78,7 @@ def __init__(self, *args, **kwargs): macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): +if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", diff --git a/test/test_buffer.py b/test/test_buffer.py index 62507cf4..04470584 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -6,7 +6,6 @@ from msgpack import packb, unpackb -@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported") def test_unpack_buffer(): from array import array diff --git a/test/test_case.py b/test/test_case.py index a0a3c5ad..1c4e3229 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -134,4 +134,4 @@ def test_match(): def test_unicode(): - assert unpackb(packb(u"foobar"), use_list=1) == u"foobar" + assert unpackb(packb("foobar"), use_list=1) == "foobar" diff --git a/test/test_except.py b/test/test_except.py index 5544f2bc..745ebecb 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -53,7 +53,7 @@ def test_invalidvalue(): def test_strict_map_key(): - valid = {u"unicode": 1, b"bytes": 2} + valid = {"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) assert valid == unpackb(packed, raw=False, strict_map_key=True) diff --git a/test/test_extension.py b/test/test_extension.py index 6b365751..dfbe4350 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -55,10 +55,7 @@ def ext_hook(code, data): print("ext_hook called", code, data) assert code == 123 obj = array.array("d") - try: - obj.frombytes(data) - except AttributeError: # PY2 - obj.fromstring(data) + obj.frombytes(data) return obj obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])] @@ -67,20 +64,14 @@ def ext_hook(code, data): assert obj == obj2 -import sys - -if sys.version > "3": - long = int - - def test_overriding_hooks(): def default(obj): - if isinstance(obj, long): + if isinstance(obj, int): return {"__type__": "long", "__data__": str(obj)} else: return obj - obj = {"testval": long(1823746192837461928374619)} + obj = {"testval": 1823746192837461928374619} refobj = {"testval": default(obj["testval"])} refout = msgpack.packb(refobj) assert isinstance(refout, (str, bytes)) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index a0939a69..63beab10 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -7,11 +7,6 @@ import sys -pytestmark = pytest.mark.skipif( - sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol" -) - - def make_array(f, data): a = array(f) a.frombytes(data) diff --git a/test/test_pack.py b/test/test_pack.py index a51d84c9..65c9cb17 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -80,9 +80,6 @@ def testPackByteArrays(): check(td) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreUnicodeErrors(): re = unpackb( packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" @@ -96,9 +93,6 @@ def testStrictUnicodeUnpack(): unpackb(packed, raw=False, use_list=1) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreErrorsPack(): re = unpackb( packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 253228e7..af84a2ff 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -4,9 +4,6 @@ import msgpack from msgpack.ext import Timestamp -if sys.version_info[0] > 2: - from msgpack.ext import _utc - def test_timestamp(): # timestamp32 @@ -85,33 +82,33 @@ def test_timestamp_to(): assert t.to_unix_nano() == 42000014000 -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_timestamp_datetime(): t = Timestamp(42, 14) - assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + utc = datetime.timezone.utc + assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_unpack_datetime(): t = Timestamp(42, 14) + utc = datetime.timezone.utc packed = msgpack.packb(t) unpacked = msgpack.unpackb(packed, timestamp=3) - assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_unpack_before_epoch(): - t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + t_in = datetime.datetime(1960, 1, 1, tzinfo=utc) packed = msgpack.packb(t_in, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == t_in -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime(): t = Timestamp(42, 14000) dt = t.to_datetime() - assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) packed2 = msgpack.packb(t) @@ -131,10 +128,10 @@ def test_pack_datetime(): assert msgpack.unpackb(packed) is None -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_issue451(): # https://github.com/msgpack/msgpack-python/issues/451 - dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) assert packed == b"\xd6\xff\xf4\x86eL" @@ -142,7 +139,6 @@ def test_issue451(): assert dt == unpacked -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime_without_tzinfo(): dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) with pytest.raises(ValueError, match="where tzinfo=None"): @@ -152,7 +148,8 @@ def test_pack_datetime_without_tzinfo(): packed = msgpack.packb(dt, datetime=True, default=lambda x: None) assert packed == msgpack.packb(None) - dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == dt diff --git a/test/test_unpack.py b/test/test_unpack.py index aa4c01f8..c7141024 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -70,7 +70,7 @@ def _hook(self, code, data): def test_unpacker_tell(): - objects = 1, 2, u"abc", u"def", u"ghi" + objects = 1, 2, "abc", "def", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" positions = 1, 2, 6, 10, 14 unpacker = Unpacker(BytesIO(packed)) @@ -80,7 +80,7 @@ def test_unpacker_tell(): def test_unpacker_tell_read_bytes(): - objects = 1, u"abc", u"ghi" + objects = 1, "abc", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" raw_data = b"\x02", b"\xa3def", b"" lenghts = 1, 4, 999 diff --git a/tox.ini b/tox.ini index 29c256d1..1ef2d189 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,7 @@ [tox] envlist = - py27-pure, {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, - py27-x86, py34-x86, isolated_build = true @@ -19,17 +17,6 @@ commands= setenv= pure: MSGPACK_PUREPYTHON=x -[testenv:py27-x86] -basepython=python2.7-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - [testenv:py34-x86] basepython=python3.4-x86 deps= From c8d0751fe3375a5e8005b5edf955cd5904aaec2f Mon Sep 17 00:00:00 2001 From: Evgeny Markov Date: Tue, 23 May 2023 18:41:08 +0200 Subject: [PATCH 16/63] Drop Python 3.6 support (#543) The following steps have been taken: 1. Black was updated to latest version. The code has been formatted with the new version. 2. The pyupgrade utility is installed. This helped to remove all the code that was needed to support Python < 3.7. Fix #541. Co-authored-by: Inada Naoki --- Makefile | 8 ++++- msgpack/__init__.py | 1 - msgpack/ext.py | 21 ++++--------- msgpack/fallback.py | 70 ++++++++++++----------------------------- pyproject.toml | 5 +++ requirements.txt | 8 +++-- setup.cfg | 1 - setup.py | 9 ++---- test/test_buffer.py | 1 - test/test_case.py | 7 ++--- test/test_except.py | 1 - test/test_extension.py | 7 ++--- test/test_format.py | 5 +-- test/test_limits.py | 2 -- test/test_memoryview.py | 1 - test/test_newspec.py | 2 -- test/test_obj.py | 1 - test/test_pack.py | 12 +++---- test/test_seq.py | 1 - test/test_sequnpack.py | 1 - test/test_stricttype.py | 12 +++---- test/test_subtype.py | 1 - test/test_unpack.py | 2 +- 23 files changed, 60 insertions(+), 119 deletions(-) diff --git a/Makefile b/Makefile index 415dcfdd..e4f22da4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,16 @@ +PYTHON_SOURCES = msgpack test setup.py + .PHONY: all all: cython python setup.py build_ext -i -f .PHONY: black black: - black -S msgpack/ test/ setup.py + black $(PYTHON_SOURCES) + +.PHONY: pyupgrade +pyupgrade: + @find $(PYTHON_SOURCES) -name '*.py' -type f -exec pyupgrade --py37-plus '{}' \; .PHONY: cython cython: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6c10dc23..638236fe 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,4 +1,3 @@ -# coding: utf-8 from .exceptions import * from .ext import ExtType, Timestamp diff --git a/msgpack/ext.py b/msgpack/ext.py index 07f96a58..97942949 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,4 +1,3 @@ -# coding: utf-8 from collections import namedtuple import datetime import sys @@ -15,10 +14,10 @@ def __new__(cls, code, data): raise TypeError("data must be bytes") if not 0 <= code <= 127: raise ValueError("code must be 0~127") - return super(ExtType, cls).__new__(cls, code, data) + return super().__new__(cls, code, data) -class Timestamp(object): +class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python @@ -47,24 +46,18 @@ def __init__(self, seconds, nanoseconds=0): if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): - raise ValueError( - "nanoseconds must be a non-negative integer less than 999999999." - ) + raise ValueError("nanoseconds must be a non-negative integer less than 999999999.") self.seconds = seconds self.nanoseconds = nanoseconds def __repr__(self): """String representation of Timestamp.""" - return "Timestamp(seconds={0}, nanoseconds={1})".format( - self.seconds, self.nanoseconds - ) + return f"Timestamp(seconds={self.seconds}, nanoseconds={self.nanoseconds})" def __eq__(self, other): """Check for equality with another Timestamp object""" if type(other) is self.__class__: - return ( - self.seconds == other.seconds and self.nanoseconds == other.nanoseconds - ) + return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds return False def __ne__(self, other): @@ -164,9 +157,7 @@ def to_datetime(self): :rtype: datetime. """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( - seconds=self.to_unix() - ) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) @staticmethod def from_datetime(dt): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 618c3622..ac1eaf4c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,23 +4,6 @@ import struct -if sys.version_info < (3, 5): - # Ugly hack... - RecursionError = RuntimeError - - def _is_recursionerror(e): - return ( - len(e.args) == 1 - and isinstance(e.args[0], str) - and e.args[0].startswith("maximum recursion depth exceeded") - ) - -else: - - def _is_recursionerror(e): - return True - - if hasattr(sys, "pypy_version_info"): # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. @@ -32,7 +15,7 @@ def _is_recursionerror(e): from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class StringIO(object): + class StringIO: def __init__(self, s=b""): if s: self.builder = StringBuilder(len(s)) @@ -109,10 +92,8 @@ def unpackb(packed, **kwargs): ret = unpacker._unpack() except OutOfData: raise ValueError("Unpack failed: incomplete input") - except RecursionError as e: - if _is_recursionerror(e): - raise StackError - raise + except RecursionError: + raise StackError if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -151,7 +132,7 @@ def unpackb(packed, **kwargs): } -class Unpacker(object): +class Unpacker: """Streaming unpacker. Arguments: @@ -334,9 +315,7 @@ def __init__( if object_pairs_hook is not None and not callable(object_pairs_hook): raise TypeError("`object_pairs_hook` is not callable") if object_hook is not None and object_pairs_hook is not None: - raise TypeError( - "object_pairs_hook and object_hook are mutually " "exclusive" - ) + raise TypeError("object_pairs_hook and object_hook are mutually exclusive") if not callable(ext_hook): raise TypeError("`ext_hook` is not callable") @@ -428,20 +407,18 @@ def _read_header(self): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") elif b == 0xC0: obj = None elif b == 0xC2: @@ -457,7 +434,7 @@ def _read_header(self): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError(f"{n} exceeds max_bin_len({self._max_bin_len})") obj = self._read(n) elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -465,7 +442,7 @@ def _read_header(self): L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError(f"{L} exceeds max_ext_len({self._max_ext_len})") obj = self._read(L) elif 0xCA <= b <= 0xD3: size, fmt = _MSGPACK_HEADERS[b] @@ -478,9 +455,7 @@ def _read_header(self): elif 0xD4 <= b <= 0xD8: size, fmt, typ = _MSGPACK_HEADERS[b] if self._max_ext_len < size: - raise ValueError( - "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) - ) + raise ValueError(f"{size} exceeds max_ext_len({self._max_ext_len})") self._reserve(size + 1) n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 @@ -493,7 +468,7 @@ def _read_header(self): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -501,16 +476,14 @@ def _read_header(self): (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj @@ -549,17 +522,14 @@ def _unpack(self, execute=EX_CONSTRUCT): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in range(n) + (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n) ) else: ret = {} for _ in range(n): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): - raise ValueError( - "%s is not allowed for map key" % str(type(key)) - ) + raise ValueError("%s is not allowed for map key" % str(type(key))) if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) @@ -634,7 +604,7 @@ def tell(self): return self._stream_offset -class Packer(object): +class Packer: """ MessagePack Packer @@ -844,9 +814,9 @@ def _pack( continue if self._datetime and check(obj, _DateTime): - raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise ValueError(f"Cannot serialize {obj!r} where tzinfo=None") - raise TypeError("Cannot serialize %r" % (obj,)) + raise TypeError(f"Cannot serialize {obj!r}") def pack(self, obj): try: @@ -933,7 +903,7 @@ def _pack_map_header(self, n): def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) - for (k, v) in pairs: + for k, v in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) diff --git a/pyproject.toml b/pyproject.toml index 195795f0..86fae1c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,8 @@ requires = [ "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ["py37"] +skip_string_normalization = true diff --git a/requirements.txt b/requirements.txt index 9f3c1a0d..88b5eb97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ -# Also declared in pyproject.toml, if updating here please also update there +# Also declared in pyproject.toml, if updating here please also update there. Cython~=0.29.30 -# dev only tools. no need to add pyproject -black==22.3.0 +# Tools required only for development. No need to add it to pyproject.toml file. +black==23.3.0 +pytest==7.3.1 +pyupgrade==3.3.2 diff --git a/setup.cfg b/setup.cfg index e3782842..d6888fcf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 diff --git a/setup.py b/setup.py index 15ba7741..1cd1e8eb 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import os import sys @@ -25,7 +24,7 @@ class NoCython(Exception): def cythonize(src): - sys.stderr.write("cythonize: %r\n" % (src,)) + sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -36,11 +35,7 @@ def ensure_source(src): if not have_cython: raise NoCython cythonize(pyx) - elif ( - os.path.exists(pyx) - and os.stat(src).st_mtime < os.stat(pyx).st_mtime - and have_cython - ): + elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: cythonize(pyx) return src diff --git a/test/test_buffer.py b/test/test_buffer.py index 04470584..7ee674ae 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import sys import pytest diff --git a/test/test_case.py b/test/test_case.py index 1c4e3229..c4c615e3 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb def check(length, obj, use_bin_type=True): v = packb(obj, use_bin_type=use_bin_type) - assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v)) + assert len(v) == length, f"{obj!r} length should be {length!r} but get {len(v)!r}" assert unpackb(v, use_list=0, raw=not use_bin_type) == obj @@ -120,11 +119,11 @@ def test_match(): ), ({}, b"\x80"), ( - dict([(x, x) for x in range(15)]), + {x: x for x in range(15)}, b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e", ), ( - dict([(x, x) for x in range(16)]), + {x: x for x in range(16)}, b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f", ), ] diff --git a/test/test_except.py b/test/test_except.py index 745ebecb..8c0a9766 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData diff --git a/test/test_extension.py b/test/test_extension.py index dfbe4350..9e5e6aad 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,3 @@ -from __future__ import print_function import array import msgpack from msgpack import ExtType @@ -17,9 +16,7 @@ def p(s): assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16 assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8 assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16 - assert ( - p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 - ) # ext 32 + assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 # ext 32 def test_unpack_ext_type(): @@ -49,7 +46,7 @@ def default(obj): except AttributeError: data = obj.tostring() return ExtType(typecode, data) - raise TypeError("Unknown type object %r" % (obj,)) + raise TypeError(f"Unknown type object {obj!r}") def ext_hook(code, data): print("ext_hook called", code, data) diff --git a/test/test_format.py b/test/test_format.py index fbbc3f98..c06c87dc 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import unpackb @@ -25,9 +24,7 @@ def testFixRaw(): def testFixMap(): - check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}} - ) + check(b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}) def testUnsignedInt(): diff --git a/test/test_limits.py b/test/test_limits.py index 4314c2c0..533bc112 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals import pytest from msgpack import ( diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 63beab10..eaadef73 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import pytest from array import array diff --git a/test/test_newspec.py b/test/test_newspec.py index b7da486e..a6f4251b 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from msgpack import packb, unpackb, ExtType diff --git a/test/test_obj.py b/test/test_obj.py index 86c557cd..d3f870d9 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 65c9cb17..2753e469 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict from io import BytesIO @@ -81,9 +79,7 @@ def testPackByteArrays(): def testIgnoreUnicodeErrors(): - re = unpackb( - packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" - ) + re = unpackb(packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore") assert re == "abcdef" @@ -108,8 +104,8 @@ def testDecodeBinary(): def testPackFloat(): - assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0) - assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0) + assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(">f", 1.0) + assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(">d", 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): @@ -154,7 +150,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.seek(0) unpacker = Unpacker(bio, strict_map_key=False) for size in sizes: - assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) + assert unpacker.unpack() == {i: i * 2 for i in range(size)} def test_odict(): diff --git a/test/test_seq.py b/test/test_seq.py index 0d5d8065..def6630d 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import msgpack diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index c091076b..6b138aad 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io from msgpack import Unpacker, BufferFull from msgpack import pack, packb diff --git a/test/test_stricttype.py b/test/test_stricttype.py index fe9ec6cd..9ffaff25 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from collections import namedtuple from msgpack import packb, unpackb, ExtType @@ -10,7 +8,7 @@ def test_namedtuple(): def default(o): if isinstance(o, T): return dict(o._asdict()) - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) unpacked = unpackb(packed, raw=False) @@ -23,7 +21,7 @@ def test_tuple(): def default(o): if isinstance(o, tuple): return {"__type__": "tuple", "value": list(o)} - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") def convert(o): if o.get("__type__") == "tuple": @@ -44,9 +42,7 @@ def test_tuple_ext(): def default(o): if isinstance(o, tuple): # Convert to list and pack - payload = packb( - list(o), strict_types=True, use_bin_type=True, default=default - ) + payload = packb(list(o), strict_types=True, use_bin_type=True, default=default) return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) raise TypeError(repr(o)) @@ -54,7 +50,7 @@ def convert(code, payload): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple return tuple(unpackb(payload, raw=False, ext_hook=convert)) - raise ValueError("Unknown Ext code {}".format(code)) + raise ValueError(f"Unknown Ext code {code}") data = packb(t, strict_types=True, use_bin_type=True, default=default) expected = unpackb(data, raw=False, ext_hook=convert) diff --git a/test/test_subtype.py b/test/test_subtype.py index d91d4553..d5a9adb3 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb from collections import namedtuple diff --git a/test/test_unpack.py b/test/test_unpack.py index c7141024..bf3f960d 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -52,7 +52,7 @@ def hook(x): def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): - super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False) + super().__init__(ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: From e5249f877c18c88a0009f21097d7e48819579e60 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 02:53:58 +0900 Subject: [PATCH 17/63] ci: add Python 3.12 and drop 3.7 --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88781025..76fcf27e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.11", "3.10", "3.9", "3.8", "3.7"] + py: ["3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -23,6 +23,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + allow-prereleases: true cache: "pip" - name: Build From 427736bbcc5553b0754616b58154ed26733103b6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 11:11:04 +0900 Subject: [PATCH 18/63] try Cython 3.0 (#548) --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86fae1c7..dc8bbee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.30", + "Cython~=3.0.0", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 88b5eb97..e27df0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=0.29.30 +Cython~=3.0.0 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From 7cfced51501b8e0786da5d6b499331ef2f492b29 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:09:42 +0900 Subject: [PATCH 19/63] start v1.0.6 development --- msgpack/__init__.py | 2 +- setup.cfg | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 638236fe..2540120b 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,7 +6,7 @@ version = (1, 0, 5) -__version__ = "1.0.5" +__version__ = "1.0.6dev1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/setup.cfg b/setup.cfg index d6888fcf..c2e16721 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = msgpack -#version = attr: msgpack.__version__ -version = attr: msgpack.version +version = attr: msgpack.__version__ +#version = attr: msgpack.version license = Apache 2.0 author = Inada Naoki author_email = songofacandy@gmail.com @@ -17,11 +17,11 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From 715126c67b2339381f5ad02f45d8fe367400c749 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:20:05 +0900 Subject: [PATCH 20/63] CI: update cibuildwheel to v2.15.0 (#551) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 6cf2fe94..c1eb2ed6 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.12.0 + uses: pypa/cibuildwheel@v2.15.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 7b75b4f36819c77a12518929fecc09d94e82f5bd Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 31 Aug 2023 05:56:24 +0200 Subject: [PATCH 21/63] sphinx-related work (#554) fixes #510 --- .github/workflows/docs.yaml | 32 ++++++++++++++++++++++++++++++++ docs/_static/README.txt | 1 + docs/api.rst | 8 ++++---- docs/conf.py | 2 +- msgpack/_packer.pyx | 3 ++- msgpack/_unpacker.pyx | 6 +++--- msgpack/ext.py | 4 ++-- msgpack/fallback.py | 9 +++++---- tox.ini | 9 +++++++++ 9 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/docs.yaml create mode 100644 docs/_static/README.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..a393c6b8 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,32 @@ +name: docs + +on: ["push", "pull_request"] + +jobs: + docs: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + shell: bash + run: | + pip install -r requirements.txt + make cython + pip install . + + - name: Sphinx Documentation Generator + run: | + pip install tox + tox -e sphinx diff --git a/docs/_static/README.txt b/docs/_static/README.txt new file mode 100644 index 00000000..1c70594f --- /dev/null +++ b/docs/_static/README.txt @@ -0,0 +1 @@ +Sphinx will copy the contents of docs/_static/ directory to the build location. diff --git a/docs/api.rst b/docs/api.rst index 93827e19..f5dfbbd2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,19 +5,19 @@ API reference .. autofunction:: pack -:func:`dump` is alias for :func:`pack` +``dump()`` is an alias for :func:`pack` .. autofunction:: packb -:func:`dumps` is alias for :func:`packb` +``dumps()`` is an alias for :func:`packb` .. autofunction:: unpack -:func:`load` is alias for :func:`unpack` +``load()`` is an alias for :func:`unpack` .. autofunction:: unpackb -:func:`loads` is alias for :func:`unpackb` +``loads()`` is an alias for :func:`unpackb` .. autoclass:: Packer :members: diff --git a/docs/conf.py b/docs/conf.py index 91ce77f0..1c1895c7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +#sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 074b39fc..3c398672 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -71,7 +71,8 @@ cdef class Packer(object): Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d5dc5ea5..56126f43 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -217,7 +217,7 @@ cdef class Unpacker(object): :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -241,12 +241,12 @@ cdef class Unpacker(object): :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) diff --git a/msgpack/ext.py b/msgpack/ext.py index 97942949..f7f2d77d 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -120,7 +120,7 @@ def from_unix(unix_sec): """Create a Timestamp from posix timestamp in seconds. :param unix_float: Posix timestamp in seconds. - :type unix_float: int or float. + :type unix_float: int or float """ seconds = int(unix_sec // 1) nanoseconds = int((unix_sec % 1) * 10**9) @@ -154,7 +154,7 @@ def to_unix_nano(self): def to_datetime(self): """Get the timestamp as a UTC datetime. - :rtype: datetime. + :rtype: `datetime.datetime` """ utc = datetime.timezone.utc return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ac1eaf4c..84b2617a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -139,7 +139,7 @@ class Unpacker: :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -163,12 +163,12 @@ class Unpacker: :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) @@ -616,7 +616,8 @@ class Packer: Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/tox.ini b/tox.ini index 1ef2d189..369eddcf 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, py34-x86, + sphinx, isolated_build = true [testenv] @@ -27,3 +28,11 @@ commands= python -c 'import sys; print(hex(sys.maxsize))' python -c 'from msgpack import _cmsgpack' py.test + + +[testenv:sphinx] +changedir = docs +deps = + sphinx +commands = + sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 423c6df265d0f964733b31a7e835fe91e4b8ea89 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 5 Sep 2023 03:51:04 +0200 Subject: [PATCH 22/63] move project metadata to pyproject.toml (#555) also: replace flake8 by ruff. --- docs/conf.py | 4 ++-- msgpack/__init__.py | 1 - msgpack/ext.py | 8 ++++---- msgpack/fallback.py | 2 +- pyproject.toml | 45 +++++++++++++++++++++++++++++++++++++++++ setup.cfg | 32 ----------------------------- setup.py | 2 -- test/test_buffer.py | 2 -- test/test_memoryview.py | 2 -- test/test_obj.py | 4 ++-- test/test_pack.py | 4 +--- test/test_seq.py | 2 +- test/test_subtype.py | 2 +- test/test_timestamp.py | 1 - 14 files changed, 57 insertions(+), 54 deletions(-) delete mode 100644 setup.cfg diff --git a/docs/conf.py b/docs/conf.py index 1c1895c7..6eb472af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,11 +11,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +#import os +#import sys #sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2540120b..9a96c982 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,7 +2,6 @@ from .ext import ExtType, Timestamp import os -import sys version = (1, 0, 5) diff --git a/msgpack/ext.py b/msgpack/ext.py index f7f2d77d..02c2c430 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,5 @@ from collections import namedtuple import datetime -import sys import struct @@ -20,8 +19,9 @@ def __new__(cls, code, data): class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. - When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python - msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. + When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. + When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and + unpack `Timestamp`. This class is immutable: Do not override seconds and nanoseconds. """ @@ -39,7 +39,7 @@ def __init__(self, seconds, nanoseconds=0): Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999. Default is 0. - Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. + Note: Negative times (before the UNIX epoch) are represented as neg. seconds + pos. ns. """ if not isinstance(seconds, int): raise TypeError("seconds must be an integer") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 84b2617a..a174162a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -530,7 +530,7 @@ def _unpack(self, execute=EX_CONSTRUCT): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) - if type(key) is str: + if isinstance(key, str): key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: diff --git a/pyproject.toml b/pyproject.toml index dc8bbee3..f37d213c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,52 @@ requires = [ ] build-backend = "setuptools.build_meta" +[project] +name = "msgpack" +dynamic = ["version"] +license = {text="Apache 2.0"} +authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] +description = "MessagePack serializer" +readme = "README.md" +#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] +#requires-python = ">=3.8" +classifiers = [ +# "Development Status :: 5 - Production/Stable", +# "Operating System :: OS Independent", +# "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", +] + +[project.urls] +Homepage = "https://msgpack.org/" +Documentation = "https://msgpack-python.readthedocs.io/" +Repository = "https://github.com/msgpack/msgpack-python/" +Tracker = "https://github.com/msgpack/msgpack-python/issues" +#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" + +[tool.setuptools.dynamic] +version = {attr = "msgpack.__version__"} + [tool.black] line-length = 100 target-version = ["py37"] skip_string_normalization = true + +[tool.ruff] +line-length = 100 +target-version = "py38" +ignore = [] + +[tool.ruff.per-file-ignores] +"msgpack/__init__.py" = ["F401", "F403"] +"msgpack/fallback.py" = ["E731"] +"test/test_seq.py" = ["E501"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c2e16721..00000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[metadata] -name = msgpack -version = attr: msgpack.__version__ -#version = attr: msgpack.version -license = Apache 2.0 -author = Inada Naoki -author_email = songofacandy@gmail.com -description = MessagePack serializer -long_description = file: README.md -long_description_content_type = text/markdown -url = https://msgpack.org/ - -project_urls = - Documentation = https://msgpack-python.readthedocs.io/ - Source = https://github.com/msgpack/msgpack-python - Tracker = https://github.com/msgpack/msgpack-python/issues - -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License - -[flake8] -max_line_length = 100 - diff --git a/setup.py b/setup.py index 1cd1e8eb..7a34c8cf 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python -import io import os import sys -from glob import glob from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist diff --git a/test/test_buffer.py b/test/test_buffer.py index 7ee674ae..a3db339c 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import sys -import pytest from msgpack import packb, unpackb diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eaadef73..dc319a63 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,9 +1,7 @@ #!/usr/bin/env python -import pytest from array import array from msgpack import packb, unpackb -import sys def make_array(f, data): diff --git a/test/test_obj.py b/test/test_obj.py index d3f870d9..f78bf426 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -33,7 +33,7 @@ def test_decode_pairs_hook(): prod_sum = 1 * 2 + 3 * 4 unpacked = unpackb( packed, - object_pairs_hook=lambda l: sum(k * v for k, v in l), + object_pairs_hook=lambda lst: sum(k * v for k, v in lst), use_list=1, strict_map_key=False, ) @@ -48,7 +48,7 @@ def test_only_one_obj_hook(): def test_bad_hook(): with raises(TypeError): packed = packb([3, 1 + 2j], default=lambda o: o) - unpacked = unpackb(packed, use_list=1) + unpackb(packed, use_list=1) def _arr_to_str(arr): diff --git a/test/test_pack.py b/test/test_pack.py index 2753e469..42325378 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -3,12 +3,10 @@ from collections import OrderedDict from io import BytesIO import struct -import sys import pytest -from pytest import raises, xfail -from msgpack import packb, unpackb, Unpacker, Packer, pack +from msgpack import packb, unpackb, Unpacker, Packer def check(data, use_list=False): diff --git a/test/test_seq.py b/test/test_seq.py index def6630d..16d9dde4 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): read_count = 0 for idx, o in enumerate(unpacker): - assert type(o) == bytes + assert isinstance(o, bytes) assert o == gen_binary_data(idx) read_count += 1 diff --git a/test/test_subtype.py b/test/test_subtype.py index d5a9adb3..0d1c41af 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from msgpack import packb, unpackb +from msgpack import packb from collections import namedtuple diff --git a/test/test_timestamp.py b/test/test_timestamp.py index af84a2ff..db5cc57a 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,4 @@ import pytest -import sys import datetime import msgpack from msgpack.ext import Timestamp From ef15f4a62c25114bec1db91aa4006ae2d3a9fb53 Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 7 Sep 2023 14:25:07 +0200 Subject: [PATCH 23/63] add a basic .readthedocs.yaml file (#558) --- .readthedocs.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..8b5aaf16 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py From 41d6239c0a3bfb1daabe1a45ffdbecf4e9aa5469 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 12 Sep 2023 19:51:12 +0200 Subject: [PATCH 24/63] fix .readthedocs.yaml, fixes #559 (#560) --- .readthedocs.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b5aaf16..7447895d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,6 +7,17 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - build-essential + jobs: + pre_install: + - pip install -r requirements.txt + - make cython + +python: + install: + - method: pip + path: . sphinx: configuration: docs/conf.py From 4e10c10aaa8350f23e4b85d27ff131f7b4fd13e2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 13 Sep 2023 18:40:04 +0900 Subject: [PATCH 25/63] prepare for 1.0.6rc1 (#557) --- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- pyproject.toml | 12 ++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 53547996..bf345ddb 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.6rc1 +======== + +Release Date: 2023-09-13 + +* Add Python 3.12 wheels (#517) +* Remove Python 2.7, 3.6, and 3.7 support + + 1.0.5 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 9a96c982..2e201330 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 5) -__version__ = "1.0.6dev1" +version = (1, 0, 6, "rc", 1) +__version__ = "1.0.6rc1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/pyproject.toml b/pyproject.toml index f37d213c..a63009a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" -#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] -#requires-python = ">=3.8" +keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] +requires-python = ">=3.8" classifiers = [ -# "Development Status :: 5 - Production/Stable", -# "Operating System :: OS Independent", -# "Programming Language :: Python", + "Development Status :: 5 - Production/Stable", + "Operating System :: OS Independent", + "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -37,7 +37,7 @@ Homepage = "https://msgpack.org/" Documentation = "https://msgpack-python.readthedocs.io/" Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" -#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From e1d3d5d5c386b8b2fa99c812b4648f6532cab032 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 15 Sep 2023 12:02:06 +0900 Subject: [PATCH 26/63] update actions (#563) --- .github/workflows/black.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 1e28b7b5..e0917926 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Black Code Formatter run: | diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a393c6b8..80bbba7d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76fcf27e..4eb8849d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index c1eb2ed6..0412a385 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 From b1b0edaeedd073f21023ec01a60bfa9da077ad2b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 21 Sep 2023 14:58:37 +0900 Subject: [PATCH 27/63] release v1.0.6 (#564) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index bf345ddb..bad51aaf 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.6rc1 -======== +1.0.6 +===== -Release Date: 2023-09-13 +Release Date: 2023-09-21 * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2e201330..781bcdf4 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 6, "rc", 1) -__version__ = "1.0.6rc1" +version = (1, 0, 6) +__version__ = "1.0.6" if os.environ.get("MSGPACK_PUREPYTHON"): From ecf03748c7241a0fb6bef733c7e5d2d68179b670 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:03:16 +0900 Subject: [PATCH 28/63] remove inline macro for msvc (#567) --- msgpack/pack.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 1e849acc..2453428c 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -26,10 +26,6 @@ extern "C" { #endif -#ifdef _MSC_VER -#define inline __inline -#endif - typedef struct msgpack_packer { char *buf; size_t length; From acd068439233b8f04543c4ee81c18c8dbb681aba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:25:10 +0900 Subject: [PATCH 29/63] do not fallback on build error (#568) --- setup.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 7a34c8cf..a13bd81d --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ class NoCython(Exception): def cythonize(src): + if not have_cython: + raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -29,31 +31,15 @@ def cythonize(src): def ensure_source(src): pyx = os.path.splitext(src)[0] + ".pyx" - if not os.path.exists(src): - if not have_cython: - raise NoCython + if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: cythonize(pyx) - elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: - cythonize(pyx) - return src class BuildExt(build_ext): def build_extension(self, ext): - try: - ext.sources = list(map(ensure_source, ext.sources)) - except NoCython: - print("WARNING") - print("Cython is required for building extension from checkout.") - print("Install Cython >= 0.16 or install msgpack from PyPI.") - print("Falling back to pure Python implementation.") - return - try: - return build_ext.build_extension(self, ext) - except Exception as e: - print("WARNING: Failed to compile extension modules.") - print("msgpack uses fallback pure python implementation.") - print(e) + for src in ext.sources: + ensure_source(src) + return build_ext.build_extension(self, ext) # Cython is required for sdist From 2982e9ff729eae150d67ee608fdf1d01d93d8e3f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 17:31:52 +0900 Subject: [PATCH 30/63] release v1.0.7 (#569) --- .gitignore | 1 + ChangeLog.rst | 13 +++++++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 800f1c22..8a06e267 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ MANIFEST build/* dist/* .tox +.python-version *.pyc *.pyo *.so diff --git a/ChangeLog.rst b/ChangeLog.rst index bad51aaf..ca74ebe3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,8 +1,21 @@ +1.0.7 +===== + +Release Date: 2023-09-28 + +* Fix build error of extension module on Windows. (#567) +* ``setup.py`` doesn't skip build error of extension module. (#568) + + 1.0.6 ===== Release Date: 2023-09-21 +.. note:: + v1.0.6 Wheels for Windows don't contain extension module. + Please upgrade to v1.0.7 or newer. + * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 781bcdf4..60a088cc 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 6) -__version__ = "1.0.6" +version = (1, 0, 7) +__version__ = "1.0.7" if os.environ.get("MSGPACK_PUREPYTHON"): From c78026102c981d734c6d8fec6b2790ee31212f27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 23:34:32 +0900 Subject: [PATCH 31/63] doc: use sphinx-rtd-theme (#575) --- .github/workflows/docs.yaml | 1 - .readthedocs.yaml | 1 + docs/conf.py | 4 +--- docs/requirements.txt | 2 ++ tox.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 80bbba7d..8c8298a0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -24,7 +24,6 @@ jobs: run: | pip install -r requirements.txt make cython - pip install . - name: Sphinx Documentation Generator run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7447895d..88d87182 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,6 +18,7 @@ python: install: - method: pip path: . + - requirements: docs/requirements.txt sphinx: configuration: docs/conf.py diff --git a/docs/conf.py b/docs/conf.py index 6eb472af..ab0ad3c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # msgpack documentation build configuration file, created by # sphinx-quickstart on Sun Feb 24 14:20:50 2013. # @@ -91,7 +89,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "sphinxdoc" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..8d45d0b6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx~=7.2 +sphinx-rtd-theme~=1.3.0 diff --git a/tox.ini b/tox.ini index 369eddcf..49364be9 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,6 @@ commands= [testenv:sphinx] changedir = docs deps = - sphinx + -r docs/requirements.txt commands = sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 140864249fd0f67dffaeceeb168ffe9cdf6f1964 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 20 Dec 2023 20:46:04 +0900 Subject: [PATCH 32/63] exclude C/Cython files from wheel (#577) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a63009a7..121b1fd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,10 @@ Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +[tool.setuptools] +# Do not install C/C++/Cython source files +include-package-data = false + [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From 039022cecb04b62a29afb8260b81f57a937aaaaa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 19:24:06 +0900 Subject: [PATCH 33/63] update Cython (#581) --- .github/workflows/wheel.yml | 10 +++++++++- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 0412a385..4a8847d0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -3,6 +3,7 @@ on: push: branches: [main] create: + workflow_dispatch: jobs: build_wheels: @@ -23,7 +24,7 @@ jobs: platforms: arm64 - name: Set up Python 3.x - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" @@ -43,6 +44,13 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* + - name: Build pure Python wheel + env: + MSGPACK_PUREPYTHON: "1" + run: | + pip install build + python -m build -w -o wheelhouse + - name: Upload Wheels to artifact uses: actions/upload-artifact@v1 with: diff --git a/pyproject.toml b/pyproject.toml index 121b1fd9..f9af967b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=3.0.0", + "Cython~=3.0.8", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index e27df0f4..839dc5f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.0 +Cython~=3.0.8 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From bf7bf88ad0b3cd7a8cd74e8251521fde743e9af9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:09:55 +0900 Subject: [PATCH 34/63] ci: update workflows (#582) --- .github/workflows/test.yml | 14 +++++++++++++- .github/workflows/wheel.yml | 6 +++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eb8849d..1faeb0ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true @@ -42,3 +42,15 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + + - name: build packages + shell: bash + run: | + pip install build + python -m build + + - name: upload packages + uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}-${{ matrix.py }} + path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 4a8847d0..e91325be 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -36,7 +36,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.16.5 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -52,7 +52,7 @@ jobs: python -m build -w -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: - name: Wheels + name: wheels-${{ matrix.os }} path: wheelhouse From 9aedf8ed7f632044d42984e9710fefbd97023f71 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:35:28 +0900 Subject: [PATCH 35/63] Release v1.0.8 (#583) --- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index ca74ebe3..2408bc9f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.8 +===== + +Release Date: 2024-03-01 + +* Update Cython to 3.0.8. This fixes memory leak when iterating + ``Unpacker`` object on Python 3.12. +* Do not include C/Cython files in binary wheels. + + 1.0.7 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 60a088cc..919b86f1 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os -version = (1, 0, 7) -__version__ = "1.0.7" +version = (1, 0, 8) +__version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): From e77672200bf89f2093b96a4abd0e4eeb253975b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?hakan=20aky=C3=BCrek?= Date: Sat, 20 Apr 2024 00:46:30 +0200 Subject: [PATCH 36/63] Avoid using floating points during timestamp-datetime conversions (#591) --- msgpack/ext.py | 6 ++++-- test/test_timestamp.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 02c2c430..3940fe0f 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -157,7 +157,9 @@ def to_datetime(self): :rtype: `datetime.datetime` """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( + seconds=self.seconds, microseconds=self.nanoseconds // 1000 + ) @staticmethod def from_datetime(dt): @@ -165,4 +167,4 @@ def from_datetime(dt): :rtype: Timestamp """ - return Timestamp.from_unix(dt.timestamp()) + return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index db5cc57a..f9bc8353 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -86,6 +86,21 @@ def test_timestamp_datetime(): utc = datetime.timezone.utc assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) + ts = datetime.datetime(2024, 4, 16, 8, 43, 9, 420317, tzinfo=utc) + ts2 = datetime.datetime(2024, 4, 16, 8, 43, 9, 420318, tzinfo=utc) + + assert ( + Timestamp.from_datetime(ts2).nanoseconds - Timestamp.from_datetime(ts).nanoseconds == 1000 + ) + + ts3 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4256) + ts4 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4257) + assert ( + Timestamp.from_datetime(ts4).nanoseconds - Timestamp.from_datetime(ts3).nanoseconds == 1000 + ) + + assert Timestamp.from_datetime(ts).to_datetime() == ts + def test_unpack_datetime(): t = Timestamp(42, 14) From 2eca765533106cc4bbb44ecc565b1034ac50f91b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 15:17:54 +0900 Subject: [PATCH 37/63] use ruff instead of black (#598) --- .github/workflows/docs.yaml | 17 +++++---- .github/workflows/{black.yaml => lint.yaml} | 19 +++++------ Makefile | 14 ++++++-- docs/requirements.txt | 4 +-- msgpack/fallback.py | 1 + pyproject.toml | 4 +-- requirements.txt | 7 +--- test/test_memoryview.py | 2 +- test/test_pack.py | 2 +- test/test_read_size.py | 1 + tox.ini | 38 --------------------- 11 files changed, 38 insertions(+), 71 deletions(-) rename .github/workflows/{black.yaml => lint.yaml} (56%) delete mode 100644 tox.ini diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8c8298a0..08fc2f46 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -10,22 +10,25 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' architecture: 'x64' - - - name: Checkout - uses: actions/checkout@v4 + cache: "pip" + cache-dependency-path: | + requirements.txt + docs/requirements.txt - name: Build - shell: bash run: | pip install -r requirements.txt make cython - name: Sphinx Documentation Generator run: | - pip install tox - tox -e sphinx + pip install -r docs/requirements.txt + make docs diff --git a/.github/workflows/black.yaml b/.github/workflows/lint.yaml similarity index 56% rename from .github/workflows/black.yaml rename to .github/workflows/lint.yaml index e0917926..198cf7b5 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/lint.yaml @@ -1,25 +1,22 @@ -name: Black +name: lint on: ["push", "pull_request"] jobs: - black: + lint: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - architecture: 'x64' - - name: Checkout uses: actions/checkout@v4 - - name: Black Code Formatter + - name: ruff check + run: | + pipx run ruff check --diff msgpack/ test/ setup.py + + - name: ruff format run: | - pip install black==22.3.0 - black -S --diff --check msgpack/ test/ setup.py + pipx run ruff format --diff msgpack/ test/ setup.py diff --git a/Makefile b/Makefile index e4f22da4..5c1863c7 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,17 @@ PYTHON_SOURCES = msgpack test setup.py all: cython python setup.py build_ext -i -f -.PHONY: black -black: - black $(PYTHON_SOURCES) +.PHONY: format +format: + pipx run ruff format $(PYTHON_SOURCES) + +.PHONY: lint +lint: + pipx run ruff check $(PYTHON_SOURCES) + +.PHONY: doc +doc: + cd docs && sphinx-build -n -v -W --keep-going -b html -d doctrees . html .PHONY: pyupgrade pyupgrade: diff --git a/docs/requirements.txt b/docs/requirements.txt index 8d45d0b6..26002de4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx~=7.2 -sphinx-rtd-theme~=1.3.0 +sphinx~=7.3.7 +sphinx-rtd-theme~=2.0.0 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a174162a..ea4c4ced 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,4 +1,5 @@ """Fallback pure Python implementation of msgpack""" + from datetime import datetime as _DateTime import sys import struct diff --git a/pyproject.toml b/pyproject.toml index f9af967b..d99375ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ skip_string_normalization = true [tool.ruff] line-length = 100 target-version = "py38" -ignore = [] +lint.ignore = [] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "msgpack/__init__.py" = ["F401", "F403"] "msgpack/fallback.py" = ["E731"] "test/test_seq.py" = ["E501"] diff --git a/requirements.txt b/requirements.txt index 839dc5f1..1164a941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,2 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.8 - -# Tools required only for development. No need to add it to pyproject.toml file. -black==23.3.0 -pytest==7.3.1 -pyupgrade==3.3.2 +Cython~=3.0.10 diff --git a/test/test_memoryview.py b/test/test_memoryview.py index dc319a63..eff4bca0 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -95,4 +95,4 @@ def test_multidim_memoryview(): view = memoryview(b"\00" * 6) data = view.cast(view.format, (3, 2)) packed = packb(data) - assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' + assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00" diff --git a/test/test_pack.py b/test/test_pack.py index 42325378..4a0ef403 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -89,7 +89,7 @@ def testStrictUnicodeUnpack(): def testIgnoreErrorsPack(): re = unpackb( - packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), + packb("abc\udc80\udcffdef", use_bin_type=True, unicode_errors="ignore"), raw=False, use_list=1, ) diff --git a/test/test_read_size.py b/test/test_read_size.py index 33a7e7dd..a7d61fd9 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,4 +1,5 @@ """Test Unpacker's read_array_header and read_map_header methods""" + from msgpack import packb, Unpacker, OutOfData UnexpectedTypeException = ValueError diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 49364be9..00000000 --- a/tox.ini +++ /dev/null @@ -1,38 +0,0 @@ -[tox] -envlist = - {py35,py36,py37,py38}-{c,pure}, - {pypy,pypy3}-pure, - py34-x86, - sphinx, -isolated_build = true - -[testenv] -deps= - pytest - -changedir=test -commands= - c,x86: python -c 'from msgpack import _cmsgpack' - c,x86: py.test - pure: py.test -setenv= - pure: MSGPACK_PUREPYTHON=x - -[testenv:py34-x86] -basepython=python3.4-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - - -[testenv:sphinx] -changedir = docs -deps = - -r docs/requirements.txt -commands = - sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 0602baf3ea7fb597d1c78a90980071e03a536836 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 18:20:09 +0900 Subject: [PATCH 38/63] update Cython and setuptools (#599) --- DEVELOP.md | 8 -------- pyproject.toml | 7 +++---- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index 9c823c34..27adf8c0 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,13 +1,5 @@ # Developer's note -## Wheels - -Wheels for macOS and Linux are built on Travis and AppVeyr, in -[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository. - -Wheels for Windows are built on Github Actions in this repository. - - ### Build ``` diff --git a/pyproject.toml b/pyproject.toml index d99375ac..6254f06f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,8 @@ [build-system] requires = [ - # Also declared in requirements.txt, if updating here please also update - # there - "Cython~=3.0.8", - "setuptools >= 35.0.2", + # Also declared in requirements.txt, if updating here please also update there + "Cython~=3.0.10", + "setuptools >= 69.5.1", ] build-backend = "setuptools.build_meta" From 3e9a2a7419714c294be0590aab24f2dc040581f5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:01:48 +0900 Subject: [PATCH 39/63] Stop using c++ (#600) Python 3.13a6+ & C++ & Cython cause compile error on some compilers. --- .github/workflows/test.yml | 2 +- Makefile | 2 +- msgpack/_unpacker.pyx | 2 +- msgpack/pack.h | 2 + msgpack/unpack_container_header.h | 51 +++++++++++++++++++ msgpack/unpack_template.h | 85 ++++++++----------------------- setup.py | 6 +-- 7 files changed, 81 insertions(+), 69 deletions(-) create mode 100644 msgpack/unpack_container_header.h diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1faeb0ca..530238c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/Makefile b/Makefile index 5c1863c7..3ce178f2 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ pyupgrade: .PHONY: cython cython: - cython --cplus msgpack/_cmsgpack.pyx + cython msgpack/_cmsgpack.pyx .PHONY: test test: cython diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 56126f43..2771e7bf 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -35,7 +35,7 @@ cdef extern from "unpack.h": PyObject* timestamp_t PyObject *giga; PyObject *utc; - char *unicode_errors + const char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len Py_ssize_t max_array_len diff --git a/msgpack/pack.h b/msgpack/pack.h index 2453428c..901fad7b 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,6 +24,8 @@ #ifdef __cplusplus extern "C" { +#else +#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h new file mode 100644 index 00000000..c14a3c2b --- /dev/null +++ b/msgpack/unpack_container_header.h @@ -0,0 +1,51 @@ +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8b9fcc19..cce29e7a 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -75,8 +75,7 @@ static inline void unpack_clear(unpack_context *ctx) Py_CLEAR(ctx->stack[0].obj); } -template -static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -386,6 +385,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef construct_cb } +#undef NEXT_CS #undef SWITCH_RANGE_BEGIN #undef SWITCH_RANGE #undef SWITCH_RANGE_DEFAULT @@ -397,68 +397,27 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef again_fixed_trail_if_zero #undef start_container -template -static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; - -#define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; -#ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: -#else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: -#endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; +static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(1, ctx, data, len, off); +} +static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(0, ctx, data, len, off); } -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END - -static const execute_fn unpack_construct = &unpack_execute; -static const execute_fn unpack_skip = &unpack_execute; -static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; - -#undef NEXT_CS +#define unpack_container_header read_array_header +#define fixed_offset 0x90 +#define var_offset 0xdc +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset + +#define unpack_container_header read_map_header +#define fixed_offset 0x80 +#define var_offset 0xde +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset /* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/setup.py b/setup.py index a13bd81d..dc14a26e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def cythonize(src): if not have_cython: raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src], cplus=True) + cython_compiler.compile([src]) def ensure_source(src): @@ -51,17 +51,17 @@ def __init__(self, *args, **kwargs): libraries = [] macros = [] +ext_modules = [] if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] -ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", - sources=["msgpack/_cmsgpack.cpp"], + sources=["msgpack/_cmsgpack.c"], libraries=libraries, include_dirs=["."], define_macros=macros, From b389ccf2f72355e23836ced193b555401508ef81 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:10:37 +0900 Subject: [PATCH 40/63] update README (#561) --- README.md | 116 +++++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 61f99e1f..d3247bdf 100644 --- a/README.md +++ b/README.md @@ -10,53 +10,6 @@ It lets you exchange data among multiple languages like JSON. But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. - -## Very important notes for existing users - -### PyPI package name - -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. - -When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before -`pip install -U msgpack`. - - -### Compatibility with the old format - -You can use `use_bin_type=False` option to pack `bytes` -object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. - -You can unpack old msgpack format using `raw=True` option. -It unpacks str (raw) type in msgpack into Python bytes. - -See note below for detail. - - -### Major breaking changes in msgpack 1.0 - -* Python 2 - - * The extension module does not support Python 2 anymore. - The pure Python implementation (`msgpack.fallback`) is used for Python 2. - -* Packer - - * `use_bin_type=True` by default. bytes are encoded in bin type in msgpack. - **If you are still using Python 2, you must use unicode for all string types.** - You can use `use_bin_type=False` to encode into old msgpack format. - * `encoding` option is removed. UTF-8 is used always. - -* Unpacker - - * `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. - * `encoding` option is removed. You can use `raw=True` to support old format. - * Default value of `max_buffer_size` is changed from 0 to 100 MiB. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. - - ## Install ``` @@ -65,12 +18,9 @@ $ pip install msgpack ### Pure Python implementation -The extension module in msgpack (`msgpack._cmsgpack`) does not support -Python 2 and PyPy. - -But msgpack provides a pure Python implementation (`msgpack.fallback`) -for PyPy and Python 2. +The extension module in msgpack (`msgpack._cmsgpack`) does not support PyPy. +But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows @@ -82,10 +32,6 @@ Without extension, using pure Python implementation on CPython runs slowly. ## How to use -NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users -using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them. - - ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. @@ -97,16 +43,16 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with ```pycon >>> import msgpack ->>> msgpack.packb([1, 2, 3], use_bin_type=True) +>>> msgpack.packb([1, 2, 3]) '\x93\x01\x02\x03' ->>> msgpack.unpackb(_, raw=False) +>>> msgpack.unpackb(_) [1, 2, 3] ``` `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) +>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) ``` @@ -127,11 +73,11 @@ from io import BytesIO buf = BytesIO() for i in range(100): - buf.write(msgpack.packb(i, use_bin_type=True)) + buf.write(msgpack.packb(i)) buf.seek(0) -unpacker = msgpack.Unpacker(buf, raw=False) +unpacker = msgpack.Unpacker(buf) for unpacked in unpacker: print(unpacked) ``` @@ -162,8 +108,8 @@ def encode_datetime(obj): return obj -packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) -this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +packed_dict = msgpack.packb(useful_dict, default=encode_datetime) +this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) ``` `Unpacker`'s `object_hook` callback receives a dict; the @@ -191,8 +137,8 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... return ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) ->>> packed = msgpack.packb(data, default=default, use_bin_type=True) ->>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) +>>> packed = msgpack.packb(data, default=default) +>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) >>> data == unpacked True ``` @@ -210,7 +156,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type +### string and binary type in old msgpack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -263,3 +209,41 @@ You can use `gc.disable()` when unpacking large message. List is the default sequence type of Python. But tuple is lighter than list. You can use `use_list=False` while unpacking when performance is important. + + +## Major breaking changes in the history + +### msgpack 0.5 + +Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. + +When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before +`pip install -U msgpack`. + + +### msgpack 1.0 + +* Python 2 support + + * The extension module does not support Python 2 anymore. + The pure Python implementation (`msgpack.fallback`) is used for Python 2. + + * msgpack 1.0.6 drops official support of Python 2.7, as pip and + GitHub Action (setup-python) no longer support Python 2.7. + +* Packer + + * Packer uses `use_bin_type=True` by default. + Bytes are encoded in bin type in msgpack. + * The `encoding` option is removed. UTF-8 is used always. + +* Unpacker + + * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string + and decode them to Python str (unicode) object. + * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). + * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + You need to pass `max_buffer_size=0` if you have large but safe data. + * Default value of `strict_map_key` is changed to True to avoid hashdos. + You need to pass `strict_map_key=False` if you have data which contain map keys + which type is not bytes or str. From 526ec9c923c4867c96537c1f09783fe59432f737 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:49:22 +0900 Subject: [PATCH 41/63] update cibuildwheel to 2.17 (#601) --- .github/workflows/docs.yaml | 1 - .github/workflows/wheel.yml | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 08fc2f46..b696b926 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' - architecture: 'x64' cache: "pip" cache-dependency-path: | requirements.txt diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index e91325be..d57e0586 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,29 +14,25 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 with: - platforms: arm64 + platforms: all - - name: Set up Python 3.x - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" - - - name: Prepare + - name: Cythonize shell: bash run: | pip install -r requirements.txt make cython - name: Build - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -45,6 +41,7 @@ jobs: CIBW_SKIP: pp* - name: Build pure Python wheel + if: runner.os == 'Linux' env: MSGPACK_PUREPYTHON: "1" run: | From 52f8bc2e557ca6684f2f73d129da8317a88bc431 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 5 May 2024 23:14:27 +0900 Subject: [PATCH 42/63] implement buffer protocol (#602) Fix #479 --- msgpack/_packer.pyx | 10 +++++++--- msgpack/buff_converter.h | 8 -------- 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 msgpack/buff_converter.h diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 3c398672..c2018806 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -44,8 +44,6 @@ cdef extern from "pack.h": int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) -cdef extern from "buff_converter.h": - object buff_to_buff(char *, Py_ssize_t) cdef int DEFAULT_RECURSE_LIMIT=511 cdef long long ITEM_LIMIT = (2**32)-1 @@ -371,4 +369,10 @@ cdef class Packer(object): def getbuffer(self): """Return view of internal buffer.""" - return buff_to_buff(self.pk.buf, self.pk.length) + return memoryview(self) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h deleted file mode 100644 index 86b4196d..00000000 --- a/msgpack/buff_converter.h +++ /dev/null @@ -1,8 +0,0 @@ -#include "Python.h" - -/* cython does not support this preprocessor check => write it in raw C */ -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyMemoryView_FromMemory(buff, size, PyBUF_READ); -} From a97b31437d3301fa4c29b7813dabee0e690756e8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:13:59 +0900 Subject: [PATCH 43/63] Remove unused code (#603) --- msgpack/_packer.pyx | 7 -- msgpack/pack.h | 3 +- msgpack/pack_template.h | 238 ++-------------------------------------- 3 files changed, 8 insertions(+), 240 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c2018806..c7eeda11 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -16,8 +16,6 @@ from .ext import ExtType, Timestamp cdef extern from "Python.h": int PyMemoryView_Check(object obj) - char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL - cdef extern from "pack.h": struct msgpack_packer: @@ -26,11 +24,9 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long(msgpack_packer* pk, long d) int msgpack_pack_long_long(msgpack_packer* pk, long long d) int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) int msgpack_pack_float(msgpack_packer* pk, float d) @@ -184,9 +180,6 @@ cdef class Packer(object): continue else: raise OverflowError("Integer value out of range") - elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - longval = o - ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): if self.use_float: fval = o diff --git a/msgpack/pack.h b/msgpack/pack.h index 901fad7b..688eab86 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -21,11 +21,10 @@ #include "sysdep.h" #include #include +#include #ifdef __cplusplus extern "C" { -#else -#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 7d479b6d..b8959f02 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -37,18 +37,6 @@ * Integer */ -#define msgpack_pack_real_uint8(x, d) \ -do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ -} while(0) - #define msgpack_pack_real_uint16(x, d) \ do { \ if(d < (1<<7)) { \ @@ -123,18 +111,6 @@ do { \ } \ } while(0) -#define msgpack_pack_real_int8(x, d) \ -do { \ - if(d < -(1<<5)) { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } \ -} while(0) - #define msgpack_pack_real_int16(x, d) \ do { \ if(d < -(1<<5)) { \ @@ -264,49 +240,6 @@ do { \ } while(0) -static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) -{ - msgpack_pack_real_uint8(x, d); -} - -static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) -{ - msgpack_pack_real_uint16(x, d); -} - -static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) -{ - msgpack_pack_real_uint32(x, d); -} - -static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) -{ - msgpack_pack_real_uint64(x, d); -} - -static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) -{ - msgpack_pack_real_int8(x, d); -} - -static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) -{ - msgpack_pack_real_int16(x, d); -} - -static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) -{ - msgpack_pack_real_int32(x, d); -} - -static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) -{ - msgpack_pack_real_int64(x, d); -} - - -//#ifdef msgpack_pack_inline_func_cint - static inline int msgpack_pack_short(msgpack_packer* x, short d) { #if defined(SIZEOF_SHORT) @@ -372,192 +305,37 @@ if(sizeof(int) == 2) { static inline int msgpack_pack_long(msgpack_packer* x, long d) { #if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 +#if SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #elif defined(LONG_MAX) -#if LONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LONG_MAX == 0x7fffffffL +#if LONG_MAX == 0x7fffffffL msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #else -if(sizeof(long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} + if (sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); + } else { + msgpack_pack_real_int64(x, d); + } #endif } static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#elif defined(LLONG_MAX) -#if LLONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LLONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(long long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long long) == 4) { - msgpack_pack_real_int32(x, d); -} else { msgpack_pack_real_int64(x, d); } -#endif -} - -static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d) -{ -#if defined(SIZEOF_SHORT) -#if SIZEOF_SHORT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(USHRT_MAX) -#if USHRT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif USHRT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned short) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned short) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) -{ -#if defined(SIZEOF_INT) -#if SIZEOF_INT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(UINT_MAX) -#if UINT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif UINT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned int) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) -{ -#if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULONG_MAX) -#if ULONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULLONG_MAX) -#if ULLONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULLONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { msgpack_pack_real_uint64(x, d); } -#endif -} - -//#undef msgpack_pack_inline_func_cint -//#endif - /* @@ -810,11 +588,9 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin #undef TAKE8_32 #undef TAKE8_64 -#undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 #undef msgpack_pack_real_uint32 #undef msgpack_pack_real_uint64 -#undef msgpack_pack_real_int8 #undef msgpack_pack_real_int16 #undef msgpack_pack_real_int32 #undef msgpack_pack_real_int64 From bf2413f915474841aafee2c98321dc465e5f0a3e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:30:07 +0900 Subject: [PATCH 44/63] ignore msgpack/*.c --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a06e267..341be631 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist/* *.so *~ msgpack/__version__.py +msgpack/*.c msgpack/*.cpp *.egg-info /venv From 72e65feb0e02449fa191346f26b54d3842ab7e69 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:49:12 +0900 Subject: [PATCH 45/63] packer: add `buf_size` option (#604) And change the default buffer size to 256KiB. Signed-off-by: Rodrigo Tobar Co-authored-by: Rodrigo Tobar --- msgpack/_packer.pyx | 13 +++++++++---- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 27 +++++---------------------- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c7eeda11..99557d37 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -53,7 +53,7 @@ cdef inline int PyBytesLike_CheckExact(object o): return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) -cdef class Packer(object): +cdef class Packer: """ MessagePack Packer @@ -97,6 +97,11 @@ cdef class Packer(object): :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. + + :param int buf_size: + The size of the internal buffer. (default: 256*1024) + Useful if serialisation size can be correctly estimated, + avoid unnecessary reallocations. """ cdef msgpack_packer pk cdef object _default @@ -107,8 +112,7 @@ cdef class Packer(object): cdef bint autoreset cdef bint datetime - def __cinit__(self): - cdef int buf_size = 1024*1024 + def __cinit__(self, buf_size=256*1024, **_kwargs): self.pk.buf = PyMem_Malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") @@ -117,7 +121,8 @@ cdef class Packer(object): def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, - bint strict_types=False, bint datetime=False, unicode_errors=None): + bint strict_types=False, bint datetime=False, unicode_errors=None, + buf_size=256*1024): self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2771e7bf..34ff3304 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -210,7 +210,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise ValueError("Unpack failed: error = %d" % (ret,)) -cdef class Unpacker(object): +cdef class Unpacker: """Streaming unpacker. Arguments: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ea4c4ced..cbf0d30e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -232,6 +232,7 @@ class Unpacker: def __init__( self, file_like=None, + *, read_size=0, use_list=True, raw=False, @@ -650,32 +651,13 @@ class Packer: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. - Example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like) - for o in unpacker: - process(o) - - Example of streaming deserialize from socket:: - - unpacker = Unpacker() - while True: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - process(o) - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``OutOfData`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. + :param int buf_size: + Internal buffer size. This option is used only for C implementation. """ def __init__( self, + *, default=None, use_single_float=False, autoreset=True, @@ -683,6 +665,7 @@ def __init__( strict_types=False, datetime=False, unicode_errors=None, + buf_size=None, ): self._strict_types = strict_types self._use_float = use_single_float From 3da5818a3a96a4bd39f25a483ee3c22ecb626e9d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:12:46 +0900 Subject: [PATCH 46/63] update readme (#605) --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d3247bdf..61a03c60 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,7 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with [1, 2, 3] ``` -`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: - -```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) -(1, 2, 3) -``` - -You should always specify the `use_list` keyword argument for backward compatibility. -See performance issues relating to `use_list option`_ below. - -Read the docstring for other options. +Read the docstring for options. ### Streaming unpacking @@ -116,6 +106,9 @@ this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) `object_pairs_hook` callback may instead be used to receive a list of key-value pairs. +NOTE: msgpack can encode datetime with tzinfo into standard ext type for now. +See `datetime` option in `Packer` docstring. + ### Extended types From e1068087e0bb3aebd77c4f2407bf9c429f631787 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:13:12 +0900 Subject: [PATCH 47/63] cython: better exception handling (#606) - use `except -1` instead of manual error handling - use `PyUnicode_AsUTF8AndSize()` - use `_pack()` and `_pack_inner()` instead of `while True:` --- msgpack/_packer.pyx | 302 +++++++++++++++++++------------------------- msgpack/pack.h | 21 --- 2 files changed, 131 insertions(+), 192 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 99557d37..ad532215 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -24,21 +24,20 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_nil(msgpack_packer* pk) - int msgpack_pack_true(msgpack_packer* pk) - int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long_long(msgpack_packer* pk, long long d) - int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) - int msgpack_pack_float(msgpack_packer* pk, float d) - int msgpack_pack_double(msgpack_packer* pk, double d) - int msgpack_pack_array(msgpack_packer* pk, size_t l) - int msgpack_pack_map(msgpack_packer* pk, size_t l) - int msgpack_pack_raw(msgpack_packer* pk, size_t l) - int msgpack_pack_bin(msgpack_packer* pk, size_t l) - int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) - int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) - int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); - int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) + int msgpack_pack_nil(msgpack_packer* pk) except -1 + int msgpack_pack_true(msgpack_packer* pk) except -1 + int msgpack_pack_false(msgpack_packer* pk) except -1 + int msgpack_pack_long_long(msgpack_packer* pk, long long d) except -1 + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) except -1 + int msgpack_pack_float(msgpack_packer* pk, float d) except -1 + int msgpack_pack_double(msgpack_packer* pk, double d) except -1 + int msgpack_pack_array(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_map(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_bin(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) except -1 + int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) except -1 + int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1 cdef int DEFAULT_RECURSE_LIMIT=511 @@ -119,6 +118,10 @@ cdef class Packer: self.pk.buf_size = buf_size self.pk.length = 0 + def __dealloc__(self): + PyMem_Free(self.pk.buf) + self.pk.buf = NULL + def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -139,155 +142,127 @@ cdef class Packer: else: self.unicode_errors = self._berrors - def __dealloc__(self): - PyMem_Free(self.pk.buf) - self.pk.buf = NULL - - cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + # returns -2 when default should(o) be called + cdef int _pack_inner(self, object o, bint will_default, int nest_limit) except -1: cdef long long llval cdef unsigned long long ullval cdef unsigned long ulval - cdef long longval - cdef float fval - cdef double dval - cdef char* rawval - cdef int ret - cdef dict d + cdef const char* rawval cdef Py_ssize_t L - cdef int default_used = 0 - cdef bint strict_types = self.strict_types + cdef bool strict_types = self.strict_types cdef Py_buffer view - if nest_limit < 0: - raise ValueError("recursion limit exceeded.") - - while True: - if o is None: - ret = msgpack_pack_nil(&self.pk) - elif o is True: - ret = msgpack_pack_true(&self.pk) - elif o is False: - ret = msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): - # PyInt_Check(long) is True for Python 3. - # So we should test long before int. - try: - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) - except OverflowError as oe: - if not default_used and self._default is not None: - o = self._default(o) - default_used = True - continue - else: - raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - if self.use_float: - fval = o - ret = msgpack_pack_float(&self.pk, fval) + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + try: + if o > 0: + ullval = o + msgpack_pack_unsigned_long_long(&self.pk, ullval) else: - dval = o - ret = msgpack_pack_double(&self.pk, dval) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - L = Py_SIZE(o) - if L > ITEM_LIMIT: - PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - rawval = o - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.unicode_errors == NULL: - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") + llval = o + msgpack_pack_long_long(&self.pk, llval) + except OverflowError as oe: + if will_default: + return -2 else: - o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) - L = Py_SIZE(o) - if L > ITEM_LIMIT: - raise ValueError("unicode string is too large") - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - rawval = o - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o): - d = o - L = len(d) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in d.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif not strict_types and PyDict_Check(o): - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in o.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - # This should be before Tuple because ExtType is namedtuple. - longval = o.code - rawval = o.data - L = len(o.data) - if L > ITEM_LIMIT: - raise ValueError("EXT data is too large") - ret = msgpack_pack_ext(&self.pk, longval, L) - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif type(o) is Timestamp: - llval = o.seconds - ulval = o.nanoseconds - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + raise OverflowError("Integer value out of range") + elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + if self.use_float: + msgpack_pack_float(&self.pk, o) + else: + msgpack_pack_double(&self.pk, o) + elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) + rawval = o + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + if self.unicode_errors == NULL: + rawval = PyUnicode_AsUTF8AndSize(o, &L) + if L >ITEM_LIMIT: + raise ValueError("unicode string is too large") + else: + o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) L = Py_SIZE(o) if L > ITEM_LIMIT: - raise ValueError("list is too large") - ret = msgpack_pack_array(&self.pk, L) - if ret == 0: - for v in o: - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyMemoryView_Check(o): - if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise ValueError("could not get buffer for memoryview") - L = view.len - if L > ITEM_LIMIT: - PyBuffer_Release(&view); - raise ValueError("memoryview is too large") - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, view.buf, L) + raise ValueError("unicode string is too large") + rawval = o + msgpack_pack_raw(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + L = len(o) + if L > ITEM_LIMIT: + raise ValueError("dict is too large") + msgpack_pack_map(&self.pk, L) + for k, v in o.items(): + self._pack(k, nest_limit) + self._pack(v, nest_limit) + elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + # This should be before Tuple because ExtType is namedtuple. + rawval = o.data + L = len(o.data) + if L > ITEM_LIMIT: + raise ValueError("EXT data is too large") + msgpack_pack_ext(&self.pk, o.code, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif type(o) is Timestamp: + llval = o.seconds + ulval = o.nanoseconds + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + raise ValueError("list is too large") + msgpack_pack_array(&self.pk, L) + for v in o: + self._pack(v, nest_limit) + elif PyMemoryView_Check(o): + PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) + L = view.len + if L > ITEM_LIMIT: PyBuffer_Release(&view); - elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: - delta = o - epoch - if not PyDelta_CheckExact(delta): - raise ValueError("failed to calculate delta") - llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) - ulval = timedelta_microseconds(delta) * 1000 - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif not default_used and self._default: + raise ValueError("memoryview is too large") + try: + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, view.buf, L) + finally: + PyBuffer_Release(&view); + elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: + delta = o - epoch + if not PyDelta_CheckExact(delta): + raise ValueError("failed to calculate delta") + llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) + ulval = timedelta_microseconds(delta) * 1000 + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif will_default: + return -2 + elif self.datetime and PyDateTime_CheckExact(o): + # this should be later than will_default + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) + else: + PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) + + cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + cdef int ret + if nest_limit < 0: + raise ValueError("recursion limit exceeded.") + nest_limit -= 1 + if self._default is not None: + ret = self._pack_inner(o, 1, nest_limit) + if ret == -2: o = self._default(o) - default_used = 1 - continue - elif self.datetime and PyDateTime_CheckExact(o): - PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: - PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - return ret + return ret + return self._pack_inner(o, 0, nest_limit) - cpdef pack(self, object obj): + def pack(self, object obj): cdef int ret try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) @@ -308,11 +283,7 @@ cdef class Packer: def pack_array_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_array(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -321,11 +292,7 @@ cdef class Packer: def pack_map_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_map(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -338,17 +305,10 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - if ret == 0: - for k, v in pairs: - ret = self._pack(k) - if ret != 0: break - ret = self._pack(v) - if ret != 0: break - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, len(pairs)) + for k, v in pairs: + self._pack(k) + self._pack(v) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 diff --git a/msgpack/pack.h b/msgpack/pack.h index 688eab86..edf3a3fe 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -64,27 +64,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ #include "pack_template.h" -// return -2 when o is too long -static inline int -msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) -{ - assert(PyUnicode_Check(o)); - - Py_ssize_t len; - const char* buf = PyUnicode_AsUTF8AndSize(o, &len); - if (buf == NULL) - return -1; - - if (len > limit) { - return -2; - } - - int ret = msgpack_pack_raw(pk, len); - if (ret) return ret; - - return msgpack_pack_raw_body(pk, buf, len); -} - #ifdef __cplusplus } #endif From e0f0e145f15364819bac80bd0808834a9df0065e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 03:33:48 +0900 Subject: [PATCH 48/63] better error checks (#607) * check buffer exports * add error messages --- msgpack/_packer.pyx | 48 +++++++++++++++++++++++++++++++++------------ test/test_buffer.py | 24 +++++++++++++++++++++-- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index ad532215..402b6946 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -106,6 +106,7 @@ cdef class Packer: cdef object _default cdef object _berrors cdef const char *unicode_errors + cdef size_t exports # number of exported buffers cdef bint strict_types cdef bint use_float cdef bint autoreset @@ -117,10 +118,16 @@ cdef class Packer: raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size self.pk.length = 0 + self.exports = 0 def __dealloc__(self): PyMem_Free(self.pk.buf) self.pk.buf = NULL + assert self.exports == 0 + + cdef _check_exports(self): + if self.exports > 0: + raise BufferError("Existing exports of data: Packer cannot be changed") def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, @@ -149,8 +156,8 @@ cdef class Packer: cdef unsigned long ulval cdef const char* rawval cdef Py_ssize_t L - cdef bool strict_types = self.strict_types cdef Py_buffer view + cdef bint strict = self.strict_types if o is None: msgpack_pack_nil(&self.pk) @@ -158,7 +165,7 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + elif PyLong_CheckExact(o) if strict else PyLong_Check(o): try: if o > 0: ullval = o @@ -171,19 +178,19 @@ cdef class Packer: return -2 else: raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + elif PyFloat_CheckExact(o) if strict else PyFloat_Check(o): if self.use_float: msgpack_pack_float(&self.pk, o) else: msgpack_pack_double(&self.pk, o) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + elif PyBytesLike_CheckExact(o) if strict else PyBytesLike_Check(o): L = Py_SIZE(o) if L > ITEM_LIMIT: PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) rawval = o msgpack_pack_bin(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + elif PyUnicode_CheckExact(o) if strict else PyUnicode_Check(o): if self.unicode_errors == NULL: rawval = PyUnicode_AsUTF8AndSize(o, &L) if L >ITEM_LIMIT: @@ -196,7 +203,7 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + elif PyDict_CheckExact(o) if strict else PyDict_Check(o): L = len(o) if L > ITEM_LIMIT: raise ValueError("dict is too large") @@ -204,7 +211,7 @@ cdef class Packer: for k, v in o.items(): self._pack(k, nest_limit) self._pack(v, nest_limit) - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + elif type(o) is ExtType if strict else isinstance(o, ExtType): # This should be before Tuple because ExtType is namedtuple. rawval = o.data L = len(o.data) @@ -216,7 +223,7 @@ cdef class Packer: llval = o.seconds ulval = o.nanoseconds msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)): L = Py_SIZE(o) if L > ITEM_LIMIT: raise ValueError("list is too large") @@ -264,6 +271,7 @@ cdef class Packer: def pack(self, object obj): cdef int ret + self._check_exports() try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) except: @@ -277,12 +285,16 @@ cdef class Packer: return buf def pack_ext_type(self, typecode, data): + self._check_exports() + if len(data) > ITEM_LIMIT: + raise ValueError("ext data too large") msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("array too large") msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -290,8 +302,9 @@ cdef class Packer: return buf def pack_map_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("map too learge") msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -305,7 +318,11 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - msgpack_pack_map(&self.pk, len(pairs)) + self._check_exports() + size = len(pairs) + if size > ITEM_LIMIT: + raise ValueError("map too large") + msgpack_pack_map(&self.pk, size) for k, v in pairs: self._pack(k) self._pack(v) @@ -319,6 +336,7 @@ cdef class Packer: This method is useful only when autoreset=False. """ + self._check_exports() self.pk.length = 0 def bytes(self): @@ -326,11 +344,15 @@ cdef class Packer: return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) def getbuffer(self): - """Return view of internal buffer.""" + """Return memoryview of internal buffer. + + Note: Packer now supports buffer protocol. You can use memoryview(packer). + """ return memoryview(self) def __getbuffer__(self, Py_buffer *buffer, int flags): PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + self.exports += 1 def __releasebuffer__(self, Py_buffer *buffer): - pass + self.exports -= 1 diff --git a/test/test_buffer.py b/test/test_buffer.py index a3db339c..2165eb50 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +from pytest import raises -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Packer def test_unpack_buffer(): @@ -27,3 +27,23 @@ def test_unpack_memoryview(): assert [b"foo", b"bar"] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) + + +def test_packer_getbuffer(): + packer = Packer(autoreset=False) + packer.pack_array_header(2) + packer.pack(42) + packer.pack("hello") + buffer = packer.getbuffer() + assert isinstance(buffer, memoryview) + assert bytes(buffer) == b"\x92*\xa5hello" + + if Packer.__module__ == "msgpack._cmsgpack": # only for Cython + # cython Packer supports buffer protocol directly + assert bytes(packer) == b"\x92*\xa5hello" + + with raises(BufferError): + packer.pack(42) + buffer.release() + packer.pack(42) + assert bytes(packer) == b"\x92*\xa5hello*" From 33e0e86f4e66e6bb029f118e8f5e166b55e828f7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 11:46:31 +0900 Subject: [PATCH 49/63] Cleanup code and pyproject (#608) * use isort * fallback: use BytesIO instead of StringIO. We had dropped Python 2 already. --- Makefile | 4 ++-- msgpack/__init__.py | 12 +++++----- msgpack/ext.py | 2 +- msgpack/fallback.py | 52 ++++++++++++++++++----------------------- pyproject.toml | 17 +++++--------- setup.py | 4 ++-- test/test_buffer.py | 2 +- test/test_except.py | 5 ++-- test/test_extension.py | 1 + test/test_limits.py | 8 +++---- test/test_memoryview.py | 1 + test/test_newspec.py | 2 +- test/test_obj.py | 1 + test/test_pack.py | 4 ++-- test/test_read_size.py | 2 +- test/test_seq.py | 6 ++--- test/test_sequnpack.py | 7 +++--- test/test_stricttype.py | 3 ++- test/test_subtype.py | 3 ++- test/test_timestamp.py | 4 +++- test/test_unpack.py | 11 ++++----- 21 files changed, 73 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index 3ce178f2..51f3e0ef 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,11 @@ all: cython .PHONY: format format: - pipx run ruff format $(PYTHON_SOURCES) + ruff format $(PYTHON_SOURCES) .PHONY: lint lint: - pipx run ruff check $(PYTHON_SOURCES) + ruff check $(PYTHON_SOURCES) .PHONY: doc doc: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 919b86f1..e796efb0 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,20 +1,20 @@ -from .exceptions import * -from .ext import ExtType, Timestamp - +# ruff: noqa: F401 import os +from .exceptions import * # noqa: F403 +from .ext import ExtType, Timestamp version = (1, 0, 8) __version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb else: try: - from ._cmsgpack import Packer, unpackb, Unpacker + from ._cmsgpack import Packer, Unpacker, unpackb except ImportError: - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb def pack(o, stream, **kwargs): diff --git a/msgpack/ext.py b/msgpack/ext.py index 3940fe0f..9694819a 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,6 @@ -from collections import namedtuple import datetime import struct +from collections import namedtuple class ExtType(namedtuple("ExtType", "code data")): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index cbf0d30e..b02e47cf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,28 +1,22 @@ """Fallback pure Python implementation of msgpack""" -from datetime import datetime as _DateTime -import sys import struct - +import sys +from datetime import datetime as _DateTime if hasattr(sys, "pypy_version_info"): - # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own - # StringBuilder is fastest. from __pypy__ import newlist_hint + from __pypy__.builders import BytesBuilder - try: - from __pypy__.builders import BytesBuilder as StringBuilder - except ImportError: - from __pypy__.builders import StringBuilder - USING_STRINGBUILDER = True + _USING_STRINGBUILDER = True - class StringIO: + class BytesIO: def __init__(self, s=b""): if s: - self.builder = StringBuilder(len(s)) + self.builder = BytesBuilder(len(s)) self.builder.append(s) else: - self.builder = StringBuilder() + self.builder = BytesBuilder() def write(self, s): if isinstance(s, memoryview): @@ -35,17 +29,17 @@ def getvalue(self): return self.builder.build() else: - USING_STRINGBUILDER = False - from io import BytesIO as StringIO + from io import BytesIO - newlist_hint = lambda size: [] + _USING_STRINGBUILDER = False + def newlist_hint(size): + return [] -from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError +from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError from .ext import ExtType, Timestamp - EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 @@ -335,6 +329,7 @@ def feed(self, next_bytes): # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython self._buffer.extend(view) + view.release() def _consume(self): """Gets rid of the used parts of the buffer.""" @@ -671,12 +666,11 @@ def __init__( self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type - self._buffer = StringIO() + self._buffer = BytesIO() self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" - if default is not None: - if not callable(default): - raise TypeError("default must be callable") + if default is not None and not callable(default): + raise TypeError("default must be callable") self._default = default def _pack( @@ -807,18 +801,18 @@ def pack(self, obj): try: self._pack(obj) except: - self._buffer = StringIO() # force reset + self._buffer = BytesIO() # force reset raise if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_array_header(self, n): @@ -827,7 +821,7 @@ def pack_array_header(self, n): self._pack_array_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_header(self, n): @@ -836,7 +830,7 @@ def pack_map_header(self, n): self._pack_map_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_ext_type(self, typecode, data): @@ -925,11 +919,11 @@ def reset(self): This method is useful only when autoreset=False. """ - self._buffer = StringIO() + self._buffer = BytesIO() def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER: + if _USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/pyproject.toml b/pyproject.toml index 6254f06f..f36c7f40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,17 +45,12 @@ include-package-data = false [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} -[tool.black] -line-length = 100 -target-version = ["py37"] -skip_string_normalization = true - [tool.ruff] line-length = 100 target-version = "py38" -lint.ignore = [] - -[tool.ruff.lint.per-file-ignores] -"msgpack/__init__.py" = ["F401", "F403"] -"msgpack/fallback.py" = ["E731"] -"test/test_seq.py" = ["E501"] +lint.select = [ + "E", # pycodestyle + "F", # Pyflakes + "I", # isort + #"UP", pyupgrade +] diff --git a/setup.py b/setup.py index dc14a26e..eaca7460 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ #!/usr/bin/env python import os import sys -from setuptools import setup, Extension + +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist - PYPY = hasattr(sys, "pypy_version_info") diff --git a/test/test_buffer.py b/test/test_buffer.py index 2165eb50..2c5a14c5 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ from pytest import raises -from msgpack import packb, unpackb, Packer +from msgpack import Packer, packb, unpackb def test_unpack_buffer(): diff --git a/test/test_except.py b/test/test_except.py index 8c0a9766..b77ac800 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +import datetime + from pytest import raises -from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData -import datetime +from msgpack import FormatError, OutOfData, StackError, Unpacker, packb, unpackb class DummyException(Exception): diff --git a/test/test_extension.py b/test/test_extension.py index 9e5e6aad..aaf0fd92 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,5 @@ import array + import msgpack from msgpack import ExtType diff --git a/test/test_limits.py b/test/test_limits.py index 533bc112..9b92b4d9 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -2,14 +2,14 @@ import pytest from msgpack import ( - packb, - unpackb, - Packer, - Unpacker, ExtType, + Packer, PackOverflowError, PackValueError, + Unpacker, UnpackValueError, + packb, + unpackb, ) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eff4bca0..0a2a6f53 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from array import array + from msgpack import packb, unpackb diff --git a/test/test_newspec.py b/test/test_newspec.py index a6f4251b..9e2f9be5 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,4 +1,4 @@ -from msgpack import packb, unpackb, ExtType +from msgpack import ExtType, packb, unpackb def test_str8(): diff --git a/test/test_obj.py b/test/test_obj.py index f78bf426..23be06d5 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from pytest import raises + from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 4a0ef403..374d1549 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,12 +1,12 @@ #!/usr/bin/env python +import struct from collections import OrderedDict from io import BytesIO -import struct import pytest -from msgpack import packb, unpackb, Unpacker, Packer +from msgpack import Packer, Unpacker, packb, unpackb def check(data, use_list=False): diff --git a/test/test_read_size.py b/test/test_read_size.py index a7d61fd9..0f6c1b50 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,6 +1,6 @@ """Test Unpacker's read_array_header and read_map_header methods""" -from msgpack import packb, Unpacker, OutOfData +from msgpack import OutOfData, Unpacker, packb UnexpectedTypeException = ValueError diff --git a/test/test_seq.py b/test/test_seq.py index 16d9dde4..8dee4620 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python - +# ruff: noqa: E501 +# ignore line length limit for long comments import io -import msgpack +import msgpack binarydata = bytes(bytearray(range(256))) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6b138aad..0f895d7d 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import io -from msgpack import Unpacker, BufferFull -from msgpack import pack, packb -from msgpack.exceptions import OutOfData + from pytest import raises +from msgpack import BufferFull, Unpacker, pack, packb +from msgpack.exceptions import OutOfData + def test_partialdata(): unpacker = Unpacker() diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 9ffaff25..72776a2c 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,6 @@ from collections import namedtuple -from msgpack import packb, unpackb, ExtType + +from msgpack import ExtType, packb, unpackb def test_namedtuple(): diff --git a/test/test_subtype.py b/test/test_subtype.py index 0d1c41af..a911578c 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -from msgpack import packb from collections import namedtuple +from msgpack import packb + class MyList(list): pass diff --git a/test/test_timestamp.py b/test/test_timestamp.py index f9bc8353..831141a1 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,7 @@ -import pytest import datetime + +import pytest + import msgpack from msgpack.ext import Timestamp diff --git a/test/test_unpack.py b/test/test_unpack.py index bf3f960d..b17c3c53 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,12 +1,9 @@ -from io import BytesIO import sys -from msgpack import Unpacker, packb, OutOfData, ExtType -from pytest import raises, mark +from io import BytesIO + +from pytest import mark, raises -try: - from itertools import izip as zip -except ImportError: - pass +from msgpack import ExtType, OutOfData, Unpacker, packb def test_unpack_array_header_from_file(): From 9cea8b6da23ce66f0e78f017c96adcc447deb09a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 20:49:23 +0900 Subject: [PATCH 50/63] Release v1.1.0rc1 (#609) --- ChangeLog.rst | 20 +++++++++++++++++++- msgpack/__init__.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2408bc9f..9b16e41f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +1.1.0rc1 +======== + +Release Date: 2024-05-07 + +* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13. +* Stop using C++ mode in Cython to reduce compile error on some compilers. +* ``Packer()`` has ``buf_size`` option to specify initial size of + internal buffer to reduce reallocation. +* The default internal buffer size of ``Packer()`` is reduced from + 1MiB to 256KiB to optimize for common use cases. Use ``buf_size`` + if you are packing large data. +* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become + more accurate by avoiding floating point calculations. (#591) +* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability. +* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only + arguments to improve compatibility with the Cython implementation. + 1.0.8 ===== @@ -130,7 +148,7 @@ Important changes * unpacker: Default value of input limits are smaller than before to avoid DoS attack. If you need to handle large data, you need to specify limits manually. (#319) -* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into +* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into ``UnpackValueError``. If you want to catch all exception during unpack, you need to use ``try ... except Exception`` with minimum try code block. (#323, #233) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index e796efb0..c2525444 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 0, 8) -__version__ = "1.0.8" +version = (1, 1, 0, "rc1") +__version__ = "1.1.0rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 0b1c47b06b55d91c00c9f7153c4a9440ea878886 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 22:01:54 +0900 Subject: [PATCH 51/63] do not install cython as build dependency (#610) User can not cythonize during `pip install msgpack`. So remove cython from build dependency. If user need to use another Cython, user should download sdist, unzip, manually cythonize, and `pip install .`. --- pyproject.toml | 6 +----- setup.py | 45 --------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f36c7f40..d041d4c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,5 @@ [build-system] -requires = [ - # Also declared in requirements.txt, if updating here please also update there - "Cython~=3.0.10", - "setuptools >= 69.5.1", -] +requires = ["setuptools >= 69.5.1"] build-backend = "setuptools.build_meta" [project] diff --git a/setup.py b/setup.py index eaca7460..4029e9ed 100644 --- a/setup.py +++ b/setup.py @@ -3,52 +3,9 @@ import sys from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") - -class NoCython(Exception): - pass - - -try: - import Cython.Compiler.Main as cython_compiler - - have_cython = True -except ImportError: - have_cython = False - - -def cythonize(src): - if not have_cython: - raise Exception("Cython is required for building from checkout") - sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src]) - - -def ensure_source(src): - pyx = os.path.splitext(src)[0] + ".pyx" - - if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: - cythonize(pyx) - - -class BuildExt(build_ext): - def build_extension(self, ext): - for src in ext.sources: - ensure_source(src) - return build_ext.build_extension(self, ext) - - -# Cython is required for sdist -class Sdist(sdist): - def __init__(self, *args, **kwargs): - cythonize("msgpack/_cmsgpack.pyx") - sdist.__init__(self, *args, **kwargs) - - libraries = [] macros = [] ext_modules = [] @@ -69,9 +26,7 @@ def __init__(self, *args, **kwargs): ) del libraries, macros - setup( - cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], ) From 6e11368f5d54f7d4878dc209717495c37be03c68 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:35:16 +0900 Subject: [PATCH 52/63] update Cython to 3.0.11 (#617) --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1164a941..b677f068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -# Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.10 +Cython~=3.0.11 From 9e26d80ab2a02221d3ca36cc0b5ca2268f391204 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:56:01 +0900 Subject: [PATCH 53/63] update cibuildwheel to 2.20.0 (#618) --- .github/workflows/wheel.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d57e0586..01d0bbd1 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -40,13 +40,11 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Build pure Python wheel + - name: Build sdist if: runner.os == 'Linux' - env: - MSGPACK_PUREPYTHON: "1" run: | pip install build - python -m build -w -o wheelhouse + python -m build -s -o wheelhouse - name: Upload Wheels to artifact uses: actions/upload-artifact@v4 From 9d0c7f2f9cab21c3e39d23001cd9d2034824ee61 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 20:36:26 +0900 Subject: [PATCH 54/63] Release v1.1.0rc2 (#619) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 01d0bbd1..50157223 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -38,7 +38,7 @@ jobs: CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: pp* + CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist if: runner.os == 'Linux' diff --git a/ChangeLog.rst b/ChangeLog.rst index 9b16e41f..47328ffb 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0rc2 +======== + +Release Date: 2024-08-19 + +* Update Cython to 3.0.11 for better Python 3.13 support. +* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + 1.1.0rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index c2525444..a72e974c 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc1") -__version__ = "1.1.0rc1" +version = (1, 1, 0, "rc2") +__version__ = "1.1.0rc2" if os.environ.get("MSGPACK_PUREPYTHON"): From 20a2b8eaa26d76169049ce150e10d1d2aa37d3ab Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 21 Aug 2024 01:56:00 -0400 Subject: [PATCH 55/63] use `PyLong_*` instead of `PyInt_*` (#620) 9af421163cb8081414be347038dee7a82b29e8dd in Cython removed back-compatibility `#define`. --- msgpack/unpack.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 23aa6220..58a2f4f5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -47,7 +47,7 @@ static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong((long)d); + PyObject *p = PyLong_FromLong((long)d); if (!p) return -1; *o = p; @@ -61,7 +61,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromSize_t((size_t)d); + PyObject *p = PyLong_FromSize_t((size_t)d); if (!p) return -1; *o = p; @@ -74,7 +74,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } if (!p) return -1; @@ -84,7 +84,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong(d); + PyObject *p = PyLong_FromLong(d); if (!p) return -1; *o = p; @@ -107,7 +107,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac if (d > LONG_MAX || d < LONG_MIN) { p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } *o = p; return 0; From 4587393b1ae2c9ebbd5bc93005b8aea2c8050b27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 10 Sep 2024 01:58:00 +0900 Subject: [PATCH 56/63] release v1.1.0 (#622) --- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 47328ffb..863c6b2c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0 +===== + +Release Date: 2024-09-10 + +* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with + future Cython. (#620) + 1.1.0rc2 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a72e974c..b6151054 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc2") -__version__ = "1.1.0rc2" +version = (1, 1, 0) +__version__ = "1.1.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0eeabfb453844b441a4a77097b3d5aa0cb6645b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Tue, 8 Oct 2024 03:04:56 -0600 Subject: [PATCH 57/63] Add Python 3.13 trove classifier (#626) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d041d4c0..e24f2b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 868aa2cd83f39237deb957c68ce7232422a5950b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 31 May 2025 12:45:06 +0900 Subject: [PATCH 58/63] update Cython to 3.1.1 (#637) --- .github/workflows/test.yml | 9 +++++++-- pyproject.toml | 12 +++++++----- requirements.txt | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 530238c9..23d221c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -26,10 +26,15 @@ jobs: allow-prereleases: true cache: "pip" - - name: Build + - name: Prepare shell: bash run: | + pip install -U pip pip install -r requirements.txt pytest + + - name: Build + shell: bash + run: | make cython pip install . diff --git a/pyproject.toml b/pyproject.toml index e24f2b88..b1628322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ [build-system] -requires = ["setuptools >= 69.5.1"] +# 75.3.0 is the latest version supporting Python 3.8 +requires = ["setuptools >= 75.3.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] +# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. +# https://github.com/msgpack/msgpack-python/pull/637 license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" @@ -14,18 +17,17 @@ requires-python = ">=3.8" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", + "Topic :: File Formats", + "Intended Audience :: Developers", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index b677f068..78a2f38f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.0.11 +Cython~=3.1.1 From cdc764450370ff80e7c83edbe8d015f08f6fb9b3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Jun 2025 16:56:44 +0900 Subject: [PATCH 59/63] update cibuildwheel to v2.23.3 (#638) --- .github/workflows/wheel.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 50157223..d97de1dd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -9,17 +9,12 @@ jobs: build_wheels: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + # macos-13 is for intel + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} steps: - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -32,12 +27,10 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist From fe9e620a607702b31476f092ad01a387cff4cfbd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Jun 2025 14:46:53 +0900 Subject: [PATCH 60/63] upload to PyPI on create a release (#639) --- .github/workflows/wheel.yml | 48 ++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d97de1dd..686d7dd0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -1,8 +1,10 @@ -name: Build Wheels +name: Build sdist and Wheels on: push: branches: [main] - create: + release: + types: + - published workflow_dispatch: jobs: @@ -34,7 +36,7 @@ jobs: CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.arch == 'X64' run: | pip install build python -m build -s -o wheelhouse @@ -44,3 +46,43 @@ jobs: with: name: wheels-${{ matrix.os }} path: wheelhouse + + # combine all wheels into one artifact + combine_wheels: + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-all + path: dist + + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml + upload_pypi: + needs: [build_wheels] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + #with: + # To test: repository-url: https://test.pypi.org/legacy/ From e6445d3b922ca0b9bc82695dd9d1c1529763095a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Jun 2025 09:38:38 +0900 Subject: [PATCH 61/63] v1.1.1rc1 --- ChangeLog.rst | 7 +++++++ docs/conf.py | 6 +++--- msgpack/__init__.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 863c6b2c..0f67502c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1rc1 +======== + +Release Date: 2025-06-06 + +* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + 1.1.0 ===== diff --git a/docs/conf.py b/docs/conf.py index ab0ad3c8..28116cd6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#import os -#import sys -#sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b6151054..b8d83da8 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0) -__version__ = "1.1.0" +version = (1, 1, 1, "rc1") +__version__ = "1.1.1rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 42f056f3cfaf2e3ec220db2f864e7613d433ad48 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 13 Jun 2025 15:41:08 +0900 Subject: [PATCH 62/63] v1.1.1 --- ChangeLog.rst | 7 +++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 0f67502c..418c444c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1 +===== + +Release Date: 2025-06-13 + +* No change from 1.1.1rc1. + 1.1.1rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b8d83da8..ad68271d 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 1, "rc1") -__version__ = "1.1.1rc1" +version = (1, 1, 1) +__version__ = "1.1.1" if os.environ.get("MSGPACK_PUREPYTHON"): From d9873dab049c60d5fd553223b9be10f1dcb56929 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 26 Jul 2025 10:59:05 +0900 Subject: [PATCH 63/63] ci: update cibuildwheel and drop Python 3.8 (#642) --- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23d221c8..60894797 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 686d7dd0..12600363 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -29,11 +29,11 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.1.1 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_SKIP: "pp* cp38-*" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/requirements.txt b/requirements.txt index 78a2f38f..5d2e20b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.1.1 +Cython~=3.1.2