diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 530238c9..6b1664ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,27 +9,33 @@ jobs: test: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] - + os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] + py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] + exclude: + - os: windows-11-arm + py: "3.10" runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.py }} allow-prereleases: true cache: "pip" + - name: Prepare + shell: bash + run: | + python -m pip install -r requirements.txt pytest + - name: Build shell: bash run: | - pip install -r requirements.txt pytest make cython pip install . @@ -46,8 +52,7 @@ jobs: - name: build packages shell: bash run: | - pip install build - python -m build + python -m build -nv - name: upload packages uses: actions/upload-artifact@v4 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 50157223..531abbc3 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -1,27 +1,24 @@ -name: Build Wheels +name: Build sdist and Wheels on: push: branches: [main] - create: + release: + types: + - published workflow_dispatch: jobs: build_wheels: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + # macos-13 is for intel + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} steps: - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.x" cache: "pip" @@ -32,16 +29,14 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" - name: Build sdist - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.arch == 'X64' run: | pip install build python -m build -s -o wheelhouse @@ -51,3 +46,43 @@ jobs: with: name: wheels-${{ matrix.os }} path: wheelhouse + + # combine all wheels into one artifact + combine_wheels: + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-all + path: dist + + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml + upload_pypi: + needs: [build_wheels] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + #with: + # To test: repository-url: https://test.pypi.org/legacy/ diff --git a/ChangeLog.rst b/ChangeLog.rst index 863c6b2c..beeab15c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,30 @@ +1.1.2 +===== + +Release Date: 2025-10-08 + +This release does not change source code. It updates only building wheels: + +* Update Cython to v3.1.4 +* Update cibuildwheel to v3.2.0 +* Drop Python 3.8 +* Add Python 3.14 +* Add windows-arm + +1.1.1 +===== + +Release Date: 2025-06-13 + +* No change from 1.1.1rc1. + +1.1.1rc1 +======== + +Release Date: 2025-06-06 + +* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + 1.1.0 ===== diff --git a/MANIFEST.in b/MANIFEST.in index 57d84a4c..6317706e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING include README.md -recursive-include msgpack *.h *.c *.pyx *.cpp +recursive-include msgpack *.h *.c *.pyx recursive-include test *.py diff --git a/README.md b/README.md index 61a03c60..1f063242 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) -## What's this +## What is this? [MessagePack](https://msgpack.org/) is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. @@ -25,9 +25,9 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows -When you can't use a binary distribution, you need to install Visual Studio -or Windows SDK on Windows. -Without extension, using pure Python implementation on CPython runs slowly. +If you can't use a binary distribution, you need to install Visual Studio +or the Windows SDK on Windows. +Without the extension, the pure Python implementation on CPython runs slowly. ## How to use @@ -35,11 +35,11 @@ Without extension, using pure Python implementation on CPython runs slowly. ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. -msgpack provides `dumps` and `loads` as an alias for compatibility with +msgpack provides `dumps` and `loads` as aliases for compatibility with `json` and `pickle`. -`pack` and `dump` packs to a file-like object. -`unpack` and `load` unpacks from a file-like object. +`pack` and `dump` pack to a file-like object. +`unpack` and `load` unpack from a file-like object. ```pycon >>> import msgpack @@ -73,7 +73,7 @@ for unpacked in unpacker: ``` -### Packing/unpacking of custom data type +### Packing/unpacking of custom data types It is also possible to pack/unpack custom data types. Here is an example for `datetime.datetime`. @@ -140,8 +140,8 @@ True ### Advanced unpacking control As an alternative to iteration, `Unpacker` objects provide `unpack`, -`skip`, `read_array_header` and `read_map_header` methods. The former two -read an entire message from the stream, respectively de-serialising and returning +`skip`, `read_array_header`, and `read_map_header` methods. The former two +read an entire message from the stream, respectively deserializing and returning the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. @@ -149,7 +149,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type in old msgpack spec +### String and binary types in the old MessagePack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -167,7 +167,7 @@ and `raw=True` options. ### ext type -To use the **ext** type, pass `msgpack.ExtType` object to packer. +To use the **ext** type, pass a `msgpack.ExtType` object to the packer. ```pycon >>> import msgpack @@ -181,26 +181,26 @@ You can use it with `default` and `ext_hook`. See below. ### Security -To unpacking data received from unreliable source, msgpack provides +When unpacking data received from an unreliable source, msgpack provides two security options. `max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size. -It is used to limit the preallocated list size too. +It is also used to limit preallocated list sizes. `strict_map_key` (default: `True`) limits the type of map keys to bytes and str. -While msgpack spec doesn't limit the types of the map keys, -there is a risk of the hashdos. +While the MessagePack spec doesn't limit map key types, +there is a risk of a hash DoS. If you need to support other types for map keys, use `strict_map_key=False`. ### Performance tips -CPython's GC starts when growing allocated object. -This means unpacking may cause useless GC. -You can use `gc.disable()` when unpacking large message. +CPython's GC starts when the number of allocated objects grows. +This means unpacking may trigger unnecessary GC. +You can use `gc.disable()` when unpacking a large message. -List is the default sequence type of Python. -But tuple is lighter than list. +A list is the default sequence type in Python. +However, a tuple is lighter than a list. You can use `use_list=False` while unpacking when performance is important. @@ -208,7 +208,7 @@ You can use `use_list=False` while unpacking when performance is important. ### msgpack 0.5 -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. +The package name on PyPI was changed from `msgpack-python` to `msgpack` in 0.5. When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before `pip install -U msgpack`. @@ -218,25 +218,25 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be * Python 2 support - * The extension module does not support Python 2 anymore. + * The extension module no longer supports Python 2. The pure Python implementation (`msgpack.fallback`) is used for Python 2. * msgpack 1.0.6 drops official support of Python 2.7, as pip and - GitHub Action (setup-python) no longer support Python 2.7. + GitHub Action "setup-python" no longer supports Python 2.7. * Packer * Packer uses `use_bin_type=True` by default. - Bytes are encoded in bin type in msgpack. - * The `encoding` option is removed. UTF-8 is used always. + Bytes are encoded in the bin type in MessagePack. + * The `encoding` option is removed. UTF-8 is always used. * Unpacker - * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. + * Unpacker uses `raw=False` by default. It assumes str values are valid UTF-8 strings + and decodes them to Python str (Unicode) objects. * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). - * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + * The default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attacks. You need to pass `max_buffer_size=0` if you have large but safe data. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. + * The default value of `strict_map_key` is changed to True to avoid hash DoS. + You need to pass `strict_map_key=False` if you have data that contain map keys + whose type is neither bytes nor str. diff --git a/docs/conf.py b/docs/conf.py index ab0ad3c8..28116cd6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#import os -#import sys -#sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b6151054..f3266b70 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0) -__version__ = "1.1.0" +version = (1, 1, 2) +__version__ = "1.1.2" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 1faaac3a..9680b31e 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,5 +1,6 @@ -# coding: utf-8 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +#cython: freethreading_compatible = True +import cython from cpython.datetime cimport import_datetime, datetime_new import_datetime() diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 402b6946..94d1462c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from cpython.datetime cimport ( @@ -129,6 +127,7 @@ cdef class Packer: if self.exports > 0: raise BufferError("Existing exports of data: Packer cannot be changed") + @cython.critical_section def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -269,6 +268,7 @@ cdef class Packer: return ret return self._pack_inner(o, 0, nest_limit) + @cython.critical_section def pack(self, object obj): cdef int ret self._check_exports() @@ -284,6 +284,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_ext_type(self, typecode, data): self._check_exports() if len(data) > ITEM_LIMIT: @@ -291,6 +292,7 @@ cdef class Packer: msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) + @cython.critical_section def pack_array_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -301,6 +303,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -311,6 +314,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_pairs(self, object pairs): """ Pack *pairs* as msgpack map type. @@ -331,6 +335,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def reset(self): """Reset internal buffer. @@ -339,6 +344,7 @@ cdef class Packer: self._check_exports() self.pk.length = 0 + @cython.critical_section def bytes(self): """Return internal buffer contents as bytes object""" return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 34ff3304..f0cf96d7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject @@ -324,6 +322,7 @@ cdef class Unpacker: PyMem_Free(self.buf) self.buf = NULL + @cython.critical_section def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, @@ -384,6 +383,7 @@ cdef class Unpacker: max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) + @cython.critical_section def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" cdef Py_buffer pybuff @@ -484,6 +484,7 @@ cdef class Unpacker: else: raise ValueError("Unpack failed: error = %d" % (ret,)) + @cython.critical_section def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" cdef Py_ssize_t nread @@ -496,6 +497,7 @@ cdef class Unpacker: self.stream_offset += nread return ret + @cython.critical_section def unpack(self): """Unpack one object @@ -503,6 +505,7 @@ cdef class Unpacker: """ return self._unpack(unpack_construct) + @cython.critical_section def skip(self): """Read and ignore one object, returning None @@ -510,6 +513,7 @@ cdef class Unpacker: """ return self._unpack(unpack_skip) + @cython.critical_section def read_array_header(self): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents. @@ -518,6 +522,7 @@ cdef class Unpacker: """ return self._unpack(read_array_header) + @cython.critical_section def read_map_header(self): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs. @@ -526,6 +531,7 @@ cdef class Unpacker: """ return self._unpack(read_map_header) + @cython.critical_section def tell(self): """Returns the current position of the Unpacker in bytes, i.e., the number of bytes that were read from the input, also the starting @@ -536,6 +542,7 @@ cdef class Unpacker: def __iter__(self): return self + @cython.critical_section def __next__(self): return self._unpack(unpack_construct, 1) diff --git a/pyproject.toml b/pyproject.toml index e24f2b88..c69d5a7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,23 @@ [build-system] -requires = ["setuptools >= 69.5.1"] +requires = ["setuptools >= 78.1.1"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] -license = {text="Apache 2.0"} +license = "Apache-2.0" authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", + "Topic :: File Formats", + "Intended Audience :: Developers", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", ] [project.urls] @@ -44,7 +36,7 @@ version = {attr = "msgpack.__version__"} [tool.ruff] line-length = 100 -target-version = "py38" +target-version = "py310" lint.select = [ "E", # pycodestyle "F", # Pyflakes diff --git a/requirements.txt b/requirements.txt index b677f068..9e4643b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -Cython~=3.0.11 +Cython==3.2.1 +setuptools==78.1.1 +build diff --git a/test/test_buffer.py b/test/test_buffer.py index 2c5a14c5..ca097222 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_bytearray(): obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_unpack_memoryview(): @@ -26,7 +26,7 @@ def test_unpack_memoryview(): obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_packer_getbuffer():