From 685f46e9546e04c4fae11f2486ff1f5da3af8ebc Mon Sep 17 00:00:00 2001 From: Quentin Chateau Date: Fri, 27 Sep 2019 20:34:35 +0200 Subject: [PATCH] Windows, OSX, Python 3.5, less limitations, bugfixes --- .ci/build-linux-sdist.sh | 13 + .ci/build-manylinux-docker.sh | 26 ++ .ci/build-manylinux.sh | 6 + .ci/build-osx.sh | 30 ++ .ci/build-windows.sh | 30 ++ .ci/coverage.sh | 10 + .ci/deploy-osx.sh | 5 + .ci/deploy.sh | 5 + .ci/performance.sh | 6 + .ci/test-linux.sh | 6 + .travis.yml | 112 +++--- MANIFEST.in | 1 + README.md | 52 ++- cbitstruct/_cbitstruct.c | 574 +++++++++++++++------------ cbitstruct/clinic/_cbitstruct.c.35.h | 83 +++- cbitstruct/clinic/_cbitstruct.c.36.h | 89 ++++- cbitstruct/clinic/_cbitstruct.c.37.h | 89 ++++- cbitstruct/tests/test_api.py | 179 +++++++++ cbitstruct/tests/test_bitstruct.py | 7 +- cbitstruct/tests/test_perf.py | 6 +- dev-requirements.txt | 4 +- setup.py | 16 +- travis/build-wheels.sh | 22 - travis/deploy.sh | 4 - 24 files changed, 1007 insertions(+), 368 deletions(-) create mode 100644 .ci/build-linux-sdist.sh create mode 100755 .ci/build-manylinux-docker.sh create mode 100644 .ci/build-manylinux.sh create mode 100644 .ci/build-osx.sh create mode 100644 .ci/build-windows.sh create mode 100644 .ci/coverage.sh create mode 100644 .ci/deploy-osx.sh create mode 100644 .ci/deploy.sh create mode 100644 .ci/performance.sh create mode 100644 .ci/test-linux.sh create mode 100644 MANIFEST.in create mode 100644 cbitstruct/tests/test_api.py delete mode 100755 travis/build-wheels.sh delete mode 100644 travis/deploy.sh diff --git a/.ci/build-linux-sdist.sh b/.ci/build-linux-sdist.sh new file mode 100644 index 0000000..0502659 --- /dev/null +++ b/.ci/build-linux-sdist.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -x + +python -m pip install -U -r dev-requirements.txt +python setup.py sdist + +# test wheel, move out of here otherwise pip thinks +# cbitstruct is already installed +mkdir testdir +cd testdir +python -m pip install cbitstruct --no-index -f ../dist/ +python -m nose cbitstruct +cd .. diff --git a/.ci/build-manylinux-docker.sh b/.ci/build-manylinux-docker.sh new file mode 100755 index 0000000..0afcd86 --- /dev/null +++ b/.ci/build-manylinux-docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e -x + +# Show gcc version +gcc --version + +# Compile wheels +for PYVER in $PY_VERSIONS; do + PYNUM=$(echo $PYVER | sed 's/\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1\2/') + PYBIN=$(echo /opt/python/cp${PYNUM}*/bin) + "${PYBIN}/pip" install -r /io/dev-requirements.txt + "${PYBIN}/pip" wheel /io/ -w dist/ +done + +# Bundle external shared libraries into the wheels +for whl in dist/*.whl; do + auditwheel repair "$whl" --plat $PLAT -w /io/dist/ +done + +# Install packages and test +for PYVER in $PY_VERSIONS; do + PYNUM=$(echo $PYVER | sed 's/\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1\2/') + PYBIN=$(echo /opt/python/cp${PYNUM}*/bin) + "${PYBIN}/pip" install cbitstruct --no-index -f /io/dist + (cd "$HOME"; "${PYBIN}/nosetests" cbitstruct) +done diff --git a/.ci/build-manylinux.sh b/.ci/build-manylinux.sh new file mode 100644 index 0000000..338414e --- /dev/null +++ b/.ci/build-manylinux.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e -x + +docker pull $DOCKER_IMAGE +docker run --rm -e PLAT=$PLAT -e PY_VERSIONS="$PY_VERSIONS" -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/.ci/build-manylinux-docker.sh +ls dist/ diff --git a/.ci/build-osx.sh b/.ci/build-osx.sh new file mode 100644 index 0000000..d9b8efa --- /dev/null +++ b/.ci/build-osx.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e -x + +export PATH="/Users/travis/.pyenv/shims:${PATH}" + +brew update > /dev/null +brew install openssl readline +brew outdated pyenv || brew upgrade pyenv +brew install pyenv-virtualenv + +for PY_VERSION in $PY_VERSIONS; do + pyenv install --skip-existing $PY_VERSION + pyenv global $PY_VERSION + + python -m pip install -U pip + python -m pip install -U -r dev-requirements.txt + python --version + python -m pip --version + python setup.py bdist_wheel + + # test wheel, move out of here otherwise pip thinks + # cbitstruct is already installed + mkdir -p testdir + cd testdir + python -m pip install cbitstruct --no-index -f ../dist/ + python -m nose cbitstruct + cd .. +done + +ls dist/ \ No newline at end of file diff --git a/.ci/build-windows.sh b/.ci/build-windows.sh new file mode 100644 index 0000000..bb4d8fb --- /dev/null +++ b/.ci/build-windows.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e -x + +BASE_PATH=$PATH + +for PY_VERSION in $PY_VERSIONS; do + PYNUM=$(echo $PY_VERSION | sed 's/\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1\2/') + + export PATH=/c/Python$PYNUM:/c/Python$PYNUM/Scripts:$BASE_PATH + + choco install python --version $PY_VERSION --allow-downgrade + python -m pip install -U pip + python -m pip install -U -r dev-requirements.txt + + python --version + python -m pip --version + + python setup.py bdist_wheel + + # test the wheel, need to move our of the current dir + # otherwise pip thinks cbitstruct is already installed + mkdir -p testdir + cd testdir + python -m pip install cbitstruct --no-index -f ../dist/ + python -m nose cbitstruct + python -m pip uninstall -y cbitstruct + cd .. +done + +ls dist/ diff --git a/.ci/coverage.sh b/.ci/coverage.sh new file mode 100644 index 0000000..0091bfb --- /dev/null +++ b/.ci/coverage.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -x + +export COVERAGE=1 + +python -m pip install -U -r dev-requirements.txt +python -m pip install -U cpp-coveralls +python setup.py install +python -m nose cbitstruct +coveralls --exclude clinic --gcov-options '\-lp' diff --git a/.ci/deploy-osx.sh b/.ci/deploy-osx.sh new file mode 100644 index 0000000..e4fc9ea --- /dev/null +++ b/.ci/deploy-osx.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e -x + +python -m pip install -U pip twine wheel setuptools +python -m twine upload --repository-url "$PYPI_REPOSITORY_URL" -u "$PYPI_USERNAME" -p "$PYPI_PASSWORD" dist/* diff --git a/.ci/deploy.sh b/.ci/deploy.sh new file mode 100644 index 0000000..e4fc9ea --- /dev/null +++ b/.ci/deploy.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e -x + +python -m pip install -U pip twine wheel setuptools +python -m twine upload --repository-url "$PYPI_REPOSITORY_URL" -u "$PYPI_USERNAME" -p "$PYPI_PASSWORD" dist/* diff --git a/.ci/performance.sh b/.ci/performance.sh new file mode 100644 index 0000000..5916611 --- /dev/null +++ b/.ci/performance.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e -x + +python -m pip install -U -r dev-requirements.txt +python setup.py install +python cbitstruct/tests/test_perf.py diff --git a/.ci/test-linux.sh b/.ci/test-linux.sh new file mode 100644 index 0000000..601c1c5 --- /dev/null +++ b/.ci/test-linux.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e -x + +python -m pip install -U -r dev-requirements.txt +python setup.py install +python -m nose cbitstruct diff --git a/.travis.yml b/.travis.yml index f7caa86..4f8a172 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,75 +1,93 @@ language: python -python: "3.7" - +python: 3.7 env: global: + - PY_VERSIONS="3.7.4 3.6.8 3.5.4" - PYPI_REPOSITORY_URL=https://upload.pypi.org/legacy/ - PYPI_USERNAME=__token__ - secure: "jfAnPdxjTDw/mFbM2rmqjHaBtGpA+Rr+goCB5/Jp6RZ5N2M60nJliw8B5v7RwZDy//nSIdbTBU7cF/7WjEPwzrHUn5LQsS13ThzoS6KE7NUPa77TOxEpHXr4MJsyeSQMindEbAMeYP+lffI8/ISQ85evIbbSn6XzZSN/YBpQCTUGHWViqKIlpT70GYfRTeEJ0YvXxcJHKXoMuG7N80n8DW88FlfuI2age29cAe80pNn2yCZww2ZH0kG62HGQhzVv4v/uvxLLTaDP/JndYQyBnw6y5gesT2XCrgmCPgCTyVrtJsR96LjOukG2/T5etgpK0sgRwfgRk2DjKmlUqQOUm57lO+52ECaqqbaERFYe/Ub4JZ4DD05lKqm6wtQMD+fYl+TLKzGvCWayuLvxs7HqI+cz0BQQ7iynXuPUqX3xpkUDdAUzg9remmEJ1n8WG/NMHsv3Embh6zHS9RW7OUBXTELjrgJHVwjm8N0KtY063kMDArlSLNcf1gUPOx8MTOXBQWcauxmLfXMyaWrjHQ++1dWRFWjNVkUWfv+l17/ZHJHHnaJ6VpzhrZ5uku4LfEH8hgOCL6EYbO1PVXuEvFXDQX6wIFdbXaXSkx78ZgfiJf3xDqxcU1jl7+ipoCe2xkXaRkka0wg1gwCsfF53suXGIEIMRBWqrGJI9vETu5mVQDY=" - deploy: provider: script skip_cleanup: true - script: bash travis/deploy.sh + script: bash .ci/deploy.sh on: tags: true condition: $DEPLOY = 1 - jobs: include: - stage: build - name: "Manylinux x86_64" - env: DOCKER_IMAGE=quay.io/pypa/manylinux2010_x86_64 - PLAT=manylinux2010_x86_64 - DEPLOY=1 - services: - - docker - install: - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -e PLAT=$PLAT -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/travis/build-wheels.sh - - ls wheelhouse/ + name: OSX + os: osx + osx_image: xcode8.3 # macOS 10.12, oldest supported version + language: generic + env: + - DEPLOY=1 + script: bash .ci/build-osx.sh + before_deploy: + - export PATH="/Users/travis/.pyenv/shims:${PATH}" + - pyenv global 3.7.4 + before_cache: + - brew cleanup + - find /usr/local/Homebrew \! -regex ".+\.git.+" -delete + - find $HOME/.pyenv/ -name '*.pyc' -delete + cache: + directories: + - $HOME/.pyenv/ + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew + + - stage: build + name: Windows + os: windows + language: c + env: + - DEPLOY=1 + script: bash .ci/build-windows.sh + before_deploy: + - export PATH=/c/Python37:/c/Python37/Scripts:$PATH + - bash .ci/deploy.sh - stage: build - name: "Manylinux i686" - services: - - docker - env: DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 - PRE_CMD=linux32 - PLAT=manylinux1_i686 - DEPLOY=1 - install: - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -e PLAT=$PLAT -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/travis/build-wheels.sh - - ls wheelhouse/ + name: Manylinux x86_64 + services: docker + env: + - DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 + - PLAT=manylinux1_x86_64 + - DEPLOY=1 + script: bash .ci/build-manylinux.sh + + - stage: build + name: Manylinux i686 + services: docker + env: + - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 + - PRE_CMD=linux32 + - PLAT=manylinux1_i686 + - DEPLOY=1 + script: bash .ci/build-manylinux.sh - stage: build name: Coverage - env: COVERAGE=1 - install: - - pip install cpp-coveralls bitstruct - script: - - python setup.py install - - nosetests cbitstruct - - coveralls --exclude clinic --gcov-options '\-lp' + os: linux + script: bash .ci/coverage.sh - stage: build name: Performance - install: - - pip install bitstruct - script: - - python setup.py install - - python cbitstruct/tests/test_perf.py + os: linux + script: bash .ci/performance.sh + + - stage: build + name: Linux 3.8-dev + os: linux + python: 3.8-dev + script: bash .ci/test-linux.sh - stage: build - name: "Forward support" - python: "3.8-dev" - install: - - pip install bitstruct - script: - - python setup.py install - - nosetests cbitstruct + name: Source distribution + os: linux + env: + - DEPLOY=1 + script: bash .ci/build-linux-sdist.sh diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6c8dd08 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include cbitstruct/clinic/*.h diff --git a/README.md b/README.md index 36d6b02..c465887 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ Obvious increased performance comes with limitations described below. # Installation -Coming soon +```bash +pip3 install cbitstruct +``` # Documentation @@ -22,19 +24,15 @@ If you are not used to `bitstruct`, you should seriously consider using it first | Limitation | Will it be lifted ? | |------------|---------------------| -| Only tested on linux | Probably | | All types except padding are limited to 64 bits | Maybe for 'raw' and 'text' types | +| May not work on big-endian architectures | Maybe | | Exceptions differ from `bitstruct` | Probably not | -| Python >= 3.4 | Probably not | | CPython only | Probably not | -| May not work on big-endian architectures | Probably not | -| Out-of-range numbers in packing operations are not detected | Probably not | | Error messages are unclear | Will never be as clear as `bitstruct` | +| Python >= 3.5 | No | Some limitations are there because I did not get the time or motivation to lift them up. Some other are deeply rooted into this library and may never be lifted. -Note that since this library is performance oriented, I will refuse changes that degrade performance (except bugfixes). - # Performance ## Comparing to `bitstruct` @@ -43,29 +41,29 @@ The script available in `tests/test_perf.py` measures performance comparing to t Here are the result "on my machine" (Ubuntu in Virtualbox on a laptop): ``` -byteswap list of int | x 7.774 ( 8.634us -> 1.111us) -byteswap str | x 8.828 ( 12.992us -> 1.472us) -calcsize | x150.863 ( 60.566us -> 0.401us) -compiled pack | x 46.257 ( 35.994us -> 0.778us) -compiled pack_dict | x 26.440 ( 34.221us -> 1.294us) -compiled pack_into | x 35.690 ( 39.932us -> 1.119us) -compiled pack_into_dict | x 25.579 ( 38.404us -> 1.501us) -compiled unpack | x 35.285 ( 32.051us -> 0.908us) -compiled unpack_dict | x 21.984 ( 32.342us -> 1.471us) -compiled unpack_from | x 32.752 ( 31.353us -> 0.957us) -compiled unpack_from_dict | x 21.354 ( 32.016us -> 1.499us) -pack | x 84.085 ( 103.511us -> 1.231us) -pack_dict | x 47.052 ( 92.523us -> 1.966us) -pack_into | x 76.317 ( 105.994us -> 1.389us) -pack_into_dict | x 41.617 ( 96.210us -> 2.312us) -unpack | x 85.326 ( 94.035us -> 1.102us) -unpack_dict | x 41.138 ( 90.074us -> 2.190us) -unpack_from | x 82.550 ( 94.934us -> 1.150us) -unpack_from_dict | x 40.659 ( 89.523us -> 2.202us) +byteswap list of int | x 8.204 ( 9.208us -> 1.122us) +byteswap str | x 6.433 ( 9.689us -> 1.506us) +calcsize | x149.423 ( 61.967us -> 0.415us) +compiled pack | x 43.227 ( 34.758us -> 0.804us) +compiled pack_dict | x 26.490 ( 34.951us -> 1.319us) +compiled pack_into | x 32.017 ( 39.522us -> 1.234us) +compiled pack_into_dict | x 26.817 ( 38.984us -> 1.454us) +compiled unpack | x 34.454 ( 31.814us -> 0.923us) +compiled unpack_dict | x 23.534 ( 34.071us -> 1.448us) +compiled unpack_from | x 27.170 ( 31.884us -> 1.174us) +compiled unpack_from_dict | x 22.600 ( 33.927us -> 1.501us) +pack | x 78.314 ( 105.593us -> 1.348us) +pack_dict | x 52.916 ( 106.748us -> 2.017us) +pack_into | x 82.233 ( 119.950us -> 1.459us) +pack_into_dict | x 45.214 ( 111.338us -> 2.462us) +unpack | x 82.712 ( 93.686us -> 1.133us) +unpack_dict | x 41.064 ( 91.473us -> 2.228us) +unpack_from | x 81.678 ( 95.729us -> 1.172us) +unpack_from_dict | x 40.379 ( 90.430us -> 2.240us) ``` *Disclaimer:* these results may and will vary largely depending on the number of elements and types you pack/unpack. This script is provided as-is, and I will gladly accept an improved script providing more reliable results. ## The dict API -The `dict` API is marginally slower than the traditional one. As the packing/unpacking performance is quite high, the overhead of performing dictionary lookups and hashing significantly increas pack and unpacking duration. \ No newline at end of file +The `dict` API is marginally slower than the traditional one. As the packing/unpacking performance is quite high, the overhead of performing dictionary lookups and hashing significantly increas pack and unpacking duration. diff --git a/cbitstruct/_cbitstruct.c b/cbitstruct/_cbitstruct.c index 9f75556..7428556 100644 --- a/cbitstruct/_cbitstruct.c +++ b/cbitstruct/_cbitstruct.c @@ -1,5 +1,6 @@ #include #include +#include #include #define PY_SSIZE_T_CLEAN @@ -428,13 +429,34 @@ static void c_unpack( /* Python conversions */ +static bool unsigned_in_range(uint64_t n, int bits) +{ + if (bits == 64) { + return true; + } + return n < (1ull << bits); +} + +static bool signed_in_range(int64_t n, int bits) +{ + if (bits == 64) { + return true; + } + if (n > 0) { + return n < (1ll << (bits - 1)); + } + else { + return -n <= (1ll << (bits - 1)); + } +} + static bool python_to_parsed_elements( ParsedElement* elements, - PyObject* tuple, + PyObject** data, + Py_ssize_t data_size, CompiledFormat fmt) { - assert(PyTuple_Check(tuple)); - assert(PyTuple_GET_SIZE(tuple) >= fmt.ndescs); + assert(data_size >= fmt.ndescs); int n = 0; for (int i = 0; i < fmt.ndescs; ++i) { @@ -447,7 +469,7 @@ static bool python_to_parsed_elements( continue; } - PyObject* v = PyTuple_GET_ITEM(tuple, n++); + PyObject* v = data[n++]; switch (desc->type) { case 'u': @@ -456,6 +478,9 @@ static bool python_to_parsed_elements( #else el->uint64 = PyLong_AsUnsignedLongLong(v); #endif // SIZEOF_LONG >= 8 + if (!unsigned_in_range(el->uint64, desc->bits)) { + PyErr_SetString(PyExc_TypeError, "integer is out of range"); + } break; case 's': #if SIZEOF_LONG >= 8 @@ -463,6 +488,9 @@ static bool python_to_parsed_elements( #else el->int64 = PyLong_AsLongLong(v); #endif // SIZEOF_LONG >= 8 + if (!signed_in_range(el->int64, desc->bits)) { + PyErr_SetString(PyExc_TypeError, "integer is out of range"); + } break; case 'f': #if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 6 @@ -591,6 +619,168 @@ static PyObject* parsed_elements_to_python(ParsedElement* elements, CompiledForm return result; } +// Modified version of PyArg_ParseTupleAndKeywords +// to parse 'n' arguments from args or kwargs and return +// the number of arguments parsed from 'args' +static Py_ssize_t PyArg_ParseTupleAndKeywordsFirstN( + PyObject* args, + PyObject* kwargs, + const char* format, + char* keywords[], + int n, + ...) +{ + va_list varargs; + va_start(varargs, n); + + Py_ssize_t nkwargs = kwargs ? PyObject_Length(kwargs) : 0; + Py_ssize_t n_actual_args = n - nkwargs; + + PyObject* actual_args = PyTuple_GetSlice(args, 0, n_actual_args); + if (!actual_args) { + PyErr_NoMemory(); + goto exit; + } + + if (PyArg_VaParseTupleAndKeywords( + actual_args, kwargs, format, keywords, varargs)) { + } + Py_DECREF(actual_args); + +exit: + va_end(varargs); + return n_actual_args; +} + +static bool PopFillPadding(PyObject* kwargs) +{ + // get the fill_padding value and remove + // it from kwargs as it makes parsing painful + bool fill_padding = true; + if (kwargs) { + PyObject* py_fill_padding = PyDict_GetItemString(kwargs, FILL_PADDING); + if (py_fill_padding) { + fill_padding = PyObject_IsTrue(py_fill_padding); + PyDict_DelItemString(kwargs, FILL_PADDING); + } + } + return fill_padding; +} + +static PyObject* CompiledFormat_pack_raw( + CompiledFormat compiled_fmt, + PyObject** data, + Py_ssize_t n_data) +{ + assert(PyTuple_Check(args)); + + ParsedElement elements_stack[SMALL_FORMAT_OPTIMIZATION]; + ParsedElement* elements = elements_stack; + bool use_stack = compiled_fmt.ndescs <= SMALL_FORMAT_OPTIMIZATION; + PyObject* bytes = NULL; + + int expected_size = compiled_fmt.ndescs - compiled_fmt.npadding; + if (n_data < expected_size) { + PyErr_Format( + PyExc_TypeError, + "pack() expected %d arguments (got %ld)", + expected_size, + n_data); + return NULL; + } + + if (!use_stack) { + elements = PyMem_RawMalloc(compiled_fmt.ndescs * sizeof(ParsedElement)); + if (!elements) { + PyErr_NoMemory(); + return NULL; + } + } + + if (!python_to_parsed_elements(elements, data, n_data, compiled_fmt)) { + PyErr_SetString(PyExc_TypeError, "failed to parse arguments"); + goto exit; + } + + int nbytes = (compiled_fmt.nbits + 7) / 8; + bytes = PyBytes_FromStringAndSize(NULL, nbytes); + if (!bytes) { + PyErr_NoMemory(); + goto exit; + } + + PyBytes_AS_STRING(bytes)[nbytes - 1] = 0; + c_pack((uint8_t*)PyBytes_AS_STRING(bytes), elements, compiled_fmt, 0, true); + +exit: + if (!use_stack) { + PyMem_RawFree(elements); + } + + return bytes; +} + +static PyObject* CompiledFormat_pack_into_raw( + CompiledFormat compiled_fmt, + Py_buffer* buffer, + Py_ssize_t offset, + PyObject** data_args, + Py_ssize_t n_data_args, + bool fill_padding) +{ + ParsedElement elements_stack[SMALL_FORMAT_OPTIMIZATION]; + ParsedElement* elements = elements_stack; + bool use_stack = compiled_fmt.ndescs <= SMALL_FORMAT_OPTIMIZATION; + PyObject* return_value = NULL; + + int expected_size = compiled_fmt.ndescs - compiled_fmt.npadding; + if (n_data_args < expected_size) { + PyErr_Format( + PyExc_TypeError, + "expected %d data arguments (got %ld)", + expected_size, + n_data_args); + goto exit; + } + + if (!PyBuffer_IsContiguous(buffer, 'C')) { + PyErr_Format(PyExc_TypeError, "required a contiguous buffer"); + goto exit; + } + + int nbytes = (compiled_fmt.nbits + 7) / 8; + if (buffer->len < nbytes) { + PyErr_Format( + PyExc_TypeError, "required a buffer of at least %d bytes", nbytes); + goto exit; + } + + if (!use_stack) { + elements = PyMem_RawMalloc(compiled_fmt.ndescs * sizeof(ParsedElement)); + if (!elements) { + PyErr_NoMemory(); + goto exit; + } + } + + if (!python_to_parsed_elements(elements, data_args, n_data_args, compiled_fmt)) { + // python_to_parsed_elements should set the exception + goto exit; + } + + c_pack((uint8_t*)buffer->buf, elements, compiled_fmt, offset, fill_padding); + + return_value = Py_None; + Py_INCREF(Py_None); + +exit: + if (!use_stack && elements) { + PyMem_RawFree(elements); + } + + return return_value; +} + /* Python methods */ // clang-format off @@ -600,10 +790,12 @@ class CompiledFormat "PyCompiledFormatObject *" "&PyStructType" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=470ab77e2b50e7be]*/ // clang-format on +// clang-format off typedef struct { - PyObject_HEAD; + PyObject_HEAD CompiledFormat compiled_fmt; } PyCompiledFormatObject; +// clang-format on // clang-format off /*[clinic input] @@ -695,54 +887,9 @@ PyDoc_STRVAR(CompiledFormat_pack__doc__, // clang-format on static PyObject* CompiledFormat_pack(PyCompiledFormatObject* self, PyObject* args) { - assert(PyTuple_Check(args)); - - ParsedElement elements_stack[SMALL_FORMAT_OPTIMIZATION]; - ParsedElement* elements = elements_stack; - bool use_stack = self->compiled_fmt.ndescs <= SMALL_FORMAT_OPTIMIZATION; - PyObject* bytes = NULL; - - int expected_size = self->compiled_fmt.ndescs - self->compiled_fmt.npadding; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (nargs < expected_size) { - PyErr_Format( - PyExc_TypeError, - "pack expected %d arguments (got %ld)", - expected_size, - nargs); - return NULL; - } - - if (!use_stack) { - elements = PyMem_RawMalloc( - self->compiled_fmt.ndescs * sizeof(ParsedElement)); - if (!elements) { - PyErr_NoMemory(); - return NULL; - } - } - - if (!python_to_parsed_elements(elements, args, self->compiled_fmt)) { - PyErr_SetString(PyExc_TypeError, "failed to parse arguments"); - goto exit; - } - - int nbytes = (self->compiled_fmt.nbits + 7) / 8; - bytes = PyBytes_FromStringAndSize(NULL, nbytes); - if (!bytes) { - PyErr_NoMemory(); - goto exit; - } - - PyBytes_AS_STRING(bytes)[nbytes - 1] = 0; - c_pack((uint8_t*)PyBytes_AS_STRING(bytes), elements, self->compiled_fmt, 0, true); - -exit: - if (!use_stack) { - PyMem_RawFree(elements); - } - - return bytes; + PyObject** data = PySequence_Fast_ITEMS(args); + Py_ssize_t n_data = PyTuple_GET_SIZE(args); + return CompiledFormat_pack_raw(self->compiled_fmt, data, n_data); } // clang-format off @@ -759,93 +906,34 @@ static PyObject* CompiledFormat_pack_into( PyObject* args, PyObject* kwargs) { - assert(PyTuple_Check(args)); - assert(PyDict_Check(kwargs)); - - ParsedElement elements_stack[SMALL_FORMAT_OPTIMIZATION]; - ParsedElement* elements = elements_stack; - bool use_stack = self->compiled_fmt.ndescs <= SMALL_FORMAT_OPTIMIZATION; PyObject* return_value = NULL; - // +2 are target buffer, offset - int expected_size = 2 + self->compiled_fmt.ndescs - - self->compiled_fmt.npadding; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (nargs < expected_size) { - PyErr_Format( - PyExc_TypeError, - "pack_into expected %d arguments (got %ld)", - expected_size, - nargs); - return NULL; - } + bool fill_padding = PopFillPadding(kwargs); - PyObject* data_args = NULL; Py_buffer buffer = {NULL, NULL}; - int nbytes = (self->compiled_fmt.nbits + 7) / 8; - - if (PyObject_GetBuffer(PyTuple_GET_ITEM(args, 0), &buffer, PyBUF_SIMPLE)) { - // PyObject_GetBuffer must set the exception - goto exit; - } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - PyErr_Format(PyExc_TypeError, "pack_into expects a contiguous buffer"); - goto exit; - } - if (buffer.len < nbytes) { - PyErr_Format( - PyExc_TypeError, - "pack_into requires a buffer of at least %d bytes", - nbytes); - goto exit; - } + Py_ssize_t offset = 0; + + static char* _keywords[] = {"buf", "offset", NULL}; + // custom (and vague) error message as all other 'pack_into' + // versions are processed by this function. Using the default + // error message would give bad information to the user + Py_ssize_t n_args_parsed = PyArg_ParseTupleAndKeywordsFirstN( + args, kwargs, "y*n:pack_into", _keywords, 2, &buffer, &offset); + + Py_ssize_t n_args = PyTuple_GET_SIZE(args); + PyObject** data = PySequence_Fast_ITEMS(args); + + return_value = CompiledFormat_pack_into_raw( + self->compiled_fmt, + &buffer, + offset, + data + n_args_parsed, + n_args - n_args_parsed, + fill_padding); - long offset = PyLong_AsLong(PyTuple_GET_ITEM(args, 1)); - if (offset < 0 && PyErr_Occurred()) { - goto exit; - } - - bool fill_padding = true; - if (kwargs) { - PyObject* py_fill_padding = PyDict_GetItemString(kwargs, FILL_PADDING); - fill_padding = !py_fill_padding || PyObject_IsTrue(py_fill_padding); - } - - if (!use_stack) { - elements = PyMem_RawMalloc( - self->compiled_fmt.ndescs * sizeof(ParsedElement)); - if (!elements) { - PyErr_NoMemory(); - goto exit; - } - } - - data_args = PyTuple_GetSlice(args, 2, nargs); - if (!data_args) { - PyErr_NoMemory(); - goto exit; - } - if (!python_to_parsed_elements(elements, data_args, self->compiled_fmt)) { - // python_to_parsed_elements should set the exception - goto exit; - } - - c_pack((uint8_t*)buffer.buf, elements, self->compiled_fmt, offset, fill_padding); - - return_value = Py_None; - Py_INCREF(Py_None); - -exit: - if (data_args) { - Py_DECREF(data_args); - } - if (!use_stack && elements) { - PyMem_RawFree(elements); - } if (buffer.obj) { PyBuffer_Release(&buffer); } - return return_value; } @@ -887,7 +975,7 @@ CompiledFormat_unpack_from_impl(PyCompiledFormatObject *self, bool use_stack = self->compiled_fmt.ndescs <= SMALL_FORMAT_OPTIMIZATION; if (!PyBuffer_IsContiguous(data, 'C')) { - PyErr_Format(PyExc_TypeError, "unpack expects a contiguous buffer"); + PyErr_Format(PyExc_TypeError, "unpack() expects a contiguous buffer"); return NULL; } @@ -895,7 +983,7 @@ CompiledFormat_unpack_from_impl(PyCompiledFormatObject *self, if (data->len < nbytes) { PyErr_Format( PyExc_TypeError, - "unpack requires a buffer of at least %d bytes", + "unpack() requires a buffer of at least %d bytes", nbytes); return NULL; } @@ -1055,58 +1143,56 @@ CompiledFormatDict_pack_impl(PyCompiledFormatDictObject *self, } // clang-format off -PyDoc_STRVAR(CompiledFormatDict_pack_into__doc__, -"pack_into($self, buf, offset, data, **kwargs)\n" -"--\n" -"\n" -"Pack data into a bytes object, starting at bit offset given by the\n" -"offset argument. An optional 'fill_padding=False' argument can be given\n" -"to keep padding bits from 'buf' as-is."); +/*[clinic input] +CompiledFormatDict.pack_into + + buf: Py_buffer + offset: Py_ssize_t + data: object + * + fill_padding: bool = True + +Pack data into a bytes object, starting at bit offset given by the offset argument. + +With fill_padding=False, passing bits in 'buf' will not be modified. +[clinic start generated code]*/ + +static PyObject * +CompiledFormatDict_pack_into_impl(PyCompiledFormatDictObject *self, + Py_buffer *buf, Py_ssize_t offset, + PyObject *data, int fill_padding) +/*[clinic end generated code: output=ee246de261e9c699 input=290a9a4a3e3ed942]*/ // clang-format on -static PyObject* CompiledFormatDict_pack_into( - PyCompiledFormatDictObject* self, - PyObject* args, - PyObject* kwargs) { assert(PyTuple_Check(args)); PyObject* return_value = NULL; - PyObject* extended_args = NULL; - - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (nargs != 3) { - PyErr_SetString(PyExc_TypeError, "pack_into takes three arguments"); - goto exit; - } Py_ssize_t nnames = PySequence_Fast_GET_SIZE(self->names); PyObject** names = PySequence_Fast_ITEMS(self->names); - extended_args = PyTuple_GetSlice(args, 0, 2); - if (!extended_args) { + PyObject* data_tuple = PyTuple_New(nnames); + if (!data_tuple) { PyErr_NoMemory(); goto exit; } - if (_PyTuple_Resize(&extended_args, 2 + nnames)) { - // _PyTuple_Resize sets an exception - goto exit; - } for (int i = 0; i < nnames; ++i) { - PyObject* v = PyObject_GetItem(PyTuple_GET_ITEM(args, 2), names[i]); + PyObject* v = PyObject_GetItem(data, names[i]); if (!v) { // PyObject_GetItem sets KeyError goto exit; } - PyTuple_SET_ITEM(extended_args, 2 + i, v); + PyTuple_SET_ITEM(data_tuple, i, v); } - return_value = CompiledFormat_pack_into( - (PyCompiledFormatObject*)self, extended_args, kwargs); + PyObject** data_array = PySequence_Fast_ITEMS(data_tuple); + return_value = CompiledFormat_pack_into_raw( + self->super.compiled_fmt, buf, offset, data_array, nnames, fill_padding); exit: - if (extended_args) { - Py_DECREF(extended_args); + if (data_tuple) { + Py_DECREF(data_tuple); } return return_value; @@ -1208,12 +1294,7 @@ static struct PyMethodDef CompiledFormatDict_methods[] = { COMPILEDFORMATDICT_PACK_METHODDEF COMPILEDFORMATDICT_UNPACK_METHODDEF COMPILEDFORMATDICT_UNPACK_FROM_METHODDEF - { - "pack_into", - (PyCFunction)CompiledFormatDict_pack_into, - METH_VARARGS|METH_KEYWORDS, - CompiledFormatDict_pack_into__doc__ - }, + COMPILEDFORMATDICT_PACK_INTO_METHODDEF {NULL, NULL} }; @@ -1240,36 +1321,27 @@ PyDoc_STRVAR(pack__doc__, "\n" "Pack args into a bytes object according to fmt"); // clang-format on -static PyObject* pack(PyObject* module, PyObject* args) +static PyObject* pack(PyObject* module, PyObject* args, PyObject* kwargs) { - assert(PyTuple_Check(args)); + PyObject* return_value = NULL; + const char* fmt = NULL; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (nargs < 1) { - PyErr_SetString(PyExc_TypeError, "pack expects at least one argument"); - return NULL; - } + static char* _keywords[] = {"fmt", NULL}; + Py_ssize_t n_args_parsed = PyArg_ParseTupleAndKeywordsFirstN( + args, kwargs, "s:pack", _keywords, 1, &fmt); - PyObject* return_value = NULL; PyCompiledFormatObject self; + memset(&self, 0, sizeof(self)); - const char* fmt = PyUnicode_AsUTF8(PyTuple_GET_ITEM(args, 0)); - if (!fmt) { - // PyUnicode_AsUTF8 has set the exception - goto exit; - } if (CompiledFormat___init___impl(&self, fmt)) { // CompiledFormat___init___impl has set the exception goto exit; } - PyObject* obj_args = PyTuple_GetSlice(args, 1, nargs); - if (!obj_args) { - PyErr_NoMemory(); - goto exit; - } - return_value = CompiledFormat_pack(&self, obj_args); - Py_DECREF(obj_args); + Py_ssize_t n_args = PyTuple_GET_SIZE(args); + PyObject** data = PySequence_Fast_ITEMS(args); + return_value = CompiledFormat_pack_raw( + self.compiled_fmt, data + n_args_parsed, n_args - n_args_parsed); exit: CompiledFormat_deinit(&self); @@ -1287,34 +1359,34 @@ PyDoc_STRVAR(pack_into__doc__, // clang-format on static PyObject* pack_into(PyObject* module, PyObject* args, PyObject* kwargs) { - assert(PyTuple_Check(args)); + PyObject* return_value = NULL; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t offset = 0; + const char* fmt = NULL; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - if (nargs < 1) { - PyErr_SetString( - PyExc_TypeError, "pack_into expects at least one argument"); - return NULL; - } + bool fill_padding = PopFillPadding(kwargs); + + static char* _keywords[] = {"fmt", "buf", "offset", NULL}; + Py_ssize_t n_args_parsed = PyArg_ParseTupleAndKeywordsFirstN( + args, kwargs, "sy*n:pack_into", _keywords, 3, &fmt, &buffer, &offset); - PyObject* return_value = NULL; PyCompiledFormatObject self; - const char* fmt = PyUnicode_AsUTF8(PyTuple_GET_ITEM(args, 0)); - if (!fmt) { - // PyUnicode_AsUTF8 has set the exception - goto exit; - } + memset(&self, 0, sizeof(self)); + if (CompiledFormat___init___impl(&self, fmt)) { // CompiledFormat___init___impl has set the exception goto exit; } - PyObject* obj_args = PyTuple_GetSlice(args, 1, nargs); - if (!obj_args) { - PyErr_NoMemory(); - goto exit; - } - return_value = CompiledFormat_pack_into(&self, obj_args, kwargs); - Py_DECREF(obj_args); + Py_ssize_t n_args = PyTuple_GET_SIZE(args); + PyObject** data = PySequence_Fast_ITEMS(args); + return_value = CompiledFormat_pack_into_raw( + self.compiled_fmt, + &buffer, + offset, + data + n_args_parsed, + n_args - n_args_parsed, + fill_padding); exit: CompiledFormat_deinit(&self); @@ -1343,6 +1415,7 @@ pack_dict_impl(PyObject *module, const char *fmt, PyObject *names, { PyObject* return_value = NULL; PyCompiledFormatDictObject self; + memset(&self, 0, sizeof(self)); if (CompiledFormatDict___init___impl(&self, fmt, names)) { // CompiledFormatDict___init___impl has set the exception @@ -1356,52 +1429,45 @@ pack_dict_impl(PyObject *module, const char *fmt, PyObject *names, } // clang-format off -PyDoc_STRVAR(pack_into_dict__doc__, -"pack_into_dict(fmt, names, buf, offset, data, **kwargs)\n" -"--\n" -"\n" -"Pack data into a bytes object, starting at bit offset given by the\n" -"offset argument. Values are taken from the dict 'data' in the order\n" -"given by the list 'names'. An optional 'fill_padding=False' argument\n" -"can be given to keep padding bits from 'buf' as-is."); +/*[clinic input] +pack_into_dict + + fmt: str + names: object + buf: Py_buffer + offset: Py_ssize_t + data: object + * + fill_padding: bool = True + +Pack data into a bytes object, starting at bit offset given by the offset argument. + +With fill_padding=False, passing bits in 'buf' will not be modified. +[clinic start generated code]*/ + +static PyObject * +pack_into_dict_impl(PyObject *module, const char *fmt, PyObject *names, + Py_buffer *buf, Py_ssize_t offset, PyObject *data, + int fill_padding) +/*[clinic end generated code: output=619b415fc187011b input=e72dec46484ec66f]*/ // clang-format on -static PyObject* pack_into_dict(PyObject* module, PyObject* args, PyObject* kwargs) { assert(PyTuple_Check(args)); - PyCompiledFormatDictObject self; - PyObject* cstor_args = NULL; PyObject* return_value = NULL; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - - if (nargs != 5) { - PyErr_SetString(PyExc_TypeError, "pack_into_dict takes 5 arguments"); - goto exit; - } + PyCompiledFormatDictObject self; + memset(&self, 0, sizeof(self)); - cstor_args = PyTuple_GetSlice(args, 0, 2); - if (!cstor_args) { - PyErr_NoMemory(); - goto exit; - } - if (CompiledFormatDict___init__((PyObject*)&self, cstor_args, NULL)) { - // CompiledFormatDict___init__ has set the exception + if (CompiledFormatDict___init___impl(&self, fmt, names)) { + // CompiledFormatDict___init___impl has set the exception goto exit; } - PyObject* obj_args = PyTuple_GetSlice(args, 2, nargs); - if (!obj_args) { - PyErr_NoMemory(); - goto exit; - } - return_value = CompiledFormatDict_pack_into(&self, obj_args, kwargs); - Py_DECREF(obj_args); + return_value = + CompiledFormatDict_pack_into_impl(&self, buf, offset, data, fill_padding); exit: CompiledFormatDict_deinit(&self); - if (cstor_args) { - Py_DECREF(cstor_args); - } return return_value; } @@ -1446,6 +1512,7 @@ unpack_from_impl(PyObject *module, const char *fmt, Py_buffer *data, { PyObject* return_value = NULL; PyCompiledFormatObject self; + memset(&self, 0, sizeof(self)); if (CompiledFormat___init___impl(&self, fmt)) { // CompiledFormat___init___impl has set the exception @@ -1502,6 +1569,7 @@ unpack_from_dict_impl(PyObject *module, const char *fmt, PyObject *names, { PyObject* return_value = NULL; PyCompiledFormatDictObject self; + memset(&self, 0, sizeof(self)); if (CompiledFormatDict___init___impl(&self, fmt, names)) { // CompiledFormatDict___init___impl has set the exception @@ -1559,6 +1627,7 @@ calcsize_impl(PyObject *module, const char *fmt) { Py_ssize_t return_value = -1; PyCompiledFormatObject self; + memset(&self, 0, sizeof(self)); if (CompiledFormat___init___impl(&self, fmt)) { // CompiledFormat___init___impl has set the exception @@ -1599,7 +1668,7 @@ byteswap_impl(PyObject *module, PyObject *fmt, Py_buffer *data, Py_ssize_t length = -1; if (!PyBuffer_IsContiguous(data, 'C')) { - PyErr_Format(PyExc_TypeError, "byteswap expects a contiguous buffer"); + PyErr_Format(PyExc_TypeError, "byteswap() expects a contiguous buffer"); goto exit; } @@ -1608,8 +1677,8 @@ byteswap_impl(PyObject *module, PyObject *fmt, Py_buffer *data, goto exit; } - return_value = - PyBytes_FromStringAndSize(data->buf + offset, data->len - offset); + return_value = PyBytes_FromStringAndSize( + ((const char*)data->buf) + offset, data->len - offset); if (!return_value) { PyErr_NoMemory(); goto exit; @@ -1649,7 +1718,7 @@ byteswap_impl(PyObject *module, PyObject *fmt, Py_buffer *data, if (sum > PyBytes_Size(return_value)) { PyErr_Format( PyExc_TypeError, - "byteswap requires a buffer of at least %d bytes", + "byteswap() requires a buffer of at least %d bytes", sum); goto exit; } @@ -1674,7 +1743,7 @@ static struct PyMethodDef py_module_functions[] = { { "pack", (PyCFunction)pack, - METH_VARARGS, + METH_VARARGS|METH_KEYWORDS, pack__doc__ }, { @@ -1684,12 +1753,7 @@ static struct PyMethodDef py_module_functions[] = { pack_into__doc__ }, PACK_DICT_METHODDEF - { - "pack_into_dict", - (PyCFunction)pack_into_dict, - METH_VARARGS|METH_KEYWORDS, - pack_into_dict__doc__ - }, + PACK_INTO_DICT_METHODDEF UNPACK_METHODDEF UNPACK_FROM_METHODDEF UNPACK_DICT_METHODDEF diff --git a/cbitstruct/clinic/_cbitstruct.c.35.h b/cbitstruct/clinic/_cbitstruct.c.35.h index 35aed7c..0a42ef0 100644 --- a/cbitstruct/clinic/_cbitstruct.c.35.h +++ b/cbitstruct/clinic/_cbitstruct.c.35.h @@ -187,6 +187,45 @@ CompiledFormatDict_pack(PyCompiledFormatDictObject *self, PyObject *args, PyObje return return_value; } +PyDoc_STRVAR(CompiledFormatDict_pack_into__doc__, +"pack_into($self, /, buf, offset, data, *, fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define COMPILEDFORMATDICT_PACK_INTO_METHODDEF \ + {"pack_into", (PyCFunction)CompiledFormatDict_pack_into, METH_VARARGS|METH_KEYWORDS, CompiledFormatDict_pack_into__doc__}, + +static PyObject * +CompiledFormatDict_pack_into_impl(PyCompiledFormatDictObject *self, + Py_buffer *buf, Py_ssize_t offset, + PyObject *data, int fill_padding); + +static PyObject * +CompiledFormatDict_pack_into(PyCompiledFormatDictObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"buf", "offset", "data", "fill_padding", NULL}; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*nO|$p:pack_into", _keywords, + &buf, &offset, &data, &fill_padding)) + goto exit; + return_value = CompiledFormatDict_pack_into_impl(self, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) + PyBuffer_Release(&buf); + + return return_value; +} + PyDoc_STRVAR(CompiledFormatDict_unpack__doc__, "unpack($self, /, data)\n" "--\n" @@ -291,6 +330,48 @@ pack_dict(PyObject *module, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(pack_into_dict__doc__, +"pack_into_dict($module, /, fmt, names, buf, offset, data, *,\n" +" fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define PACK_INTO_DICT_METHODDEF \ + {"pack_into_dict", (PyCFunction)pack_into_dict, METH_VARARGS|METH_KEYWORDS, pack_into_dict__doc__}, + +static PyObject * +pack_into_dict_impl(PyObject *module, const char *fmt, PyObject *names, + Py_buffer *buf, Py_ssize_t offset, PyObject *data, + int fill_padding); + +static PyObject * +pack_into_dict(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"fmt", "names", "buf", "offset", "data", "fill_padding", NULL}; + const char *fmt; + PyObject *names; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "sOy*nO|$p:pack_into_dict", _keywords, + &fmt, &names, &buf, &offset, &data, &fill_padding)) + goto exit; + return_value = pack_into_dict_impl(module, fmt, names, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) + PyBuffer_Release(&buf); + + return return_value; +} + PyDoc_STRVAR(unpack__doc__, "unpack($module, /, fmt, data)\n" "--\n" @@ -536,4 +617,4 @@ byteswap(PyObject *module, PyObject *args, PyObject *kwargs) return return_value; } -/*[clinic end generated code: output=6fdde2d37d3c1023 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c102aaba2bc6b295 input=a9049054013a1b77]*/ diff --git a/cbitstruct/clinic/_cbitstruct.c.36.h b/cbitstruct/clinic/_cbitstruct.c.36.h index 31ec5a5..62fd0e7 100644 --- a/cbitstruct/clinic/_cbitstruct.c.36.h +++ b/cbitstruct/clinic/_cbitstruct.c.36.h @@ -200,6 +200,48 @@ CompiledFormatDict_pack(PyCompiledFormatDictObject *self, PyObject **args, Py_ss return return_value; } +PyDoc_STRVAR(CompiledFormatDict_pack_into__doc__, +"pack_into($self, /, buf, offset, data, *, fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define COMPILEDFORMATDICT_PACK_INTO_METHODDEF \ + {"pack_into", (PyCFunction)CompiledFormatDict_pack_into, METH_FASTCALL, CompiledFormatDict_pack_into__doc__}, + +static PyObject * +CompiledFormatDict_pack_into_impl(PyCompiledFormatDictObject *self, + Py_buffer *buf, Py_ssize_t offset, + PyObject *data, int fill_padding); + +static PyObject * +CompiledFormatDict_pack_into(PyCompiledFormatDictObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"buf", "offset", "data", "fill_padding", NULL}; + static _PyArg_Parser _parser = {"y*nO|$p:pack_into", _keywords, 0}; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &buf, &offset, &data, &fill_padding)) { + goto exit; + } + return_value = CompiledFormatDict_pack_into_impl(self, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) { + PyBuffer_Release(&buf); + } + + return return_value; +} + PyDoc_STRVAR(CompiledFormatDict_unpack__doc__, "unpack($self, /, data)\n" "--\n" @@ -312,6 +354,51 @@ pack_dict(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames return return_value; } +PyDoc_STRVAR(pack_into_dict__doc__, +"pack_into_dict($module, /, fmt, names, buf, offset, data, *,\n" +" fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define PACK_INTO_DICT_METHODDEF \ + {"pack_into_dict", (PyCFunction)pack_into_dict, METH_FASTCALL, pack_into_dict__doc__}, + +static PyObject * +pack_into_dict_impl(PyObject *module, const char *fmt, PyObject *names, + Py_buffer *buf, Py_ssize_t offset, PyObject *data, + int fill_padding); + +static PyObject * +pack_into_dict(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"fmt", "names", "buf", "offset", "data", "fill_padding", NULL}; + static _PyArg_Parser _parser = {"sOy*nO|$p:pack_into_dict", _keywords, 0}; + const char *fmt; + PyObject *names; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &fmt, &names, &buf, &offset, &data, &fill_padding)) { + goto exit; + } + return_value = pack_into_dict_impl(module, fmt, names, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) { + PyBuffer_Release(&buf); + } + + return return_value; +} + PyDoc_STRVAR(unpack__doc__, "unpack($module, /, fmt, data)\n" "--\n" @@ -577,4 +664,4 @@ byteswap(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) return return_value; } -/*[clinic end generated code: output=dcfb6ee6b91c44c1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=93b815cb6fb17f58 input=a9049054013a1b77]*/ diff --git a/cbitstruct/clinic/_cbitstruct.c.37.h b/cbitstruct/clinic/_cbitstruct.c.37.h index 9cc5cbf..d52f94d 100644 --- a/cbitstruct/clinic/_cbitstruct.c.37.h +++ b/cbitstruct/clinic/_cbitstruct.c.37.h @@ -200,6 +200,48 @@ CompiledFormatDict_pack(PyCompiledFormatDictObject *self, PyObject *const *args, return return_value; } +PyDoc_STRVAR(CompiledFormatDict_pack_into__doc__, +"pack_into($self, /, buf, offset, data, *, fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define COMPILEDFORMATDICT_PACK_INTO_METHODDEF \ + {"pack_into", (PyCFunction)CompiledFormatDict_pack_into, METH_FASTCALL|METH_KEYWORDS, CompiledFormatDict_pack_into__doc__}, + +static PyObject * +CompiledFormatDict_pack_into_impl(PyCompiledFormatDictObject *self, + Py_buffer *buf, Py_ssize_t offset, + PyObject *data, int fill_padding); + +static PyObject * +CompiledFormatDict_pack_into(PyCompiledFormatDictObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"buf", "offset", "data", "fill_padding", NULL}; + static _PyArg_Parser _parser = {"y*nO|$p:pack_into", _keywords, 0}; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &buf, &offset, &data, &fill_padding)) { + goto exit; + } + return_value = CompiledFormatDict_pack_into_impl(self, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) { + PyBuffer_Release(&buf); + } + + return return_value; +} + PyDoc_STRVAR(CompiledFormatDict_unpack__doc__, "unpack($self, /, data)\n" "--\n" @@ -312,6 +354,51 @@ pack_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } +PyDoc_STRVAR(pack_into_dict__doc__, +"pack_into_dict($module, /, fmt, names, buf, offset, data, *,\n" +" fill_padding=True)\n" +"--\n" +"\n" +"Pack data into a bytes object, starting at bit offset given by the offset argument.\n" +"\n" +"With fill_padding=False, passing bits in \'buf\' will not be modified."); + +#define PACK_INTO_DICT_METHODDEF \ + {"pack_into_dict", (PyCFunction)pack_into_dict, METH_FASTCALL|METH_KEYWORDS, pack_into_dict__doc__}, + +static PyObject * +pack_into_dict_impl(PyObject *module, const char *fmt, PyObject *names, + Py_buffer *buf, Py_ssize_t offset, PyObject *data, + int fill_padding); + +static PyObject * +pack_into_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"fmt", "names", "buf", "offset", "data", "fill_padding", NULL}; + static _PyArg_Parser _parser = {"sOy*nO|$p:pack_into_dict", _keywords, 0}; + const char *fmt; + PyObject *names; + Py_buffer buf = {NULL, NULL}; + Py_ssize_t offset; + PyObject *data; + int fill_padding = 1; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &fmt, &names, &buf, &offset, &data, &fill_padding)) { + goto exit; + } + return_value = pack_into_dict_impl(module, fmt, names, &buf, offset, data, fill_padding); + +exit: + /* Cleanup for buf */ + if (buf.obj) { + PyBuffer_Release(&buf); + } + + return return_value; +} + PyDoc_STRVAR(unpack__doc__, "unpack($module, /, fmt, data)\n" "--\n" @@ -577,4 +664,4 @@ byteswap(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw return return_value; } -/*[clinic end generated code: output=7a1379e9e1df08e8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=20c77328e33739fc input=a9049054013a1b77]*/ diff --git a/cbitstruct/tests/test_api.py b/cbitstruct/tests/test_api.py new file mode 100644 index 0000000..717caa0 --- /dev/null +++ b/cbitstruct/tests/test_api.py @@ -0,0 +1,179 @@ +import unittest +import inspect +import bitstruct +import cbitstruct + + +NAMES = ["foo"] +FMT = "u32" +DICT = {"foo": 42} +DATA = b"\x00\x00\x00\x00" +BUF = bytearray(4) +ARGS = [12] +CF = cbitstruct.CompiledFormat(FMT) +CFD = cbitstruct.CompiledFormatDict(FMT, NAMES) + + +class BitstructApiTest(unittest.TestCase): + def test_compiled_format(self): + cbitstruct.CompiledFormat(fmt=FMT) + cbitstruct.CompiledFormat(FMT) + + def test_compiled_format_dict(self): + cbitstruct.CompiledFormatDict(fmt=FMT, names=NAMES) + cbitstruct.CompiledFormatDict(FMT, names=NAMES) + cbitstruct.CompiledFormatDict(FMT, NAMES) + + def test_pack(self): + cbitstruct.pack(fmt=FMT, *ARGS) + cbitstruct.pack(FMT, *ARGS) + + def test_compiled_pack(self): + CF.pack(*ARGS) + + def test_pack_into(self): + cbitstruct.pack_into(*ARGS, fmt=FMT, buf=BUF, offset=0) + cbitstruct.pack_into(FMT, *ARGS, buf=BUF, offset=0) + cbitstruct.pack_into(FMT, BUF, *ARGS, offset=0) + cbitstruct.pack_into(FMT, BUF, 0, *ARGS) + + cbitstruct.pack_into(*ARGS, fmt=FMT, buf=BUF, offset=0, fill_padding=False) + cbitstruct.pack_into(FMT, *ARGS, buf=BUF, offset=0, fill_padding=False) + cbitstruct.pack_into(FMT, BUF, *ARGS, offset=0, fill_padding=False) + cbitstruct.pack_into(FMT, BUF, 0, *ARGS, fill_padding=False) + + def test_compiled_pack_into(self): + CF.pack_into(*ARGS, buf=BUF, offset=0) + CF.pack_into(BUF, *ARGS, offset=0) + CF.pack_into(BUF, 0, *ARGS) + + CF.pack_into(*ARGS, buf=BUF, offset=0, fill_padding=False) + CF.pack_into(BUF, *ARGS, offset=0, fill_padding=False) + CF.pack_into(BUF, 0, *ARGS, fill_padding=False) + + def test_pack_dict(self): + cbitstruct.pack_dict(fmt=FMT, names=NAMES, data=DICT) + cbitstruct.pack_dict(FMT, names=NAMES, data=DICT) + cbitstruct.pack_dict(FMT, NAMES, data=DICT) + cbitstruct.pack_dict(FMT, NAMES, DICT) + + def test_compiled_pack_dict(self): + CFD.pack(data=DICT) + CFD.pack(DICT) + + def test_pack_into_dict(self): + cbitstruct.pack_into_dict(fmt=FMT, names=NAMES, buf=BUF, offset=0, data=DICT) + cbitstruct.pack_into_dict(FMT, names=NAMES, buf=BUF, offset=0, data=DICT) + cbitstruct.pack_into_dict(FMT, NAMES, buf=BUF, offset=0, data=DICT) + cbitstruct.pack_into_dict(FMT, NAMES, BUF, offset=0, data=DICT) + cbitstruct.pack_into_dict(FMT, NAMES, BUF, 0, data=DICT) + cbitstruct.pack_into_dict(FMT, NAMES, BUF, 0, DICT) + + cbitstruct.pack_into_dict( + fmt=FMT, names=NAMES, buf=BUF, offset=0, data=DICT, fill_padding=False + ) + cbitstruct.pack_into_dict( + FMT, names=NAMES, buf=BUF, offset=0, data=DICT, fill_padding=False + ) + cbitstruct.pack_into_dict( + FMT, NAMES, buf=BUF, offset=0, data=DICT, fill_padding=False + ) + cbitstruct.pack_into_dict( + FMT, NAMES, BUF, offset=0, data=DICT, fill_padding=False + ) + cbitstruct.pack_into_dict(FMT, NAMES, BUF, 0, data=DICT, fill_padding=False) + cbitstruct.pack_into_dict(FMT, NAMES, BUF, 0, DICT, fill_padding=False) + + def test_compiled_pack_into_dict(self): + CFD.pack_into(buf=BUF, offset=0, data=DICT) + CFD.pack_into(BUF, offset=0, data=DICT) + CFD.pack_into(BUF, 0, data=DICT) + CFD.pack_into(BUF, 0, DICT) + + CFD.pack_into(buf=BUF, offset=0, data=DICT, fill_padding=False) + CFD.pack_into(BUF, offset=0, data=DICT, fill_padding=False) + CFD.pack_into(BUF, 0, data=DICT, fill_padding=False) + CFD.pack_into(BUF, 0, DICT, fill_padding=False) + + def test_unpack(self): + cbitstruct.unpack(fmt=FMT, data=DATA) + cbitstruct.unpack(FMT, data=DATA) + cbitstruct.unpack(FMT, DATA) + + def test_compiled_unpack(self): + CF.unpack(data=DATA) + CF.unpack(DATA) + + def test_unpack_from(self): + cbitstruct.unpack_from(fmt=FMT, data=DATA, offset=0) + cbitstruct.unpack_from(FMT, data=DATA, offset=0) + cbitstruct.unpack_from(FMT, DATA, offset=0) + cbitstruct.unpack_from(FMT, DATA, 0) + + cbitstruct.unpack_from(fmt=FMT, data=DATA) + cbitstruct.unpack_from(FMT, data=DATA) + cbitstruct.unpack_from(FMT, DATA) + + def test_compiled_unpack_from(self): + CF.unpack_from(data=DATA, offset=0) + CF.unpack_from(DATA, offset=0) + CF.unpack_from(DATA, 0) + + CF.unpack_from(data=DATA) + CF.unpack_from(DATA) + + def test_unpack_dict(self): + cbitstruct.unpack_dict(fmt=FMT, names=NAMES, data=DATA) + cbitstruct.unpack_dict(FMT, names=NAMES, data=DATA) + cbitstruct.unpack_dict(FMT, NAMES, data=DATA) + cbitstruct.unpack_dict(FMT, NAMES, DATA) + + def test_compiled_unpack_dict(self): + CFD.unpack(data=DATA) + CFD.unpack(DATA) + + def test_unpack_from_dict(self): + cbitstruct.unpack_from_dict(fmt=FMT, names=NAMES, data=DATA, offset=0) + cbitstruct.unpack_from_dict(FMT, names=NAMES, data=DATA, offset=0) + cbitstruct.unpack_from_dict(FMT, NAMES, data=DATA, offset=0) + cbitstruct.unpack_from_dict(FMT, NAMES, DATA, offset=0) + cbitstruct.unpack_from_dict(FMT, NAMES, DATA, 0) + + cbitstruct.unpack_from_dict(fmt=FMT, names=NAMES, data=DATA) + cbitstruct.unpack_from_dict(FMT, names=NAMES, data=DATA) + cbitstruct.unpack_from_dict(FMT, NAMES, data=DATA) + cbitstruct.unpack_from_dict(FMT, NAMES, DATA) + + def test_compiled_unpack_from_dict(self): + CFD.unpack_from(data=DATA, offset=0) + CFD.unpack_from(DATA, offset=0) + CFD.unpack_from(DATA, 0) + + CFD.unpack_from(data=DATA) + CFD.unpack_from(DATA) + + def test_compile(self): + cbitstruct.compile(fmt=FMT, names=NAMES) + cbitstruct.compile(FMT, names=NAMES) + cbitstruct.compile(FMT, NAMES) + + cbitstruct.compile(fmt=FMT) + cbitstruct.compile(FMT) + + def test_calcsize(self): + cbitstruct.calcsize(fmt=FMT) + cbitstruct.calcsize(FMT) + + def test_byteswap(self): + cbitstruct.byteswap(fmt="1", data=DATA, offset=0) + cbitstruct.byteswap("1", data=DATA, offset=0) + cbitstruct.byteswap("1", DATA, offset=0) + cbitstruct.byteswap("1", DATA, 0) + + cbitstruct.byteswap(fmt="1", data=DATA) + cbitstruct.byteswap("1", data=DATA) + cbitstruct.byteswap("1", DATA) + + +if __name__ == "__main__": + unittest.main() diff --git a/cbitstruct/tests/test_bitstruct.py b/cbitstruct/tests/test_bitstruct.py index 9d08344..8dc6358 100644 --- a/cbitstruct/tests/test_bitstruct.py +++ b/cbitstruct/tests/test_bitstruct.py @@ -8,7 +8,7 @@ Error = TypeError -class BitStructTest(unittest.TestCase): +class BitstructTest(unittest.TestCase): def test_pack(self): """Pack values. @@ -492,6 +492,11 @@ def test_pack_integers_value_checks(self): pack(fmt, minimum) pack(fmt, maximum) + # Numbers out of range. + for number in [minimum - 1, maximum + 1]: + with self.assertRaises(Error) as cm: + pack(fmt, number) + def test_pack_unpack_raw(self): """Pack and unpack raw values. diff --git a/cbitstruct/tests/test_perf.py b/cbitstruct/tests/test_perf.py index f923505..73745d9 100644 --- a/cbitstruct/tests/test_perf.py +++ b/cbitstruct/tests/test_perf.py @@ -4,7 +4,6 @@ import timeit import unittest -SETUP_SCRIPT = """ import string import random import cbitstruct @@ -25,7 +24,6 @@ dbs = bitstruct.compile(fmt, names) cdbs = cbitstruct.compile(fmt, names) -""" NBS = 10000 NCBS = 100000 @@ -33,8 +31,8 @@ class PerfTest(unittest.TestCase): def generic(self, name, bitstruct, cbitstruct): - bstime = timeit.timeit(bitstruct, setup=SETUP_SCRIPT, number=NBS) / NBS - cbstime = timeit.timeit(cbitstruct, setup=SETUP_SCRIPT, number=NCBS) / NCBS + bstime = timeit.timeit(bitstruct, number=NBS, globals=globals()) / NBS + cbstime = timeit.timeit(cbitstruct, number=NCBS, globals=globals()) / NCBS improvement = bstime / cbstime print( diff --git a/dev-requirements.txt b/dev-requirements.txt index c7f4ae8..1154b08 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1,4 @@ nose -bitstruct \ No newline at end of file +bitstruct +wheel +setuptools \ No newline at end of file diff --git a/setup.py b/setup.py index 11c4eaa..b74b270 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,19 @@ #!/usr/bin/env python import os -from distutils.core import setup, Extension +import sys +from setuptools import setup, Extension -extra_compile_args = ["-std=c11"] +extra_compile_args = [] extra_link_args = [] + +if sys.platform == "win32": + extra_compile_args += [] +else: + extra_compile_args += ["-std=c11", "-Wall", "-Werror"] + + if os.environ.get("COVERAGE"): extra_compile_args += ["-g", "-O0", "-fprofile-arcs", "-ftest-coverage"] extra_link_args += ["-fprofile-arcs"] @@ -17,7 +25,7 @@ setup( name="cbitstruct", - version="0.0.1", + version="1.0.0", author="Quentin CHATEAU", author_email="quentin.chateau@gmail.com", license="GPLv3", @@ -26,7 +34,6 @@ long_description=long_description, long_description_content_type="text/markdown", classifiers=[ - "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", "Programming Language :: C", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", @@ -49,6 +56,7 @@ extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, sources=["cbitstruct/_cbitstruct.c"], + include_dirs=["cbitstruct/"], ) ], packages=["cbitstruct", "cbitstruct.tests"], diff --git a/travis/build-wheels.sh b/travis/build-wheels.sh deleted file mode 100755 index f7b85db..0000000 --- a/travis/build-wheels.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -x - -# Show gcc version -gcc --version - -# Compile wheels -for PYBIN in /opt/python/cp3*/bin; do - "${PYBIN}/pip" install -r /io/dev-requirements.txt - "${PYBIN}/pip" wheel /io/ -w wheelhouse/ -done - -# Bundle external shared libraries into the wheels -for whl in wheelhouse/*.whl; do - auditwheel repair "$whl" --plat $PLAT -w /io/wheelhouse/ -done - -# Install packages and test -for PYBIN in /opt/python/cp3*/bin/; do - "${PYBIN}/pip" install cbitstruct --no-index -f /io/wheelhouse - (cd "$HOME"; "${PYBIN}/nosetests" cbitstruct) -done diff --git a/travis/deploy.sh b/travis/deploy.sh deleted file mode 100644 index 2ebd734..0000000 --- a/travis/deploy.sh +++ /dev/null @@ -1,4 +0,0 @@ -pip install pip --upgrade -pip install twine - -twine upload --repository-url "$PYPI_REPOSITORY_URL" -u "$PYPI_USERNAME" -p "$PYPI_PASSWORD" wheelhouse/*