diff --git a/CHANGELOG.md b/CHANGELOG.md index b983493..6c1db85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## Next version (Unreleased) +- Upgrade zstd source code from v1.5.5 to [v1.5.6](https://github.com/facebook/zstd/releases/tag/v1.5.6) - Fix pyzstd_pep517 parameter name in `get_requires_for_build_wheel` - Deprecate support for Python version before 3.8 and stop building wheels for them - Minor fixes in type hints diff --git a/build_script/pyzstd_build_cffi.py b/build_script/pyzstd_build_cffi.py index c2b8a58..01f5701 100644 --- a/build_script/pyzstd_build_cffi.py +++ b/build_script/pyzstd_build_cffi.py @@ -57,6 +57,7 @@ ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_strategy, + ZSTD_c_targetCBlockSize, /* LDM mode parameters */ ZSTD_c_enableLongDistanceMatching, @@ -203,6 +204,12 @@ } #endif +#if ZSTD_VERSION_NUMBER < 10506 +typedef enum { + ZSTD_c_targetCBlockSize=130 +} PYZSTD_compat_c_targetCBlockSize; +#endif + #ifdef PYZSTD_STATIC_LINK int pyzstd_static_link = 1; #else diff --git a/docs/index.rst b/docs/index.rst index 22a687e..a180462 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -983,7 +983,7 @@ Advanced parameters Attributes of :py:class:`CParameter` class: - Compression level (:py:attr:`~CParameter.compressionLevel`) - - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`) + - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`, :py:attr:`~CParameter.targetCBlockSize`) - Long distance matching (:py:attr:`~CParameter.enableLongDistanceMatching`, :py:attr:`~CParameter.ldmHashLog`, :py:attr:`~CParameter.ldmMinMatch`, :py:attr:`~CParameter.ldmBucketSizeLog`, :py:attr:`~CParameter.ldmHashRateLog`) - Misc (:py:attr:`~CParameter.contentSizeFlag`, :py:attr:`~CParameter.checksumFlag`, :py:attr:`~CParameter.dictIDFlag`) - Multi-threaded compression (:py:attr:`~CParameter.nbWorkers`, :py:attr:`~CParameter.jobSize`, :py:attr:`~CParameter.overlapLog`) @@ -1126,6 +1126,18 @@ Advanced parameters Special: value ``0`` means "use default strategy", then the value is dynamically set, see "strat" column in `this table `_. + .. py:attribute:: targetCBlockSize + + Attempts to fit compressed block size into approximatively targetCBlockSize (in bytes). Note that it's not a guarantee, just a convergence target. + + This is helpful in low bandwidth streaming environments to improve end-to-end latency, when a client can make use of partial documents. + + Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. No target when targetCBlockSize == 0. + + Default value is ``0``. + + Only available for zstd v1.5.6+. + .. py:attribute:: enableLongDistanceMatching Enable long distance matching. diff --git a/src/__init__.pyi b/src/__init__.pyi index 8c64817..e96212f 100644 --- a/src/__init__.pyi +++ b/src/__init__.pyi @@ -37,6 +37,7 @@ class CParameter(IntEnum): minMatch: int targetLength: int strategy: int + targetCBlockSize: int enableLongDistanceMatching: int ldmHashLog: int diff --git a/src/bin_ext/pyzstd.c b/src/bin_ext/pyzstd.c index c4751c6..2898338 100644 --- a/src/bin_ext/pyzstd.c +++ b/src/bin_ext/pyzstd.c @@ -211,6 +211,9 @@ add_parameters(PyObject *module) ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch); ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength); ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy); +#if ZSTD_VERSION_NUMBER >= 10506 + ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetCBlockSize); +#endif ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching); ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog); diff --git a/src/bin_ext/pyzstd.h b/src/bin_ext/pyzstd.h index ddb8729..152ee05 100644 --- a/src/bin_ext/pyzstd.h +++ b/src/bin_ext/pyzstd.h @@ -251,6 +251,10 @@ static const ParameterInfo cp_list[] = {ZSTD_c_targetLength, "targetLength"}, {ZSTD_c_strategy, "strategy"}, +#if ZSTD_VERSION_NUMBER >= 10506 + {ZSTD_c_targetCBlockSize, "targetCBlockSize"}, +#endif + {ZSTD_c_enableLongDistanceMatching, "enableLongDistanceMatching"}, {ZSTD_c_ldmHashLog, "ldmHashLog"}, {ZSTD_c_ldmMinMatch, "ldmMinMatch"}, diff --git a/src/c/__init__.py b/src/c/__init__.py index df45ee6..c409dc2 100644 --- a/src/c/__init__.py +++ b/src/c/__init__.py @@ -55,6 +55,18 @@ def get_frame_info(frame_buffer): return _nt_frame_info(*ret_tuple) +class _UnsupportedCParameter: + def __set_name__(self, _, name): + self.name = name + + def __get__(self, *_, **__): + msg = ("%s CParameter only available when the underlying " + "zstd library's version is greater than or equal to v1.5.6. " + "At pyzstd module's run-time, zstd version is %s.") % \ + (self.name, zstd_version) + raise NotImplementedError(msg) + + class CParameter(IntEnum): """Compression parameters""" @@ -66,6 +78,10 @@ class CParameter(IntEnum): minMatch = _zstd._ZSTD_c_minMatch targetLength = _zstd._ZSTD_c_targetLength strategy = _zstd._ZSTD_c_strategy + if zstd_version_info >= (1, 5, 6): + targetCBlockSize = _zstd._ZSTD_c_targetCBlockSize + else: + targetCBlockSize = _UnsupportedCParameter() enableLongDistanceMatching = _zstd._ZSTD_c_enableLongDistanceMatching ldmHashLog = _zstd._ZSTD_c_ldmHashLog diff --git a/src/cffi/common.py b/src/cffi/common.py index 5aabdb0..e7da9bc 100644 --- a/src/cffi/common.py +++ b/src/cffi/common.py @@ -44,6 +44,19 @@ def _get_param_bounds(is_compress, key): return (bounds.lowerBound, bounds.upperBound) + +class _UnsupportedCParameter: + def __set_name__(self, _, name): + self.name = name + + def __get__(self, *_, **__): + msg = ("%s CParameter only available when the underlying " + "zstd library's version is greater than or equal to v1.5.6. " + "At pyzstd module's run-time, zstd version is %s.") % \ + (self.name, zstd_version) + raise NotImplementedError(msg) + + class CParameter(IntEnum): """Compression parameters""" @@ -55,6 +68,10 @@ class CParameter(IntEnum): minMatch = m.ZSTD_c_minMatch targetLength = m.ZSTD_c_targetLength strategy = m.ZSTD_c_strategy + if zstd_version_info >= (1, 5, 6): + targetCBlockSize = m.ZSTD_c_targetCBlockSize + else: + targetCBlockSize = _UnsupportedCParameter() enableLongDistanceMatching = m.ZSTD_c_enableLongDistanceMatching ldmHashLog = m.ZSTD_c_ldmHashLog @@ -166,6 +183,8 @@ def _set_parameter_error(is_compress, key, value): m.ZSTD_c_nbWorkers: "nbWorkers", m.ZSTD_c_jobSize: "jobSize", m.ZSTD_c_overlapLog: "overlapLog"} + if zstd_version_info >= (1, 5, 6): + COMPRESS_PARAMETERS[m.ZSTD_c_targetCBlockSize] = "targetCBlockSize" DECOMPRESS_PARAMETERS = {m.ZSTD_d_windowLogMax: "windowLogMax"} diff --git a/tests/test_zstd.py b/tests/test_zstd.py index 6ef995a..2b53de1 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -429,6 +429,11 @@ def test_CParameter(self): CParameter.minMatch CParameter.targetLength CParameter.strategy + if zstd_version_info >= (1, 5, 6): + CParameter.targetCBlockSize + else: + with self.assertRaises(NotImplementedError): + CParameter.targetCBlockSize CParameter.enableLongDistanceMatching CParameter.ldmHashLog @@ -611,6 +616,8 @@ def test_compress_parameters(self): CParameter.jobSize : 5*MB if zstd_support_multithread else 0, CParameter.overlapLog : 9 if zstd_support_multithread else 0, } + if zstd_version_info >= (1, 5, 6): + d[CParameter.targetCBlockSize] = 150 ZstdCompressor(level_or_option=d) # larger than signed int, ValueError diff --git a/zstd b/zstd index 63779c7..794ea1b 160000 --- a/zstd +++ b/zstd @@ -1 +1 @@ -Subproject commit 63779c798237346c2b245c546c40b72a5a5913fe +Subproject commit 794ea1b0afca0f020f4e57b6732332231fb23c70