zstd version 1.5.6

CHANGELOG.md

@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 
 ## Next version (Unreleased)
 
+- Upgrade zstd source code from v1.5.5 to [v1.5.6](https://github.com/facebook/zstd/releases/tag/v1.5.6)
 - Fix pyzstd_pep517 parameter name in `get_requires_for_build_wheel`
 - Deprecate support for Python version before 3.8 and stop building wheels for them
 - Minor fixes in type hints

build_script/pyzstd_build_cffi.py

@@ -57,6 +57,7 @@
     ZSTD_c_minMatch,
     ZSTD_c_targetLength,
     ZSTD_c_strategy,
+    ZSTD_c_targetCBlockSize,
 
     /* LDM mode parameters */
     ZSTD_c_enableLongDistanceMatching,
@@ -203,6 +204,12 @@
 }
 #endif
 
+#if ZSTD_VERSION_NUMBER < 10506
+typedef enum {
+    ZSTD_c_targetCBlockSize=130
+} PYZSTD_compat_c_targetCBlockSize;
+#endif
+
 #ifdef PYZSTD_STATIC_LINK
 int pyzstd_static_link = 1;
 #else

docs/index.rst

@@ -983,7 +983,7 @@ Advanced parameters
     Attributes of :py:class:`CParameter` class:
 
         - Compression level (:py:attr:`~CParameter.compressionLevel`)
-        - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`)
+        - Compress algorithm parameters (:py:attr:`~CParameter.windowLog`, :py:attr:`~CParameter.hashLog`, :py:attr:`~CParameter.chainLog`, :py:attr:`~CParameter.searchLog`, :py:attr:`~CParameter.minMatch`, :py:attr:`~CParameter.targetLength`, :py:attr:`~CParameter.strategy`, :py:attr:`~CParameter.targetCBlockSize`)
         - Long distance matching (:py:attr:`~CParameter.enableLongDistanceMatching`, :py:attr:`~CParameter.ldmHashLog`, :py:attr:`~CParameter.ldmMinMatch`, :py:attr:`~CParameter.ldmBucketSizeLog`, :py:attr:`~CParameter.ldmHashRateLog`)
         - Misc (:py:attr:`~CParameter.contentSizeFlag`, :py:attr:`~CParameter.checksumFlag`, :py:attr:`~CParameter.dictIDFlag`)
         - Multi-threaded compression (:py:attr:`~CParameter.nbWorkers`, :py:attr:`~CParameter.jobSize`, :py:attr:`~CParameter.overlapLog`)
@@ -1126,6 +1126,18 @@ Advanced parameters
 
         Special: value ``0`` means "use default strategy", then the value is dynamically set, see "strat" column in `this table <https://github.com/facebook/zstd/blob/release/lib/compress/clevels.h>`_.
 
+     .. py:attribute:: targetCBlockSize
+
+        Attempts to fit compressed block size into approximatively targetCBlockSize (in bytes). Note that it's not a guarantee, just a convergence target.
+
+        This is helpful in low bandwidth streaming environments to improve end-to-end latency, when a client can make use of partial documents.
+
+        Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. No target when targetCBlockSize == 0.
+
+        Default value is ``0``.
+
+        Only available for zstd v1.5.6+.
+
     .. py:attribute:: enableLongDistanceMatching
 
         Enable long distance matching.

src/__init__.pyi

@@ -37,6 +37,7 @@ class CParameter(IntEnum):
     minMatch: int
     targetLength: int
     strategy: int
+    targetCBlockSize: int
 
     enableLongDistanceMatching: int
     ldmHashLog: int

src/bin_ext/pyzstd.c

@@ -211,6 +211,9 @@ add_parameters(PyObject *module)
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy);
+#if ZSTD_VERSION_NUMBER >= 10506
+    ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetCBlockSize);
+#endif
 
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching);
     ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog);

src/bin_ext/pyzstd.h

@@ -251,6 +251,10 @@ static const ParameterInfo cp_list[] =
     {ZSTD_c_targetLength,     "targetLength"},
     {ZSTD_c_strategy,         "strategy"},
 
+#if ZSTD_VERSION_NUMBER >= 10506
+    {ZSTD_c_targetCBlockSize, "targetCBlockSize"},
+#endif
+
     {ZSTD_c_enableLongDistanceMatching, "enableLongDistanceMatching"},
     {ZSTD_c_ldmHashLog,       "ldmHashLog"},
     {ZSTD_c_ldmMinMatch,      "ldmMinMatch"},

src/c/__init__.py

@@ -55,6 +55,18 @@ def get_frame_info(frame_buffer):
     return _nt_frame_info(*ret_tuple)
 
 
+class _UnsupportedCParameter:
+    def __set_name__(self, _, name):
+        self.name = name
+
+    def __get__(self, *_, **__):
+        msg = ("%s CParameter only available when the underlying "
+               "zstd library's version is greater than or equal to v1.5.6. "
+               "At pyzstd module's run-time, zstd version is %s.") % \
+               (self.name, zstd_version)
+        raise NotImplementedError(msg)
+
+
 class CParameter(IntEnum):
     """Compression parameters"""
 
@@ -66,6 +78,10 @@ class CParameter(IntEnum):
     minMatch                   = _zstd._ZSTD_c_minMatch
     targetLength               = _zstd._ZSTD_c_targetLength
     strategy                   = _zstd._ZSTD_c_strategy
+    if zstd_version_info >= (1, 5, 6):
+        targetCBlockSize       = _zstd._ZSTD_c_targetCBlockSize
+    else:
+        targetCBlockSize       = _UnsupportedCParameter()
 
     enableLongDistanceMatching = _zstd._ZSTD_c_enableLongDistanceMatching
     ldmHashLog                 = _zstd._ZSTD_c_ldmHashLog

src/cffi/common.py

@@ -44,6 +44,19 @@ def _get_param_bounds(is_compress, key):
 
     return (bounds.lowerBound, bounds.upperBound)
 
+
+class _UnsupportedCParameter:
+    def __set_name__(self, _, name):
+        self.name = name
+
+    def __get__(self, *_, **__):
+        msg = ("%s CParameter only available when the underlying "
+               "zstd library's version is greater than or equal to v1.5.6. "
+               "At pyzstd module's run-time, zstd version is %s.") % \
+               (self.name, zstd_version)
+        raise NotImplementedError(msg)
+
+
 class CParameter(IntEnum):
     """Compression parameters"""
 
@@ -55,6 +68,10 @@ class CParameter(IntEnum):
     minMatch                   = m.ZSTD_c_minMatch
     targetLength               = m.ZSTD_c_targetLength
     strategy                   = m.ZSTD_c_strategy
+    if zstd_version_info >= (1, 5, 6):
+        targetCBlockSize       = m.ZSTD_c_targetCBlockSize
+    else:
+        targetCBlockSize       = _UnsupportedCParameter()
 
     enableLongDistanceMatching = m.ZSTD_c_enableLongDistanceMatching
     ldmHashLog                 = m.ZSTD_c_ldmHashLog
@@ -166,6 +183,8 @@ def _set_parameter_error(is_compress, key, value):
      m.ZSTD_c_nbWorkers:        "nbWorkers",
      m.ZSTD_c_jobSize:          "jobSize",
      m.ZSTD_c_overlapLog:       "overlapLog"}
+    if zstd_version_info >= (1, 5, 6):
+        COMPRESS_PARAMETERS[m.ZSTD_c_targetCBlockSize] = "targetCBlockSize"
 
     DECOMPRESS_PARAMETERS = {m.ZSTD_d_windowLogMax: "windowLogMax"}
 

tests/test_zstd.py

@@ -429,6 +429,11 @@ def test_CParameter(self):
         CParameter.minMatch
         CParameter.targetLength
         CParameter.strategy
+        if zstd_version_info >= (1, 5, 6):
+            CParameter.targetCBlockSize
+        else:
+            with self.assertRaises(NotImplementedError):
+                CParameter.targetCBlockSize
 
         CParameter.enableLongDistanceMatching
         CParameter.ldmHashLog
@@ -611,6 +616,8 @@ def test_compress_parameters(self):
              CParameter.jobSize : 5*MB if zstd_support_multithread else 0,
              CParameter.overlapLog : 9 if zstd_support_multithread else 0,
              }
+        if zstd_version_info >= (1, 5, 6):
+            d[CParameter.targetCBlockSize] = 150
         ZstdCompressor(level_or_option=d)
 
         # larger than signed int, ValueError