From 2c600de3cd196f71e9a84bf7f544ef5897f2652f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 07:40:53 +0100 Subject: [PATCH 01/10] Add test for zlib.compress wbits --- Lib/test/test_zlib.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 7f30cac64f71b9..252e8360e0189f 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -825,6 +825,12 @@ def test_wbits(self): dco = zlib.decompressobj(32 + 15) self.assertEqual(dco.decompress(gzip), HAMLET_SCENE) + for wbits in (-15, 15, 31): + self.assertEqual( + zlib.decompress( + zlib.compress(HAMLET_SCENE, wbits=wbits), wbits=wbits + ), HAMLET_SCENE) + def choose_lines(source, number, seed=None, generator=random): """Return a list of number lines randomly chosen from the source""" From b15bd4828d2edc51afbdefc7b027a4abcebe0f76 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 08:07:36 +0100 Subject: [PATCH 02/10] Add wbits argument to zlib.compress --- Modules/clinic/zlibmodule.c.h | 30 ++++++++++++++++++++++++------ Modules/zlibmodule.c | 6 ++++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 14e955db64e729..a4da381e65a5d5 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -17,18 +17,19 @@ PyDoc_STRVAR(zlib_compress__doc__, {"compress", (PyCFunction)(void(*)(void))zlib_compress, METH_FASTCALL|METH_KEYWORDS, zlib_compress__doc__}, static PyObject * -zlib_compress_impl(PyObject *module, Py_buffer *data, int level); +zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits); static PyObject * zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"", "level", NULL}; + static const char * const _keywords[] = {"", "level", "wbits", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "compress", 0}; PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int level = Z_DEFAULT_COMPRESSION; + int wbits = MAX_WBITS; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); if (!args) { @@ -44,12 +45,29 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec if (!noptargs) { goto skip_optional_pos; } - level = _PyLong_AsInt(args[1]); - if (level == -1 && PyErr_Occurred()) { - goto exit; + if (args[1]) { + level = _PyLong_AsInt(args[1]); + if (level == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + { + int ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = _PyLong_AsInt(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + wbits = ival; } skip_optional_pos: - return_value = zlib_compress_impl(module, &data, level); + return_value = zlib_compress_impl(module, &data, level, wbits); exit: /* Cleanup for data */ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a537087d19d835..b9040750c7771f 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -205,12 +205,14 @@ zlib.compress / level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION Compression level, in 0-9 or -1. + wbits: int(c_default="MAX_WBITS") = MAX_WBITS + The window buffer size and container format. Returns a bytes object containing compressed data. [clinic start generated code]*/ static PyObject * -zlib_compress_impl(PyObject *module, Py_buffer *data, int level) +zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) /*[clinic end generated code: output=d80906d73f6294c8 input=638d54b6315dbed3]*/ { PyObject *RetVal = NULL; @@ -227,7 +229,7 @@ zlib_compress_impl(PyObject *module, Py_buffer *data, int level) zst.zalloc = PyZlib_Malloc; zst.zfree = PyZlib_Free; zst.next_in = ibuf; - int err = deflateInit(&zst, level); + int err = deflateInit2(&zst, level, DEFLATED, wbits, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); switch (err) { case Z_OK: From ea55f570aea9b0e75f595f131eaaa3f72e9b72c4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 09:26:06 +0100 Subject: [PATCH 03/10] Use clinic to generate input --- Modules/clinic/zlibmodule.c.h | 26 ++++++++++---------------- Modules/zlibmodule.c | 2 +- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index a4da381e65a5d5..e2a5fccd36c54f 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -3,7 +3,7 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(zlib_compress__doc__, -"compress($module, data, /, level=Z_DEFAULT_COMPRESSION)\n" +"compress($module, data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS)\n" "--\n" "\n" "Returns a bytes object containing compressed data.\n" @@ -11,7 +11,9 @@ PyDoc_STRVAR(zlib_compress__doc__, " data\n" " Binary data to be compressed.\n" " level\n" -" Compression level, in 0-9 or -1."); +" Compression level, in 0-9 or -1.\n" +" wbits\n" +" The window buffer size and container format."); #define ZLIB_COMPRESS_METHODDEF \ {"compress", (PyCFunction)(void(*)(void))zlib_compress, METH_FASTCALL|METH_KEYWORDS, zlib_compress__doc__}, @@ -25,13 +27,13 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *return_value = NULL; static const char * const _keywords[] = {"", "level", "wbits", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "compress", 0}; - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer data = {NULL, NULL}; int level = Z_DEFAULT_COMPRESSION; int wbits = MAX_WBITS; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); if (!args) { goto exit; } @@ -54,17 +56,9 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec goto skip_optional_pos; } } - { - int ival = -1; - PyObject *iobj = _PyNumber_Index(args[2]); - if (iobj != NULL) { - ival = _PyLong_AsInt(iobj); - Py_DECREF(iobj); - } - if (ival == -1 && PyErr_Occurred()) { - goto exit; - } - wbits = ival; + wbits = _PyLong_AsInt(args[2]); + if (wbits == -1 && PyErr_Occurred()) { + goto exit; } skip_optional_pos: return_value = zlib_compress_impl(module, &data, level, wbits); @@ -821,4 +815,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=6736bae59fab268b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e3e8a6142ea045a7 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b9040750c7771f..b6301392212412 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -213,7 +213,7 @@ Returns a bytes object containing compressed data. static PyObject * zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) -/*[clinic end generated code: output=d80906d73f6294c8 input=638d54b6315dbed3]*/ +/*[clinic end generated code: output=46bd152fadd66df2 input=c4d06ee5782a7e3f]*/ { PyObject *RetVal = NULL; Py_ssize_t obuflen = DEF_BUF_SIZE; From ff24df370f9b78140078672401e421b7c438d496 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 09:35:07 +0100 Subject: [PATCH 04/10] Update documentation for zlib --- Doc/library/zlib.rst | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ec60ea24db6627..1d2fa1bb34e9f9 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -47,19 +47,43 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data, /, level=-1) +.. function:: compress(data, /, level=-1, wbits=MAX_WBITS) Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; + + *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). + + .. _compress-wbits: + + The *wbits* argument controls the size of the history buffer (or the + "window size") used when compressing data, and whether a header and + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): + + * +9 to +15: The base-two logarithm of the window size, which + therefore ranges between 512 and 32768. Larger values produce + better compression at the expense of greater memory usage. The + resulting output will include a zlib-specific header and trailer. + + * −9 to −15: Uses the absolute value of *wbits* as the + window size logarithm, while producing a raw output stream with no + header or trailing checksum. + + * +25 to +31 = 16 + (9 to 15): Uses the low 4 bits of the value as the + window size logarithm, while including a basic :program:`gzip` header + and trailing checksum in the output. + Raises the :exc:`error` exception if any error occurs. .. versionchanged:: 3.6 *level* can now be used as a keyword parameter. + .. versionchanged:: 3.10 + *wbits* parameter added. .. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) @@ -76,23 +100,9 @@ The available exception and functions in this module are: *method* is the compression algorithm. Currently, the only supported value is :const:`DEFLATED`. - The *wbits* argument controls the size of the history buffer (or the - "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values, - defaulting to ``15`` (MAX_WBITS): - - * +9 to +15: The base-two logarithm of the window size, which - therefore ranges between 512 and 32768. Larger values produce - better compression at the expense of greater memory usage. The - resulting output will include a zlib-specific header and trailer. - - * −9 to −15: Uses the absolute value of *wbits* as the - window size logarithm, while producing a raw output stream with no - header or trailing checksum. - - * +25 to +31 = 16 + (9 to 15): Uses the low 4 bits of the value as the - window size logarithm, while including a basic :program:`gzip` header - and trailing checksum in the output. + The *wbits* parameter controls the size of the history buffer (or the + "window size"), and what header and trailer format will be used It has + the same meaning as `described for compress() <#compress-wbits>`__. The *memLevel* argument controls the amount of memory used for the internal compression state. Valid values range from ``1`` to ``9``. From 07cc0adc77471b6385f2079eafcad419805081c3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 09:41:09 +0100 Subject: [PATCH 05/10] Add blurb news entry for zlib.compress wbits parameter --- .../next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst diff --git a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst new file mode 100644 index 00000000000000..fb8878e1d302cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst @@ -0,0 +1,5 @@ +``zlib.compress`` now accepts a wbits parameter which allows users to +compress data as a raw deflate block without zlib headers and trailers in +one go. Previously this required instantiating a ``zlib.compressobj``. It +also provides a faster alternative to ``gzip.compress`` when wbits=31 is +used. From 60e799a586979ef95eec5ce37083cbd81d95e462 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 24 Mar 2021 09:58:40 +0100 Subject: [PATCH 06/10] Fix doc typo --- Doc/library/zlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 1d2fa1bb34e9f9..c0a139b9941365 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,7 +51,7 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; + *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default From eb52f20ffa5d746512cafcf45848d0d2a881423c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Jun 2021 08:19:32 +0200 Subject: [PATCH 07/10] Remove unnecessary whitespace, add punctionation and complete sentences. --- Doc/library/zlib.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index c0a139b9941365..793c90f3c4e7a4 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -50,7 +50,6 @@ The available exception and functions in this module are: .. function:: compress(data, /, level=-1, wbits=MAX_WBITS) Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. @@ -82,8 +81,9 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *level* can now be used as a keyword parameter. - .. versionchanged:: 3.10 - *wbits* parameter added. + .. versionchanged:: 3.11 + The *wbits* parameter is now available to set window bits and + compression type. .. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) @@ -101,7 +101,7 @@ The available exception and functions in this module are: :const:`DEFLATED`. The *wbits* parameter controls the size of the history buffer (or the - "window size"), and what header and trailer format will be used It has + "window size"), and what header and trailer format will be used. It has the same meaning as `described for compress() <#compress-wbits>`__. The *memLevel* argument controls the amount of memory used for the From a19f434d0d92ceed1fc3896028322549e9e208bd Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Jun 2021 08:22:32 +0200 Subject: [PATCH 08/10] Break line to comply with PEP-7 --- Modules/zlibmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b6301392212412..32132baf5d083e 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -229,7 +229,8 @@ zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) zst.zalloc = PyZlib_Malloc; zst.zfree = PyZlib_Free; zst.next_in = ibuf; - int err = deflateInit2(&zst, level, DEFLATED, wbits, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); + int err = deflateInit2(&zst, level, DEFLATED, wbits, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY); switch (err) { case Z_OK: From 8bc7333d45ab49e10ab6aa7a79d25d6665793e2a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Jun 2021 08:30:01 +0200 Subject: [PATCH 09/10] Update blurb to include :func: reference --- .../next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst index fb8878e1d302cb..3f0063962d6ce4 100644 --- a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst +++ b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst @@ -1,4 +1,4 @@ -``zlib.compress`` now accepts a wbits parameter which allows users to +:func:``zlib.compress`` now accepts a wbits parameter which allows users to compress data as a raw deflate block without zlib headers and trailers in one go. Previously this required instantiating a ``zlib.compressobj``. It also provides a faster alternative to ``gzip.compress`` when wbits=31 is From efe434e5f446d6e02b700f1071ef8e9990bb19c7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Jun 2021 09:10:55 +0200 Subject: [PATCH 10/10] Remove erroneous double backticks --- .../next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst index 3f0063962d6ce4..e6fc88f45eea5e 100644 --- a/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst +++ b/Misc/NEWS.d/next/Library/2021-03-24-09-40-02.bpo-43612.vMGZ4y.rst @@ -1,4 +1,4 @@ -:func:``zlib.compress`` now accepts a wbits parameter which allows users to +:func:`zlib.compress` now accepts a wbits parameter which allows users to compress data as a raw deflate block without zlib headers and trailers in one go. Previously this required instantiating a ``zlib.compressobj``. It also provides a faster alternative to ``gzip.compress`` when wbits=31 is