Lerking/micropython
1
0
Fork 0
mirror of https://github.com/micropython/micropython.git synced 2026-01-05 11:40:18 +01:00
Code Issues Actions 27 Packages Projects Releases Wiki Activity

extmod/moddeflate: Change default window size.

Browse Source 
The primary purpose of this commit is to make decompress default to
wbits=15 when the format is gzip (or auto format with gzip detected). The
idea is that someone decompressing a gzip stream should be able to use the
default `deflate.DeflateIO(f)` and it will "just work" for any input
stream, even though it uses a lot of memory.

This is done by making uzlib report gzip files as having wbits set to 15
in their header (where it previously only set the wbits out parameter for
zlib files), and then fixing up the logic in `deflateio_init_read`.

Updates the documentation to match.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
This commit is contained in:
Jim Mussared
2023-08-31 14:29:49 +10:00
parent 81c19d93bc
commit 32db4c58f7
3 changed files with 68 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
extmod/moddeflate.c
View File

@@ -54,6 +54,8 @@ typedef enum {
DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP,
} deflateio_format_t;
// This is used when the wbits is unset in the DeflateIO constructor. Default
// to the smallest window size (faster compression, less RAM usage, etc).
const int DEFLATEIO_DEFAULT_WBITS = 8;
typedef struct {
@@ -114,24 +116,32 @@ STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) {
// Don't modify self->window_bits as it may also be used for write.
int wbits = self->window_bits;
// Parse the header if we're in NONE/ZLIB/GZIP modes.
if (self->format != DEFLATEIO_FORMAT_RAW) {
int header_wbits = wbits;
if (self->format == DEFLATEIO_FORMAT_RAW) {
if (wbits == 0) {
// The docs recommends always setting wbits explicitly when using
// RAW, but we still allow a default.
wbits = DEFLATEIO_DEFAULT_WBITS;
}
} else {
// Parse the header if we're in NONE/ZLIB/GZIP modes.
int header_wbits;
int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits);
if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
if (header_type < 0) {
// Stream header was invalid.
return false;
}
if (wbits == 0 && header_wbits < 15) {
// If the header specified something lower than the default, then
// use that instead.
if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) {
// Not what we expected.
return false;
}
// header_wbits will either be 15 (gzip) or 8-15 (zlib).
if (wbits == 0 || header_wbits < wbits) {
// If the header specified something lower, then use that instead.
// No point doing a bigger allocation than we need to.
wbits = header_wbits;
}
}
if (wbits == 0) {
wbits = DEFLATEIO_DEFAULT_WBITS;
}
size_t window_len = 1 << wbits;
self->read->window = m_new(uint8_t, window_len);
@@ -163,6 +173,7 @@ STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) {
int wbits = self->window_bits;
if (wbits == 0) {
// Same default wbits for all formats.
wbits = DEFLATEIO_DEFAULT_WBITS;
}
size_t window_len = 1 << wbits;