Visitar URL original
zlib: clear unconsumed_tail when a bounded decompress reaches stream … · RustPython/RustPython@3805038 · GitHub
Skip to content

Commit 3805038

Browse files
JMLX42JMLX42
authored
zlib: clear unconsumed_tail when a bounded decompress reaches stream end (#8908)
* zlib: clear unconsumed_tail when a bounded decompress reaches stream end When a max_length-bounded Decompress.decompress() call reached the end of the stream, the bytes after the stream went to unused_data, but unconsumed_tail kept the previous call's tail, including input the call had already consumed. A later flush() then moved those stale bytes into unused_data. Empty unconsumed_tail at end of stream, as CPython's save_unconsumed_input() does since gh-158169. CPython releases before that change report the bytes after the stream in unconsumed_tail, so the snippet accepts either value when it runs under CPython. Fixes #8906 Assisted-by: Claude Code:claude-opus-5-5 * zlib: bind the unconsumed input with an if expression Choose between the empty slice and the rest of the input in one expression instead of reassigning a mutable binding. Assisted-by: Claude Code:claude-opus-5-5 * zlib: accept both valid tails in the snippet on every implementation The snippet no longer branches on sys.implementation. RustPython and CPython after gh-158169 give an empty tail; CPython before it gives the bytes after the stream. The stale tail of gh-8906 fails either way. Assisted-by: Claude Code:claude-opus-5-5
1 parent e8919b9 commit 3805038

2 files changed

Lines changed: 45 additions & 11 deletions

File tree

‎crates/common/src/compression/zlib.rs‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -670,17 +670,16 @@ impl Decompressor {
670670
};
671671
let consumed = (d.total_in() - prev_in) as usize;
672672

673-
// save unused input
674-
let unconsumed = &data[consumed..];
675-
if !unconsumed.is_empty() {
676-
if stream_end {
677-
unused_data.extend_from_slice(unconsumed);
678-
} else {
679-
*unconsumed_tail = unconsumed.to_vec();
680-
}
681-
} else if !unconsumed_tail.is_empty() {
682-
unconsumed_tail.clear();
683-
}
673+
// save unused input: past the end of the stream it is unused_data and
674+
// unconsumed_tail is emptied, otherwise it is the new unconsumed_tail
675+
let unconsumed: &[u8] = if stream_end {
676+
unused_data.extend_from_slice(&data[consumed..]);
677+
&[]
678+
} else {
679+
&data[consumed..]
680+
};
681+
unconsumed_tail.clear();
682+
unconsumed_tail.extend_from_slice(unconsumed);
684683

685684
Ok((ret, stream_end))
686685
}
@@ -1005,4 +1004,22 @@ mod tests {
10051004
assert_eq!(d.decompress(&encoded, None).unwrap(), b"later input");
10061005
assert!(d.eof());
10071006
}
1007+
1008+
#[test]
1009+
fn stream_end_clears_unconsumed_tail() {
1010+
let source = b"x".repeat(20000);
1011+
let data = [compress(&source, -1, MAX_WBITS).unwrap(), b"NEXT".to_vec()].concat();
1012+
let mut d = Decompressor::new(MAX_WBITS, None).unwrap();
1013+
let first = d.decompress(&data, Some(100)).unwrap();
1014+
assert!(!d.eof());
1015+
let tail = d.unconsumed_tail().to_vec();
1016+
let rest = d.decompress(&tail, Some(100_000)).unwrap();
1017+
assert!(d.eof());
1018+
assert_eq!([first, rest].concat(), source);
1019+
assert_eq!(d.unconsumed_tail(), b"");
1020+
assert_eq!(d.unused_data(), b"NEXT");
1021+
assert_eq!(d.flush(DEF_BUF_SIZE).unwrap(), b"");
1022+
assert_eq!(d.unconsumed_tail(), b"");
1023+
assert_eq!(d.unused_data(), b"NEXT");
1024+
}
10081025
}

‎extra_tests/snippets/stdlib_zlib.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,20 @@
6262

6363
assert_raises(zlib.error, lambda: zlib.compress(b"123", -40))
6464
assert_raises(zlib.error, lambda: zlib.compress(b"123", 10))
65+
66+
# A max_length-bounded call that reaches the end of the stream keeps no
67+
# stale input in unconsumed_tail (gh-8906). The tail is empty, or, as in
68+
# CPython before gh-158169, the bytes after the stream.
69+
source = b"x" * 20000
70+
d = zlib.decompressobj()
71+
out = d.decompress(zlib.compress(source) + b"NEXT", 100)
72+
assert not d.eof
73+
out += d.decompress(d.unconsumed_tail, 100000)
74+
assert d.eof
75+
assert out == source
76+
assert d.unused_data == b"NEXT"
77+
tail = d.unconsumed_tail
78+
assert tail in (b"", b"NEXT")
79+
assert d.flush() == b""
80+
assert d.unconsumed_tail == tail
81+
assert d.unused_data == b"NEXT" + tail

0 commit comments

Comments
 (0)