diff --git a/examples/compress.rs b/examples/compress.rs index 4ecb7d2..b9772ee 100644 --- a/examples/compress.rs +++ b/examples/compress.rs @@ -12,6 +12,11 @@ fn get_data() -> std::result::Result, std::io::Error> { fn main() { let data = get_data().unwrap(); for _ in 0..1000 { - let _v = igzip::compress(Cursor::new(&data), igzip::CompressionLevel::Three, true).unwrap(); + let _v = igzip::compress( + Cursor::new(&data), + igzip::CompressionLevel::Three, + isal::igzip::Codec::Gzip, + ) + .unwrap(); } } diff --git a/src/igzip/mod.rs b/src/igzip/mod.rs index 4a0c7ba..d8a232b 100644 --- a/src/igzip/mod.rs +++ b/src/igzip/mod.rs @@ -11,18 +11,27 @@ use isal_sys::igzip_lib as isal; /// Buffer size pub const BUF_SIZE: usize = 16 * 1024; +/// Flavor of De/Compression to use if using the `isal::igzip::Encoder/Decoder` directly +/// and not the thin wrappers like `GzDecoder/GzEncoder` and similar. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum Codec { + Gzip = isal::IGZIP_GZIP, + Deflate = isal::IGZIP_DEFLATE, +} + /// Compress `input` directly into `output`. This is the fastest possible compression available. #[inline(always)] pub fn compress_into( input: &[u8], output: &mut [u8], level: CompressionLevel, - is_gzip: bool, + codec: Codec, ) -> Result { let mut zstream = ZStream::new(level, ZStreamKind::Stateless); zstream.stream.flush = FlushFlags::NoFlush as _; - zstream.stream.gzip_flag = is_gzip as _; + zstream.stream.gzip_flag = codec as _; zstream.stream.end_of_stream = 1; // read input into buffer @@ -42,10 +51,10 @@ pub fn compress_into( pub fn compress( input: R, level: CompressionLevel, - is_gzip: bool, + codec: Codec, ) -> Result> { let mut out = vec![]; - let mut encoder = read::Encoder::new(input, level, is_gzip); + let mut encoder = read::Encoder::new(input, level, codec); io::copy(&mut encoder, &mut out)?; Ok(out) } @@ -54,7 +63,7 @@ pub fn compress( #[inline(always)] pub fn decompress(input: R) -> Result> { let mut out = vec![]; - let mut decoder = read::Decoder::new(input); + let mut decoder = read::Decoder::new(input, Codec::Gzip); io::copy(&mut decoder, &mut out)?; Ok(out) } @@ -313,7 +322,7 @@ pub mod tests { // Default testing data pub fn gen_large_data() -> Vec { - (0..(BUF_SIZE * 2) + 1) + (0..(BUF_SIZE * 20) + 1) .map(|_| b"oh what a beautiful morning, oh what a beautiful day!!".to_vec()) .flat_map(|v| v) .collect() @@ -334,7 +343,12 @@ pub mod tests { fn basic_compress_into() -> Result<()> { let data = get_data()?; let mut output = vec![0_u8; data.len()]; // assume compression isn't worse than input len. - let n_bytes = compress_into(data.as_slice(), &mut output, CompressionLevel::Three, true)?; + let n_bytes = compress_into( + data.as_slice(), + &mut output, + CompressionLevel::Three, + Codec::Gzip, + )?; println!( "n_bytes: {} - {:?}", n_bytes, @@ -346,7 +360,7 @@ pub mod tests { #[test] fn basic_compress() -> Result<()> { let data = get_data()?; - let output = compress(Cursor::new(data), CompressionLevel::Three, true)?; + let output = compress(Cursor::new(data), CompressionLevel::Three, Codec::Gzip)?; println!( "n_bytes: {:?}", &output[..std::cmp::min(output.len() - 1, 100)] @@ -384,7 +398,7 @@ pub mod tests { fn larger_decompress() -> Result<()> { /* Decompress data which is larger than BUF_SIZE */ let data = get_data()?; - let compressed = compress(Cursor::new(&data), CompressionLevel::Three, true)?; + let compressed = compress(Cursor::new(&data), CompressionLevel::Three, Codec::Gzip)?; let decompressed = decompress(Cursor::new(compressed))?; assert!(same_same(&data, &decompressed)); Ok(()) @@ -407,7 +421,7 @@ pub mod tests { #[test] fn basic_round_trip() -> Result<()> { let data = b"hello, world!"; - let compressed = compress(Cursor::new(&data), CompressionLevel::Three, true)?; + let compressed = compress(Cursor::new(&data), CompressionLevel::Three, Codec::Gzip)?; let decompressed = decompress(Cursor::new(compressed))?; assert_eq!(decompressed, data); Ok(()) @@ -417,14 +431,15 @@ pub mod tests { fn basic_round_trip_into() -> Result<()> { let data = b"hello, world!".to_vec(); - let compressed_len = compress(Cursor::new(&data), CompressionLevel::Three, true)?.len(); + let compressed_len = + compress(Cursor::new(&data), CompressionLevel::Three, Codec::Gzip)?.len(); let decompressed_len = data.len(); let mut compressed = vec![0; compressed_len]; let mut decompressed = vec![0; decompressed_len]; // compress_into - let n_bytes = compress_into(&data, &mut compressed, CompressionLevel::Three, true)?; + let n_bytes = compress_into(&data, &mut compressed, CompressionLevel::Three, Codec::Gzip)?; assert_eq!(n_bytes, compressed_len); // decompress_into @@ -440,14 +455,15 @@ pub mod tests { fn large_round_trip_into() -> Result<()> { let data = gen_large_data(); - let compressed_len = compress(Cursor::new(&data), CompressionLevel::Three, true)?.len(); + let compressed_len = + compress(Cursor::new(&data), CompressionLevel::Three, Codec::Gzip)?.len(); let decompressed_len = data.len(); let mut compressed = vec![0; compressed_len]; let mut decompressed = vec![0; decompressed_len]; // compress_into - let n_bytes = compress_into(&data, &mut compressed, CompressionLevel::Three, true)?; + let n_bytes = compress_into(&data, &mut compressed, CompressionLevel::Three, Codec::Gzip)?; assert!(n_bytes < data.len()); // decompress_into diff --git a/src/igzip/read.rs b/src/igzip/read.rs index efc2202..b04beef 100644 --- a/src/igzip/read.rs +++ b/src/igzip/read.rs @@ -3,6 +3,46 @@ use crate::igzip::*; use mem::MaybeUninit; use std::io; +/// Deflate decompression +/// Basically a wrapper to `Encoder` which sets the codec for you. +pub struct DeflateEncoder { + inner: Encoder, +} + +impl DeflateEncoder { + pub fn new(reader: R, level: CompressionLevel) -> Self { + Self { + inner: Encoder::new(reader, level, Codec::Deflate), + } + } +} + +impl io::Read for DeflateEncoder { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + +/// Deflate decompression +/// Basically a wrapper to `Decoder` which sets the codec for you. +pub struct DeflateDecoder { + inner: Decoder, +} + +impl DeflateDecoder { + pub fn new(reader: R) -> Self { + Self { + inner: Decoder::new(reader, Codec::Deflate), + } + } +} + +impl io::Read for DeflateDecoder { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + /// Streaming compression for input streams implementing `std::io::Read`. /// /// Notes @@ -14,10 +54,10 @@ use std::io; /// ------- /// ``` /// use std::{io, io::Read}; -/// use isal::igzip::{read::Encoder, CompressionLevel, decompress}; +/// use isal::igzip::{read::Encoder, CompressionLevel, decompress, Codec}; /// let data = b"Hello, World!".to_vec(); /// -/// let mut encoder = Encoder::new(data.as_slice(), CompressionLevel::Three, true); +/// let mut encoder = Encoder::new(data.as_slice(), CompressionLevel::Three, Codec::Gzip); /// let mut compressed = vec![]; /// /// // Numbeer of compressed bytes written to `output` @@ -38,7 +78,7 @@ pub struct Encoder { impl Encoder { /// Create a new `Encoder` which implements the `std::io::Read` trait. - pub fn new(reader: R, level: CompressionLevel, is_gzip: bool) -> Encoder { + pub fn new(reader: R, level: CompressionLevel, codec: Codec) -> Encoder { let in_buf = [0_u8; BUF_SIZE]; let out_buf = Vec::with_capacity(BUF_SIZE); @@ -46,7 +86,7 @@ impl Encoder { zstream.stream.end_of_stream = 0; zstream.stream.flush = FlushFlags::SyncFlush as _; - zstream.stream.gzip_flag = is_gzip as _; + zstream.stream.gzip_flag = codec as _; Self { inner: reader, @@ -133,11 +173,11 @@ impl io::Read for Encoder { /// ------- /// ``` /// use std::{io, io::Read}; -/// use isal::igzip::{read::Decoder, CompressionLevel, compress}; +/// use isal::igzip::{read::Decoder, CompressionLevel, compress, Codec}; /// let data = b"Hello, World!".to_vec(); /// -/// let compressed = compress(data.as_slice(), CompressionLevel::Three, true).unwrap(); -/// let mut decoder = Decoder::new(compressed.as_slice()); +/// let compressed = compress(data.as_slice(), CompressionLevel::Three, Codec::Gzip).unwrap(); +/// let mut decoder = Decoder::new(compressed.as_slice(), Codec::Gzip); /// let mut decompressed = vec![]; /// /// // Numbeer of compressed bytes written to `output` @@ -152,12 +192,13 @@ pub struct Decoder { out_buf: Vec, dsts: usize, dste: usize, + codec: Codec, } impl Decoder { - pub fn new(reader: R) -> Decoder { + pub fn new(reader: R, codec: Codec) -> Decoder { let mut zst = InflateState::new(); - zst.0.crc_flag = isal::IGZIP_GZIP; + zst.0.crc_flag = codec as _; Self { inner: reader, @@ -166,6 +207,7 @@ impl Decoder { out_buf: Vec::with_capacity(BUF_SIZE), dste: 0, dsts: 0, + codec, } } @@ -201,12 +243,15 @@ impl io::Read for Decoder { Ok(count) } else { // Read out next buf len worth to compress; filling intermediate out_buf + debug_assert_eq!(self.zst.0.avail_in, 0); self.zst.0.avail_in = self.inner.read(&mut self.in_buf)? as _; self.zst.0.next_in = self.in_buf.as_mut_ptr(); let mut n_bytes = 0; while self.zst.0.avail_in != 0 { - if self.zst.block_state() == isal::isal_block_state_ISAL_BLOCK_NEW_HDR { + if self.codec == Codec::Gzip + && self.zst.block_state() == isal::isal_block_state_ISAL_BLOCK_NEW_HDR + { // Read this member's gzip header let mut gz_hdr: MaybeUninit = MaybeUninit::uninit(); unsafe { isal::isal_gzip_header_init(gz_hdr.as_mut_ptr()) }; @@ -214,6 +259,7 @@ impl io::Read for Decoder { read_gzip_header(&mut self.zst.0, &mut gz_hdr)?; } + // TODO: I'm pretty sure we can remove out_buf // decompress member loop { self.out_buf.resize(n_bytes + BUF_SIZE, 0); @@ -226,12 +272,28 @@ impl io::Read for Decoder { n_bytes += BUF_SIZE - self.zst.0.avail_out as usize; let state = self.zst.block_state(); - if state == isal::isal_block_state_ISAL_BLOCK_CODED - || state == isal::isal_block_state_ISAL_BLOCK_TYPE0 - || state == isal::isal_block_state_ISAL_BLOCK_HDR - || state == isal::isal_block_state_ISAL_BLOCK_FINISH - { - break; + match self.codec { + Codec::Deflate => { + if state == isal::isal_block_state_ISAL_BLOCK_FINISH { + break; + + // refill avail in, still actively decoding but reached end of input + } else if state == isal::isal_block_state_ISAL_BLOCK_CODED + && self.zst.0.avail_in == 0 + { + self.zst.0.avail_in = self.inner.read(&mut self.in_buf)? as _; + self.zst.0.next_in = self.in_buf.as_mut_ptr(); + } + } + Codec::Gzip => { + if state == isal::isal_block_state_ISAL_BLOCK_CODED + || state == isal::isal_block_state_ISAL_BLOCK_TYPE0 + || state == isal::isal_block_state_ISAL_BLOCK_HDR + || state == isal::isal_block_state_ISAL_BLOCK_FINISH + { + break; + } + } } } if self.zst.0.block_state == isal::isal_block_state_ISAL_BLOCK_FINISH { @@ -255,15 +317,21 @@ mod tests { use std::io::{self, Cursor}; #[test] - fn large_roundtrip() { - let input = gen_large_data(); - let mut encoder = Encoder::new(Cursor::new(&input), CompressionLevel::Three, true); + fn roundtrip_small() { + roundtrip(b"foobar") + } + #[test] + fn roundtrip_large() { + roundtrip(&gen_large_data()) + } + fn roundtrip(input: &[u8]) { + let mut encoder = Encoder::new(Cursor::new(&input), CompressionLevel::Three, Codec::Gzip); let mut output = vec![]; let n = io::copy(&mut encoder, &mut output).unwrap(); - assert!(n < input.len() as u64); + assert_eq!(n as usize, output.len()); - let mut decoder = Decoder::new(Cursor::new(output)); + let mut decoder = Decoder::new(Cursor::new(output), Codec::Gzip); let mut decompressed = vec![]; let nbytes = io::copy(&mut decoder, &mut decompressed).unwrap(); @@ -272,9 +340,15 @@ mod tests { } #[test] - fn basic_compress() -> Result<()> { - let input = b"hello, world!"; - let mut encoder = Encoder::new(Cursor::new(input), CompressionLevel::Three, true); + fn basic_compress_small() -> Result<()> { + basic_compress(b"foobar") + } + #[test] + fn basic_compress_large() -> Result<()> { + basic_compress(&gen_large_data()) + } + fn basic_compress(input: &[u8]) -> Result<()> { + let mut encoder = Encoder::new(Cursor::new(input), CompressionLevel::Three, Codec::Gzip); let mut output = vec![]; let n = io::copy(&mut encoder, &mut output)? as usize; @@ -295,7 +369,7 @@ mod tests { } } - let mut encoder = Encoder::new(Cursor::new(&input), CompressionLevel::Three, true); + let mut encoder = Encoder::new(Cursor::new(&input), CompressionLevel::Three, Codec::Gzip); let mut output = vec![]; let n = io::copy(&mut encoder, &mut output)? as usize; @@ -306,11 +380,17 @@ mod tests { } #[test] - fn basic_decompress() -> Result<()> { - let input = b"hello, world!"; - let compressed = compress(Cursor::new(input), CompressionLevel::Three, true)?; + fn basic_decompress_small() -> Result<()> { + basic_decompress(b"foobar") + } + #[test] + fn basic_decompress_large() -> Result<()> { + basic_decompress(&gen_large_data()) + } + fn basic_decompress(input: &[u8]) -> Result<()> { + let compressed = compress(Cursor::new(input), CompressionLevel::Three, Codec::Gzip)?; - let mut decoder = Decoder::new(compressed.as_slice()); + let mut decoder = Decoder::new(compressed.as_slice(), Codec::Gzip); let mut decompressed = vec![]; let n = io::copy(&mut decoder, &mut decompressed)? as usize; @@ -330,9 +410,9 @@ mod tests { } } - let compressed = compress(input.as_slice(), CompressionLevel::Three, true)?; + let compressed = compress(input.as_slice(), CompressionLevel::Three, Codec::Gzip)?; - let mut decoder = Decoder::new(compressed.as_slice()); + let mut decoder = Decoder::new(compressed.as_slice(), Codec::Gzip); let mut decompressed = vec![]; let n = io::copy(&mut decoder, &mut decompressed)? as usize; @@ -343,11 +423,16 @@ mod tests { } #[test] - fn flate2_gzip_compat_encoder_out() { - let data = gen_large_data(); - + fn flate2_gzip_compat_encoder_out_small() { + flate2_gzip_compat_encoder_out(b"foobar") + } + #[test] + fn flate2_gzip_compat_encoder_out_large() { + flate2_gzip_compat_encoder_out(&gen_large_data()) + } + fn flate2_gzip_compat_encoder_out(data: &[u8]) { // our encoder - let mut encoder = Encoder::new(data.as_slice(), CompressionLevel::Three, true); + let mut encoder = Encoder::new(data, CompressionLevel::Three, Codec::Gzip); let mut compressed = vec![]; io::copy(&mut encoder, &mut compressed).unwrap(); @@ -360,20 +445,69 @@ mod tests { } #[test] - fn flate2_gzip_compat_decoder_out() { - let data = gen_large_data(); + fn flate2_gzip_compat_decoder_out_small() { + flate2_gzip_compat_decoder_out(b"foobar") + } + #[test] + fn flate2_gzip_compat_decoder_out_large() { + flate2_gzip_compat_decoder_out(&gen_large_data()) + } + fn flate2_gzip_compat_decoder_out(data: &[u8]) { + // their encoder + let mut encoder = flate2::read::GzEncoder::new(data, flate2::Compression::fast()); + let mut compressed = vec![]; + io::copy(&mut encoder, &mut compressed).unwrap(); + // our decoder + let mut decoder = Decoder::new(compressed.as_slice(), Codec::Gzip); + let mut decompressed = vec![]; + io::copy(&mut decoder, &mut decompressed).unwrap(); + + assert!(same_same(&data, &decompressed)); + } + + #[test] + fn flate2_deflate_compat_encoder_out_small() { + flate2_deflate_compat_encoder_out(b"foobar") + } + #[test] + fn flate2_deflate_compat_encoder_out_large() { + flate2_deflate_compat_encoder_out(&gen_large_data()) + } + fn flate2_deflate_compat_encoder_out(data: &[u8]) { + // our encoder + let mut encoder = DeflateEncoder::new(data, CompressionLevel::Three); + let mut compressed = vec![]; + io::copy(&mut encoder, &mut compressed).unwrap(); + + // their decoder + let mut decoder = flate2::read::DeflateDecoder::new(compressed.as_slice()); + let mut decompressed = vec![]; + io::copy(&mut decoder, &mut decompressed).unwrap(); + + assert!(same_same(&data, &decompressed)); + } + + #[test] + fn flate2_deflate_compat_decoder_out_small() { + flate2_deflate_compat_decoder_out(b"foobar") + } + #[test] + fn flate2_deflate_compat_decoder_out_large() { + flate2_deflate_compat_decoder_out(&gen_large_data()) + } + fn flate2_deflate_compat_decoder_out(data: &[u8]) { // their encoder - let mut encoder = - flate2::read::GzEncoder::new(data.as_slice(), flate2::Compression::fast()); + let mut encoder = flate2::read::DeflateEncoder::new(data, flate2::Compression::fast()); let mut compressed = vec![]; io::copy(&mut encoder, &mut compressed).unwrap(); // our decoder - let mut decoder = Decoder::new(compressed.as_slice()); + let mut decoder = DeflateDecoder::new(compressed.as_slice()); let mut decompressed = vec![]; io::copy(&mut decoder, &mut decompressed).unwrap(); + assert_eq!(data.len(), decompressed.len()); assert!(same_same(&data, &decompressed)); } } diff --git a/src/igzip/write.rs b/src/igzip/write.rs index 5cd8817..d3c6c22 100644 --- a/src/igzip/write.rs +++ b/src/igzip/write.rs @@ -3,6 +3,52 @@ use crate::igzip::*; use std::io; use std::io::Write; +/// Deflate compression +/// Basically a wrapper to `Encoder` which sets the codec for you. +pub struct DeflateEncoder { + inner: Encoder, +} + +impl DeflateEncoder { + pub fn new(writer: W, level: CompressionLevel) -> Self { + Self { + inner: Encoder::new(writer, level, Codec::Deflate), + } + } +} + +impl io::Write for DeflateEncoder { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// Deflate decompression +/// Basically a wrapper to `Decoder` which sets the codec for you. +pub struct DeflateDecoder { + inner: Decoder, +} + +impl DeflateDecoder { + pub fn new(writer: W) -> Self { + Self { + inner: Decoder::new(writer, Codec::Deflate), + } + } +} + +impl io::Write for DeflateDecoder { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + /// Streaming compression for input streams implementing `std::io::Write`. /// /// Notes @@ -14,12 +60,12 @@ use std::io::Write; /// ------- /// ``` /// use std::{io, io::Write}; -/// use isal::igzip::{write::Encoder, CompressionLevel, decompress}; +/// use isal::igzip::{write::Encoder, CompressionLevel, decompress, Codec}; /// /// let data = b"Hello, World!".to_vec(); /// let mut compressed = vec![]; /// -/// let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, true); +/// let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, Codec::Gzip); /// /// // Numbeer of compressed bytes written to `output` /// io::copy(&mut io::Cursor::new(&data), &mut encoder).unwrap(); @@ -43,14 +89,14 @@ pub struct Encoder { impl Encoder { /// Create a new `Encoder` which implements the `std::io::Read` trait. - pub fn new(writer: W, level: CompressionLevel, is_gzip: bool) -> Encoder { + pub fn new(writer: W, level: CompressionLevel, codec: Codec) -> Encoder { let out_buf = Vec::with_capacity(BUF_SIZE); let mut zstream = ZStream::new(level, ZStreamKind::Stateful); zstream.stream.end_of_stream = 0; zstream.stream.flush = FlushFlags::NoFlush as _; - zstream.stream.gzip_flag = is_gzip as _; + zstream.stream.gzip_flag = codec as _; Self { inner: writer, @@ -167,13 +213,13 @@ impl io::Write for Encoder { /// ------- /// ``` /// use std::{io, io::Write}; -/// use isal::igzip::{write::Decoder, CompressionLevel, compress}; +/// use isal::igzip::{write::Decoder, CompressionLevel, compress, Codec}; /// let data = b"Hello, World!".to_vec(); /// -/// let compressed = compress(io::Cursor::new(data.as_slice()), CompressionLevel::Three, true).unwrap(); +/// let compressed = compress(io::Cursor::new(data.as_slice()), CompressionLevel::Three, Codec::Gzip).unwrap(); /// /// let mut decompressed = vec![]; -/// let mut decoder = Decoder::new(&mut decompressed); +/// let mut decoder = Decoder::new(&mut decompressed, Codec::Gzip); /// /// // Numbeer of compressed bytes written to `output` /// let n = io::copy(&mut io::Cursor::new(&compressed), &mut decoder).unwrap(); @@ -186,12 +232,13 @@ pub struct Decoder { out_buf: Vec, dsts: usize, dste: usize, + codec: Codec, } impl Decoder { - pub fn new(writer: W) -> Decoder { + pub fn new(writer: W, codec: Codec) -> Decoder { let mut zst = InflateState::new(); - zst.0.crc_flag = isal::IGZIP_GZIP; + zst.0.crc_flag = codec as _; Self { inner: writer, @@ -199,6 +246,7 @@ impl Decoder { out_buf: Vec::with_capacity(BUF_SIZE), dste: 0, dsts: 0, + codec, } } @@ -228,12 +276,15 @@ impl io::Write for Decoder { fn write(&mut self, buf: &[u8]) -> io::Result { // Check if there is data left in out_buf, otherwise refill; if end state, return 0 // Read out next buf len worth to compress; filling intermediate out_buf + debug_assert_eq!(self.zst.0.avail_in, 0); self.zst.0.avail_in = buf.len() as _; self.zst.0.next_in = buf.as_ptr() as *mut _; let mut n_bytes = 0; while self.zst.0.avail_in > 0 { - if self.zst.block_state() == isal::isal_block_state_ISAL_BLOCK_NEW_HDR { + if self.codec == Codec::Gzip + && self.zst.block_state() == isal::isal_block_state_ISAL_BLOCK_NEW_HDR + { // Read this member's gzip header let mut gz_hdr: mem::MaybeUninit = mem::MaybeUninit::uninit(); @@ -254,12 +305,25 @@ impl io::Write for Decoder { n_bytes += BUF_SIZE - self.zst.0.avail_out as usize; let state = self.zst.block_state(); - if state == isal::isal_block_state_ISAL_BLOCK_CODED - || state == isal::isal_block_state_ISAL_BLOCK_TYPE0 - || state == isal::isal_block_state_ISAL_BLOCK_HDR - || state == isal::isal_block_state_ISAL_BLOCK_FINISH - { - break; + match self.codec { + Codec::Deflate => { + // On block finished we're done done w/ the block, + // on block coded, we need to move onto the next input buffer + if state == isal::isal_block_state_ISAL_BLOCK_FINISH + || state == isal::isal_block_state_ISAL_BLOCK_CODED + { + break; + } + } + Codec::Gzip => { + if state == isal::isal_block_state_ISAL_BLOCK_CODED + || state == isal::isal_block_state_ISAL_BLOCK_TYPE0 + || state == isal::isal_block_state_ISAL_BLOCK_HDR + || state == isal::isal_block_state_ISAL_BLOCK_FINISH + { + break; + } + } } } if self.zst.0.block_state == isal::isal_block_state_ISAL_BLOCK_FINISH { @@ -274,7 +338,12 @@ impl io::Write for Decoder { Ok(buf.len()) } fn flush(&mut self) -> io::Result<()> { - Ok(()) + loop { + if self.write_from_out_buf()? == 0 { + break; + } + } + self.inner.flush() } } @@ -287,11 +356,16 @@ pub mod tests { use std::io::Write; #[test] - fn test_encoder_basic() { - let data = gen_large_data(); - + fn test_encoder_basic_small() { + test_encoder_basic(&gen_large_data()) + } + #[test] + fn test_encoder_basic_large() { + test_encoder_basic(&gen_large_data()) + } + fn test_encoder_basic(data: &[u8]) { let mut compressed = vec![]; - let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, true); + let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, Codec::Gzip); let nbytes = io::copy(&mut io::Cursor::new(&data), &mut encoder).unwrap(); // Footer isn't written until .flush is called @@ -319,7 +393,7 @@ pub mod tests { let second = b"bar"; let mut compressed = vec![]; - let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, true); + let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, Codec::Gzip); encoder.write_all(first).unwrap(); encoder.flush().unwrap(); @@ -334,14 +408,20 @@ pub mod tests { } #[test] - fn test_decoder_basic() { - let data = gen_large_data(); - + fn test_decoder_basic_small() { + test_decoder_basic(b"foobar") + } + #[test] + fn test_decoder_basic_large() { + test_decoder_basic(&gen_large_data()) + } + fn test_decoder_basic(data: &[u8]) { let compressed = - crate::igzip::compress(io::Cursor::new(&data), CompressionLevel::Three, true).unwrap(); + crate::igzip::compress(io::Cursor::new(&data), CompressionLevel::Three, Codec::Gzip) + .unwrap(); let mut decompressed = vec![]; - let mut decoder = Decoder::new(&mut decompressed); + let mut decoder = Decoder::new(&mut decompressed, Codec::Gzip); let nbytes = io::copy(&mut io::Cursor::new(&compressed), &mut decoder).unwrap(); assert_eq!(nbytes, compressed.len() as u64); assert!(same_same(&decompressed, &data)); @@ -352,15 +432,23 @@ pub mod tests { let first = b"foo"; let second = b"bar"; - let mut compressed = - crate::igzip::compress(io::Cursor::new(&first), CompressionLevel::Three, true).unwrap(); + let mut compressed = crate::igzip::compress( + io::Cursor::new(&first), + CompressionLevel::Three, + Codec::Gzip, + ) + .unwrap(); compressed.extend( - crate::igzip::compress(io::Cursor::new(&second), CompressionLevel::Three, true) - .unwrap(), + crate::igzip::compress( + io::Cursor::new(&second), + CompressionLevel::Three, + Codec::Gzip, + ) + .unwrap(), ); let mut decompressed = vec![]; - let mut decoder = Decoder::new(&mut decompressed); + let mut decoder = Decoder::new(&mut decompressed, Codec::Gzip); let nbytes = io::copy(&mut io::Cursor::new(&compressed), &mut decoder).unwrap(); assert_eq!(nbytes, compressed.len() as _); @@ -368,13 +456,18 @@ pub mod tests { } #[test] - fn flate2_gzip_compat_encoder_out() { - let data = gen_large_data(); - + fn flate2_gzip_compat_encoder_out_small() { + flate2_gzip_compat_encoder_out(b"foobar") + } + #[test] + fn flate2_gzip_compat_encoder_out_large() { + flate2_gzip_compat_encoder_out(&gen_large_data()) + } + fn flate2_gzip_compat_encoder_out(data: &[u8]) { // our encoder let mut compressed = vec![]; { - let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, true); + let mut encoder = Encoder::new(&mut compressed, CompressionLevel::Three, Codec::Gzip); io::copy(&mut Cursor::new(&data), &mut encoder).unwrap(); encoder.flush().unwrap(); } @@ -391,9 +484,14 @@ pub mod tests { } #[test] - fn flate2_gzip_compat_decoder_out() { - let data = gen_large_data(); - + fn flate2_gzip_compat_decoder_out_small() { + flate2_gzip_compat_decoder_out(b"foobar"); + } + #[test] + fn flate2_gzip_compat_decoder_out_large() { + flate2_gzip_compat_decoder_out(&gen_large_data()); + } + fn flate2_gzip_compat_decoder_out(data: &[u8]) { // their encoder let mut compressed = vec![]; { @@ -406,11 +504,74 @@ pub mod tests { // our decoder let mut decompressed = vec![]; { - let mut decoder = Decoder::new(&mut decompressed); + let mut decoder = Decoder::new(&mut decompressed, Codec::Gzip); + io::copy(&mut Cursor::new(&compressed), &mut decoder).unwrap(); + decoder.flush().unwrap(); + } + + assert!(same_same(&data, &decompressed)); + } + + #[test] + fn flate2_deflate_compat_encoder_out_small() { + flate2_deflate_compat_encoder_out(b"foobar"); + } + #[test] + fn flate2_deflate_compat_encoder_out_large() { + flate2_deflate_compat_encoder_out(&gen_large_data()); + } + fn flate2_deflate_compat_encoder_out(data: &[u8]) { + // our encoder + let mut compressed = vec![]; + { + let mut encoder = DeflateEncoder::new(&mut compressed, CompressionLevel::Three); + io::copy(&mut Cursor::new(&data), &mut encoder).unwrap(); + encoder.flush().unwrap(); // TODO: impl flush on drop + } + + // their decoder + let mut decompressed = vec![]; + { + let mut decoder = flate2::write::DeflateDecoder::new(&mut decompressed); io::copy(&mut Cursor::new(&compressed), &mut decoder).unwrap(); decoder.flush().unwrap(); } assert!(same_same(&data, &decompressed)); } + + #[test] + fn flate2_deflate_compat_decoder_out_small() { + flate2_deflate_compat_decoder_out(b"foobar"); + } + #[test] + fn flate2_deflate_compat_decoder_out_large() { + flate2_deflate_compat_decoder_out(&gen_large_data()); + } + fn flate2_deflate_compat_decoder_out(data: &[u8]) { + // their encoder + let mut compressed = vec![]; + { + let mut encoder = + flate2::write::DeflateEncoder::new(&mut compressed, flate2::Compression::fast()); + io::copy(&mut Cursor::new(&data), &mut encoder).unwrap(); + encoder.flush().unwrap(); + } + + // our decoder + let mut decompressed = vec![]; + { + let mut decoder = DeflateDecoder::new(&mut decompressed); + io::copy(&mut Cursor::new(&compressed), &mut decoder).unwrap(); + decoder.flush().unwrap(); // TODO: impl flush on drop + } + + println!( + "data.len() - decompressed.len() = {}", + data.len() - decompressed.len() + ); + + assert_eq!(data.len(), decompressed.len()); + assert!(same_same(&data, &decompressed)); + } }