Skip to content

Commit 4ccc5b3

Browse files
NobodyXuNobodyXu
authored
Implement new traits DecodeV2/EncodeV2 (#398)
* Implement new traits `DecodeV2`/`EncodeV2` This new traits: - remove generics so it is trait object safe and avoid duplicate mono - use new `WriteBuffer<'_>` so we can add uninitialized buffer support to it in the future easily. * make WriteBuffer cdoc looks better Signed-off-by: Jiahao XU <[email protected]> * Fix documentation formatting for buffer layout of WriteBuffer Corrected code block formatting in documentation. Signed-off-by: Jiahao XU <[email protected]> --------- Signed-off-by: Jiahao XU <[email protected]>
1 parent 2c2af42 commit 4ccc5b3

File tree

26 files changed

+590
-482
lines changed

26 files changed

+590
-482
lines changed

‎crates/compression-codecs/src/brotli/decoder.rs‎

Lines changed: 26 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,45 @@
1-
use crate::Decode;
2-
use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult, BrotliState};
3-
use compression_core::util::PartialBuffer;
1+
use crate::DecodeV2;
2+
use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult};
3+
use compression_core::util::{PartialBuffer, WriteBuffer};
44
use std::{fmt, io};
55

6+
type BrotliState = brotli::BrotliState<StandardAlloc, StandardAlloc, StandardAlloc>;
7+
68
pub struct BrotliDecoder {
79
// `BrotliState` is very large (over 2kb) which is why we're boxing it.
8-
state: Box<BrotliState<StandardAlloc, StandardAlloc, StandardAlloc>>,
10+
state: Box<BrotliState>,
911
}
1012

1113
impl Default for BrotliDecoder {
1214
fn default() -> Self {
1315
Self {
14-
state: Box::new(BrotliState::new(
15-
StandardAlloc::default(),
16-
StandardAlloc::default(),
17-
StandardAlloc::default(),
18-
)),
16+
state: Box::new(Self::new_brotli_state()),
1917
}
2018
}
2119
}
2220

2321
impl BrotliDecoder {
22+
fn new_brotli_state() -> BrotliState {
23+
BrotliState::new(
24+
StandardAlloc::default(),
25+
StandardAlloc::default(),
26+
StandardAlloc::default(),
27+
)
28+
}
29+
2430
pub fn new() -> Self {
2531
Self::default()
2632
}
2733

2834
fn decode(
2935
&mut self,
30-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
31-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
36+
input: &mut PartialBuffer<&[u8]>,
37+
output: &mut WriteBuffer<'_>,
3238
) -> io::Result<BrotliResult> {
39+
output.initialize_unwritten();
40+
3341
let in_buf = input.unwritten();
34-
let out_buf = output.unwritten_mut();
42+
let out_buf = output.unwritten_initialized_mut();
3543

3644
let mut input_len = 0;
3745
let mut output_len = 0;
@@ -57,20 +65,16 @@ impl BrotliDecoder {
5765
}
5866
}
5967

60-
impl Decode for BrotliDecoder {
68+
impl DecodeV2 for BrotliDecoder {
6169
fn reinit(&mut self) -> io::Result<()> {
62-
self.state = Box::new(BrotliState::new(
63-
StandardAlloc::default(),
64-
StandardAlloc::default(),
65-
StandardAlloc::default(),
66-
));
70+
*self.state = Self::new_brotli_state();
6771
Ok(())
6872
}
6973

7074
fn decode(
7175
&mut self,
72-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
73-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
76+
input: &mut PartialBuffer<&[u8]>,
77+
output: &mut WriteBuffer<'_>,
7478
) -> io::Result<bool> {
7579
match self.decode(input, output)? {
7680
BrotliResult::ResultSuccess => Ok(true),
@@ -79,21 +83,15 @@ impl Decode for BrotliDecoder {
7983
}
8084
}
8185

82-
fn flush(
83-
&mut self,
84-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
85-
) -> io::Result<bool> {
86+
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
8687
match self.decode(&mut PartialBuffer::new(&[][..]), output)? {
8788
BrotliResult::ResultSuccess | BrotliResult::NeedsMoreInput => Ok(true),
8889
BrotliResult::NeedsMoreOutput => Ok(false),
8990
BrotliResult::ResultFailure => unreachable!(),
9091
}
9192
}
9293

93-
fn finish(
94-
&mut self,
95-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
96-
) -> io::Result<bool> {
94+
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
9795
match self.decode(&mut PartialBuffer::new(&[][..]), output)? {
9896
BrotliResult::ResultSuccess => Ok(true),
9997
BrotliResult::NeedsMoreOutput => Ok(false),

‎crates/compression-codecs/src/brotli/encoder.rs‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
use crate::{brotli::params::EncoderParams, Encode};
1+
use crate::{brotli::params::EncoderParams, EncodeV2};
22
use brotli::enc::{
33
backward_references::BrotliEncoderParams,
44
encode::{BrotliEncoderOperation, BrotliEncoderStateStruct},
55
StandardAlloc,
66
};
7-
use compression_core::util::PartialBuffer;
7+
use compression_core::util::{PartialBuffer, WriteBuffer};
88
use std::{fmt, io};
99

1010
pub struct BrotliEncoder {
@@ -21,12 +21,14 @@ impl BrotliEncoder {
2121

2222
fn encode(
2323
&mut self,
24-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
25-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
24+
input: &mut PartialBuffer<&[u8]>,
25+
output: &mut WriteBuffer<'_>,
2626
op: BrotliEncoderOperation,
2727
) -> io::Result<()> {
28+
output.initialize_unwritten();
29+
2830
let in_buf = input.unwritten();
29-
let out_buf = output.unwritten_mut();
31+
let out_buf = output.unwritten_initialized_mut();
3032

3133
let mut input_len = 0;
3234
let mut output_len = 0;
@@ -52,11 +54,11 @@ impl BrotliEncoder {
5254
}
5355
}
5456

55-
impl Encode for BrotliEncoder {
57+
impl EncodeV2 for BrotliEncoder {
5658
fn encode(
5759
&mut self,
58-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
59-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
60+
input: &mut PartialBuffer<&[u8]>,
61+
output: &mut WriteBuffer<'_>,
6062
) -> io::Result<()> {
6163
self.encode(
6264
input,
@@ -65,10 +67,7 @@ impl Encode for BrotliEncoder {
6567
)
6668
}
6769

68-
fn flush(
69-
&mut self,
70-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
71-
) -> io::Result<bool> {
70+
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
7271
self.encode(
7372
&mut PartialBuffer::new(&[][..]),
7473
output,
@@ -78,10 +77,7 @@ impl Encode for BrotliEncoder {
7877
Ok(!self.state.has_more_output())
7978
}
8079

81-
fn finish(
82-
&mut self,
83-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
84-
) -> io::Result<bool> {
80+
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
8581
self.encode(
8682
&mut PartialBuffer::new(&[][..]),
8783
output,

‎crates/compression-codecs/src/bzip2/decoder.rs‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
use crate::Decode;
1+
use crate::DecodeV2;
22
use bzip2::{Decompress, Status};
3-
use compression_core::util::PartialBuffer;
3+
use compression_core::util::{PartialBuffer, WriteBuffer};
44
use std::{fmt, io};
55

66
pub struct BzDecoder {
@@ -33,15 +33,17 @@ impl BzDecoder {
3333

3434
fn decode(
3535
&mut self,
36-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
37-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
36+
input: &mut PartialBuffer<&[u8]>,
37+
output: &mut WriteBuffer<'_>,
3838
) -> io::Result<Status> {
39+
output.initialize_unwritten();
40+
3941
let prior_in = self.decompress.total_in();
4042
let prior_out = self.decompress.total_out();
4143

4244
let status = self
4345
.decompress
44-
.decompress(input.unwritten(), output.unwritten_mut())
46+
.decompress(input.unwritten(), output.unwritten_initialized_mut())
4547
.map_err(io::Error::other)?;
4648

4749
input.advance((self.decompress.total_in() - prior_in) as usize);
@@ -51,16 +53,16 @@ impl BzDecoder {
5153
}
5254
}
5355

54-
impl Decode for BzDecoder {
56+
impl DecodeV2 for BzDecoder {
5557
fn reinit(&mut self) -> io::Result<()> {
5658
self.decompress = Decompress::new(false);
5759
Ok(())
5860
}
5961

6062
fn decode(
6163
&mut self,
62-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
63-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
64+
input: &mut PartialBuffer<&[u8]>,
65+
output: &mut WriteBuffer<'_>,
6466
) -> io::Result<bool> {
6567
match self.decode(input, output)? {
6668
// Decompression went fine, nothing much to report.
@@ -84,27 +86,21 @@ impl Decode for BzDecoder {
8486
}
8587
}
8688

87-
fn flush(
88-
&mut self,
89-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
90-
) -> io::Result<bool> {
89+
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
9190
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
9291

9392
loop {
94-
let old_len = output.written().len();
93+
let old_len = output.written_len();
9594
self.decode(&mut PartialBuffer::new(&[][..]), output)?;
96-
if output.written().len() == old_len {
95+
if output.written_len() == old_len {
9796
break;
9897
}
9998
}
10099

101-
Ok(!output.unwritten().is_empty())
100+
Ok(!output.has_no_spare_space())
102101
}
103102

104-
fn finish(
105-
&mut self,
106-
_output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
107-
) -> io::Result<bool> {
103+
fn finish(&mut self, _output: &mut WriteBuffer<'_>) -> io::Result<bool> {
108104
Ok(true)
109105
}
110106
}

‎crates/compression-codecs/src/bzip2/encoder.rs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
use crate::{bzip2::params::Bzip2EncoderParams, Encode};
1+
use crate::{bzip2::params::Bzip2EncoderParams, EncodeV2};
22
use bzip2::{Action, Compress, Compression, Status};
3-
use compression_core::util::PartialBuffer;
3+
use compression_core::util::{PartialBuffer, WriteBuffer};
44
use std::{fmt, io};
55

66
pub struct BzEncoder {
@@ -48,16 +48,22 @@ impl BzEncoder {
4848

4949
fn encode(
5050
&mut self,
51-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
52-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
51+
input: &mut PartialBuffer<&[u8]>,
52+
output: &mut WriteBuffer<'_>,
5353
action: Action,
5454
) -> io::Result<Status> {
55+
output.initialize_unwritten();
56+
5557
let prior_in = self.compress.total_in();
5658
let prior_out = self.compress.total_out();
5759

5860
let status = self
5961
.compress
60-
.compress(input.unwritten(), output.unwritten_mut(), action)
62+
.compress(
63+
input.unwritten(),
64+
output.unwritten_initialized_mut(),
65+
action,
66+
)
6167
.map_err(io::Error::other)?;
6268

6369
input.advance((self.compress.total_in() - prior_in) as usize);
@@ -67,11 +73,11 @@ impl BzEncoder {
6773
}
6874
}
6975

70-
impl Encode for BzEncoder {
76+
impl EncodeV2 for BzEncoder {
7177
fn encode(
7278
&mut self,
73-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
74-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
79+
input: &mut PartialBuffer<&[u8]>,
80+
output: &mut WriteBuffer<'_>,
7581
) -> io::Result<()> {
7682
match self.encode(input, output, Action::Run)? {
7783
// Decompression went fine, nothing much to report.
@@ -95,10 +101,7 @@ impl Encode for BzEncoder {
95101
}
96102
}
97103

98-
fn flush(
99-
&mut self,
100-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
101-
) -> io::Result<bool> {
104+
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
102105
match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Flush)? {
103106
// Decompression went fine, nothing much to report.
104107
Status::Ok => unreachable!(),
@@ -121,10 +124,7 @@ impl Encode for BzEncoder {
121124
}
122125
}
123126

124-
fn finish(
125-
&mut self,
126-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
127-
) -> io::Result<bool> {
127+
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result<bool> {
128128
match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Finish)? {
129129
// Decompression went fine, nothing much to report.
130130
Status::Ok => Ok(false),

‎crates/compression-codecs/src/deflate/decoder.rs‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
use crate::{Decode, FlateDecoder};
2-
use compression_core::util::PartialBuffer;
1+
use crate::{DecodeV2, FlateDecoder};
2+
use compression_core::util::{PartialBuffer, WriteBuffer};
33
use std::io::Result;
44

55
#[derive(Debug)]
@@ -21,31 +21,25 @@ impl DeflateDecoder {
2121
}
2222
}
2323

24-
impl Decode for DeflateDecoder {
24+
impl DecodeV2 for DeflateDecoder {
2525
fn reinit(&mut self) -> Result<()> {
2626
self.inner.reinit()?;
2727
Ok(())
2828
}
2929

3030
fn decode(
3131
&mut self,
32-
input: &mut PartialBuffer<impl AsRef<[u8]>>,
33-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
32+
input: &mut PartialBuffer<&[u8]>,
33+
output: &mut WriteBuffer<'_>,
3434
) -> Result<bool> {
3535
self.inner.decode(input, output)
3636
}
3737

38-
fn flush(
39-
&mut self,
40-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
41-
) -> Result<bool> {
38+
fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
4239
self.inner.flush(output)
4340
}
4441

45-
fn finish(
46-
&mut self,
47-
output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
48-
) -> Result<bool> {
42+
fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result<bool> {
4943
self.inner.finish(output)
5044
}
5145
}

0 commit comments

Comments
 (0)