File tree Expand file tree Collapse file tree 2 files changed +7
-9
lines changed
Expand file tree Collapse file tree 2 files changed +7
-9
lines changed Original file line number Diff line number Diff line change @@ -266,18 +266,14 @@ impl<W: Write + Send> ArrowWriter<W> {
266266 let row_group_writer_factory =
267267 ArrowRowGroupWriterFactory :: new ( & file_writer, arrow_schema. clone ( ) ) ;
268268
269- let cdc_chunkers = match props_ptr. content_defined_chunking ( ) {
270- Some ( opts) => {
271- let chunkers = file_writer
269+ let cdc_chunkers = props_ptr. content_defined_chunking ( ) . map ( |opts| {
270+ file_writer
272271 . schema_descr ( )
273272 . columns ( )
274273 . iter ( )
275274 . map ( |desc| ContentDefinedChunker :: new ( desc, opts) )
276- . collect :: < Result < Vec < _ > > > ( ) ?;
277- Some ( chunkers)
278- }
279- None => None ,
280- } ;
275+ . collect :: < Result < Vec < _ > > > ( )
276+ } ) . transpose ( ) ?;
281277
282278 Ok ( Self {
283279 writer : file_writer,
Original file line number Diff line number Diff line change @@ -83,10 +83,12 @@ pub struct CdcOptions {
8383 pub min_chunk_size : usize ,
8484 /// Maximum chunk size in bytes, default is 1024 KiB.
8585 /// The chunker will create a new chunk whenever the chunk size exceeds this value.
86- /// Note that the parquet writer has a related `data_page_size_limit` property that
86+ /// Note that the parquet writer has a related [ `data_page_size_limit`] property that
8787 /// controls the maximum size of a parquet data page after encoding. While setting
8888 /// `data_page_size_limit` to a smaller value than `max_chunk_size` doesn't affect
8989 /// the chunking effectiveness, it results in more small parquet data pages.
90+ ///
91+ /// [`data_page_size_limit`]: WriterPropertiesBuilder::set_data_page_size_limit
9092 pub max_chunk_size : usize ,
9193 /// Number of bit adjustment to the gearhash mask in order to center the chunk size
9294 /// around the average size more aggressively, default is 0.
You can’t perform that action at this time.
0 commit comments