File tree Expand file tree Collapse file tree 4 files changed +13
-9
lines changed Expand file tree Collapse file tree 4 files changed +13
-9
lines changed Original file line number Diff line number Diff line change @@ -58,6 +58,12 @@ use crate::{
58
58
mod staging;
59
59
mod streams;
60
60
61
+ /// File extension for "finish"ed arrow files in staging
62
+ const ARROW_FILE_EXTENSION : & str = "data.arrows" ;
63
+
64
+ /// File extension for un"finish"ed arrow files in staging
65
+ const ARROW_PART_FILE_EXTENSION : & str = "part.arrows" ;
66
+
61
67
/// Name of a Stream
62
68
/// NOTE: this used to be a struct, flattened out for simplicity
63
69
pub type LogStream = String ;
Original file line number Diff line number Diff line change 20
20
pub mod reader;
21
21
pub mod writer;
22
22
23
- /// File extension for "finish"ed arrow files in staging
24
- const ARROW_FILE_EXTENSION : & str = "data.arrows" ;
25
-
26
- /// File extension for un"finish"ed arrow files in staging
27
- const ARROW_PART_FILE_EXTENSION : & str = "part.arrows" ;
28
-
29
23
#[ derive( Debug , thiserror:: Error ) ]
30
24
pub enum StagingError {
31
25
#[ error( "Unable to create recordbatch stream" ) ]
Original file line number Diff line number Diff line change @@ -31,9 +31,10 @@ use arrow_schema::Schema;
31
31
use arrow_select:: concat:: concat_batches;
32
32
use itertools:: Itertools ;
33
33
34
+ use crate :: parseable:: { ARROW_FILE_EXTENSION , ARROW_PART_FILE_EXTENSION } ;
34
35
use crate :: utils:: arrow:: adapt_batch;
35
36
36
- use super :: { StagingError , ARROW_FILE_EXTENSION , ARROW_PART_FILE_EXTENSION } ;
37
+ use super :: StagingError ;
37
38
38
39
/// Context regarding `.arrows` file being persisted onto disk
39
40
pub struct DiskWriter {
Original file line number Diff line number Diff line change @@ -61,7 +61,7 @@ use super::{
61
61
writer:: { DiskWriter , Writer } ,
62
62
StagingError ,
63
63
} ,
64
- LogStream ,
64
+ LogStream , ARROW_FILE_EXTENSION ,
65
65
} ;
66
66
67
67
#[ derive( Debug , thiserror:: Error ) ]
@@ -170,7 +170,10 @@ impl Stream {
170
170
let paths = dir
171
171
. flatten ( )
172
172
. map ( |file| file. path ( ) )
173
- . filter ( |file| file. extension ( ) . is_some_and ( |ext| ext. eq ( "arrows" ) ) )
173
+ . filter ( |path| {
174
+ path. file_name ( )
175
+ . is_some_and ( |f| f. to_string_lossy ( ) . ends_with ( ARROW_FILE_EXTENSION ) )
176
+ } )
174
177
. sorted_by_key ( |f| f. metadata ( ) . unwrap ( ) . modified ( ) . unwrap ( ) )
175
178
. collect ( ) ;
176
179
You can’t perform that action at this time.
0 commit comments