@@ -28,6 +28,18 @@ fn string_datum_strategy() -> impl Strategy<Value = Datum> {
2828 any :: < String > ( ) . prop_map ( Datum :: String )
2929}
3030
31+ fn string_datum_strategy_for_whitespace ( ) -> impl Strategy < Value = Datum > {
32+ prop:: collection:: vec (
33+ prop_oneof ! [
34+ prop:: char :: range( ' ' , ';' ) , // space through ;
35+ prop:: char :: range( '=' , '=' ) , // =
36+ prop:: char :: range( '?' , '~' ) , // ? through ~
37+ ] ,
38+ 0 ..20 ,
39+ )
40+ . prop_map ( |v| Datum :: String ( v. into_iter ( ) . collect ( ) ) )
41+ }
42+
3143fn boolean_datum_strategy ( ) -> impl Strategy < Value = Datum > {
3244 any :: < bool > ( ) . prop_map ( Datum :: Boolean )
3345}
@@ -51,54 +63,117 @@ fn time_datum_strategy() -> impl Strategy<Value = Datum> {
5163 } )
5264}
5365
54- fn datum_strategy ( ) -> impl Strategy < Value = Datum > {
55- prop_oneof ! [
56- string_datum_strategy( ) ,
57- boolean_datum_strategy( ) ,
58- number_datum_strategy( ) ,
59- date_datum_strategy( ) ,
60- time_datum_strategy( ) ,
61- ]
66+ fn datum_strategy ( whitespace : bool ) -> impl Strategy < Value = Datum > {
67+ if whitespace {
68+ prop_oneof ! [
69+ string_datum_strategy_for_whitespace( ) ,
70+ boolean_datum_strategy( ) ,
71+ number_datum_strategy( ) ,
72+ date_datum_strategy( ) ,
73+ time_datum_strategy( ) ,
74+ ]
75+ . boxed ( )
76+ } else {
77+ prop_oneof ! [
78+ string_datum_strategy( ) ,
79+ boolean_datum_strategy( ) ,
80+ number_datum_strategy( ) ,
81+ date_datum_strategy( ) ,
82+ time_datum_strategy( ) ,
83+ ]
84+ . boxed ( )
85+ }
6286}
6387
64- fn record_strategy ( ) -> impl Strategy < Value = Record > {
65- prop:: collection:: hash_map ( field_name_strategy ( ) , datum_strategy ( ) , 0 ..=10 )
66- . prop_map ( |fields| {
67- let mut record = Record :: new ( ) ;
68- for ( name, datum) in fields {
69- let _ = record. insert ( name, datum) ;
70- }
71- record
72- } )
88+ fn record_strategy ( whitespace : bool ) -> impl Strategy < Value = Record > {
89+ prop:: collection:: hash_map (
90+ field_name_strategy ( ) ,
91+ datum_strategy ( whitespace) ,
92+ 0 ..=10 ,
93+ )
94+ . prop_map ( |fields| {
95+ let mut record = Record :: new ( ) ;
96+ for ( name, datum) in fields {
97+ let _ = record. insert ( name, datum) ;
98+ }
99+ record
100+ } )
73101}
74102
75- fn header_strategy ( ) -> impl Strategy < Value = Record > {
76- prop:: collection:: hash_map ( field_name_strategy ( ) , datum_strategy ( ) , 0 ..=3 )
77- . prop_map ( |fields| {
78- let mut record = Record :: new_header ( ) ;
79- for ( name, datum) in fields {
80- let _ = record. insert ( name, datum) ;
81- }
82- record
83- } )
103+ fn header_strategy ( whitespace : bool ) -> impl Strategy < Value = Record > {
104+ prop:: collection:: hash_map (
105+ field_name_strategy ( ) ,
106+ datum_strategy ( whitespace) ,
107+ 0 ..=3 ,
108+ )
109+ . prop_map ( |fields| {
110+ let mut record = Record :: new_header ( ) ;
111+ for ( name, datum) in fields {
112+ let _ = record. insert ( name, datum) ;
113+ }
114+ record
115+ } )
84116}
85117
86118fn output_types_strategy ( ) -> impl Strategy < Value = OutputTypes > {
87119 prop_oneof ! [ Just ( OutputTypes :: Always ) , Just ( OutputTypes :: OnlyNonString ) , ]
88120}
89121
90- async fn test_roundtrip (
91- header : Record , records : Vec < Record > , output_types : OutputTypes ,
92- ) {
122+ fn whitespace_injection_strategy ( )
123+ -> impl Strategy < Value = Vec < ( usize , Vec < u8 > ) > > {
124+ prop:: collection:: vec (
125+ (
126+ 0usize ..=5 ,
127+ prop:: collection:: vec (
128+ prop:: sample:: select ( vec ! [ b' ' , b'\n' , b'\t' ] ) ,
129+ 1 ..=3 ,
130+ ) ,
131+ ) ,
132+ 3 ,
133+ )
134+ }
135+
136+ async fn write_records (
137+ header : & Record , records : & [ Record ] , output_types : OutputTypes ,
138+ whitespace_injections : Option < & [ ( usize , Vec < u8 > ) ] > ,
139+ ) -> Vec < u8 > {
93140 let mut buf = Vec :: new ( ) ;
94141 let mut sink = RecordSink :: with_types ( & mut buf, output_types) ;
95142
96143 sink. send ( header. clone ( ) ) . await . unwrap ( ) ;
97- for record in & records {
144+ for record in records {
98145 sink. send ( record. clone ( ) ) . await . unwrap ( ) ;
99146 }
100147 sink. close ( ) . await . unwrap ( ) ;
101148
149+ let Some ( whitespace_injections) = whitespace_injections else {
150+ return buf;
151+ } ;
152+
153+ let mut result = Vec :: new ( ) ;
154+ let mut injections = whitespace_injections. iter ( ) . cycle ( ) ;
155+ let mut tags_to_skip = 0 ;
156+
157+ for & byte in & buf {
158+ if byte == b'<' {
159+ if tags_to_skip == 0 {
160+ let ( skip, ws) = injections. next ( ) . unwrap ( ) ;
161+ result. extend_from_slice ( ws) ;
162+ tags_to_skip = * skip;
163+ } else {
164+ tags_to_skip -= 1 ;
165+ }
166+ }
167+ result. push ( byte) ;
168+ }
169+ result
170+ }
171+
172+ async fn test_roundtrip (
173+ header : Record , records : Vec < Record > , output_types : OutputTypes ,
174+ ) {
175+ let buf = write_records ( & header, & records, output_types, None ) . await ;
176+
102177 let mut stream = RecordStream :: new ( & buf[ ..] , true ) ;
103178 let parsed_header = stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
104179 assert ! ( parsed_header. is_header( ) ) ;
@@ -143,14 +218,7 @@ fn assert_records_equal_coerced(parsed: &Record, original: &Record) {
143218}
144219
145220async fn test_roundtrip_never ( header : Record , records : Vec < Record > ) {
146- let mut buf = Vec :: new ( ) ;
147- let mut sink = RecordSink :: with_types ( & mut buf, OutputTypes :: Never ) ;
148-
149- sink. send ( header. clone ( ) ) . await . unwrap ( ) ;
150- for record in & records {
151- sink. send ( record. clone ( ) ) . await . unwrap ( ) ;
152- }
153- sink. close ( ) . await . unwrap ( ) ;
221+ let buf = write_records ( & header, & records, OutputTypes :: Never , None ) . await ;
154222
155223 let mut stream = RecordStream :: new ( & buf[ ..] , true ) ;
156224 let parsed_header = stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
@@ -164,6 +232,30 @@ async fn test_roundtrip_never(header: Record, records: Vec<Record>) {
164232 assert ! ( stream. next( ) . await . is_none( ) ) ;
165233}
166234
235+ async fn test_whitespace (
236+ header : Record , records : Vec < Record > , output_types : OutputTypes ,
237+ whitespace_injections : Vec < ( usize , Vec < u8 > ) > ,
238+ ) {
239+ let buf = write_records (
240+ & header,
241+ & records,
242+ output_types,
243+ Some ( & whitespace_injections) ,
244+ )
245+ . await ;
246+
247+ let mut stream = RecordStream :: new ( & buf[ ..] , true ) ;
248+ let parsed_header = stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
249+ assert ! ( parsed_header. is_header( ) ) ;
250+ assert_eq ! ( parsed_header, header) ;
251+ for record in records {
252+ let parsed = stream. next ( ) . await . unwrap ( ) . unwrap ( ) ;
253+ assert ! ( !parsed. is_header( ) ) ;
254+ assert_eq ! ( parsed, record) ;
255+ }
256+ assert ! ( stream. next( ) . await . is_none( ) ) ;
257+ }
258+
167259async fn test_truncation_error_position ( records : Vec < Record > ) {
168260 let mut buf = Vec :: new ( ) ;
169261 let mut sink = RecordSink :: with_types ( & mut buf, OutputTypes :: Never ) ;
@@ -197,24 +289,36 @@ proptest! {
197289
198290 #[ test]
199291 fn roundtrip_typed(
200- header in header_strategy( ) ,
201- records in prop:: collection:: vec( record_strategy( ) , 0 ..=20 ) ,
292+ header in header_strategy( false ) ,
293+ records in prop:: collection:: vec( record_strategy( false ) , 0 ..=20 ) ,
202294 output_types in output_types_strategy( )
203295 ) {
204296 tokio_test:: block_on( test_roundtrip( header, records, output_types) ) ;
205297 }
206298
207299 #[ test]
208300 fn roundtrip_untyped(
209- header in header_strategy( ) ,
210- records in prop:: collection:: vec( record_strategy( ) , 0 ..=20 )
301+ header in header_strategy( false ) ,
302+ records in prop:: collection:: vec( record_strategy( false ) , 0 ..=20 )
211303 ) {
212304 tokio_test:: block_on( test_roundtrip_never( header, records) ) ;
213305 }
214306
307+ #[ test]
308+ fn roundtrip_with_whitespace(
309+ header in header_strategy( true ) ,
310+ records in prop:: collection:: vec( record_strategy( true ) , 0 ..=20 ) ,
311+ output_types in output_types_strategy( ) ,
312+ whitespace in whitespace_injection_strategy( )
313+ ) {
314+ let test =
315+ test_whitespace( header, records, output_types, whitespace) ;
316+ tokio_test:: block_on( test) ;
317+ }
318+
215319 #[ test]
216320 fn truncation_error_position(
217- records in prop:: collection:: vec( record_strategy( ) , 1 ..=2 )
321+ records in prop:: collection:: vec( record_strategy( false ) , 1 ..=2 )
218322 ) {
219323 tokio_test:: block_on( test_truncation_error_position( records) ) ;
220324 }
0 commit comments