@@ -33,8 +33,12 @@ pub enum JsonFlattenError {
33
33
FieldNotPartOfLog ( String ) ,
34
34
#[ error( "Ingestion failed as field {0} is empty or 'null'" ) ]
35
35
FieldEmptyOrNull ( String ) ,
36
- #[ error( "Ingestion failed as field {0} has a floating point value" ) ]
37
- FieldIsFloatingPoint ( String ) ,
36
+ #[ error( "Ingestion failed as field {0} is an object" ) ]
37
+ FieldIsObject ( String ) ,
38
+ #[ error( "Ingestion failed as field {0} is an array" ) ]
39
+ FieldIsArray ( String ) ,
40
+ #[ error( "Ingestion failed as field {0} contains a period in the value" ) ]
41
+ FieldContainsPeriod ( String ) ,
38
42
#[ error( "Ingestion failed as field {0} is not a string" ) ]
39
43
FieldNotString ( String ) ,
40
44
#[ error( "Field {0} is not in the correct datetime format" ) ]
@@ -86,19 +90,8 @@ pub fn flatten(
86
90
Ok ( ( ) )
87
91
}
88
92
89
- // Checks if a JSON value is null or empty
90
- fn is_null_or_empty ( value : & Value ) -> bool {
91
- match value {
92
- Value :: Null => true ,
93
- Value :: Object ( o) if o. is_empty ( ) => true ,
94
- Value :: Array ( a) if a. is_empty ( ) => true ,
95
- Value :: String ( s) if s. is_empty ( ) => true ,
96
- _ => false ,
97
- }
98
- }
99
-
100
- // Validates the presence and content of custom partition fields
101
- // that it is not null, empty, or a floating-point number
93
+ // Validates the presence and content of custom partition fields, that it is
94
+ // not null, empty, an object , an array, or contain a `.` when serialized
102
95
pub fn validate_custom_partition (
103
96
value : & Map < String , Value > ,
104
97
custom_partition : Option < & String > ,
@@ -116,14 +109,31 @@ pub fn validate_custom_partition(
116
109
) ) ;
117
110
} ;
118
111
119
- if is_null_or_empty ( field_value) {
120
- return Err ( JsonFlattenError :: FieldEmptyOrNull ( trimmed_field. to_owned ( ) ) ) ;
121
- }
122
-
123
- if field_value. is_f64 ( ) {
124
- return Err ( JsonFlattenError :: FieldIsFloatingPoint (
125
- trimmed_field. to_owned ( ) ,
126
- ) ) ;
112
+ // The field should not be null, empty, an object, an array or contain a `.` in the value
113
+ match field_value {
114
+ Value :: Null => {
115
+ return Err ( JsonFlattenError :: FieldEmptyOrNull ( trimmed_field. to_owned ( ) ) )
116
+ }
117
+ Value :: String ( s) if s. is_empty ( ) => {
118
+ return Err ( JsonFlattenError :: FieldEmptyOrNull ( trimmed_field. to_owned ( ) ) )
119
+ }
120
+ Value :: Object ( _) => {
121
+ return Err ( JsonFlattenError :: FieldIsObject ( trimmed_field. to_owned ( ) ) )
122
+ }
123
+ Value :: Array ( _) => {
124
+ return Err ( JsonFlattenError :: FieldIsArray ( trimmed_field. to_owned ( ) ) )
125
+ }
126
+ Value :: String ( s) if s. contains ( '.' ) => {
127
+ return Err ( JsonFlattenError :: FieldContainsPeriod (
128
+ trimmed_field. to_owned ( ) ,
129
+ ) )
130
+ }
131
+ Value :: Number ( n) if n. is_f64 ( ) => {
132
+ return Err ( JsonFlattenError :: FieldContainsPeriod (
133
+ trimmed_field. to_owned ( ) ,
134
+ ) )
135
+ }
136
+ _ => { }
127
137
}
128
138
}
129
139
@@ -332,7 +342,7 @@ pub fn convert_to_array(flattened: Vec<Value>) -> Result<Value, JsonFlattenError
332
342
mod tests {
333
343
use crate :: utils:: json:: flatten:: flatten_array_objects;
334
344
335
- use super :: flatten;
345
+ use super :: { flatten, JsonFlattenError } ;
336
346
use serde_json:: { json, Map , Value } ;
337
347
338
348
#[ test]
@@ -514,4 +524,78 @@ mod tests {
514
524
assert_eq ! ( map. get( "key.q.x" ) . unwrap( ) , & json!( [ [ 1 , 2 ] , [ 1 ] , null] ) ) ;
515
525
assert_eq ! ( map. get( "key.r" ) . unwrap( ) , & json!( [ null, 2 , 3 ] ) ) ;
516
526
}
527
+
528
+ #[ test]
529
+ fn acceptable_value_custom_parition_test ( ) {
530
+ let mut value = json ! ( {
531
+ "a" : 1 ,
532
+ } ) ;
533
+ assert ! ( flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . is_ok( ) ) ;
534
+
535
+ let mut value = json ! ( {
536
+ "a" : true ,
537
+ } ) ;
538
+ assert ! ( flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . is_ok( ) ) ;
539
+
540
+ let mut value = json ! ( {
541
+ "a" : "yes" ,
542
+ } ) ;
543
+ assert ! ( flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . is_ok( ) ) ;
544
+
545
+ let mut value = json ! ( {
546
+ "a" : -1 ,
547
+ } ) ;
548
+ assert ! ( flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . is_ok( ) ) ;
549
+ }
550
+
551
+ #[ test]
552
+ fn unacceptable_value_custom_partition_test ( ) {
553
+ let mut value = json ! ( {
554
+ "a" : null,
555
+ } ) ;
556
+ matches ! (
557
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
558
+ JsonFlattenError :: FieldEmptyOrNull ( _)
559
+ ) ;
560
+
561
+ let mut value = json ! ( {
562
+ "a" : "" ,
563
+ } ) ;
564
+ matches ! (
565
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
566
+ JsonFlattenError :: FieldEmptyOrNull ( _)
567
+ ) ;
568
+
569
+ let mut value = json ! ( {
570
+ "a" : { "b" : 1 } ,
571
+ } ) ;
572
+ matches ! (
573
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
574
+ JsonFlattenError :: FieldIsObject ( _)
575
+ ) ;
576
+
577
+ let mut value = json ! ( {
578
+ "a" : [ "b" , "c" ] ,
579
+ } ) ;
580
+ matches ! (
581
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
582
+ JsonFlattenError :: FieldIsArray ( _)
583
+ ) ;
584
+
585
+ let mut value = json ! ( {
586
+ "a" : "b.c" ,
587
+ } ) ;
588
+ matches ! (
589
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
590
+ JsonFlattenError :: FieldContainsPeriod ( _)
591
+ ) ;
592
+
593
+ let mut value = json ! ( {
594
+ "a" : 1.0 ,
595
+ } ) ;
596
+ matches ! (
597
+ flatten( & mut value, "_" , None , None , Some ( & "a" . to_string( ) ) , true ) . unwrap_err( ) ,
598
+ JsonFlattenError :: FieldContainsPeriod ( _)
599
+ ) ;
600
+ }
517
601
}
0 commit comments