Skip to content

Commit f44b737

Browse files
author
Devdutt Shenoi
committed
fix: custom partition should not be an array or object as well
1 parent d7419a3 commit f44b737

File tree

1 file changed

+108
-24
lines changed

1 file changed

+108
-24
lines changed

src/utils/json/flatten.rs

Lines changed: 108 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,12 @@ pub enum JsonFlattenError {
3333
FieldNotPartOfLog(String),
3434
#[error("Ingestion failed as field {0} is empty or 'null'")]
3535
FieldEmptyOrNull(String),
36-
#[error("Ingestion failed as field {0} has a floating point value")]
37-
FieldIsFloatingPoint(String),
36+
#[error("Ingestion failed as field {0} is an object")]
37+
FieldIsObject(String),
38+
#[error("Ingestion failed as field {0} is an array")]
39+
FieldIsArray(String),
40+
#[error("Ingestion failed as field {0} contains a period in the value")]
41+
FieldContainsPeriod(String),
3842
#[error("Ingestion failed as field {0} is not a string")]
3943
FieldNotString(String),
4044
#[error("Field {0} is not in the correct datetime format")]
@@ -86,19 +90,8 @@ pub fn flatten(
8690
Ok(())
8791
}
8892

89-
// Checks if a JSON value is null or empty
90-
fn is_null_or_empty(value: &Value) -> bool {
91-
match value {
92-
Value::Null => true,
93-
Value::Object(o) if o.is_empty() => true,
94-
Value::Array(a) if a.is_empty() => true,
95-
Value::String(s) if s.is_empty() => true,
96-
_ => false,
97-
}
98-
}
99-
100-
// Validates the presence and content of custom partition fields
101-
// that it is not null, empty, or a floating-point number
93+
// Validates the presence and content of custom partition fields, that it is
94+
// not null, empty, an object , an array, or contain a `.` when serialized
10295
pub fn validate_custom_partition(
10396
value: &Map<String, Value>,
10497
custom_partition: Option<&String>,
@@ -116,14 +109,31 @@ pub fn validate_custom_partition(
116109
));
117110
};
118111

119-
if is_null_or_empty(field_value) {
120-
return Err(JsonFlattenError::FieldEmptyOrNull(trimmed_field.to_owned()));
121-
}
122-
123-
if field_value.is_f64() {
124-
return Err(JsonFlattenError::FieldIsFloatingPoint(
125-
trimmed_field.to_owned(),
126-
));
112+
// The field should not be null, empty, an object, an array or contain a `.` in the value
113+
match field_value {
114+
Value::Null => {
115+
return Err(JsonFlattenError::FieldEmptyOrNull(trimmed_field.to_owned()))
116+
}
117+
Value::String(s) if s.is_empty() => {
118+
return Err(JsonFlattenError::FieldEmptyOrNull(trimmed_field.to_owned()))
119+
}
120+
Value::Object(_) => {
121+
return Err(JsonFlattenError::FieldIsObject(trimmed_field.to_owned()))
122+
}
123+
Value::Array(_) => {
124+
return Err(JsonFlattenError::FieldIsArray(trimmed_field.to_owned()))
125+
}
126+
Value::String(s) if s.contains('.') => {
127+
return Err(JsonFlattenError::FieldContainsPeriod(
128+
trimmed_field.to_owned(),
129+
))
130+
}
131+
Value::Number(n) if n.is_f64() => {
132+
return Err(JsonFlattenError::FieldContainsPeriod(
133+
trimmed_field.to_owned(),
134+
))
135+
}
136+
_ => {}
127137
}
128138
}
129139

@@ -332,7 +342,7 @@ pub fn convert_to_array(flattened: Vec<Value>) -> Result<Value, JsonFlattenError
332342
mod tests {
333343
use crate::utils::json::flatten::flatten_array_objects;
334344

335-
use super::flatten;
345+
use super::{flatten, JsonFlattenError};
336346
use serde_json::{json, Map, Value};
337347

338348
#[test]
@@ -514,4 +524,78 @@ mod tests {
514524
assert_eq!(map.get("key.q.x").unwrap(), &json!([[1, 2], [1], null]));
515525
assert_eq!(map.get("key.r").unwrap(), &json!([null, 2, 3]));
516526
}
527+
528+
#[test]
529+
fn acceptable_value_custom_parition_test() {
530+
let mut value = json!({
531+
"a": 1,
532+
});
533+
assert!(flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).is_ok());
534+
535+
let mut value = json!({
536+
"a": true,
537+
});
538+
assert!(flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).is_ok());
539+
540+
let mut value = json!({
541+
"a": "yes",
542+
});
543+
assert!(flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).is_ok());
544+
545+
let mut value = json!({
546+
"a": -1,
547+
});
548+
assert!(flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).is_ok());
549+
}
550+
551+
#[test]
552+
fn unacceptable_value_custom_partition_test() {
553+
let mut value = json!({
554+
"a": null,
555+
});
556+
matches!(
557+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
558+
JsonFlattenError::FieldEmptyOrNull(_)
559+
);
560+
561+
let mut value = json!({
562+
"a": "",
563+
});
564+
matches!(
565+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
566+
JsonFlattenError::FieldEmptyOrNull(_)
567+
);
568+
569+
let mut value = json!({
570+
"a": {"b": 1},
571+
});
572+
matches!(
573+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
574+
JsonFlattenError::FieldIsObject(_)
575+
);
576+
577+
let mut value = json!({
578+
"a": ["b", "c"],
579+
});
580+
matches!(
581+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
582+
JsonFlattenError::FieldIsArray(_)
583+
);
584+
585+
let mut value = json!({
586+
"a": "b.c",
587+
});
588+
matches!(
589+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
590+
JsonFlattenError::FieldContainsPeriod(_)
591+
);
592+
593+
let mut value = json!({
594+
"a": 1.0,
595+
});
596+
matches!(
597+
flatten(&mut value, "_", None, None, Some(&"a".to_string()), true).unwrap_err(),
598+
JsonFlattenError::FieldContainsPeriod(_)
599+
);
600+
}
517601
}

0 commit comments

Comments
 (0)