Timestamp field can now be defined at any field mapping level (#3537)

* Fix #3531 - Timestamp field can now be defined at any field mapping level.

* Update index config doc.

* Apply suggestions from code review

Co-authored-by: Adrien Guillo <[email protected]>

* Add new tests to cover edges cases and building from a yaml doc mapper.

---------

Co-authored-by: Adrien Guillo <[email protected]>

Author: fmassot

docs/configuration/index-config.md

@@ -88,15 +88,17 @@ The doc mapping defines how a document and the fields it contains are stored and
 | `field_mappings` | Collection of field mapping, each having its own data type (text, binary, datetime, bool, i64, u64, f64).   | `[]` |
 | `mode`        | Defines how quickwit should handle document fields that are not present in the `field_mappings`. In particular, the "dynamic" mode makes it possible to use quickwit in a schemaless manner. (See [mode](#mode)) | `lenient`
 | `dynamic_mapping` | This parameter is only allowed when `mode` is set to `dynamic`. It then defines whether dynamically mapped fields should be indexed, stored, etc.  | (See [mode](#mode))
-| `tag_fields` | Collection of fields already defined in `field_mappings` whose values will be stored as part of the `tags` metadata. [Learn more about tags](../overview/concepts/querying.md#tag-pruning). | `[]` |
+| `tag_fields` | Collection of fields* already defined in `field_mappings` whose values will be stored as part of the `tags` metadata. [Learn more about tags](../overview/concepts/querying.md#tag-pruning). | `[]` |
 | `store_source` | Whether or not the original JSON document is stored or not in the index.   | `false` |
-| `timestamp_field`      | Timestamp field used for sharding documents in splits. The field has to be of type `datetime`. [Learn more about time sharding](./../overview/architecture.md).  | `None` |
+| `timestamp_field`      | Timestamp field* used for sharding documents in splits. The field has to be of type `datetime`. [Learn more about time sharding](./../overview/architecture.md).  | `None` |
  `partition_key`   |  If set, quickwit will route documents into different splits depending on the field name declared as the `partition_key`. | `null` |
 | `max_num_partitions`  | Limits the number of splits created through partitioning. (See [Partitioning](../overview/concepts/querying.md#partitioning))  |    `200` |
 
+*: tags fields and timestamp field are expressed as a path from the root of the JSON object to the given field. If a field name contains a `.` character, it needs to be escaped with a `\` character.
+
 ### Field types
 
-Each field has a type that indicates the kind of data it contains, such as integer on 64 bits or text.
+Each field[^1] has a type that indicates the kind of data it contains, such as integer on 64 bits or text.
 Quickwit supports the following raw types [`text`](#text-type), [`i64`](#numeric-types-i64-u64-and-f64-type), [`u64`](#numeric-types-i64-u64-and-f64-type), [`f64`](#numeric-types-i64-u64-and-f64-type), [`datetime`](#datetime-type), [`bool`](#bool-type), [`ip`](#ip-type), and [`bytes`](#bytes-type), and also supports composite types such as array and object. Behind the scenes, Quickwit is using tantivy field types, don't hesitate to look at [tantivy documentation](https://github.com/tantivy-search/tantivy) if you want to go into the details.
 
 ### Raw types

quickwit/Cargo.lock

@@ -33,10 +33,11 @@ typetag = { workspace = true }
 utoipa = { workspace = true }
 
 [dev-dependencies]
-proptest = { workspace = true }
 criterion = { workspace = true }
 matches = { workspace = true }
+proptest = { workspace = true }
 quickwit-proto = { workspace = true }
+serde_yaml = { workspace = true }
 time = { workspace = true }
 
 [features]

quickwit/quickwit-doc-mapper/Cargo.toml

@@ -117,31 +117,22 @@ impl DefaultDocMapper {
     }
 }
 
-fn validate_timestamp_field_if_any(builder: &DefaultDocMapperBuilder) -> anyhow::Result<()> {
-    let Some(timestamp_field_name) = builder.timestamp_field.as_ref() else {
-        return Ok(());
+fn validate_timestamp_field(
+    timestamp_field_path: &str,
+    mapping_root_node: &MappingNode,
+) -> anyhow::Result<()> {
+    let Some(timestamp_field_type) = mapping_root_node.find_field_mapping_type(timestamp_field_path) else {
+        bail!("Could not find timestamp field `{timestamp_field_path}` in field mappings.");
     };
-    let Some(timestamp_field_entry) = builder.field_mappings.iter().find(|mapping| {
-                &mapping.name == timestamp_field_name
-            }) else {
-                bail!("Missing timestamp field in field mappings: `{}`", timestamp_field_name);
-            };
-    if let FieldMappingType::DateTime(date_time_option, cardinality) =
-        &timestamp_field_entry.mapping_type
-    {
+    if let FieldMappingType::DateTime(date_time_option, cardinality) = &timestamp_field_type {
         if cardinality != &Cardinality::SingleValue {
-            bail!(
-                "Multiple values are forbidden for  the timestamp field \
-                 (`{timestamp_field_name}`)."
-            );
+            bail!("Timestamp field `{timestamp_field_path}` should be single-valued.");
         }
         if !date_time_option.fast {
-            bail!("The timestamp field `{timestamp_field_name}`is required to be a fast field.");
+            bail!("Timestamp field `{timestamp_field_path}` should be a fast field.");
         }
     } else {
-        bail!(
-            "The timestamp field `{timestamp_field_name}` is required to have the datetime type."
-        );
+        bail!("Timestamp field `{timestamp_field_path}` should be a datetime field.");
     }
     Ok(())
 }
@@ -159,7 +150,9 @@ impl TryFrom<DefaultDocMapperBuilder> for DefaultDocMapper {
             None
         };
 
-        validate_timestamp_field_if_any(&builder)?;
+        if let Some(timestamp_field_path) = builder.timestamp_field.as_ref() {
+            validate_timestamp_field(timestamp_field_path, &field_mappings)?;
+        };
 
         let dynamic_field = if let Mode::Dynamic(json_options) = &mode {
             Some(schema_builder.add_json_field(DYNAMIC_FIELD_NAME, json_options.clone()))
@@ -584,20 +577,84 @@ mod tests {
     }
 
     #[test]
-    fn test_fail_to_build_doc_mapper_with_non_datetime_timestamp_field() {
+    fn test_timestamp_field_in_object_is_valid() {
+        serde_json::from_str::<DefaultDocMapper>(
+            r#"{
+            "field_mappings": [
+                {
+                    "name": "some_obj",
+                    "type": "object",
+                    "field_mappings": [
+                        {
+                            "name": "timestamp",
+                            "type": "datetime",
+                            "fast": true
+                        }
+                    ]
+                }
+            ],
+            "timestamp_field": "some_obj.timestamp"
+        }"#,
+        )
+        .unwrap();
+
+        serde_yaml::from_str::<DefaultDocMapper>(
+            r#"
+            field_mappings:
+              - name: some_obj
+                type: object
+                field_mappings:
+                  - name: timestamp
+                    type: datetime
+                    fast: true
+            timestamp_field: some_obj.timestamp
+        "#,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn test_timestamp_field_with_dots_in_its_name_is_valid() {
+        serde_json::from_str::<DefaultDocMapper>(
+            r#"{
+            "field_mappings": [
+                {
+                    "name": "my.timestamp",
+                    "type": "datetime",
+                    "fast": true
+                }
+            ],
+            "timestamp_field": "my\\.timestamp"
+        }"#,
+        )
+        .unwrap();
+
+        serde_yaml::from_str::<DefaultDocMapper>(
+            r#"
+            field_mappings:
+              - name: my.timestamp
+                type: datetime
+                fast: true
+            timestamp_field: "my\\.timestamp"
+        "#,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn test_fail_to_build_doc_mapper_with_timestamp_field_with_multivalues_cardinality() {
         let doc_mapper = r#"{
-            "default_search_fields": [],
             "timestamp_field": "timestamp",
             "tag_fields": [],
             "field_mappings": [
                 {
                     "name": "timestamp",
-                    "type": "text"
+                    "type": "array<i64>"
                 }
             ]
         }"#;
         let builder = serde_json::from_str::<DefaultDocMapperBuilder>(doc_mapper).unwrap();
-        let expected_msg = "The timestamp field `timestamp` is required to have the datetime type.";
+        let expected_msg = "Timestamp field `timestamp` should be a datetime field.";
         assert_eq!(&builder.try_build().unwrap_err().to_string(), &expected_msg);
     }
 
@@ -616,7 +673,7 @@ mod tests {
             ]
         }"#;
         let builder = serde_json::from_str::<DefaultDocMapperBuilder>(doc_mapper).unwrap();
-        let expected_msg = "The timestamp field `timestamp`is required to be a fast field.";
+        let expected_msg = "Timestamp field `timestamp` should be a fast field.";
         assert_eq!(&builder.try_build().unwrap_err().to_string(), &expected_msg);
     }
 
@@ -689,7 +746,7 @@ mod tests {
         }"#;
 
         let builder = serde_json::from_str::<DefaultDocMapperBuilder>(doc_mapper).unwrap();
-        let expected_msg = "Multiple values are forbidden for  the timestamp field (`timestamp`).";
+        let expected_msg = "Timestamp field `timestamp` should be single-valued.";
         assert_eq!(&builder.try_build().unwrap_err().to_string(), expected_msg);
     }
 
@@ -802,7 +859,7 @@ mod tests {
     }
 
     #[test]
-    fn test_partion_key_in_tags() {
+    fn test_partition_key_in_tags() {
         let doc_mapper = r#"{
             "default_search_fields": [],
             "timestamp_field": null,
@@ -838,7 +895,7 @@ mod tests {
     }
 
     #[test]
-    fn test_partion_key_in_tags_without_explicit_tags() {
+    fn test_partition_key_in_tags_without_explicit_tags() {
         let doc_mapper = r#"{
             "default_search_fields": [],
             "timestamp_field": null,
@@ -872,6 +929,42 @@ mod tests {
         assert_eq!(tag_fields, vec!["city", "division", "service",]);
     }
 
+    #[test]
+    fn test_build_doc_mapper_with_tag_field_with_dots_in_its_name() {
+        let doc_mapper = r#"{
+            "default_search_fields": [],
+            "tag_fields": ["my\\.city\\.id"],
+            "field_mappings": [
+                {
+                    "name": "my.city.id",
+                    "type": "u64"
+                }
+            ]
+        }"#;
+        serde_json::from_str::<DefaultDocMapper>(doc_mapper).unwrap();
+    }
+
+    #[test]
+    fn test_build_doc_mapper_with_tag_field_in_object() {
+        let doc_mapper = r#"{
+            "default_search_fields": [],
+            "tag_fields": ["location.city"],
+            "field_mappings": [
+                {
+                    "name": "location",
+                    "type": "object",
+                    "field_mappings": [
+                        {
+                            "name": "city",
+                            "type": "u64"
+                        }
+                    ]
+                }
+            ]
+        }"#;
+        serde_json::from_str::<DefaultDocMapper>(doc_mapper).unwrap();
+    }
+
     #[test]
     fn test_fail_to_build_doc_mapper_with_wrong_tag_fields_types() -> anyhow::Result<()> {
         let doc_mapper_one = r#"{
@@ -1280,4 +1373,65 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn test_find_field_mapping_type() {
+        let mapper = serde_json::from_str::<DefaultDocMapper>(
+            r#"{
+            "field_mappings": [
+                {
+                    "name": "some_obj",
+                    "type": "object",
+                    "field_mappings": [
+                        {
+                            "name": "timestamp",
+                            "type": "datetime",
+                            "fast": true
+                        },
+                        {
+                            "name": "object2",
+                            "type": "object",
+                            "field_mappings": [
+                                {
+                                    "name": "id",
+                                    "type": "u64"
+                                },
+                                {
+                                    "name": "my.id",
+                                    "type": "u64"
+                                }
+                            ]
+                        }
+                    ]
+                },
+                {
+                    "name": "my.timestamp",
+                    "type": "datetime",
+                    "fast": true
+                }
+            ]
+        }"#,
+        )
+        .unwrap();
+        mapper
+            .field_mappings
+            .find_field_mapping_type("some_obj.timestamp")
+            .unwrap();
+        mapper
+            .field_mappings
+            .find_field_mapping_type("some_obj.object2.id")
+            .unwrap();
+        mapper
+            .field_mappings
+            .find_field_mapping_type("some_obj.object2")
+            .unwrap();
+        mapper
+            .field_mappings
+            .find_field_mapping_type("some_obj.object2.my\\.id")
+            .unwrap();
+        mapper
+            .field_mappings
+            .find_field_mapping_type("my\\.timestamp")
+            .unwrap();
+    }
 }