feat(es-compat): support regexp shorthand, expose concatenate fields, map text to keyword in _mapping (quickwit-oss#6208)

* feat(es-compat): support regexp shorthand format, expose concatenate fields, and map text to keyword in _mapping

Elasticsearch's `regexp` query accepts two formats:
- Shorthand: `{"regexp": {"field": "pattern"}}`
- Full: `{"regexp": {"field": {"value": "pattern", "case_insensitive": true}}}`

Quickwit only supported the full form, causing queries from ES-compatible
connectors (e.g. Trino ES connector) to fail with a deserialization error.
This adds support for the shorthand format via `#[serde(untagged)]` enum
deserialization.

Additionally, in the `_mapping` endpoint:
- `Text` fields are now reported as `keyword` type. This enables filter
  pushdown (e.g. `LIKE` predicates) from connectors that only push down
  filters for `keyword`-typed fields.
- `Concatenate` fields are now exposed as `keyword` type instead of being
  hidden. This allows connectors to discover and query these fields.

Made-with: Cursor

* fix(es-compat): replace manual Default impl with derive to fix clippy lint

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* address PR review: inline custom Deserialize for RegexQueryParams, add keyword comment

- Replace inner enum + serde(from) with a custom Deserialize impl directly
  on RegexQueryParams, as suggested by reviewer
- Add comment explaining why text fields are mapped to keyword in the
  ES-compat _mapping response

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* refactor: simplify RegexQueryParams to a plain untagged enum

Replace the custom Deserialize visitor with a simple #[serde(untagged)]
enum that handles both shorthand and full regexp query formats directly.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: congx4

quickwit/quickwit-query/src/elastic_query_dsl/regex_query.rs

@@ -18,22 +18,43 @@ use crate::elastic_query_dsl::ConvertibleToQueryAst;
 use crate::elastic_query_dsl::one_field_map::OneFieldMap;
 use crate::query_ast::{QueryAst, RegexQuery as AstRegexQuery};
 
-#[derive(Deserialize, Debug, Default, Eq, PartialEq, Clone)]
-#[serde(deny_unknown_fields)]
-pub struct RegexQueryParams {
-    value: String,
-    #[serde(default)]
-    case_insensitive: bool,
+/// Elasticsearch supports two formats for regexp queries:
+/// - Shorthand: `{"regexp": {"field": "pattern"}}`
+/// - Full:      `{"regexp": {"field": {"value": "pattern", "case_insensitive": true}}}`
+#[derive(Deserialize, Debug, Eq, PartialEq, Clone)]
+#[serde(untagged)]
+pub enum RegexQueryParams {
+    Full {
+        #[serde(rename = "value")]
+        pattern: String,
+        #[serde(default)]
+        case_insensitive: bool,
+    },
+    Shorthand(String),
+}
+
+impl RegexQueryParams {
+    fn into_tuple(self) -> (String, bool) {
+        match self {
+            RegexQueryParams::Full {
+                pattern,
+                case_insensitive,
+            } => (pattern, case_insensitive),
+            RegexQueryParams::Shorthand(pattern) => (pattern, false),
+        }
+    }
 }
 
 pub type RegexQuery = OneFieldMap<RegexQueryParams>;
 
 impl ConvertibleToQueryAst for RegexQuery {
     fn convert_to_query_ast(self) -> anyhow::Result<QueryAst> {
-        let regex = if self.value.case_insensitive {
-            format!("(?i){}", self.value.value)
+        let (pattern, case_insensitive) = self.value.into_tuple();
+
+        let regex = if case_insensitive {
+            format!("(?i){pattern}")
         } else {
-            self.value.value.clone()
+            pattern
         };
         Ok(AstRegexQuery {
             field: self.field,
@@ -42,3 +63,38 @@ impl ConvertibleToQueryAst for RegexQuery {
         .into())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_regex_query_shorthand_format() {
+        let json = serde_json::json!({"service": ".*logs.*"});
+        let query: RegexQuery = serde_json::from_value(json).unwrap();
+        assert_eq!(query.field, "service");
+        let (pattern, case_insensitive) = query.value.into_tuple();
+        assert_eq!(pattern, ".*logs.*");
+        assert!(!case_insensitive);
+    }
+
+    #[test]
+    fn test_regex_query_full_format() {
+        let json = serde_json::json!({"service": {"value": ".*logs.*", "case_insensitive": true}});
+        let query: RegexQuery = serde_json::from_value(json).unwrap();
+        assert_eq!(query.field, "service");
+        let (pattern, case_insensitive) = query.value.into_tuple();
+        assert_eq!(pattern, ".*logs.*");
+        assert!(case_insensitive);
+    }
+
+    #[test]
+    fn test_regex_query_full_format_default_case() {
+        let json = serde_json::json!({"service": {"value": ".*logs.*"}});
+        let query: RegexQuery = serde_json::from_value(json).unwrap();
+        assert_eq!(query.field, "service");
+        let (pattern, case_insensitive) = query.value.into_tuple();
+        assert_eq!(pattern, ".*logs.*");
+        assert!(!case_insensitive);
+    }
+}

quickwit/quickwit-serve/src/elasticsearch_api/model/mappings.rs

@@ -99,7 +99,11 @@ fn build_properties(field_mappings: &[FieldMappingEntry]) -> HashMap<String, Fie
 
 fn field_mapping_from_entry(entry: &FieldMappingEntry) -> Option<FieldMapping> {
     match &entry.mapping_type {
-        FieldMappingType::Text(..) => Some(FieldMapping::Leaf { typ: "text" }),
+        // Quickwit text fields behave like ES keyword fields: they support exact
+        // match, prefix, and regexp queries. Reporting them as "keyword" enables
+        // downstream connectors (e.g. Trino ES connector) to push down filters and
+        // LIKE predicates, which they only do for keyword-typed fields.
+        FieldMappingType::Text(..) => Some(FieldMapping::Leaf { typ: "keyword" }),
         FieldMappingType::I64(..) => Some(FieldMapping::Leaf { typ: "long" }),
         FieldMappingType::U64(..) => Some(FieldMapping::Leaf { typ: "long" }),
         FieldMappingType::F64(..) => Some(FieldMapping::Leaf { typ: "double" }),
@@ -115,7 +119,7 @@ fn field_mapping_from_entry(entry: &FieldMappingEntry) -> Option<FieldMapping> {
                 properties,
             })
         }
-        FieldMappingType::Concatenate(_) => None,
+        FieldMappingType::Concatenate(_) => Some(FieldMapping::Leaf { typ: "keyword" }),
     }
 }
 
@@ -178,7 +182,7 @@ mod tests {
         let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap();
         let mapping = field_mapping_from_entry(&entry).unwrap();
         let serialized = serde_json::to_value(&mapping).unwrap();
-        assert_eq!(serialized, json!({ "type": "text" }));
+        assert_eq!(serialized, json!({ "type": "keyword" }));
     }
 
     #[test]
@@ -209,21 +213,23 @@ mod tests {
                 "type": "object",
                 "properties": {
                     "id": { "type": "long" },
-                    "label": { "type": "text" }
+                    "label": { "type": "keyword" }
                 }
             })
         );
     }
 
     #[test]
-    fn test_field_mapping_from_entry_concatenate_skipped() {
+    fn test_field_mapping_from_entry_concatenate_exposed_as_keyword() {
         let entry_json = json!({
             "name": "concat_field",
             "type": "concatenate",
             "concatenate_fields": ["field_a", "field_b"]
         });
         let entry: FieldMappingEntry = serde_json::from_value(entry_json).unwrap();
-        assert!(field_mapping_from_entry(&entry).is_none());
+        let mapping = field_mapping_from_entry(&entry).unwrap();
+        let serialized = serde_json::to_value(&mapping).unwrap();
+        assert_eq!(serialized, json!({ "type": "keyword" }));
     }
 
     #[test]
@@ -251,7 +257,7 @@ mod tests {
         let props = build_properties(&entries);
         let to_json = |fm: &FieldMapping| serde_json::to_value(fm).unwrap();
 
-        assert_eq!(to_json(&props["title"]), json!({ "type": "text" }));
+        assert_eq!(to_json(&props["title"]), json!({ "type": "keyword" }));
         assert_eq!(to_json(&props["count"]), json!({ "type": "long" }));
         assert_eq!(to_json(&props["unsigned"]), json!({ "type": "long" }));
         assert_eq!(to_json(&props["score"]), json!({ "type": "double" }));
@@ -263,7 +269,7 @@ mod tests {
 
         let meta = to_json(&props["metadata"]);
         assert_eq!(meta["type"], "object");
-        assert_eq!(meta["properties"]["source"]["type"], "text");
+        assert_eq!(meta["properties"]["source"]["type"], "keyword");
     }
 
     #[test]