Skip to content

Commit 1102f89

Browse files
nzxwangpront
nzxwang
and
pront
authored
enhancement(enriching): add optional wildcard search parameter (#23074)
* add wildcard parameter * update unit tests from 1d232fb * unit tests * format * update docs * update docs with examples * changelog * rename changelog * bugfix and finds_row_with_wildcard without index * use Value instead of String * reset row_equals matching wildcard logic * f * update indexed signature only * fix composite hash key bug in indexed * revert accidental name cahnge * cargo format * update example title to wildcard match * 3 sections for searching enrichment table * indexed_with_wildcard * cargo fmt * run ./website/scripts/cue.sh fmt * fix * fix docs --------- Co-authored-by: Pavlos Rontidis <[email protected]>
1 parent 31756e0 commit 1102f89

File tree

14 files changed

+384
-45
lines changed

14 files changed

+384
-45
lines changed

‎changelog.d/23074_enrichment_function_single_bounded_date_range_search.enhancement.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support an optional wildcard parameter where a match will succeed if the field value equals either the wildcard or the actual comparison value.
2+
3+
authors: nzxwang

‎lib/enrichment/src/find_enrichment_table_records.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ fn find_enrichment_table_records(
1212
enrichment_tables: &TableSearch,
1313
table: &str,
1414
case_sensitive: Case,
15+
wildcard: Option<Value>,
1516
condition: &[Condition],
1617
index: Option<IndexHandle>,
1718
) -> Resolved {
@@ -34,6 +35,7 @@ fn find_enrichment_table_records(
3435
case_sensitive,
3536
condition,
3637
select.as_ref().map(|select| select.as_ref()),
38+
wildcard.as_ref(),
3739
index,
3840
)?
3941
.into_iter()
@@ -71,6 +73,11 @@ impl Function for FindEnrichmentTableRecords {
7173
kind: kind::BOOLEAN,
7274
required: false,
7375
},
76+
Parameter {
77+
keyword: "wildcard",
78+
kind: kind::BYTES,
79+
required: false,
80+
},
7481
]
7582
}
7683

@@ -112,6 +119,7 @@ impl Function for FindEnrichmentTableRecords {
112119
let select = arguments.optional("select");
113120

114121
let case_sensitive = is_case_sensitive(&arguments, state)?;
122+
let wildcard = arguments.optional("wildcard");
115123
let index = Some(
116124
add_index(registry, &table, case_sensitive, &condition)
117125
.map_err(|err| Box::new(err) as Box<_>)?,
@@ -123,6 +131,7 @@ impl Function for FindEnrichmentTableRecords {
123131
index,
124132
select,
125133
case_sensitive,
134+
wildcard,
126135
enrichment_tables: registry.as_readonly(),
127136
}
128137
.as_expr())
@@ -136,6 +145,7 @@ pub struct FindEnrichmentTableRecordsFn {
136145
index: Option<IndexHandle>,
137146
select: Option<Box<dyn Expression>>,
138147
case_sensitive: Case,
148+
wildcard: Option<Box<dyn Expression>>,
139149
enrichment_tables: TableSearch,
140150
}
141151

@@ -158,6 +168,11 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn {
158168

159169
let table = &self.table;
160170
let case_sensitive = self.case_sensitive;
171+
let wildcard = self
172+
.wildcard
173+
.as_ref()
174+
.map(|array| array.resolve(ctx))
175+
.transpose()?;
161176
let index = self.index;
162177
let enrichment_tables = &self.enrichment_tables;
163178

@@ -166,6 +181,7 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn {
166181
enrichment_tables,
167182
table,
168183
case_sensitive,
184+
wildcard,
169185
&condition,
170186
index,
171187
)
@@ -199,6 +215,7 @@ mod tests {
199215
index: Some(IndexHandle(999)),
200216
select: None,
201217
case_sensitive: Case::Sensitive,
218+
wildcard: None,
202219
enrichment_tables: registry.as_readonly(),
203220
};
204221

‎lib/enrichment/src/get_enrichment_table_record.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ fn get_enrichment_table_record(
1212
enrichment_tables: &TableSearch,
1313
table: &str,
1414
case_sensitive: Case,
15+
wildcard: Option<Value>,
1516
condition: &[Condition],
1617
index: Option<IndexHandle>,
1718
) -> Resolved {
@@ -27,11 +28,13 @@ fn get_enrichment_table_record(
2728
}),
2829
})
2930
.transpose()?;
31+
3032
let data = enrichment_tables.find_table_row(
3133
table,
3234
case_sensitive,
3335
condition,
3436
select.as_ref().map(|select| select.as_ref()),
37+
wildcard.as_ref(),
3538
index,
3639
)?;
3740

@@ -67,6 +70,11 @@ impl Function for GetEnrichmentTableRecord {
6770
kind: kind::BOOLEAN,
6871
required: false,
6972
},
73+
Parameter {
74+
keyword: "wildcard",
75+
kind: kind::BYTES,
76+
required: false,
77+
},
7078
]
7179
}
7280

@@ -104,6 +112,7 @@ impl Function for GetEnrichmentTableRecord {
104112
let select = arguments.optional("select");
105113

106114
let case_sensitive = is_case_sensitive(&arguments, state)?;
115+
let wildcard = arguments.optional("wildcard");
107116
let index = Some(
108117
add_index(registry, &table, case_sensitive, &condition)
109118
.map_err(|err| Box::new(err) as Box<_>)?,
@@ -115,6 +124,7 @@ impl Function for GetEnrichmentTableRecord {
115124
index,
116125
select,
117126
case_sensitive,
127+
wildcard,
118128
enrichment_tables: registry.as_readonly(),
119129
}
120130
.as_expr())
@@ -127,6 +137,7 @@ pub struct GetEnrichmentTableRecordFn {
127137
condition: BTreeMap<KeyString, expression::Expr>,
128138
index: Option<IndexHandle>,
129139
select: Option<Box<dyn Expression>>,
140+
wildcard: Option<Box<dyn Expression>>,
130141
case_sensitive: Case,
131142
enrichment_tables: TableSearch,
132143
}
@@ -150,6 +161,11 @@ impl FunctionExpression for GetEnrichmentTableRecordFn {
150161

151162
let table = &self.table;
152163
let case_sensitive = self.case_sensitive;
164+
let wildcard = self
165+
.wildcard
166+
.as_ref()
167+
.map(|array| array.resolve(ctx))
168+
.transpose()?;
153169
let index = self.index;
154170
let enrichment_tables = &self.enrichment_tables;
155171

@@ -158,6 +174,7 @@ impl FunctionExpression for GetEnrichmentTableRecordFn {
158174
enrichment_tables,
159175
table,
160176
case_sensitive,
177+
wildcard,
161178
&condition,
162179
index,
163180
)
@@ -191,6 +208,7 @@ mod tests {
191208
index: Some(IndexHandle(999)),
192209
select: None,
193210
case_sensitive: Case::Sensitive,
211+
wildcard: None,
194212
enrichment_tables: registry.as_readonly(),
195213
};
196214

‎lib/enrichment/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ pub trait Table: DynClone {
5757
case: Case,
5858
condition: &'a [Condition<'a>],
5959
select: Option<&[String]>,
60+
wildcard: Option<&Value>,
6061
index: Option<IndexHandle>,
6162
) -> Result<ObjectMap, String>;
6263

@@ -68,6 +69,7 @@ pub trait Table: DynClone {
6869
case: Case,
6970
condition: &'a [Condition<'a>],
7071
select: Option<&[String]>,
72+
wildcard: Option<&Value>,
7173
index: Option<IndexHandle>,
7274
) -> Result<Vec<ObjectMap>, String>;
7375

‎lib/enrichment/src/tables.rs‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use std::{
3535
};
3636

3737
use arc_swap::ArcSwap;
38-
use vrl::value::ObjectMap;
38+
use vrl::value::{ObjectMap, Value};
3939

4040
use super::{Condition, IndexHandle, Table};
4141
use crate::Case;
@@ -216,13 +216,14 @@ impl TableSearch {
216216
case: Case,
217217
condition: &'a [Condition<'a>],
218218
select: Option<&[String]>,
219+
wildcard: Option<&Value>,
219220
index: Option<IndexHandle>,
220221
) -> Result<ObjectMap, String> {
221222
let tables = self.0.load();
222223
if let Some(ref tables) = **tables {
223224
match tables.get(table) {
224225
None => Err(format!("table {} not loaded", table)),
225-
Some(table) => table.find_table_row(case, condition, select, index),
226+
Some(table) => table.find_table_row(case, condition, select, wildcard, index),
226227
}
227228
} else {
228229
Err("finish_load not called".to_string())
@@ -238,13 +239,14 @@ impl TableSearch {
238239
case: Case,
239240
condition: &'a [Condition<'a>],
240241
select: Option<&[String]>,
242+
wildcard: Option<&Value>,
241243
index: Option<IndexHandle>,
242244
) -> Result<Vec<ObjectMap>, String> {
243245
let tables = self.0.load();
244246
if let Some(ref tables) = **tables {
245247
match tables.get(table) {
246248
None => Err(format!("table {} not loaded", table)),
247-
Some(table) => table.find_table_rows(case, condition, select, index),
249+
Some(table) => table.find_table_rows(case, condition, select, wildcard, index),
248250
}
249251
} else {
250252
Err("finish_load not called".to_string())
@@ -337,6 +339,7 @@ mod tests {
337339
value: Value::from("thang"),
338340
}],
339341
None,
342+
None,
340343
None
341344
)
342345
);
@@ -378,6 +381,7 @@ mod tests {
378381
value: Value::from("thang"),
379382
}],
380383
None,
384+
None,
381385
None
382386
)
383387
);
@@ -442,7 +446,7 @@ mod tests {
442446
tables
443447
.get("dummy1")
444448
.unwrap()
445-
.find_table_row(Case::Sensitive, &Vec::new(), None, None)
449+
.find_table_row(Case::Sensitive, &Vec::new(), None, None, None)
446450
.unwrap()
447451
.get("field")
448452
.cloned()
@@ -455,7 +459,7 @@ mod tests {
455459
tables
456460
.get("dummy2")
457461
.unwrap()
458-
.find_table_row(Case::Sensitive, &Vec::new(), None, None)
462+
.find_table_row(Case::Sensitive, &Vec::new(), None, None, None)
459463
.unwrap()
460464
.get("thing")
461465
.cloned()

‎lib/enrichment/src/test_util.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ impl Table for DummyEnrichmentTable {
3939
_case: Case,
4040
_condition: &[Condition],
4141
_select: Option<&[String]>,
42+
_wildcard: Option<&Value>,
4243
_index: Option<IndexHandle>,
4344
) -> Result<ObjectMap, String> {
4445
Ok(self.data.clone())
@@ -49,6 +50,7 @@ impl Table for DummyEnrichmentTable {
4950
_case: Case,
5051
_condition: &[Condition],
5152
_select: Option<&[String]>,
53+
_wildcard: Option<&Value>,
5254
_index: Option<IndexHandle>,
5355
) -> Result<Vec<ObjectMap>, String> {
5456
Ok(vec![self.data.clone()])

‎lib/vector-vrl/tests/src/test_enrichment.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ impl enrichment::Table for TestEnrichmentTable {
1010
_case: enrichment::Case,
1111
_condition: &'a [enrichment::Condition<'a>],
1212
_select: Option<&[String]>,
13+
_wildcard: Option<&Value>,
1314
_index: Option<enrichment::IndexHandle>,
1415
) -> Result<ObjectMap, String> {
1516
let mut result = ObjectMap::new();
@@ -25,6 +26,7 @@ impl enrichment::Table for TestEnrichmentTable {
2526
_case: enrichment::Case,
2627
_condition: &'a [enrichment::Condition<'a>],
2728
_select: Option<&[String]>,
29+
_wildcard: Option<&Value>,
2830
_index: Option<enrichment::IndexHandle>,
2931
) -> Result<Vec<ObjectMap>, String> {
3032
let mut result1 = ObjectMap::new();

0 commit comments

Comments
 (0)