Skip to content

Commit fee8b9e

Browse files
sixdouglassixdouglas
committed
fix: CEL filtering expression
Trying multiple things here: - bump CEL version - add test in `MessageFiltersTest` on different fields of different types - add `CelValidationTest' and 'CelTypedValidationTest' to try to understand how CEL works
1 parent 0ad8695 commit fee8b9e

File tree

5 files changed

+222
-8
lines changed

5 files changed

+222
-8
lines changed

‎api/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,6 @@
215215
<artifactId>spring-boot-starter-actuator</artifactId>
216216
</dependency>
217217

218-
<dependency>
219-
<groupId>org.antlr</groupId>
220-
<artifactId>antlr4-runtime</artifactId>
221-
<version>${antlr4-maven-plugin.version}</version>
222-
</dependency>
223-
224218
<dependency>
225219
<groupId>org.opendatadiscovery</groupId>
226220
<artifactId>oddrn-generator-java</artifactId>

‎api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package io.kafbat.ui.emitter;
2+
3+
import com.google.common.collect.ImmutableCollection;
4+
import com.google.common.collect.ImmutableSet;
5+
import dev.cel.common.CelAbstractSyntaxTree;
6+
import dev.cel.common.CelOptions;
7+
import dev.cel.common.CelValidationException;
8+
import dev.cel.common.CelValidationResult;
9+
import dev.cel.common.types.CelType;
10+
import dev.cel.common.types.CelTypeProvider;
11+
import dev.cel.common.types.SimpleType;
12+
import dev.cel.common.types.StructType;
13+
import dev.cel.compiler.CelCompiler;
14+
import dev.cel.compiler.CelCompilerFactory;
15+
import dev.cel.extensions.CelExtensions;
16+
import dev.cel.parser.CelStandardMacro;
17+
import dev.cel.runtime.CelEvaluationException;
18+
import dev.cel.runtime.CelRuntime;
19+
import dev.cel.runtime.CelRuntimeFactory;
20+
import org.junit.jupiter.params.ParameterizedTest;
21+
import org.junit.jupiter.params.provider.CsvSource;
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
import java.util.Optional;
25+
26+
import static org.assertj.core.api.Assertions.assertThat;
27+
28+
class CelTypedValidationTest {
29+
30+
private final StructType recordContentType = StructType.create(
31+
"MessageContentDTO",
32+
ImmutableSet.of("lastname", "firstname", "age"),
33+
fieldName -> switch (fieldName) {
34+
case "lastname", "firstname" ->
35+
Optional.of(SimpleType.STRING);
36+
case "age" ->
37+
Optional.of(SimpleType.INT);
38+
default ->
39+
Optional.empty();
40+
}
41+
);
42+
43+
private final StructType recordType = StructType.create(
44+
"MessageDTO",
45+
ImmutableSet.of("value"),
46+
fieldName -> Optional.of(recordContentType)
47+
);
48+
49+
private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
50+
.setOptions(CelOptions.DEFAULT)
51+
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
52+
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
53+
.addVar("record", recordType)
54+
.setResultType(SimpleType.BOOL)
55+
.setTypeProvider(new CelTypeProvider() {
56+
@Override
57+
public ImmutableCollection<CelType> types() {
58+
return ImmutableSet.of(recordType);
59+
}
60+
61+
@Override
62+
public Optional<CelType> findType(String typeName) {
63+
if ("MessageDTO".equals(typeName)) {
64+
return Optional.of(recordType);
65+
}
66+
if ("MessageContentDTO".equals(typeName)) {
67+
return Optional.of(recordContentType);
68+
}
69+
return Optional.empty();
70+
}
71+
})
72+
.build();
73+
74+
private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();
75+
76+
@ParameterizedTest
77+
@CsvSource({
78+
"record.value.age == 24, true",
79+
"record.value.age >= 24, true",
80+
"record.value.age > 24, true",
81+
"has(record.value.age), true",
82+
"has(record.value.ages), false",
83+
"record.value.age < 50, true",
84+
"record.value.age < 24, true"
85+
})
86+
void typedExpressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
87+
Map<String, Integer> objectKeys = new HashMap<>();
88+
objectKeys.put("age", 24);
89+
Map<String, Object> valueKeys = new HashMap<>();
90+
valueKeys.put("value", objectKeys);
91+
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
92+
recordKeys.put("record", valueKeys);
93+
94+
CelValidationResult celValidationResult = celCompiler.compile(expression);
95+
CelAbstractSyntaxTree ast = celValidationResult.getAst();
96+
CelRuntime.Program program = celRuntime.createProgram(ast);
97+
var programResult = program.eval(recordKeys);
98+
assertThat(programResult)
99+
.describedAs("The result of the assertion was incorrect")
100+
.isNotNull()
101+
.isEqualTo(expected)
102+
;
103+
}
104+
}

‎api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package io.kafbat.ui.emitter;
2+
3+
import com.google.common.collect.ImmutableCollection;
4+
import com.google.common.collect.ImmutableSet;
5+
import dev.cel.common.CelAbstractSyntaxTree;
6+
import dev.cel.common.CelOptions;
7+
import dev.cel.common.CelValidationException;
8+
import dev.cel.common.CelValidationResult;
9+
import dev.cel.common.types.CelType;
10+
import dev.cel.common.types.CelTypeProvider;
11+
import dev.cel.common.types.SimpleType;
12+
import dev.cel.common.types.StructType;
13+
import dev.cel.compiler.CelCompiler;
14+
import dev.cel.compiler.CelCompilerFactory;
15+
import dev.cel.extensions.CelExtensions;
16+
import dev.cel.parser.CelStandardMacro;
17+
import dev.cel.runtime.CelEvaluationException;
18+
import dev.cel.runtime.CelRuntime;
19+
import dev.cel.runtime.CelRuntimeFactory;
20+
import org.junit.jupiter.params.ParameterizedTest;
21+
import org.junit.jupiter.params.provider.CsvSource;
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
import java.util.Optional;
25+
26+
import static org.assertj.core.api.Assertions.assertThat;
27+
28+
class CelValidationTest {
29+
private final Map<String, CelType> fields = Map.of(
30+
"value", SimpleType.DYN
31+
);
32+
33+
private final ImmutableSet<String> names = ImmutableSet
34+
.<String>builder()
35+
.addAll(fields.keySet())
36+
.build();
37+
38+
private final StructType recordType = StructType.create(
39+
"MessageDTO",
40+
names,
41+
fieldName -> Optional.ofNullable(fields.get(fieldName))
42+
);
43+
44+
private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
45+
.setOptions(CelOptions.DEFAULT)
46+
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
47+
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
48+
.addVar("record", recordType)
49+
.setResultType(SimpleType.BOOL)
50+
.setTypeProvider(new CelTypeProvider() {
51+
@Override
52+
public ImmutableCollection<CelType> types() {
53+
return ImmutableSet.of(recordType);
54+
}
55+
56+
@Override
57+
public Optional<CelType> findType(String typeName) {
58+
return "MessageDTO".equals(typeName) ? Optional.of(recordType) : Optional.empty();
59+
}
60+
})
61+
.build();
62+
63+
private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();
64+
65+
@ParameterizedTest
66+
@CsvSource({
67+
"record.value.lastname == null, true",
68+
"record.value.firstname == 'Paul', true",
69+
"size(record.value.firstname) == 4, true",
70+
"record.value.age == 24, true",
71+
"record.value.age >= 24, true",
72+
"record.value.age > 24, true",
73+
"has(record.value.age), true",
74+
"has(record.value.ages), false",
75+
"record.value.age < 50, true",
76+
"record.value.age < 24, true"
77+
})
78+
void expressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
79+
Map<String, Object> objectKeys = new HashMap<>();
80+
objectKeys.put("lastname", null);
81+
objectKeys.put("firstname", "Paul");
82+
objectKeys.put("age", 24);
83+
Map<String, Object> valueKeys = new HashMap<>();
84+
valueKeys.put("value", objectKeys);
85+
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
86+
recordKeys.put("record", valueKeys);
87+
88+
CelValidationResult celValidationResult = celCompiler.compile(expression);
89+
CelAbstractSyntaxTree ast = celValidationResult.getAst();
90+
CelRuntime.Program program = celRuntime.createProgram(ast);
91+
var programResult = program.eval(recordKeys);
92+
assertThat(programResult)
93+
.describedAs("The result of the assertion was incorrect")
94+
.isNotNull()
95+
.isEqualTo(expected)
96+
;
97+
}
98+
}

‎api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,24 @@ void canCheckValueAsJsonObjectIfItCanBeParsedToJson() {
149149
assertFalse(f.test(msg().content("{ \"name\" : { \"second\" : \"user2\" } }")));
150150
}
151151

152+
@Test
153+
void canFilterNullValueInJsonObjectIfItCanBeParsedToJson() {
154+
var filter1 = celScriptFilter("record.value.age == 24");
155+
assertTrue(filter1.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Age filter KO");
156+
var filter2 = celScriptFilter("record.value.name == 'Paul'");
157+
assertTrue(filter2.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Name filter KO");
158+
var filter3 = celScriptFilter("record.value.name == ''");
159+
assertTrue(filter3.test(msg().content("{\"name\": \"\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Empty Name filter KO");
160+
var filter4 = celScriptFilter("record.value.name == null");
161+
assertFalse(filter4.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name in filter KO");
162+
var filter5 = celScriptFilter("record.value.address.city == \"Lille\"");
163+
assertTrue(filter5.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name filter KO");
164+
var filter6 = celScriptFilter("record.value.address.city == \"\"");
165+
assertTrue(filter6.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"\"}}")), "Null Name filter KO");
166+
var filter7 = celScriptFilter("record.value.address.city == null");
167+
assertTrue(filter7.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": null}}")), "Null Name filter KO");
168+
}
169+
152170
@Test
153171
void valueSetToContentStringIfCantBeParsedToJson() {
154172
var f = celScriptFilter("record.value == \"not json\"");

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
<git.revision>latest</git.revision>
3131

3232
<!-- Dependency versions -->
33-
<antlr4-maven-plugin.version>4.12.0</antlr4-maven-plugin.version>
33+
<antlr4-maven-plugin.version>4.13.2</antlr4-maven-plugin.version>
3434
<apache.commons.version>2.12.0</apache.commons.version>
3535
<assertj.version>3.25.3</assertj.version>
3636
<avro.version>1.11.4</avro.version>
@@ -50,7 +50,7 @@
5050
<odd-oddrn-generator.version>0.1.17</odd-oddrn-generator.version>
5151
<odd-oddrn-client.version>0.1.39</odd-oddrn-client.version>
5252
<org.json.version>20240303</org.json.version>
53-
<dev.cel.version>0.3.0</dev.cel.version>
53+
<dev.cel.version>0.7.0</dev.cel.version>
5454
<guava.version>33.3.1-jre</guava.version>
5555
<!-- Test dependency versions -->
5656
<junit.version>5.11.2</junit.version>

0 commit comments

Comments
 (0)