Skip to content

Commit 41af127

Browse files
Mesut-DonerMesut-Doner
committed
[flink] Fix SIMILAR TO predicate conversion in PredicateConverter
- Fix typo: child.get(0) -> children.get(0) in IS_NOT_TRUE branch - Fix missing dot before toString() in SIMILAR branch (compile error) - Fix BinaryString: pass BinaryString.fromString(pattern) to builder.like instead of a raw String - Rewrite convertSimilarToRegex as convertSimilarToLike: produce a SQL LIKE pattern (not a Java regex) so that the downstream Like function processes it correctly; use backslash as the output escape char to match Like's default escape convention - Fix escape-sequence handling: escaped _ and % become \_ / \% (literals), escaped escape char becomes the literal char itself - Throw UnsupportedExpression for SIMILAR TO-only features (character classes [...], alternation |, quantifiers * + ?, grouping ()) that have no SQL LIKE equivalent - Add unit tests: testConvertSimilarToLike covers pattern pass-through, escape sequences and unsupported-feature rejection; testSimilarExpression* tests cover the full predicate path including row-level filtering
1 parent eb31fd8 commit 41af127

2 files changed

Lines changed: 285 additions & 3 deletions

File tree

‎paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/PredicateConverter.java‎

Lines changed: 121 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,10 +194,49 @@ public Predicate visit(CallExpression call) {
194194
FieldReferenceExpression fieldRefExpr =
195195
extractFieldReference(children.get(0)).orElseThrow(UnsupportedExpression::new);
196196
return builder.equal(builder.indexOf(fieldRefExpr.getName()), Boolean.FALSE);
197+
} else if (func == BuiltInFunctionDefinitions.NOT) {
198+
// NOT predicate - negate the inner predicate
199+
Predicate innerPredicate = children.get(0).accept(this);
200+
return innerPredicate.negate().orElseThrow(UnsupportedExpression::new);
201+
} else if (func == BuiltInFunctionDefinitions.IS_NOT_TRUE) {
202+
FieldReferenceExpression fieldRefExpr =
203+
extractFieldReference(children.get(0)).orElseThrow(UnsupportedExpression::new);
204+
return builder.notEqual(builder.indexOf(fieldRefExpr.getName()), Boolean.TRUE);
205+
} else if (func == BuiltInFunctionDefinitions.NOT_BETWEEN) {
206+
FieldReferenceExpression fieldRefExpr =
207+
extractFieldReference(children.get(0)).orElseThrow(UnsupportedExpression::new);
208+
return builder.between(builder.indexOf(fieldRefExpr.getName()), children.get(1), children.get(2))
209+
.negate()
210+
.orElseThrow(UnsupportedExpression::new);
211+
} else if (func == BuiltInFunctionDefinitions.SIMILAR) {
212+
FieldReferenceExpression fieldRefExpr =
213+
extractFieldReference(children.get(0)).orElseThrow(UnsupportedExpression::new);
214+
if (fieldRefExpr
215+
.getOutputDataType()
216+
.getLogicalType()
217+
.getTypeRoot()
218+
.getFamilies()
219+
.contains(LogicalTypeFamily.CHARACTER_STRING)) {
220+
String sqlPattern =
221+
Objects.requireNonNull(
222+
extractLiteral(
223+
fieldRefExpr.getOutputDataType(), children.get(1)))
224+
.toString();
225+
String escape =
226+
children.size() <= 2
227+
? null
228+
: Objects.requireNonNull(
229+
extractLiteral(
230+
fieldRefExpr.getOutputDataType(),
231+
children.get(2)))
232+
.toString();
233+
String likePattern = convertSimilarToLike(sqlPattern, escape);
234+
return builder.like(
235+
builder.indexOf(fieldRefExpr.getName()),
236+
BinaryString.fromString(likePattern));
237+
}
197238
}
198-
199-
// TODO is_xxx, between_xxx, similar, in, not_in, not?
200-
239+
201240
throw new UnsupportedExpression();
202241
}
203242

@@ -291,6 +330,85 @@ private boolean supportsPredicate(LogicalType type) {
291330
}
292331
}
293332

333+
/**
334+
* Converts a SQL SIMILAR TO pattern to an equivalent SQL LIKE pattern so that it can be
335+
* evaluated by {@link PredicateBuilder#like}.
336+
*
337+
* <p>The conversion handles only the subset of SIMILAR TO syntax that maps directly to SQL
338+
* LIKE:
339+
* <ul>
340+
* <li>{@code %} (any-string wildcard) is preserved as-is.</li>
341+
* <li>{@code _} (single-character wildcard) is preserved as-is.</li>
342+
* <li>Escape sequences: {@code escape + '_'} and {@code escape + '%'} become their literal
343+
* equivalents, emitted as {@code \ + char} so that the downstream
344+
* {@link Like} function (which uses {@code \} as its default escape) treats them as
345+
* literals. {@code escape + escape} becomes a literal escape character.</li>
346+
* </ul>
347+
*
348+
* <p>SIMILAR TO-only features (character classes {@code [...]}, alternation {@code |},
349+
* quantifiers {@code *}, {@code +}, {@code ?}, and grouping {@code ()}) are not supported and
350+
* will cause an {@link UnsupportedExpression} to be thrown.
351+
*
352+
* @param sqlPattern the SIMILAR TO pattern string
353+
* @param escape the escape character string (single char), or {@code null} for no escaping
354+
* @return an equivalent SQL LIKE pattern (using {@code \} as the escape character)
355+
* @throws UnsupportedExpression if the pattern uses SIMILAR TO-only features
356+
*/
357+
private String convertSimilarToLike(String sqlPattern, String escape) {
358+
if (sqlPattern == null || sqlPattern.isEmpty()) {
359+
return sqlPattern;
360+
}
361+
362+
// Sentinel 0 means no escape character is defined.
363+
char escapeChar = (escape != null && !escape.isEmpty()) ? escape.charAt(0) : 0;
364+
// The output LIKE pattern will always use '\' as its escape char, because that is what
365+
// Like.sqlToRegexLike uses by default.
366+
final char outputEscape = '\\';
367+
368+
StringBuilder like = new StringBuilder();
369+
370+
for (int i = 0; i < sqlPattern.length(); i++) {
371+
char c = sqlPattern.charAt(i);
372+
373+
if (escapeChar != 0 && c == escapeChar) {
374+
// Escape sequence
375+
if (i + 1 >= sqlPattern.length()) {
376+
throw new UnsupportedExpression();
377+
}
378+
char next = sqlPattern.charAt(i + 1);
379+
if (next == '_' || next == '%') {
380+
// Escaped wildcard -> literal in the LIKE output.
381+
// Emit as outputEscape + wildcard so Like treats it as a literal.
382+
like.append(outputEscape).append(next);
383+
} else if (next == escapeChar) {
384+
// Escaped escape char -> emit as a literal character.
385+
// If the escape char itself is special to LIKE (% or _), escape it; otherwise
386+
// emit it as-is since Like only special-cases % and _.
387+
like.append(escapeChar);
388+
} else {
389+
// Unknown escape sequence - not supported
390+
throw new UnsupportedExpression();
391+
}
392+
i++;
393+
} else if (c == '%' || c == '_') {
394+
// SIMILAR TO wildcards are the same as SQL LIKE wildcards
395+
like.append(c);
396+
} else if (c == '[' || c == '|' || c == '(' || c == ')'
397+
|| c == '*' || c == '+' || c == '?') {
398+
// SIMILAR TO-only features: not representable in SQL LIKE
399+
throw new UnsupportedExpression();
400+
} else if (c == outputEscape) {
401+
// A literal backslash in the pattern needs to be escaped in the output,
402+
// since the output escape char is '\'.
403+
like.append(outputEscape).append(outputEscape);
404+
} else {
405+
like.append(c);
406+
}
407+
}
408+
409+
return like.toString();
410+
}
411+
294412
@Override
295413
public Predicate visit(ValueLiteralExpression valueLiteralExpression) {
296414
throw new UnsupportedExpression();

‎paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateConverterTest.java‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -808,6 +808,170 @@ public void testUnsupportedFieldReferenceExpression() {
808808
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
809809
}
810810

811+
// -------------------------------------------------------------------------
812+
// Tests for convertSimilarToLike
813+
// -------------------------------------------------------------------------
814+
815+
@Test
816+
public void testConvertSimilarToLike() {
817+
// '%' stays as '%' (SQL LIKE wildcard)
818+
assertThat(convertSimilarToLike("abc%", null)).isEqualTo("abc%");
819+
// '_' stays as '_' (SQL LIKE wildcard)
820+
assertThat(convertSimilarToLike("a_c", null)).isEqualTo("a_c");
821+
// Literal characters pass through unchanged
822+
assertThat(convertSimilarToLike("hello", null)).isEqualTo("hello");
823+
// '.' is just a literal dot in SIMILAR TO — kept as-is in the LIKE output
824+
assertThat(convertSimilarToLike("3.14", null)).isEqualTo("3.14");
825+
// Backslash in pattern is doubled so downstream Like sees it as a literal '\'
826+
assertThat(convertSimilarToLike("a\\b", null)).isEqualTo("a\\\\b");
827+
// Empty pattern returns empty
828+
assertThat(convertSimilarToLike("", null)).isEqualTo("");
829+
830+
// --- SIMILAR TO-only features must throw UnsupportedExpression ---
831+
assertThatThrownBy(() -> convertSimilarToLike("a|b", null))
832+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
833+
assertThatThrownBy(() -> convertSimilarToLike("(a|b)%", null))
834+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
835+
assertThatThrownBy(() -> convertSimilarToLike("a+", null))
836+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
837+
assertThatThrownBy(() -> convertSimilarToLike("[a-z]", null))
838+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
839+
840+
// --- Escape-character tests (using '=' as the escape char) ---
841+
// '=_' -> escaped underscore: output '\_' so Like treats '_' as literal
842+
assertThat(convertSimilarToLike("a=_b", "=")).isEqualTo("a\\_b");
843+
// '=%' -> escaped percent: output '\%' so Like treats '%' as literal
844+
assertThat(convertSimilarToLike("a=%b", "=")).isEqualTo("a\\%b");
845+
// '==' -> literal '=' (the escape char)
846+
assertThat(convertSimilarToLike("a==b", "=")).isEqualTo("a=b");
847+
// Trailing escape char alone must throw
848+
assertThatThrownBy(() -> convertSimilarToLike("a=", "="))
849+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
850+
// Unknown escape sequence (e.g. '=a') must throw
851+
assertThatThrownBy(() -> convertSimilarToLike("a=b", "="))
852+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
853+
}
854+
855+
/** Invokes the private {@code convertSimilarToLike} via reflection. */
856+
private static String convertSimilarToLike(String sqlPattern, String escape) {
857+
try {
858+
java.lang.reflect.Method m =
859+
PredicateConverter.class.getDeclaredMethod(
860+
"convertSimilarToLike", String.class, String.class);
861+
m.setAccessible(true);
862+
return (String) m.invoke(new PredicateConverter(BUILDER), sqlPattern, escape);
863+
} catch (java.lang.reflect.InvocationTargetException e) {
864+
Throwable cause = e.getCause();
865+
if (cause instanceof RuntimeException) {
866+
throw (RuntimeException) cause;
867+
}
868+
throw new RuntimeException(cause);
869+
} catch (Exception e) {
870+
throw new RuntimeException(e);
871+
}
872+
}
873+
874+
// -------------------------------------------------------------------------
875+
// Tests for the SIMILAR (SIMILAR TO) branch in visit(CallExpression)
876+
// -------------------------------------------------------------------------
877+
878+
@Test
879+
public void testSimilarExpressionBasic() {
880+
// 'abc%' SIMILAR TO: any string starting with 'abc'
881+
PredicateConverter converter = new PredicateConverter(RowType.of(new VarCharType()));
882+
CallExpression expr =
883+
call(
884+
BuiltInFunctionDefinitions.SIMILAR,
885+
field(0, STRING()),
886+
literal("abc%", STRING()));
887+
Predicate predicate = expr.accept(converter);
888+
889+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("abc")))).isTrue();
890+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("abcdef")))).isTrue();
891+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("ab")))).isFalse();
892+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("ABC")))).isFalse();
893+
assertThat(predicate.test(GenericRow.of((Object) null))).isFalse();
894+
}
895+
896+
@Test
897+
public void testSimilarExpressionUnderscore() {
898+
// 'a_c' SIMILAR TO: exactly 3 chars starting with 'a', ending with 'c'
899+
PredicateConverter converter = new PredicateConverter(RowType.of(new VarCharType()));
900+
CallExpression expr =
901+
call(
902+
BuiltInFunctionDefinitions.SIMILAR,
903+
field(0, STRING()),
904+
literal("a_c", STRING()));
905+
Predicate predicate = expr.accept(converter);
906+
907+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("abc")))).isTrue();
908+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("axc")))).isTrue();
909+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("ac")))).isFalse();
910+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("abbc")))).isFalse();
911+
}
912+
913+
@Test
914+
public void testSimilarExpressionAlternation() {
915+
// '(cat|dog)%' uses SIMILAR TO-only syntax -> UnsupportedExpression
916+
PredicateConverter converter = new PredicateConverter(RowType.of(new VarCharType()));
917+
assertThatThrownBy(
918+
() ->
919+
call(
920+
BuiltInFunctionDefinitions.SIMILAR,
921+
field(0, STRING()),
922+
literal("(cat|dog)%", STRING()))
923+
.accept(converter))
924+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
925+
}
926+
927+
@Test
928+
public void testSimilarExpressionCharacterClass() {
929+
// '[a-z]+' uses SIMILAR TO-only syntax -> UnsupportedExpression
930+
PredicateConverter converter = new PredicateConverter(RowType.of(new VarCharType()));
931+
assertThatThrownBy(
932+
() ->
933+
call(
934+
BuiltInFunctionDefinitions.SIMILAR,
935+
field(0, STRING()),
936+
literal("[a-z]+", STRING()))
937+
.accept(converter))
938+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
939+
}
940+
941+
@Test
942+
public void testSimilarExpressionEscapedWildcards() {
943+
// 'a=%b' SIMILAR TO with escape '=': literal 'a%b' (% is not a wildcard)
944+
// convertSimilarToLike converts this to 'a\%b', which Like then processes as 'a' + literal
945+
// '%' + 'b'
946+
PredicateConverter converter = new PredicateConverter(RowType.of(new VarCharType()));
947+
CallExpression expr =
948+
call(
949+
BuiltInFunctionDefinitions.SIMILAR,
950+
field(0, STRING()),
951+
literal("a=%b", STRING()),
952+
literal("=", STRING()));
953+
Predicate predicate = expr.accept(converter);
954+
955+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("a%b")))).isTrue();
956+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("axb")))).isFalse();
957+
assertThat(predicate.test(GenericRow.of(BinaryString.fromString("ab")))).isFalse();
958+
}
959+
960+
@Test
961+
public void testSimilarExpressionNonStringTypeThrows() {
962+
// SIMILAR on a non-string column must throw UnsupportedExpression
963+
assertThatThrownBy(
964+
() ->
965+
call(
966+
BuiltInFunctionDefinitions.SIMILAR,
967+
field(0, DataTypes.INT()),
968+
literal(5))
969+
.accept(
970+
new PredicateConverter(
971+
RowType.of(new IntType()))))
972+
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
973+
}
974+
811975
private static FieldReferenceExpression field(int i, DataType type) {
812976
return new FieldReferenceExpression("f" + i, type, Integer.MAX_VALUE, Integer.MAX_VALUE);
813977
}

0 commit comments

Comments
 (0)