From 2cb204813f80cf22cb139ff0df1da0789743c9b2 Mon Sep 17 00:00:00 2001 From: xinyual Date: Tue, 4 Nov 2025 14:27:54 +0800 Subject: [PATCH 1/7] fix anoymizer for search command Signed-off-by: xinyual --- .../java/org/opensearch/sql/ast/dsl/AstDSL.java | 2 +- .../opensearch/sql/ast/expression/SearchAnd.java | 5 +++++ .../sql/ast/expression/SearchComparison.java | 5 +++++ .../sql/ast/expression/SearchExpression.java | 7 +++++++ .../sql/ast/expression/SearchGroup.java | 5 +++++ .../opensearch/sql/ast/expression/SearchIn.java | 5 +++++ .../sql/ast/expression/SearchLiteral.java | 5 +++++ .../opensearch/sql/ast/expression/SearchNot.java | 5 +++++ .../opensearch/sql/ast/expression/SearchOr.java | 5 +++++ .../java/org/opensearch/sql/ast/tree/Search.java | 4 +++- .../CalcitePPLDateTimeBuiltinFunctionIT.java | 9 +++++++++ .../org/opensearch/sql/ppl/parser/AstBuilder.java | 2 +- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 15 +++++++++------ .../sql/ppl/utils/PPLQueryDataAnonymizerTest.java | 4 ++-- 14 files changed, 67 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 67cc893c5b0..1e8f5c1f895 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -112,7 +112,7 @@ public UnresolvedPlan describe(String tableName) { } public static UnresolvedPlan search(UnresolvedPlan input, String queryString) { - return new Search(input, queryString); + return new Search(input, queryString, null); } public UnresolvedPlan subqueryAlias(UnresolvedPlan child, String alias) { diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java index fe7be0db13e..bdfbd9fda39 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java @@ -27,6 +27,11 @@ public String toQueryString() { return left.toQueryString() + " AND " + right.toQueryString(); } + @Override + public String toAnonymizedString() { + return left.toAnonymizedString() + " AND " + right.toAnonymizedString(); + } + @Override public List getChild() { return Arrays.asList(left, right); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java index be099059546..99ff20e45a5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java @@ -63,6 +63,11 @@ public String toQueryString() { } } + @Override + public String toAnonymizedString() { + return "identifier " + operator.symbol + " ***"; + } + @Override public List getChild() { return Arrays.asList(field, value); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java index 23bf806d230..b705909445f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java @@ -17,6 +17,13 @@ public abstract class SearchExpression extends UnresolvedExpression { */ public abstract String toQueryString(); + /** + * Convert the search expression to anonymized string + * + * @return the anonymized string + */ + public abstract String toAnonymizedString(); + @Override public R accept(AbstractNodeVisitor nodeVisitor, C context) { return nodeVisitor.visitChildren(this, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java index 41b85f408ca..a2b9b02f5ba 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java @@ -26,6 +26,11 @@ public String toQueryString() { return "(" + expression.toQueryString() + ")"; } + @Override + public String toAnonymizedString() { + return "(" + expression.toAnonymizedString() + ")"; + } + @Override public List getChild() { return Collections.singletonList(expression); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java index bdbdb712a9c..8291d130dff 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java @@ -33,6 +33,11 @@ public String toQueryString() { return fieldName + ":( " + valueList + " )"; } + @Override + public String toAnonymizedString() { + return "identifier IN ***"; + } + @Override public List getChild() { List children = new ArrayList<>(); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java index f27dad34bab..460615afa64 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java @@ -55,6 +55,11 @@ public String toQueryString() { return QueryStringUtils.escapeLuceneSpecialCharacters(text); } + @Override + public String toAnonymizedString() { + return "***"; + } + @Override public List getChild() { return Collections.singletonList(literal); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java index 3e5083b7678..b9ea7b416b4 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java @@ -26,6 +26,11 @@ public String toQueryString() { return "NOT(" + expression.toQueryString() + ")"; } + @Override + public String toAnonymizedString() { + return "NOT(" + expression.toAnonymizedString() + ")"; + } + @Override public List getChild() { return Collections.singletonList(expression); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java index b5aa72807bd..1a9e95e89a2 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java @@ -27,6 +27,11 @@ public String toQueryString() { return left.toQueryString() + " OR " + right.toQueryString(); } + @Override + public String toAnonymizedString() { + return left.toAnonymizedString() + " OR " + right.toAnonymizedString(); + } + @Override public List getChild() { return Arrays.asList(left, right); diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Search.java b/core/src/main/java/org/opensearch/sql/ast/tree/Search.java index ebc74192987..441eb5f8a8c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Search.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Search.java @@ -12,6 +12,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.SearchExpression; /** * Logical plan node for Search operation. Represents search expressions that get converted to @@ -25,6 +26,7 @@ public class Search extends UnresolvedPlan { private final UnresolvedPlan child; private final String queryString; + private final SearchExpression originalExpression; @Override public List getChild() { @@ -38,6 +40,6 @@ public T accept(AbstractNodeVisitor nodeVisitor, C context) { @Override public UnresolvedPlan attach(UnresolvedPlan child) { - return new Search(child, queryString); + return new Search(child, queryString, originalExpression); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java index e87bb01ced5..1ea6b28fae0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java @@ -49,6 +49,15 @@ public void init() throws IOException { initRelativeDocs(); } + @Test + public void test() { + JSONObject actual = + executeQuery( + String.format( + "source=%s age=2", + TEST_INDEX_BANK)); + } + private static String getFormattedLocalDate() { return LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 8802dcbf3c9..81bc4780647 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -178,7 +178,7 @@ public UnresolvedPlan visitSearchFrom(SearchFromContext ctx) { // Create Search node with relation and query string Relation relation = (Relation) visitFromClause(ctx.fromClause()); - return new Search(relation, queryString); + return new Search(relation, queryString, combined); } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index f8c935175d0..36abb137416 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -41,6 +41,7 @@ import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.Or; import org.opensearch.sql.ast.expression.ParseMethod; +import org.opensearch.sql.ast.expression.SearchExpression; import org.opensearch.sql.ast.expression.Span; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.When; @@ -106,11 +107,11 @@ /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { - private static final String MASK_LITERAL = "***"; + public static final String MASK_LITERAL = "***"; - private static final String MASK_COLUMN = "identifier"; + public static final String MASK_COLUMN = "identifier"; - private static final String MASK_TABLE = "table"; + public static final String MASK_TABLE = "table"; private final AnonymizerExpressionAnalyzer expressionAnalyzer; private final Settings settings; @@ -250,9 +251,11 @@ public String visitTableFunction(TableFunction node, String context) { @Override public String visitSearch(Search node, String context) { String source = node.getChild().get(0).accept(this, context); - String queryString = node.getQueryString(); - String anonymized = queryString.replaceAll(":\\S+", ":" + MASK_LITERAL); - return StringUtils.format("%s %s", source, anonymized); + //SearchExpression expression = node.getOriginalExpression(); + + //String queryString = node.getQueryString(); + //String anonymized = queryString.replaceAll(":\\S+", ":" + MASK_LITERAL); + return StringUtils.format("%s %s", source, node.getOriginalExpression().toAnonymizedString()); } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 2f18db5c995..666414fbd39 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -33,7 +33,7 @@ public class PPLQueryDataAnonymizerTest { @Test public void testSearchCommand() { - assertEquals("source=table a:***", anonymize("search source=t a=1")); + assertEquals("source=table identifier = ***", anonymize("search source=t a=1")); } @Test @@ -797,7 +797,7 @@ private String anonymizeStatement(String query, boolean isExplain) { @Test public void testSearchWithAbsoluteTimeRange() { assertEquals( - "source=table (@timestamp:*** AND (@timestamp:***", + "source=table (identifier >= ***) AND (identifier <= ***)", anonymize("search source=t earliest='2012-12-10 15:00:00' latest=now")); } From 24bcb662e6e782beb087594433c084b89d173097 Mon Sep 17 00:00:00 2001 From: xinyual Date: Tue, 4 Nov 2025 15:42:26 +0800 Subject: [PATCH 2/7] pushdown match when only one equal in search command Signed-off-by: xinyual --- .../sql/ast/expression/SearchComparison.java | 20 +++++++++++++++++++ .../sql/ast/expression/SearchExpression.java | 9 +++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 5 +---- .../sql/calcite/remote/CalciteExplainIT.java | 9 +++++++++ .../CalcitePPLDateTimeBuiltinFunctionIT.java | 9 --------- .../explain_search_with_match_pushdown.json | 6 ++++++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 7 +++---- 7 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_search_with_match_pushdown.json diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java index 99ff20e45a5..66304bfb4cc 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java @@ -5,12 +5,15 @@ package org.opensearch.sql.ast.expression; +import static org.opensearch.sql.ast.expression.SearchComparison.Operator.EQUALS; + import java.util.Arrays; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.utils.QueryStringUtils; /** Search expression for field comparisons. */ @@ -63,6 +66,23 @@ public String toQueryString() { } } + @Override + public Function toDSLFunction() { + String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString()); + String valueStr = value.toQueryString(); + switch (operator) { + case EQUALS: + return AstDSL.function( + "match", + AstDSL.unresolvedArg("field_name", AstDSL.qualifiedName(fieldName)), + AstDSL.unresolvedArg("value", AstDSL.stringLiteral(valueStr))); + default: + return AstDSL.function( + "query_string", + AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); + } + } + @Override public String toAnonymizedString() { return "identifier " + operator.symbol + " ***"; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java index b705909445f..c17c8438c90 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ast.expression; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.dsl.AstDSL; /** Base class for search expressions that get converted to query_string syntax. */ public abstract class SearchExpression extends UnresolvedExpression { @@ -17,6 +18,14 @@ public abstract class SearchExpression extends UnresolvedExpression { */ public abstract String toQueryString(); + public Function toDSLFunction() { + Function queryStringFunc = + AstDSL.function( + "query_string", + AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); + return queryStringFunc; + } + /** * Convert the search expression to anonymized string * diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index c6a964fce17..1a8af41ff64 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -210,10 +210,7 @@ public RelNode visitSearch(Search node, CalcitePlanContext context) { // Visit the Relation child to get the scan node.getChild().get(0).accept(this, context); // Create query_string function - Function queryStringFunc = - AstDSL.function( - "query_string", - AstDSL.unresolvedArg("query", AstDSL.stringLiteral(node.getQueryString()))); + Function queryStringFunc = node.getOriginalExpression().toDSLFunction(); RexNode queryStringRex = rexVisitor.analyze(queryStringFunc, context); context.relBuilder.filter(queryStringRex); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index cff92408d4f..6717fa1e9ad 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -78,6 +78,15 @@ public void supportSearchSargPushDown_timeRange() throws IOException { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void supportSearchMatchPushDown() throws IOException { + String query = + "source=opensearch-sql_test_index_bank birthdate = '2016-12-08 00:00:00.000000000'"; + var result = explainQueryToString(query); + String expected = loadExpectedPlan("explain_search_with_match_pushdown.json"); + assertJsonEqualsIgnoreId(expected, result); + } + // Only for Calcite @Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725") public void testJoinWithCriteriaAndMaxOption() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java index 1ea6b28fae0..e87bb01ced5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java @@ -49,15 +49,6 @@ public void init() throws IOException { initRelativeDocs(); } - @Test - public void test() { - JSONObject actual = - executeQuery( - String.format( - "source=%s age=2", - TEST_INDEX_BANK)); - } - private static String getFormattedLocalDate() { return LocalDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_search_with_match_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_with_match_pushdown.json new file mode 100644 index 00000000000..fb6ef8e36ff --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_search_with_match_pushdown.json @@ -0,0 +1,6 @@ +{ + "calcite":{ + "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n LogicalFilter(condition=[match(MAP('field_name', $3), MAP('value', '\"2016\\-12\\-08 00\\:00\\:00.000000000\"':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->match(MAP('field_name', $3), MAP('value', '\"2016\\-12\\-08 00\\:00\\:00.000000000\"':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"match\":{\"birthdate\":{\"query\":\"\\\"2016\\\\-12\\\\-08 00\\\\:00\\\\:00.000000000\\\"\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + } +} \ No newline at end of file diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 36abb137416..5501a3bfdac 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -41,7 +41,6 @@ import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.Or; import org.opensearch.sql.ast.expression.ParseMethod; -import org.opensearch.sql.ast.expression.SearchExpression; import org.opensearch.sql.ast.expression.Span; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.When; @@ -251,10 +250,10 @@ public String visitTableFunction(TableFunction node, String context) { @Override public String visitSearch(Search node, String context) { String source = node.getChild().get(0).accept(this, context); - //SearchExpression expression = node.getOriginalExpression(); + // SearchExpression expression = node.getOriginalExpression(); - //String queryString = node.getQueryString(); - //String anonymized = queryString.replaceAll(":\\S+", ":" + MASK_LITERAL); + // String queryString = node.getQueryString(); + // String anonymized = queryString.replaceAll(":\\S+", ":" + MASK_LITERAL); return StringUtils.format("%s %s", source, node.getOriginalExpression().toAnonymizedString()); } From 0e601c6b3d4d22b32ba42ef52c8a4023478cce0c Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 5 Nov 2025 14:16:00 +0800 Subject: [PATCH 3/7] fix regex case Signed-off-by: xinyual --- .../sql/ast/expression/SearchComparison.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java index 66304bfb4cc..01eceb00a83 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java @@ -70,17 +70,16 @@ public String toQueryString() { public Function toDSLFunction() { String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString()); String valueStr = value.toQueryString(); - switch (operator) { - case EQUALS: - return AstDSL.function( - "match", - AstDSL.unresolvedArg("field_name", AstDSL.qualifiedName(fieldName)), - AstDSL.unresolvedArg("value", AstDSL.stringLiteral(valueStr))); - default: - return AstDSL.function( - "query_string", - AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); + if (operator == EQUALS + && !(valueStr.contains("*") + || valueStr.contains("?"))) { // for regex case, we cannot use match + return AstDSL.function( + "match", + AstDSL.unresolvedArg("field_name", AstDSL.qualifiedName(fieldName)), + AstDSL.unresolvedArg("value", AstDSL.stringLiteral(valueStr))); } + return AstDSL.function( + "query_string", AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); } @Override From 6a41ad02a8177381fcb79cf25ca439f876c22030 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 5 Nov 2025 14:46:35 +0800 Subject: [PATCH 4/7] fix UT Signed-off-by: xinyual --- .../sql/analysis/AnalyzerSearchTest.java | 10 +++++----- .../opensearch/sql/ast/tree/SearchTest.java | 18 +++++++++--------- .../CalciteRelNodeVisitorSearchSimpleTest.java | 10 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerSearchTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerSearchTest.java index aa780a8fb08..bdace062792 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerSearchTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerSearchTest.java @@ -65,7 +65,7 @@ public void testVisitSearchWithSimpleQuery() { // Arrange String queryString = "field1:value1"; Relation relation = new Relation(AstDSL.qualifiedName("test_index")); - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); LogicalRelation logicalRelation = new LogicalRelation("test_index", mockTable); FunctionExpression queryStringExpr = mock(FunctionExpression.class); @@ -103,7 +103,7 @@ public void testVisitSearchWithComplexQuery() { // Arrange String queryString = "(field1:value1 OR field2:value2) AND NOT field3:value3"; Relation relation = new Relation(AstDSL.qualifiedName("test_index")); - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); FunctionExpression queryStringExpr = mock(FunctionExpression.class); when(queryStringExpr.type()).thenReturn(ExprCoreType.BOOLEAN); @@ -134,7 +134,7 @@ public void testVisitSearchPreservesChildPlan() { // Arrange String queryString = "test:query"; UnresolvedPlan mockChild = mock(UnresolvedPlan.class); - Search searchNode = new Search(mockChild, queryString); + Search searchNode = new Search(mockChild, queryString, null); LogicalPlan mockLogicalPlan = mock(LogicalPlan.class); FunctionExpression queryStringExpr = mock(FunctionExpression.class); @@ -158,7 +158,7 @@ public void testVisitSearchWithEmptyQuery() { // Arrange String queryString = ""; Relation relation = new Relation(AstDSL.qualifiedName("test_index")); - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); FunctionExpression queryStringExpr = mock(FunctionExpression.class); @@ -193,7 +193,7 @@ public void testVisitSearchCreatesCorrectQueryStringFunction() { // Arrange String queryString = "field:\"exact phrase\" AND field2:wildcard*"; Relation relation = new Relation(AstDSL.qualifiedName("test_index")); - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); FunctionExpression queryStringExpr = mock(FunctionExpression.class); diff --git a/core/src/test/java/org/opensearch/sql/ast/tree/SearchTest.java b/core/src/test/java/org/opensearch/sql/ast/tree/SearchTest.java index c6808d502ff..b16053b41cc 100644 --- a/core/src/test/java/org/opensearch/sql/ast/tree/SearchTest.java +++ b/core/src/test/java/org/opensearch/sql/ast/tree/SearchTest.java @@ -29,7 +29,7 @@ public class SearchTest { public void setUp() { mockChild = mock(UnresolvedPlan.class); testQueryString = "field1:value1 AND field2:value2"; - search = new Search(mockChild, testQueryString); + search = new Search(mockChild, testQueryString, null); } @Test @@ -69,7 +69,7 @@ public void testAccept() { public void testEquals() { UnresolvedPlan sameChild = mockChild; String sameQueryString = testQueryString; - Search sameSearch = new Search(sameChild, sameQueryString); + Search sameSearch = new Search(sameChild, sameQueryString, null); assertEquals(search, sameSearch); assertEquals(search, search); @@ -78,14 +78,14 @@ public void testEquals() { @Test public void testNotEqualsWithDifferentChild() { UnresolvedPlan differentChild = mock(UnresolvedPlan.class); - Search differentSearch = new Search(differentChild, testQueryString); + Search differentSearch = new Search(differentChild, testQueryString, null); assertNotEquals(search, differentSearch); } @Test public void testNotEqualsWithDifferentQueryString() { - Search differentSearch = new Search(mockChild, "different:query"); + Search differentSearch = new Search(mockChild, "different:query", null); assertNotEquals(search, differentSearch); } @@ -102,14 +102,14 @@ public void testNotEqualsWithDifferentClass() { @Test public void testHashCode() { - Search sameSearch = new Search(mockChild, testQueryString); + Search sameSearch = new Search(mockChild, testQueryString, null); assertEquals(search.hashCode(), sameSearch.hashCode()); } @Test public void testHashCodeWithDifferentValues() { UnresolvedPlan differentChild = mock(UnresolvedPlan.class); - Search differentSearch = new Search(differentChild, "different:query"); + Search differentSearch = new Search(differentChild, "different:query", null); assertNotEquals(search.hashCode(), differentSearch.hashCode()); } @@ -123,7 +123,7 @@ public void testToString() { @Test public void testWithEmptyQueryString() { - Search emptySearch = new Search(mockChild, ""); + Search emptySearch = new Search(mockChild, "", null); assertEquals("", emptySearch.getQueryString()); assertEquals(mockChild, emptySearch.getChild().get(0)); } @@ -131,14 +131,14 @@ public void testWithEmptyQueryString() { @Test public void testWithComplexQueryString() { String complexQuery = "(field1:value1 OR field2:value2) AND NOT field3:value3"; - Search complexSearch = new Search(mockChild, complexQuery); + Search complexSearch = new Search(mockChild, complexQuery, null); assertEquals(complexQuery, complexSearch.getQueryString()); } @Test public void testWithSpecialCharactersInQueryString() { String specialCharsQuery = "field:\"value with spaces\" AND field2:value*"; - Search specialSearch = new Search(mockChild, specialCharsQuery); + Search specialSearch = new Search(mockChild, specialCharsQuery, null); assertEquals(specialCharsQuery, specialSearch.getQueryString()); } } diff --git a/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java b/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java index ed878ca622d..f3b2c8d5ab7 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java @@ -36,7 +36,7 @@ public void testVisitSearchRequiresContext() { // Arrange String queryString = "field:value"; Relation relation = new Relation(AstDSL.qualifiedName("test_index")); - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); // Act & Assert - should throw NPE without proper context assertThrows( @@ -53,7 +53,7 @@ public void testSearchNodeStructure() { Relation relation = new Relation(AstDSL.qualifiedName("test_index")); // Act - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); // Assert assertNotNull(searchNode); @@ -70,7 +70,7 @@ public void testSearchWithEmptyQuery() { Relation relation = new Relation(AstDSL.qualifiedName("test_index")); // Act - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); // Assert assertEquals("", searchNode.getQueryString()); @@ -84,7 +84,7 @@ public void testSearchWithComplexQuery() { Relation relation = new Relation(AstDSL.qualifiedName("test_index")); // Act - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); // Assert assertEquals(queryString, searchNode.getQueryString()); @@ -97,7 +97,7 @@ public void testSearchWithSpecialCharacters() { Relation relation = new Relation(AstDSL.qualifiedName("test_index")); // Act - Search searchNode = new Search(relation, queryString); + Search searchNode = new Search(relation, queryString, null); // Assert assertEquals(queryString, searchNode.getQueryString()); From d19c1474074715d447d31f169d9b1ec99a6beb97 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 5 Nov 2025 16:15:05 +0800 Subject: [PATCH 5/7] fix UT Signed-off-by: xinyual --- .../sql/ppl/calcite/CalcitePPLSearchTest.java | 6 ++- .../sql/ppl/parser/AstBuilderTest.java | 16 +++++-- .../ppl/parser/AstExpressionBuilderTest.java | 44 +++++++++---------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java index ddb1df95edd..ffc9916df3b 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java @@ -45,12 +45,14 @@ public void testSearchWithFilter() { String ppl = "search source=EMP DEPTNO=20"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalFilter(condition=[query_string(MAP('query', 'DEPTNO:20':VARCHAR))])\n" + "LogicalFilter(condition=[match(MAP('field_name', $7), MAP('value', '20':VARCHAR))])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT *\nFROM `scott`.`EMP`\nWHERE `query_string`(MAP ('query', 'DEPTNO:20'))"; + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `match`(MAP ('field_name', `DEPTNO`), MAP ('value', '20'))"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index a1823e4befe..49071a26ca1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -77,7 +77,9 @@ import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Timechart; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; @@ -111,7 +113,7 @@ public void setup() { @Test public void testSearchCommand() { - assertEqual("search source=t a=1", search(relation("t"), "a:1")); + assertSearchEqual("search source=t a=1", search(relation("t"), "a:1")); } @Test @@ -172,7 +174,7 @@ public void testSearchWithPrometheusQueryRangeWithNamedArguments() { @Test public void testSearchCommandString() { - assertEqual("search source=t a=\"a\"", search(relation("t"), "a:a")); + assertSearchEqual("search source=t a=\"a\"", search(relation("t"), "a:a")); } @Test @@ -183,7 +185,7 @@ public void testSearchCommandWithoutSearch() { @Test public void testSearchCommandWithFilterBeforeSource() { - assertEqual("search a=1 source=t", search(relation("t"), "a:1")); + assertSearchEqual("search a=1 source=t", search(relation("t"), "a:1")); } @Test @@ -1347,6 +1349,14 @@ protected void assertEqual(String query, Node expectedPlan) { assertEquals(expectedPlan, actualPlan); } + protected void assertSearchEqual(String query, UnresolvedPlan expectedPlan) { + Node actualPlan = plan(query); + assertTrue(actualPlan instanceof Search); + assertTrue(expectedPlan instanceof Search); + assertEquals(expectedPlan.getChild(), actualPlan.getChild()); + assertEquals(((Search) expectedPlan).getQueryString(), ((Search) actualPlan).getQueryString()); + } + protected void assertEqual(String query, String expected) { Node expectedPlan = plan(expected); assertEqual(query, expectedPlan); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 6b0e0a081f8..49c8adf0086 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -72,7 +72,7 @@ public void testLogicalNotExpr() { assertEqual( "source=t | where not a=1", filter(relation("t"), not(compare("=", field("a"), intLiteral(1))))); - assertEqual("source=t not a=1", search(relation("t"), "NOT(a:1)")); + assertSearchEqual("source=t not a=1", search(relation("t"), "NOT(a:1)")); } @Test @@ -82,7 +82,7 @@ public void testLogicalOrExpr() { filter( relation("t"), or(compare("=", field("a"), intLiteral(1)), compare("=", field("b"), intLiteral(2))))); - assertEqual("source=t a=1 or b=2", search(relation("t"), "(a:1 OR b:2)")); + assertSearchEqual("source=t a=1 or b=2", search(relation("t"), "(a:1 OR b:2)")); } @Test @@ -92,7 +92,7 @@ public void testLogicalAndExpr() { filter( relation("t"), and(compare("=", field("a"), intLiteral(1)), compare("=", field("b"), intLiteral(2))))); - assertEqual("source=t a=1 and b=2", search(relation("t"), "(a:1 AND b:2)")); + assertSearchEqual("source=t a=1 and b=2", search(relation("t"), "(a:1 AND b:2)")); } @Test @@ -102,8 +102,8 @@ public void testLogicalAndExprWithoutKeywordAnd() { filter( relation("t"), and(compare("=", field("a"), intLiteral(1)), compare("=", field("b"), intLiteral(2))))); - assertEqual("source=t a=1 b=2", search(relation("t"), "(a:1) AND (b:2)")); - assertEqual( + assertSearchEqual("source=t a=1 b=2", search(relation("t"), "(a:1) AND (b:2)")); + assertSearchEqual( "source=t a=1 b=2 c=2 text", search(relation("t"), "(a:1) AND (b:2) AND (c:2) AND (text)")); } @@ -129,7 +129,7 @@ public void testLogicalAndOr() { compare("=", field("b"), intLiteral(2))), compare("=", field("c"), intLiteral(3))), compare("=", field("d"), intLiteral(4))))); - assertEqual( + assertSearchEqual( "source=t a=1 and b=2 and c=3 or d=4", search(relation("t"), "((a:1 AND b:2) AND (c:3 OR d:4))")); } @@ -148,7 +148,7 @@ public void testLogicalParenthetic() { compare("=", field("c"), intLiteral(3)), compare("=", field("d"), intLiteral(4)))))); - assertEqual( + assertSearchEqual( "source=t (a=1 or b=2) and (c=3 or d=4)", search(relation("t"), "(((a:1 OR b:2)) AND ((c:3 OR d:4)))")); } @@ -415,14 +415,14 @@ public void testCompareExpr() { assertEqual( "source=t | where a='b'", filter(relation("t"), compare("=", field("a"), stringLiteral("b")))); - assertEqual("source=t a='b'", search(relation("t"), "a:b")); + assertSearchEqual("source=t a='b'", search(relation("t"), "a:b")); } @Test public void testCompareFieldsExpr() { assertEqual( "source=t | where a>b", filter(relation("t"), compare(">", field("a"), field("b")))); - assertEqual("source=t a>b", search(relation("t"), "a:>b")); + assertSearchEqual("source=t a>b", search(relation("t"), "a:>b")); } @Test @@ -454,7 +454,7 @@ public void testMixedEqualOperators() { @Test public void testInExpr() { - assertEqual("source=t f in (1, 2, 3)", search(relation("t"), "f:( 1 OR 2 OR 3 )")); + assertSearchEqual("source=t f in (1, 2, 3)", search(relation("t"), "f:( 1 OR 2 OR 3 )")); assertEqual( "source=t | where f in (1, 2, 3)", @@ -913,7 +913,7 @@ public void testNestedFieldNameWithSpecialChars() { @Test public void testStringLiteralExpr() { - assertEqual("source=t a=\"string\"", search(relation("t"), "a:string")); + assertSearchEqual("source=t a=\"string\"", search(relation("t"), "a:string")); assertEqual( "source=t | where a=\"string\"", filter(relation("t"), compare("=", field("a"), stringLiteral("string")))); @@ -929,12 +929,12 @@ public void testIntegerLiteralExpr() { compare("=", field("a"), intLiteral(1)), compare("=", field("b"), intLiteral(-1))))); - assertEqual("source=t a=1 b=-1", search(relation("t"), "(a:1) AND (b:-1)")); + assertSearchEqual("source=t a=1 b=-1", search(relation("t"), "(a:1) AND (b:-1)")); } @Test public void testLongLiteralExpr() { - assertEqual( + assertSearchEqual( "source=t a=1234567890123 b=-1234567890123", search(relation("t"), "(a:1234567890123) AND (b:-1234567890123)")); @@ -949,7 +949,7 @@ public void testLongLiteralExpr() { @Test public void testDoubleLiteralExpr() { - assertEqual("source=t b=0.1d", search(relation("t"), "b:0.1")); + assertSearchEqual("source=t b=0.1d", search(relation("t"), "b:0.1")); assertEqual( "source=t | where b=0.1d", filter(relation("t"), compare("=", field("b"), doubleLiteral(0.1)))); @@ -957,7 +957,7 @@ public void testDoubleLiteralExpr() { @Test public void testFloatLiteralExpr() { - assertEqual("source=t b=0.1f", search(relation("t"), "b:0.1")); + assertSearchEqual("source=t b=0.1f", search(relation("t"), "b:0.1")); assertEqual( "source=t | where b=0.1f", filter(relation("t"), compare("=", field("b"), floatLiteral(0.1f)))); @@ -965,7 +965,7 @@ public void testFloatLiteralExpr() { @Test public void testDecimalLiteralExpr() { - assertEqual("source=t b=0.1", search(relation("t"), "b:0.1")); + assertSearchEqual("source=t b=0.1", search(relation("t"), "b:0.1")); assertEqual( "source=t | where b=0.1", filter(relation("t"), compare("=", field("b"), decimalLiteral(0.1)))); @@ -973,7 +973,7 @@ public void testDecimalLiteralExpr() { @Test public void testBooleanLiteralExpr() { - assertEqual("source=t a=true", search(relation("t"), "a:true")); + assertSearchEqual("source=t a=true", search(relation("t"), "a:true")); assertEqual( "source=t | where a=true", filter(relation("t"), compare("=", field("a"), booleanLiteral(true)))); @@ -981,7 +981,7 @@ public void testBooleanLiteralExpr() { @Test public void testBackQuotedFieldNames() { - assertEqual("source=t `first name`=true", search(relation("t"), "first\\ name:true")); + assertSearchEqual("source=t `first name`=true", search(relation("t"), "first\\ name:true")); assertEqual( "source=t | where `first name`=true", filter(relation("t"), compare("=", field("first name"), booleanLiteral(true)))); @@ -1369,23 +1369,23 @@ public void testMedianAggFuncExpr() { @Test public void testTimeModifierEarliestWithNumericValue() { - assertEqual("source=t earliest=1", search(relation("t"), "@timestamp:>=1000")); + assertSearchEqual("source=t earliest=1", search(relation("t"), "@timestamp:>=1000")); - assertEqual( + assertSearchEqual( "source=t earliest=1754020061.123456", search(relation("t"), "@timestamp:>=1754020061123.456")); } @Test public void testTimeModifierLatestWithNowValue() { - assertEqual( + assertSearchEqual( "source=t earliest=now latest=now()", search(relation("t"), "(@timestamp:>=now) AND (@timestamp:<=now)")); } @Test public void testTimeModifierEarliestWithStringValue() { - assertEqual( + assertSearchEqual( "source=t earliest='2025-12-10 14:00:00'", search(relation("t"), "@timestamp:>=2025\\-12\\-10T14\\:00\\:00Z")); } From 5e22011cb781712e3572ebea935488fd3e0b5a51 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 12 Nov 2025 10:16:16 +0800 Subject: [PATCH 6/7] revert match change Signed-off-by: xinyual --- .../sql/ast/expression/SearchComparison.java | 19 ------------------- .../sql/ast/expression/SearchExpression.java | 9 --------- .../sql/calcite/CalciteRelNodeVisitor.java | 5 ++++- .../sql/calcite/remote/CalciteExplainIT.java | 9 --------- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 4 ---- .../sql/ppl/calcite/CalcitePPLSearchTest.java | 6 ++---- 6 files changed, 6 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java index 01eceb00a83..99ff20e45a5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java @@ -5,15 +5,12 @@ package org.opensearch.sql.ast.expression; -import static org.opensearch.sql.ast.expression.SearchComparison.Operator.EQUALS; - import java.util.Arrays; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; -import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.utils.QueryStringUtils; /** Search expression for field comparisons. */ @@ -66,22 +63,6 @@ public String toQueryString() { } } - @Override - public Function toDSLFunction() { - String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString()); - String valueStr = value.toQueryString(); - if (operator == EQUALS - && !(valueStr.contains("*") - || valueStr.contains("?"))) { // for regex case, we cannot use match - return AstDSL.function( - "match", - AstDSL.unresolvedArg("field_name", AstDSL.qualifiedName(fieldName)), - AstDSL.unresolvedArg("value", AstDSL.stringLiteral(valueStr))); - } - return AstDSL.function( - "query_string", AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); - } - @Override public String toAnonymizedString() { return "identifier " + operator.symbol + " ***"; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java index c17c8438c90..b705909445f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java @@ -6,7 +6,6 @@ package org.opensearch.sql.ast.expression; import org.opensearch.sql.ast.AbstractNodeVisitor; -import org.opensearch.sql.ast.dsl.AstDSL; /** Base class for search expressions that get converted to query_string syntax. */ public abstract class SearchExpression extends UnresolvedExpression { @@ -18,14 +17,6 @@ public abstract class SearchExpression extends UnresolvedExpression { */ public abstract String toQueryString(); - public Function toDSLFunction() { - Function queryStringFunc = - AstDSL.function( - "query_string", - AstDSL.unresolvedArg("query", AstDSL.stringLiteral(this.toQueryString()))); - return queryStringFunc; - } - /** * Convert the search expression to anonymized string * diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 1a8af41ff64..c6a964fce17 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -210,7 +210,10 @@ public RelNode visitSearch(Search node, CalcitePlanContext context) { // Visit the Relation child to get the scan node.getChild().get(0).accept(this, context); // Create query_string function - Function queryStringFunc = node.getOriginalExpression().toDSLFunction(); + Function queryStringFunc = + AstDSL.function( + "query_string", + AstDSL.unresolvedArg("query", AstDSL.stringLiteral(node.getQueryString()))); RexNode queryStringRex = rexVisitor.analyze(queryStringFunc, context); context.relBuilder.filter(queryStringRex); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 6717fa1e9ad..cff92408d4f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -78,15 +78,6 @@ public void supportSearchSargPushDown_timeRange() throws IOException { assertYamlEqualsIgnoreId(expected, result); } - @Test - public void supportSearchMatchPushDown() throws IOException { - String query = - "source=opensearch-sql_test_index_bank birthdate = '2016-12-08 00:00:00.000000000'"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_search_with_match_pushdown.json"); - assertJsonEqualsIgnoreId(expected, result); - } - // Only for Calcite @Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725") public void testJoinWithCriteriaAndMaxOption() throws IOException { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 5501a3bfdac..c26bb5e717b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -250,10 +250,6 @@ public String visitTableFunction(TableFunction node, String context) { @Override public String visitSearch(Search node, String context) { String source = node.getChild().get(0).accept(this, context); - // SearchExpression expression = node.getOriginalExpression(); - - // String queryString = node.getQueryString(); - // String anonymized = queryString.replaceAll(":\\S+", ":" + MASK_LITERAL); return StringUtils.format("%s %s", source, node.getOriginalExpression().toAnonymizedString()); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java index ffc9916df3b..ddb1df95edd 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLSearchTest.java @@ -45,14 +45,12 @@ public void testSearchWithFilter() { String ppl = "search source=EMP DEPTNO=20"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalFilter(condition=[match(MAP('field_name', $7), MAP('value', '20':VARCHAR))])\n" + "LogicalFilter(condition=[query_string(MAP('query', 'DEPTNO:20':VARCHAR))])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT *\n" - + "FROM `scott`.`EMP`\n" - + "WHERE `match`(MAP ('field_name', `DEPTNO`), MAP ('value', '20'))"; + "SELECT *\nFROM `scott`.`EMP`\nWHERE `query_string`(MAP ('query', 'DEPTNO:20'))"; verifyPPLToSparkSQL(root, expectedSparkSql); } From e02a035709191c7ab2d18ab6323266ce74ff5e6d Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 13 Nov 2025 17:31:34 +0800 Subject: [PATCH 7/7] fix UT by ignore the expression Signed-off-by: xinyual --- core/src/main/java/org/opensearch/sql/ast/tree/Search.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Search.java b/core/src/main/java/org/opensearch/sql/ast/tree/Search.java index 441eb5f8a8c..4187a9141e0 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Search.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Search.java @@ -20,12 +20,13 @@ */ @Getter @ToString -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false) @RequiredArgsConstructor public class Search extends UnresolvedPlan { - private final UnresolvedPlan child; - private final String queryString; + @EqualsAndHashCode.Include private final UnresolvedPlan child; + @EqualsAndHashCode.Include private final String queryString; + private final SearchExpression originalExpression; @Override