Skip to content

Commit 0f197e5

Browse files
qianheng-awsqianheng-aws
authored
Fix error when pushing down script filter with struct field (#3693)
* Fix error when pushing down script filter with struct field Signed-off-by: Heng Qian <qianheng@amazon.com> * Fix IT Signed-off-by: Heng Qian <qianheng@amazon.com> * Address comments Signed-off-by: Heng Qian <qianheng@amazon.com> * Fix IT Signed-off-by: Heng Qian <qianheng@amazon.com> --------- Signed-off-by: Heng Qian <qianheng@amazon.com>
1 parent 401c72e commit 0f197e5

File tree

8 files changed

+76
-10
lines changed

8 files changed

+76
-10
lines changed

‎core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public class ReferenceExpression implements Expression {
2525

2626
@Getter private final List<String> paths;
2727

28-
private final ExprType type;
28+
@Getter private final ExprType type;
2929

3030
/**
3131
* Constructor of ReferenceExpression.

‎integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_agg_push.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"root": {
3+
"name": "ProjectOperator",
4+
"description": {
5+
"fields": "[count(), patterns_field]"
6+
},
7+
"children": [
8+
{
9+
"name": "OpenSearchIndexScan",
10+
"description": {
11+
"request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb27CZfCltUMmOQIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZbmXyjSCpmeCAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\",\"lang\":\"opensearch_query_expression\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"
12+
},
13+
"children": []
14+
}
15+
]
16+
}
17+
}

‎integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
{
99
"name":"OpenSearchIndexScan",
1010
"description":{
11-
"request":"OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb27CZfCltUMmOQIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uFAqD3gJLjyQCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZbmXyjSCpmeCAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\",\"lang\":\"opensearch_query_expression\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"},
11+
"request":"OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb27CZfCltUMmOQIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZbmXyjSCpmeCAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\",\"lang\":\"opensearch_query_expression\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"},
1212
"children":[]
1313
}
1414
]
1515
}
16-
}
16+
}

‎integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3312.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"Push down filter with struct field":
2+
- skip:
3+
features:
4+
- headers
5+
- do:
6+
bulk:
7+
index: test
8+
refresh: true
9+
body:
10+
- '{"index": {}}'
11+
- '{"log": {"url": {"message": "/e2e/h/zap"} } }'
12+
- do:
13+
headers:
14+
Content-Type: 'application/json'
15+
ppl:
16+
body:
17+
query: 'source=test | where isNotNull(log)'
18+
- match: {"total": 1}
19+
- match: {"schema": [{"name": "log", "type": "struct"}]}
20+
- match: {"datarows": [[{"url": {"message": "/e2e/h/zap"}}]]}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,9 @@ public CalciteLogicalIndexScan pushDownFilter(Filter filter) {
101101
return newScan;
102102
} catch (Exception e) {
103103
if (LOG.isDebugEnabled()) {
104-
LOG.debug("Cannot pushdown the filter condition {}", filter.getCondition(), e);
104+
LOG.debug("Cannot pushdown the filter condition.", e);
105105
} else {
106-
LOG.info("Cannot pushdown the filter condition {}, ", filter.getCondition());
106+
LOG.info("Cannot pushdown the filter condition.");
107107
}
108108
}
109109
return null;

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder;
2828
import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder.PushDownUnSupportedException;
2929
import org.opensearch.sql.opensearch.storage.script.filter.FilterQueryBuilder;
30+
import org.opensearch.sql.opensearch.storage.script.filter.FilterQueryBuilder.ScriptQueryUnSupportedException;
3031
import org.opensearch.sql.opensearch.storage.script.sort.SortQueryBuilder;
3132
import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer;
3233
import org.opensearch.sql.planner.logical.LogicalFilter;
@@ -56,10 +57,23 @@ public OpenSearchIndexScanQueryBuilder(OpenSearchRequestBuilder requestBuilder)
5657
public boolean pushDownFilter(LogicalFilter filter) {
5758
FilterQueryBuilder queryBuilder = new FilterQueryBuilder(new DefaultExpressionSerializer());
5859
Expression queryCondition = filter.getCondition();
59-
QueryBuilder query = queryBuilder.build(queryCondition);
60-
requestBuilder.pushDownFilter(query);
61-
requestBuilder.pushDownTrackedScore(trackScoresFromOpenSearchFunction(queryCondition));
62-
return true;
60+
try {
61+
QueryBuilder query = queryBuilder.build(queryCondition);
62+
requestBuilder.pushDownFilter(query);
63+
requestBuilder.pushDownTrackedScore(trackScoresFromOpenSearchFunction(queryCondition));
64+
return true;
65+
} catch (ScriptQueryUnSupportedException e) {
66+
// Don't catch SyntaxCheckException, which should fall back to the old SQL engine.
67+
// See test {@link NestedIT::test_nested_where_with_and_conditional}, only the old SQL engine
68+
// can cover this case.
69+
// It will fail if catching SyntaxCheckException here and keep using v2 engine.
70+
if (LOG.isDebugEnabled()) {
71+
LOG.debug("Cannot pushdown the filter condition.", e);
72+
} else {
73+
LOG.info("Cannot pushdown the filter condition.");
74+
}
75+
return false;
76+
}
6377
}
6478

6579
@Override

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public ExprValue execute(
7878
});
7979
}
8080

81-
private Set<ReferenceExpression> extractFields(Expression expr) {
81+
public static Set<ReferenceExpression> extractFields(Expression expr) {
8282
Set<ReferenceExpression> fields = new HashSet<>();
8383
expr.accept(
8484
new ExpressionNodeVisitor<Object, Set<ReferenceExpression>>() {

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import com.google.common.collect.ImmutableMap;
1313
import java.util.Map;
14+
import java.util.Set;
1415
import java.util.function.BiFunction;
1516
import lombok.RequiredArgsConstructor;
1617
import org.opensearch.index.query.BoolQueryBuilder;
@@ -19,11 +20,14 @@
1920
import org.opensearch.index.query.ScriptQueryBuilder;
2021
import org.opensearch.script.Script;
2122
import org.opensearch.sql.common.antlr.SyntaxCheckException;
23+
import org.opensearch.sql.data.type.ExprCoreType;
2224
import org.opensearch.sql.expression.Expression;
2325
import org.opensearch.sql.expression.ExpressionNodeVisitor;
2426
import org.opensearch.sql.expression.FunctionExpression;
27+
import org.opensearch.sql.expression.ReferenceExpression;
2528
import org.opensearch.sql.expression.function.BuiltinFunctionName;
2629
import org.opensearch.sql.expression.function.FunctionName;
30+
import org.opensearch.sql.opensearch.storage.script.core.ExpressionScript;
2731
import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery;
2832
import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery;
2933
import org.opensearch.sql.opensearch.storage.script.filter.lucene.NestedQuery;
@@ -44,6 +48,12 @@
4448
@RequiredArgsConstructor
4549
public class FilterQueryBuilder extends ExpressionNodeVisitor<QueryBuilder, Object> {
4650

51+
public static class ScriptQueryUnSupportedException extends RuntimeException {
52+
public ScriptQueryUnSupportedException(String message) {
53+
super(message);
54+
}
55+
}
56+
4757
/** Serializer that serializes expression for build DSL query. */
4858
private final ExpressionSerializer serializer;
4959

@@ -129,6 +139,11 @@ private BoolQueryBuilder buildBoolQuery(
129139
}
130140

131141
private ScriptQueryBuilder buildScriptQuery(FunctionExpression node) {
142+
Set<ReferenceExpression> fields = ExpressionScript.extractFields(node);
143+
if (fields.stream().anyMatch(field -> field.getType() == ExprCoreType.STRUCT)) {
144+
throw new ScriptQueryUnSupportedException(
145+
"Script query does not support fields of struct type in OpenSearch.");
146+
}
132147
return new ScriptQueryBuilder(
133148
new Script(
134149
DEFAULT_SCRIPT_TYPE, EXPRESSION_LANG_NAME, serializer.serialize(node), emptyMap()));

0 commit comments

Comments
 (0)