Do not remove nested fields in resolving AllFieldsExcludeMeta (#4708)

Signed-off-by: Lantao Jin <[email protected]>

Author: LantaoJin

core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java

@@ -18,7 +18,7 @@
 public class AllFields extends UnresolvedExpression {
   public static final AllFields INSTANCE = new AllFields();
 
-  public AllFields() {}
+  protected AllFields() {}
 
   public static AllFields of() {
     return INSTANCE;

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -380,7 +380,10 @@ private RelNode handleAllFieldsProject(Project node, CalcitePlanContext context)
           "Invalid field exclusion: operation would exclude all fields from the result set");
     }
     AllFields allFields = (AllFields) node.getProjectList().getFirst();
-    tryToRemoveNestedFields(context);
+    if (!(allFields instanceof AllFieldsExcludeMeta)) {
+      // Should not remove nested fields for AllFieldsExcludeMeta.
+      tryToRemoveNestedFields(context);
+    }
     tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta);
     return context.relBuilder.peek();
   }

core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java

@@ -65,6 +65,7 @@ private static RexNode resolveInNonJoinCondition(
     log.debug("resolveInNonJoinCondition() called with nameNode={}", nameNode);
 
     return resolveLambdaVariable(nameNode, context)
+        .or(() -> resolveFieldDirectly(nameNode, context, 1))
         .or(() -> resolveFieldWithAlias(nameNode, context, 1))
         .or(() -> resolveFieldWithoutAlias(nameNode, context, 1))
         .or(() -> resolveRenamedField(nameNode, context))
@@ -88,6 +89,26 @@ private static String joinParts(List<String> parts, int start) {
     return joinParts(parts, start, parts.size() - start);
   }
 
+  private static Optional<RexNode> resolveFieldDirectly(
+      QualifiedName nameNode, CalcitePlanContext context, int inputCount) {
+    List<String> parts = nameNode.getParts();
+    log.debug(
+        "resolveFieldDirectly() called with nameNode={}, parts={}, inputCount={}",
+        nameNode,
+        parts,
+        inputCount);
+
+    List<String> currentFields = context.relBuilder.peek().getRowType().getFieldNames();
+    if (currentFields.contains(nameNode.toString())) {
+      try {
+        return Optional.of(context.relBuilder.field(nameNode.toString()));
+      } catch (IllegalArgumentException e) {
+        log.debug("resolveFieldDirectly() failed: {}", e.getMessage());
+      }
+    }
+    return Optional.empty();
+  }
+
   private static Optional<RexNode> resolveFieldWithAlias(
       QualifiedName nameNode, CalcitePlanContext context, int inputCount) {
     List<String> parts = nameNode.getParts();

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4575.yml

@@ -0,0 +1,60 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              a:
+                properties:
+                  b:
+                    properties:
+                      c:
+                        type: keyword
+              d:
+                properties:
+                  e:
+                    properties:
+                      f:
+                        type: keyword
+              num:
+                type: integer
+
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : true
+
+---
+teardown:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : false
+
+---
+"Access struct fields after join":
+  - skip:
+      features:
+        - headers
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"a": {"b": {"c": "/a/b/c"} }, "d": {"e": {"f": "/d/e/f"} }, "num": 10 }'
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=test | eval N=d.e.f | join type=inner left=l, right=r ON l.N = r.F [ source=test | rename d.e.f as F | fields F ] | fields num, N, a.b.c, d.e.f'
+  - match: {"total": 1}
+  - match: { schema: [ { "name": "num", "type": "int" }, { "name": "N", "type": "string" }, { "name": "a.b.c", "type": "string" }, { "name": "d.e.f", "type": "string" } ] }
+  - match: { datarows: [ [ 10, "/d/e/f",  "/a/b/c",  "/d/e/f"] ] }