Support kNN filter on nested metadata

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml

@@ -402,23 +402,24 @@ setup:
   - do:
       indices.refresh: {}
 
-  - do:
-      search:
-          index: nested_text
-          body:
-            knn:
-              field: paragraphs.vector
-              query_vector: [1, 2]
-              num_candidates: 10
-              k: 10
-              filter:
-                bool:
-                  must_not:
-                    exists:
-                      field: publish_date
-
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0._id: "2"}
+#  This query fails now, breaking change: as the filter matches both nested and non-nested docs .
+#  - do:
+#      search:
+#          index: nested_text
+#          body:
+#            knn:
+#              field: paragraphs.vector
+#              query_vector: [1, 2]
+#              num_candidates: 10
+#              k: 10
+#              filter:
+#                bool:
+#                  must_not:
+#                    exists:
+#                      field: publish_date
+#
+#  - match: {hits.total.value: 1}
+#  - match: {hits.hits.0._id: "2"}
 ---
 "nested Knn search with required similarity appropriately filters inner_hits":
   - requires:
@@ -561,3 +562,30 @@ setup:
   - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "FR" }
   - close_to: { hits.hits.0._score: { value: 0.0043, error: 0.0001 } }
 
+
+---
+"Test filter on nested fields with filter on both nested and parent metadata is not allowed":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ knn_filter_on_nested_fields ]
+      test_runner_features: ["capabilities", "close_to"]
+      reason: "Capability for filtering on nested fields required"
+
+  - do:
+      catch: /A filter in knn search might match both nested and non-nested documents, which is not allowed. Modify the filter to be either over the top-level or nested metadata./
+      search:
+        index: test
+        body:
+          _source: false
+          knn:
+            field: nested.vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            filter:
+              bool:
+                must:
+                  - match: { nested.language: "FR" }
+                  - term: { name: "rabbit.jpg" }
+

server/src/main/java/org/elasticsearch/index/search/NestedHelper.java

@@ -24,6 +24,8 @@
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
+import java.util.List;
+
 /** Utility class to filter parent and children clauses when building nested
  * queries. */
 public final class NestedHelper {
@@ -164,4 +166,41 @@ private static boolean mightMatchNonNestedDocs(SearchExecutionContext searchExec
         }
         return true;
     }
+
+    /**
+     * Returns true if the given query might match both nested documents and non-nested documents
+     */
+    public static boolean mightMatchMixedDocs(Query query, String nestedPath, SearchExecutionContext searchExecutionContext) {
+        // First check if the individual query might match both types, such as match_all or wildcard or meta field queries
+        boolean mightMatchNestedDocs = mightMatchNestedDocs(query, searchExecutionContext);
+        boolean mightMatchNonNestedDocs = mightMatchNonNestedDocs(query, nestedPath, searchExecutionContext);
+        if (mightMatchNestedDocs && mightMatchNonNestedDocs) {
+            return true;
+        }
+
+        if (query instanceof final BooleanQuery bq) {
+            List<BooleanClause> clauses = bq.clauses();
+            if (clauses.isEmpty()) {
+                return false;
+            }
+            boolean hasNested = false;
+            boolean hasNonNested = false;
+            for (BooleanClause clause : clauses) {
+                Query clauseQuery = clause.query();
+                boolean clauseMatchesNested = mightMatchNestedDocs(clauseQuery, searchExecutionContext);
+                boolean clauseMatchesNonNested = mightMatchNonNestedDocs(clauseQuery, nestedPath, searchExecutionContext);
+                if (clauseMatchesNested) {
+                    hasNested = true;
+                }
+                if (clauseMatchesNonNested) {
+                    hasNonNested = true;
+                }
+                if (hasNested && hasNonNested) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
 }

server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java

@@ -556,6 +556,12 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
             parentBitSet = context.bitsetFilter(parentFilter);
             ArrayList<Query> filterAdjusted = new ArrayList<>(filtersInitial.size());
             for (Query f : filtersInitial) {
+                if (NestedHelper.mightMatchMixedDocs(f, parentPath, context)) {
+                    throw new IllegalArgumentException(
+                        "A filter in knn search might match both nested and non-nested documents, which is not allowed. "
+                            + "Modify the filter to be either over the top-level or nested metadata."
+                    );
+                }
                 // If filter matches non-nested docs, we assume this is a filter over parents docs,
                 // so we will modify it accordingly: matching parents docs with join to its child docs
                 if (NestedHelper.mightMatchNonNestedDocs(f, parentPath, context)) {