Support kNN filter on nested metadata

docs/changelog/113949.yaml

@@ -0,0 +1,6 @@
+pr: 113949
+summary: Support kNN filter on nested metadata
+area: Vector Search
+type: enhancement
+issues:
+ - 106994

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -202,10 +202,19 @@ POST my-image-index/_search
 `knn` query can be used inside a nested query. The behaviour here is similar to [top level nested kNN search](docs-content://solutions/search/vector/knn.md#nested-knn-search):
 
 * kNN search over nested dense_vectors diversifies the top results over the top-level document
-* `filter`  over the top-level document metadata is supported and acts as a pre-filter
-* `filter` over `nested` field metadata is not supported
+* `filter` both over the top-level document metadata and `nested` is supported and acts as a pre-filter
+
+::::{note}
+To ensure correct results: each individual filter must be either over
+the top-level metadata or `nested` metadata. However, a single knn query
+supports multiple filters, where some filters can be over the top-level
+metadata and some over nested.
+::::
 
-A sample query can look like below:
+
+Below is a sample query with filter over nested metadata.
+For scoring parents' documents,  this query only considers vectors that
+have "paragraph.language" set to "EN".
 
 ```json
 {
@@ -214,12 +223,46 @@ A sample query can look like below:
       "path" : "paragraph",
         "query" : {
           "knn": {
-            "query_vector": [
-                0.45,
-                45
-            ],
+            "query_vector": [0.45, 0.50],
             "field": "paragraph.vector",
-            "num_candidates": 2
+            "filter": {
+              "match": {
+                "paragraph.language": "EN"
+              }
+            }
+        }
+      }
+    }
+  }
+}
+```
+
+Below is a sample query with two filters: one over nested metadata
+and another over the top level metadata. For scoring parents' documents,
+this query only considers vectors whose parent's title contain "essay"
+word and have "paragraph.language" set to "EN".
+
+```json
+{
+  "query" : {
+    "nested" : {
+      "path" : "paragraph",
+      "query" : {
+        "knn": {
+          "query_vector": [0.45, 0.50],
+          "field": "paragraph.vector",
+          "filter": [
+            {
+              "match": {
+                "paragraph.language": "EN"
+              }
+            },
+            {
+              "match": {
+                "title": "essay"
+              }
+            }
+          ]
         }
       }
     }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml

@@ -16,6 +16,8 @@ setup:
               nested:
                 type: nested
                 properties:
+                  language:
+                    type: keyword
                   paragraph_id:
                     type: keyword
                   vector:
@@ -37,8 +39,10 @@ setup:
           nested:
           - paragraph_id: 0
             vector: [230.0, 300.33, -34.8988, 15.555, -200.0]
+            language: EN
           - paragraph_id: 1
             vector: [240.0, 300, -3, 1, -20]
+            language: FR
 
   - do:
       index:
@@ -49,10 +53,13 @@ setup:
           nested:
           - paragraph_id: 0
             vector: [-0.5, 100.0, -13, 14.8, -156.0]
+            language: EN
           - paragraph_id: 2
             vector: [0, 100.0, 0, 14.8, -156.0]
+            language: EN
           - paragraph_id: 3
             vector: [0, 1.0, 0, 1.8, -15.0]
+            language: FR
 
   - do:
       index:
@@ -63,6 +70,7 @@ setup:
           nested:
             - paragraph_id: 0
               vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+              language: FR
 
   - do:
       indices.refresh: {}
@@ -394,23 +402,24 @@ setup:
   - do:
       indices.refresh: {}
 
-  - do:
-      search:
-          index: nested_text
-          body:
-            knn:
-              field: paragraphs.vector
-              query_vector: [1, 2]
-              num_candidates: 10
-              k: 10
-              filter:
-                bool:
-                  must_not:
-                    exists:
-                      field: publish_date
-
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0._id: "2"}
+#  This query fails now, breaking change: as the filter matches both nested and non-nested docs .
+#  - do:
+#      search:
+#          index: nested_text
+#          body:
+#            knn:
+#              field: paragraphs.vector
+#              query_vector: [1, 2]
+#              num_candidates: 10
+#              k: 10
+#              filter:
+#                bool:
+#                  must_not:
+#                    exists:
+#                      field: publish_date
+#
+#  - match: {hits.total.value: 1}
+#  - match: {hits.hits.0._id: "2"}
 ---
 "nested Knn search with required similarity appropriately filters inner_hits":
   - requires:
@@ -461,3 +470,122 @@ setup:
   - match: {hits.hits.0._id: "2"}
   - length: {hits.hits.0.inner_hits.nested.hits.hits: 1}
   - match: {hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0"}
+
+
+---
+"Test filter on nested fields":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ knn_filter_on_nested_fields ]
+      test_runner_features: ["capabilities", "close_to"]
+      reason: "Capability for filtering on nested fields required"
+
+  - do:
+      search:
+        index: test
+        body:
+          _source: false
+          knn:
+            boost: 2
+            field: nested.vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            num_candidates: 10
+            filter: { match: { nested.language: "EN" } }
+            inner_hits: { size: 3, "fields": [ "nested.paragraph_id", "nested.language"], _source: false }
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0.inner_hits.nested.hits.total.value: 2 }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "EN" }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.1.fields.nested.0.paragraph_id.0: "2" }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.1.fields.nested.0.language.0: "EN" }
+  - close_to: { hits.hits.0._score: { value: 0.0182, error: 0.0001 } }
+  - close_to: { hits.hits.0.inner_hits.nested.hits.hits.0._score: { value:  0.0182, error: 0.0001 } }
+  - match: { hits.hits.1._id: "1" }
+  - match: { hits.hits.1.inner_hits.nested.hits.total.value: 1 }
+  - match: { hits.hits.1.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" }
+  - match: { hits.hits.1.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "EN" }
+
+  - do:
+      search:
+        index: test
+        body:
+          _source: false
+          knn:
+            boost: 2
+            field: nested.vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            num_candidates: 10
+            filter: { match: { nested.language: "FR" } }
+            inner_hits: { size: 3, "fields": [ "nested.paragraph_id", "nested.language"], _source: false }
+
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.inner_hits.nested.hits.total.value: 1 }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "FR" }
+  - close_to: { hits.hits.0._score: { value: 0.0043, error: 0.0001 } }
+  - close_to: { hits.hits.0.inner_hits.nested.hits.hits.0._score: { value: 0.0043, error: 0.0001 } }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1.inner_hits.nested.hits.total.value: 1 }
+  - match: { hits.hits.1.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "3" }
+  - match: { hits.hits.1.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "FR" }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2.inner_hits.nested.hits.total.value: 1 }
+  - match: { hits.hits.2.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "1" }
+  - match: { hits.hits.2.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "FR" }
+
+  # filter on both nested and parent metadata with 2 different filters
+  - do:
+      search:
+        index: test
+        body:
+          _source: false
+          knn:
+            boost: 2
+            field: nested.vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            num_candidates: 10
+            filter: [{ match: { nested.language: "FR" }}, {term: {name: "rabbit.jpg"}} ]
+            inner_hits: { size: 3, "fields": [ "nested.paragraph_id", "nested.language"], _source: false }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.inner_hits.nested.hits.total.value: 1 }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" }
+  - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.language.0: "FR" }
+  - close_to: { hits.hits.0._score: { value: 0.0043, error: 0.0001 } }
+
+
+---
+"Test filter on nested fields with filter on both nested and parent metadata is not allowed":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ knn_filter_on_nested_fields ]
+      test_runner_features: ["capabilities", "close_to"]
+      reason: "Capability for filtering on nested fields required"
+
+  - do:
+      catch: /A filter in knn search might match both nested and non-nested documents, which is not allowed. Modify the filter to be either over the top-level or nested metadata./
+      search:
+        index: test
+        body:
+          _source: false
+          knn:
+            field: nested.vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            filter:
+              bool:
+                must:
+                  - match: { nested.language: "FR" }
+                  - term: { name: "rabbit.jpg" }
+