Remove old _knn_search tech preview API in v9 (#118104)

Removes the old `_knn_search` API that was never out of tech preview and
deprecated throughout the v8 cycle. 

To utilize the API, `compatible-with=8` can be utilized.

Author: benwtrent

docs/changelog/118104.yaml

@@ -0,0 +1,12 @@
+pr: 118104
+summary: Remove old `_knn_search` tech preview API in v9
+area: Vector Search
+type: breaking
+issues: []
+breaking:
+  title: Remove old `_knn_search` tech preview API in v9
+  area: REST API
+  details: The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search
+    operations, you should utilize the `_search` endpoint.
+  impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8.
+  notable: false

docs/reference/mapping/types/dense-vector.asciidoc

@@ -232,8 +232,8 @@ it will be set to the length of the first vector added to the field.
 
 `index`::
 (Optional, Boolean)
-If `true`, you can search this field using the <<knn-search-api, kNN search
-API>>. Defaults to `true`.
+If `true`, you can search this field using the <<query-dsl-knn-query, knn query>>
+or <<search-api-knn, knn in _search>> . Defaults to `true`.
 
 [[dense-vector-similarity]]
 `similarity`::

docs/reference/redirects.asciidoc

@@ -1942,3 +1942,8 @@ Refer to <<get-ip-location-database-api>>.
 === Delete geoip database configuration API
 
 Refer to <<delete-ip-location-database-api>>.
+
+[role="exclude",id="knn-search-api"]
+=== Delete _knn_search API
+
+Refer to <<search-api-knn>>.

docs/reference/search.asciidoc

@@ -50,8 +50,6 @@ include::search/async-search.asciidoc[]
 
 include::search/point-in-time-api.asciidoc[]
 
-include::search/knn-search.asciidoc[]
-
 include::search/retriever.asciidoc[]
 
 include::search/rrf.asciidoc[]

docs/reference/search/knn-search.asciidoc

@@ -1058,8 +1058,10 @@ PUT image-index
 * When using kNN search in <<modules-cross-cluster-search,{ccs}>>, the <<ccs-min-roundtrips,`ccs_minimize_roundtrips`>>
 option is not supported.
 
-* {blank}
-include::{es-ref-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm]
+* {es} uses the https://arxiv.org/abs/1603.09320[HNSW algorithm] to support
+efficient kNN search. Like most kNN algorithms, HNSW is an approximate method
+that sacrifices result accuracy for improved search speed. This means the
+results returned are not always the true _k_ closest neighbors.
 
 NOTE: Approximate kNN search always uses the
 <<dfs-query-then-fetch,`dfs_query_then_fetch`>> search type in order to gather

docs/reference/search/search-your-data/knn-search.asciidoc

@@ -215,8 +215,11 @@ setup:
 ---
 "kNN search in _knn_search endpoint":
   - skip:
-      features: [ "allowed_warnings" ]
+      features: [ "allowed_warnings", "headers" ]
   - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
       knn_search:
@@ -240,8 +243,11 @@ setup:
   - requires:
       cluster_features: "gte_v8.2.0"
       reason: 'kNN with filtering added in 8.2'
-      test_runner_features: [ "allowed_warnings" ]
+      test_runner_features: [ "allowed_warnings", "headers" ]
   - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
       knn_search:
@@ -262,6 +268,9 @@ setup:
   - match: { hits.hits.0.fields.name.0: "rabbit.jpg" }
 
   - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
       knn_search:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -55,6 +55,9 @@ setup:
       reason: 'dense_vector field usage was added in 8.1'
       test_runner_features: ["allowed_warnings"]
   - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
       knn_search:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/50_dense_vector_field_usage.yml

@@ -11,7 +11,8 @@
 
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
@@ -34,19 +35,12 @@ public class RestKnnSearchAction extends BaseRestHandler {
 
     public RestKnnSearchAction() {}
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_RELEVANCE)
-    // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access
-    // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will
-    // continue to emit deprecations warnings but will not require any special headers to access the API in v9.
-    // Please review and update the code and tests as needed. The original code remains commented out below for reference.
     @Override
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_RELEVANCE)
     public List<Route> routes() {
-
         return List.of(
-            // Route.builder(GET, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build(),
-            // Route.builder(POST, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build()
-            Route.builder(GET, "{index}/_knn_search").deprecateAndKeep(DEPRECATION_MESSAGE).build(),
-            Route.builder(POST, "{index}/_knn_search").deprecateAndKeep(DEPRECATION_MESSAGE).build()
+            Route.builder(GET, "{index}/_knn_search").deprecatedForRemoval(DEPRECATION_MESSAGE, RestApiVersion.V_8).build(),
+            Route.builder(POST, "{index}/_knn_search").deprecatedForRemoval(DEPRECATION_MESSAGE, RestApiVersion.V_8).build()
         );
     }