Merge branch 'main' into cps-spi-poc

Author: n1v0lg

docs/changelog/130382.yaml

@@ -0,0 +1,5 @@
+pr: 130382
+summary: Remove vectors from `_source` transparently
+area: "Vector Search"
+type: enhancement
+issues: []

docs/changelog/130448.yaml

@@ -0,0 +1,6 @@
+pr: 130448
+summary: Fix wildcard drop after lookup join
+area: ES|QL
+type: bug
+issues:
+ - 129561

docs/reference/elasticsearch/mapping-reference/mapping-source-field.md

@@ -308,13 +308,12 @@ PUT my-index-000001
 }
 ```
 
-::::{admonition} Think before disabling the _source field
-:class: warning
+::::{warning}
 
-Users often disable the `_source` field without thinking about the consequences, and then live to regret it. If the `_source` field isn’t available then a number of features are not supported:
+Do not disable the `_source` field, unless absolutely necessary. If you disable it, the following critical features will not be supported:
 
 * The [`update`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update), [`update_by_query`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update-by-query), and [`reindex`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-reindex) APIs.
-* In the {{kib}} [Discover](docs-content://explore-analyze/discover.md) application, field data will not be displayed.
+* Display of field data in the {{kib}} [Discover](docs-content://explore-analyze/discover.md) application.
 * On the fly [highlighting](/reference/elasticsearch/rest-apis/highlighting.md).
 * The ability to reindex from one Elasticsearch index to another, either to change mappings or analysis, or to upgrade an index to a new major version.
 * The ability to debug queries or aggregations by viewing the original document used at index time.

muted-tests.yml

@@ -539,18 +539,12 @@ tests:
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=indices.resolve_index/10_basic_resolve_index/Resolve index with hidden and closed indices}
   issue: https://github.com/elastic/elasticsearch/issues/130568
-- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/130067
 - class: org.elasticsearch.xpack.monitoring.exporter.http.HttpExporterTests
   method: testExporterWithHostOnly
   issue: https://github.com/elastic/elasticsearch/issues/130599
 - class: org.elasticsearch.xpack.monitoring.exporter.http.HttpExporterTests
   method: testCreateRestClient
   issue: https://github.com/elastic/elasticsearch/issues/130600
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/130067
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search.vectors/40_knn_search/Dimensions are dynamically set}
   issue: https://github.com/elastic/elasticsearch/issues/130626
@@ -587,6 +581,9 @@ tests:
 - class: org.elasticsearch.multiproject.test.XpackWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=dlm/10_usage/Test data stream lifecycle usage stats}
   issue: https://github.com/elastic/elasticsearch/issues/130677
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
+  method: testStopQueryLocal
+  issue: https://github.com/elastic/elasticsearch/issues/121672
 
 # Examples:
 #

qa/ccs-common-rest/build.gradle

@@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 restResources {
   restApi {
     include 'capabilities', 'cat.shards', '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch',
-      "nodes.stats", 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql'
+      "nodes.stats", 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql', 'update'
   }
   restTests {
     includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll', "indices.resolve_index"

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -91,7 +91,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.license.self_generated.type", "trial")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
-        .feature(FeatureFlag.IVF_FORMAT);
+        .feature(FeatureFlag.IVF_FORMAT)
+        .feature(FeatureFlag.SYNTHETIC_VECTORS);
 
     private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
         .name(REMOTE_CLUSTER_NAME)

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -93,6 +93,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .feature(FeatureFlag.IVF_FORMAT)
+        .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .user("test_admin", "x-pack-test-password");
 
     private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -39,6 +39,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
         .feature(FeatureFlag.IVF_FORMAT)
+        .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json

@@ -44,6 +44,14 @@
       "include_defaults":{
         "type":"boolean",
         "description":"Return all default configurations for the component template (default: false)"
+      },
+      "flat_settings":{
+        "type":"boolean",
+        "description":"Return settings in flat format (default: false)"
+      },
+      "settings_filter":{
+        "type":"string",
+        "description":"Filter out results, for example to filter out sensitive information. Supports wildcards or full settings keys"
       }
     }
   }

rest-api-spec/src/main/resources/rest-api-spec/api/get.json

@@ -68,6 +68,10 @@
         "type":"list",
         "description":"A list of fields to extract and return from the _source field"
       },
+      "_source_exclude_vectors":{
+        "type":"boolean",
+        "description":"Whether vectors should be excluded from _source"
+      },
       "version":{
         "type":"number",
         "description":"Explicit version number for concurrency control"