Merge branch 'main' into index-version-compatibility-IT

Author: elasticmachine

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -36,7 +36,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");

docs/changelog/125404.yaml

@@ -0,0 +1,5 @@
+pr: 125404
+summary: Check if the anomaly results index has been rolled over
+area: Machine Learning
+type: upgrade
+issues: []

docs/changelog/125659.yaml

@@ -0,0 +1,6 @@
+pr: 125659
+summary: Non existing synonyms sets do not fail shard recovery for indices
+area: "Analysis"
+type: bug
+issues:
+ - 125603

docs/changelog/125666.yaml

@@ -0,0 +1,5 @@
+pr: 125666
+summary: Minor-Fixes Support 7x segments as archive in 8x / 9x
+area: Search
+type: bug
+issues: []

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java

@@ -72,7 +72,7 @@ public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexC
                     );
                 } else {
                     reader = new ReaderWithOrigin(
-                        Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService),
+                        Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService, factory.lenient),
                         "[" + synonymsSet + "] synonyms_set in .synonyms index",
                         synonymsSet
                     );

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java

@@ -311,9 +311,6 @@ tests:
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=search.vectors/41_knn_search_byte_quantized/kNN search plus query}
   issue: https://github.com/elastic/elasticsearch/issues/124687
-- class: org.elasticsearch.common.metrics.ExponentiallyWeightedMovingRateTests
-  method: testEwmr_threadSafe
-  issue: https://github.com/elastic/elasticsearch/issues/124692
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test20RunWithBootstrapChecks
   issue: https://github.com/elastic/elasticsearch/issues/124940
@@ -347,9 +344,6 @@ tests:
 - class: org.elasticsearch.index.shard.StoreRecoveryTests
   method: testAddIndices
   issue: https://github.com/elastic/elasticsearch/issues/124104
-- class: org.elasticsearch.xpack.esql.parser.StatementParserTests
-  method: testInvalidJoinPatterns
-  issue: https://github.com/elastic/elasticsearch/issues/125536
 - class: org.elasticsearch.xpack.inference.registry.ModelRegistryMetadataTests
   method: testUpgrade
   issue: https://github.com/elastic/elasticsearch/issues/125554
@@ -389,6 +383,9 @@ tests:
 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
   method: test {yaml=cat.allocation/10_basic/Node forecasts}
   issue: https://github.com/elastic/elasticsearch/issues/125661
+- class: org.elasticsearch.xpack.esql.spatial.SpatialExtentAggregationNoLicenseIT
+  method: testStExtentAggregationWithPoints
+  issue: https://github.com/elastic/elasticsearch/issues/125735
 
 # Examples:
 #

muted-tests.yml

@@ -70,9 +70,6 @@ excludeList.add('aggregations/percentiles_hdr_metric/Negative values test')
 // sync_id is removed in 9.0
 excludeList.add("cat.shards/10_basic/Help")
 
-// Can't work until auto-expand replicas is 0-1 for synonyms index
-excludeList.add("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set")
-
 def clusterPath = getPath()
 
 buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->

qa/mixed-cluster/build.gradle

@@ -88,4 +88,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("indices.create/21_synthetic_source_stored/field param - keep nested array", "Synthetic source keep arrays now stores leaf arrays natively")
   task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively")
   task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed")
+  task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist")
 })

rest-api-spec/build.gradle

@@ -313,3 +313,159 @@ setup:
       indices.stats: { index: test_index }
 
   - length: { indices: 0 }
+
+---
+"Load index with non existent synonyms set":
+  - requires:
+      cluster_features: [ index.synonyms_set_lenient_on_non_existing ]
+      reason: "requires synonyms_set_lenient_on_non_existing bug fix"
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  # Synonyms are not applied
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: foo }
+
+
+  # Create synonyms set and check synonyms are applied
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        index: .synonyms
+        wait_for_status: green
+
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  # Synonyms are applied
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 2 }
+
+---
+"Load index with non existent synonyms set and lenient set to false":
+  - requires:
+      test_runner_features: [ allowed_warnings ]
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+                  lenient: false
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: false }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - length: { indices: 0 }
+
+  # Create synonyms set and check synonyms are applied
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
+  - do:
+      cluster.health:
+        index: .synonyms
+        wait_for_status: green
+
+  - do:
+      # Warning issued in previous versions
+      allowed_warnings:
+        - "The [state] field in the response to the reroute API is deprecated and will be removed in a future version. Specify ?metric=none to adopt the future behaviour."
+      cluster.reroute:
+        retry_failed: true
+
+  - do:
+      cluster.health:
+        index: test_index
+        wait_for_status: green
+
+  # Synonyms are applied
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 2 }
+