Merge branch 'main' into opex-product-fix-490

Author: alaudazzi

muted-tests.yml

@@ -213,8 +213,6 @@ tests:
 - class: org.elasticsearch.cluster.service.MasterServiceTests
   method: testThreadContext
   issue: https://github.com/elastic/elasticsearch/issues/118914
-- class: org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/118970
 - class: org.elasticsearch.aggregations.bucket.SearchCancellationIT
   method: testCancellationDuringTimeSeriesAggregation
   issue: https://github.com/elastic/elasticsearch/issues/118992

rest-api-spec/build.gradle

@@ -59,25 +59,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
-  task.skipTest("tsdb/20_mapping/exact match object type", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/25_id_generation/delete over _bulk", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/80_index_resize/split", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/noop update", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/regular update", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/search with routing", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/index with routing over _bulk", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/update over _bulk", "skip until pr/116687 gets backported")
-  task.skipTest("tsdb/90_unsupported_operations/index with routing", "skip until pr/116687 gets backported")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
-  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
-  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
-  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
-  task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0")
-  task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
-  task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
-  task.skipTest("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set", "Can't work until auto-expand replicas is 0-1 for synonyms index")
-  task.skipTest("search/90_search_after/_shard_doc sort", "restriction has been lifted in latest versions")
 })

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java

@@ -189,7 +189,7 @@ private TestStats testTrendVs(TestStats H0, double[] values, double[] weights) {
     private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) {
 
         double vStep = Double.MAX_VALUE;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -226,7 +226,7 @@ private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weigh
     private TestStats testTrendChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) {
 
         double vChange = Double.MAX_VALUE;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -349,7 +349,7 @@ private TestStats testDistributionChange(
     ) {
 
         double maxDiff = 0.0;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -378,10 +378,12 @@ private TestStats testDistributionChange(
         // before we run the tests.
         SampleData sampleData = sample(values, weights, discoveredChangePoints);
         final double[] sampleValues = sampleData.values();
-        final double[] sampleWeights = sampleData.weights();
 
         double pValue = 1;
         for (int cp : sampleData.changePoints()) {
+            if (cp == ChangeType.NO_CHANGE_POINT) {
+                continue;
+            }
             double[] x = Arrays.copyOfRange(sampleValues, 0, cp);
             double[] y = Arrays.copyOfRange(sampleValues, cp, sampleValues.length);
             double statistic = KOLMOGOROV_SMIRNOV_TEST.kolmogorovSmirnovStatistic(x, y);
@@ -451,7 +453,7 @@ private record TestStats(Type type, double pValue, double var, double nParams, i
         }
 
         TestStats(Type type, double pValue, double var, double nParams, DataStats dataStats) {
-            this(type, pValue, var, nParams, -1, dataStats);
+            this(type, pValue, var, nParams, ChangeType.NO_CHANGE_POINT, dataStats);
         }
 
         boolean accept(double pValueThreshold) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java

@@ -47,7 +47,7 @@ public InternalAggregation doReduce(InternalAggregations aggregations, Aggregati
         ChangeType change = ChangePointDetector.getChangeType(bucketValues);
 
         ChangePointBucket changePointBucket = null;
-        if (change.changePoint() >= 0) {
+        if (change.changePoint() != ChangeType.NO_CHANGE_POINT) {
             changePointBucket = extractBucket(bucketsPaths()[0], aggregations, change.changePoint()).map(
                 b -> new ChangePointBucket(b.getKey(), b.getDocCount(), b.getAggregations())
             ).orElse(null);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java

@@ -21,8 +21,10 @@
  */
 public interface ChangeType extends NamedWriteable, NamedXContentObject {
 
+    int NO_CHANGE_POINT = -1;
+
     default int changePoint() {
-        return -1;
+        return NO_CHANGE_POINT;
     }
 
     default double pValue() {

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java

@@ -17,6 +17,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.DoubleStream;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -243,4 +244,14 @@ public void testProblemDistributionChange() {
         ChangeType type = new ChangeDetector(bucketValues).detect(0.05);
         assertThat(type, instanceOf(ChangeType.DistributionChange.class));
     }
+
+    public void testUncertainNonStationary() {
+        MlAggsHelper.DoubleBucketValues bucketValues = new MlAggsHelper.DoubleBucketValues(
+            null,
+            new double[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 735, 715 }
+        );
+        ChangeType type = new ChangeDetector(bucketValues).detect(0.01);
+        assertThat(type, instanceOf(ChangeType.NonStationary.class));
+        assertThat(((ChangeType.NonStationary) type).getTrend(), equalTo("increasing"));
+    }
 }

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+public class MountFromVersion5IT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion5IT(Version version) {
+        super(version);
+    }
+
+    public void testSearchableSnapshot() throws Exception {
+        verifyCompatibility("5");
+    }
+}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+public class MountFromVersion6IT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion6IT(Version version) {
+        super(version);
+    }
+
+    public void testSearchableSnapshot() throws Exception {
+        verifyCompatibility("6");
+    }
+}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
+import org.elasticsearch.test.cluster.util.Version;
+
+import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+
+/**
+ * Test suite for Archive indices backward compatibility with N-2 versions.
+ * The test suite creates a cluster in the N-1 version, where N is the current version.
+ * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
+ * Test methods are executed after each upgrade.
+ */
+public class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
+    }
+
+    public SearchableSnapshotTestCase(Version version) {
+        super(version);
+    }
+
+    /**
+     * Overrides the snapshot-restore operation for archive-indices scenario.
+     */
+    @Override
+    public void recover(RestClient client, String repository, String snapshot, String index) throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount");
+        request.addParameter("wait_for_completion", "true");
+        request.addParameter("storage", "full_copy");
+        request.setJsonEntity(Strings.format("""
+             {
+              "index": "%s",
+              "renamed_index": "%s"
+            }""", index, index));
+        createFromResponse(client.performRequest(request));
+    }
+}