Merge branch 'main' into tests/fix-129713

Author: carlosdelest

muted-tests.yml

@@ -448,9 +448,6 @@ tests:
 - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
   method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch
   issue: https://github.com/elastic/elasticsearch/issues/128418
-- class: org.elasticsearch.xpack.esql.action.ForkIT
-  method: testProfile
-  issue: https://github.com/elastic/elasticsearch/issues/128377
 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT
   method: testTimestampFilterFromQuery
   issue: https://github.com/elastic/elasticsearch/issues/127332
@@ -568,6 +565,9 @@ tests:
 - class: org.elasticsearch.snapshots.SnapshotShutdownIT
   method: testSnapshotShutdownProgressTracker
   issue: https://github.com/elastic/elasticsearch/issues/129752
+- class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
+  method: testUpdatingFileBasedRoleAffectsAllProjects
+  issue: https://github.com/elastic/elasticsearch/issues/129775
 
 # Examples:
 #

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -90,7 +90,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         // geohex_grid requires gold license
         .setting("xpack.license.self_generated.type", "trial")
         .feature(FeatureFlag.TIME_SERIES_MODE)
-        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED);
+        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
+        .feature(FeatureFlag.IVF_FORMAT);
 
     private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
         .name(REMOTE_CLUSTER_NAME)

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -92,6 +92,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
+        .feature(FeatureFlag.IVF_FORMAT)
         .user("test_admin", "x-pack-test-password");
 
     private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -262,6 +262,15 @@ public ShardFieldStats shardFieldStats() {
         }
     }
 
+    /**
+     * @throws AlreadyClosedException if the shard is closed
+     */
+    public FieldInfos shardFieldInfos() {
+        try (var searcher = acquireSearcher("field_has_value")) {
+            return FieldInfos.getMergedFieldInfos(searcher.getIndexReader());
+        }
+    }
+
     protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves) {
         int numSegments = 0;
         int totalFields = 0;
@@ -2407,6 +2416,13 @@ public final EngineConfig getEngineConfig() {
         return engineConfig;
     }
 
+    public static SeqNoStats buildSeqNoStats(EngineConfig config, SegmentInfos infos) {
+        final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(infos.userData.entrySet());
+        long maxSeqNo = seqNoStats.maxSeqNo();
+        long localCheckpoint = seqNoStats.localCheckpoint();
+        return new SeqNoStats(maxSeqNo, localCheckpoint, config.getGlobalCheckpointSupplier().getAsLong());
+    }
+
     /**
      * Allows registering a listener for when the index shard is on a segment generation >= minGeneration.
      *

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -243,13 +243,6 @@ protected void closeNoLock(String reason, CountDownLatch closedLatch) {
         }
     }
 
-    private static SeqNoStats buildSeqNoStats(EngineConfig config, SegmentInfos infos) {
-        final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(infos.userData.entrySet());
-        long maxSeqNo = seqNoStats.maxSeqNo();
-        long localCheckpoint = seqNoStats.localCheckpoint();
-        return new SeqNoStats(maxSeqNo, localCheckpoint, config.getGlobalCheckpointSupplier().getAsLong());
-    }
-
     private static TranslogStats translogStats(final EngineConfig config, final SegmentInfos infos) throws IOException {
         assert config.getTranslogConfig().hasTranslog();
         final String translogUuid = infos.getUserData().get(Translog.TRANSLOG_UUID_KEY);

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -4308,9 +4308,9 @@ public void afterRefresh(boolean didRefresh) {
     }
 
     private FieldInfos loadFieldInfos() {
-        try (Engine.Searcher hasValueSearcher = getEngine().acquireSearcher("field_has_value")) {
-            return FieldInfos.getMergedFieldInfos(hasValueSearcher.getIndexReader());
-        } catch (AlreadyClosedException ignore) {
+        try {
+            return getEngine().shardFieldInfos();
+        } catch (AlreadyClosedException ignored) {
             // engine is closed - no update to FieldInfos is fine
         }
         return FieldInfos.EMPTY;

x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/GenerativeIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.multi_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeRestTest;
+import org.junit.ClassRule;
+
+/**
+ * This test generates random queries, runs them against the CSV test dataset and checks that they don't throw unexpected exceptions.
+ *
+ * If muted, please:
+ * <ul>
+ * <li>see the error message reported in the failure and the corresponding query (it's in the logs right before the error)</li>
+ * <li>update the corresponding issue with the query (if there is no issue for that failure yet, create one)</li>
+ * <li>add a pattern that matches the error message to {@link GenerativeRestTest#ALLOWED_ERRORS}; also link the issue</li>
+ * <li>unmute (and possibly check that the test doesn't fail anymore)</li>
+ * </ul>
+ */
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class GenerativeIT extends GenerativeRestTest {
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected boolean supportsSourceFieldMapping() {
+        return false;
+    }
+}

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java

@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.CsvTestsDataLoader;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator;
+import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.ChangePointGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.DissectGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.DropGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.EnrichGenerator;
@@ -49,6 +50,7 @@ public record QueryExecuted(String query, int depth, List<Column> outputSchema,
      * These are downstream commands, ie. that cannot appear as the first command in a query
      */
     static List<CommandGenerator> PIPE_COMMANDS = List.of(
+        ChangePointGenerator.INSTANCE,
         DissectGenerator.INSTANCE,
         DropGenerator.INSTANCE,
         EnrichGenerator.INSTANCE,
@@ -197,6 +199,10 @@ public static String randomNumericOrDateField(List<Column> previousOutput) {
         return randomName(previousOutput, Set.of("long", "integer", "double", "date"));
     }
 
+    public static String randomDateField(List<Column> previousOutput) {
+        return randomName(previousOutput, Set.of("date"));
+    }
+
     public static String randomNumericField(List<Column> previousOutput) {
         return randomName(previousOutput, Set.of("long", "integer", "double"));
     }
@@ -255,6 +261,22 @@ public static String constantExpression() {
 
     }
 
+    /**
+     * returns a random identifier or one of the existing names
+     */
+    public static String randomAttributeOrIdentifier(List<EsqlQueryGenerator.Column> previousOutput) {
+        String name;
+        if (randomBoolean()) {
+            name = EsqlQueryGenerator.randomIdentifier();
+        } else {
+            name = EsqlQueryGenerator.randomName(previousOutput);
+            if (name == null) {
+                name = EsqlQueryGenerator.randomIdentifier();
+            }
+        }
+        return name;
+    }
+
     public static String randomIdentifier() {
         // Let's create identifiers that are long enough to avoid collisions with reserved keywords.
         // There could be a smarter way (introspection on the lexer class?), but probably it's not worth the effort

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.esql.AssertWarnings;
 import org.elasticsearch.xpack.esql.CsvTestsDataLoader;
 import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator;
@@ -44,10 +45,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase {
         "The field names are too complex to process", // field_caps problem
         "must be \\[any type except counter types\\]", // TODO refine the generation of count()
 
-        // warnings
-        "Field '.*' shadowed by field at line .*",
-        "evaluation of \\[.*\\] failed, treating result as null", // TODO investigate?
-
         // Awaiting fixes for query failure
         "Unknown column \\[<all-fields-projected>\\]", // https://github.com/elastic/elasticsearch/issues/121741,
         "Plan \\[ProjectExec\\[\\[<no-fields>.* optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/125866
@@ -99,10 +96,10 @@ public void test() throws IOException {
             EsqlQueryGenerator.QueryExecuted result = execute(command, 0);
             if (result.exception() != null) {
                 checkException(result);
-                break;
+                continue;
             }
             if (checkResults(List.of(), commandGenerator, desc, null, result).success() == false) {
-                break;
+                continue;
             }
             previousResult = result;
             previousCommands.add(desc);
@@ -168,7 +165,11 @@ private void checkException(EsqlQueryGenerator.QueryExecuted query) {
     @SuppressWarnings("unchecked")
     private EsqlQueryGenerator.QueryExecuted execute(String command, int depth) {
         try {
-            Map<String, Object> a = RestEsqlTestCase.runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query(command).build());
+            Map<String, Object> a = RestEsqlTestCase.runEsql(
+                new RestEsqlTestCase.RequestObjectBuilder().query(command).build(),
+                new AssertWarnings.AllowedRegexes(List.of(Pattern.compile(".*"))),// we don't care about warnings
+                RestEsqlTestCase.Mode.SYNC
+            );
             List<EsqlQueryGenerator.Column> outputSchema = outputSchema(a);
             List<List<Object>> values = (List<List<Object>>) a.get("values");
             return new EsqlQueryGenerator.QueryExecuted(command, depth, outputSchema, values, null);

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/CommandGenerator.java

@@ -139,4 +139,22 @@ static ValidationResult expectSameColumns(List<EsqlQueryGenerator.Column> previo
 
         return VALIDATION_OK;
     }
+
+    /**
+     * The command doesn't have to produce LESS columns than the previous query
+     */
+    static ValidationResult expectAtLeastSameNumberOfColumns(
+        List<EsqlQueryGenerator.Column> previousColumns,
+        List<EsqlQueryGenerator.Column> columns
+    ) {
+        if (previousColumns.stream().anyMatch(x -> x.name().contains("<all-fields-projected>"))) {
+            return VALIDATION_OK; // known bug
+        }
+
+        if (previousColumns.size() > columns.size()) {
+            return new ValidationResult(false, "Expecting at least [" + previousColumns.size() + "] columns, got [" + columns.size() + "]");
+        }
+
+        return VALIDATION_OK;
+    }
 }