Merge branch 'main' into remove-some-copies

Author: original-brownbear

docs/changelog/127199.yaml

@@ -0,0 +1,6 @@
+pr: 127199
+summary: Disable a bugged commit
+area: ES|QL
+type: bug
+issues:
+ - 127197

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -691,26 +691,6 @@ private static void resolveIndicesForDataStreamRelatedAbstraction(
         }
     }
 
-    private static void resolveWriteIndexForDataStreams(
-        Context context,
-        DataStream dataStream,
-        Set<Index> concreteIndicesResult,
-        IndexComponentSelector selector
-    ) {
-        if (shouldIncludeRegularIndices(context.getOptions(), selector)) {
-            Index writeIndex = dataStream.getWriteIndex();
-            if (addIndex(writeIndex, null, context)) {
-                concreteIndicesResult.add(writeIndex);
-            }
-        }
-        if (shouldIncludeFailureIndices(context.getOptions(), selector)) {
-            Index failureStoreWriteIndex = dataStream.getWriteFailureIndex();
-            if (failureStoreWriteIndex != null && addIndex(failureStoreWriteIndex, null, context)) {
-                concreteIndicesResult.add(failureStoreWriteIndex);
-            }
-        }
-    }
-
     public static boolean shouldIncludeRegularIndices(IndicesOptions indicesOptions, IndexComponentSelector expressionSelector) {
         if (indicesOptions.allowSelectors()) {
             return expressionSelector == null || expressionSelector.shouldIncludeData();
@@ -873,19 +853,6 @@ public Index concreteSingleIndex(ProjectMetadata project, IndicesRequest request
         return indices[0];
     }
 
-    /**
-     * Utility method that allows to resolve an index expression to its corresponding single write index.
-     *
-     * @param state             the cluster state containing all the data to resolve to expression to a concrete index
-     * @param request           The request that defines how an alias or an index need to be resolved to a concrete index
-     *                          and the expression that can be resolved to an alias or an index name.
-     * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index
-     * @return the write index obtained as a result of the index resolution
-     */
-    public Index concreteWriteIndex(ClusterState state, IndicesRequest request) {
-        return concreteWriteIndex(projectResolver.getProjectMetadata(state), request);
-    }
-
     /**
      * Utility method that allows to resolve an index expression to its corresponding single write index.
      *
@@ -977,14 +944,6 @@ public Index concreteWriteIndex(
         return indices[0];
     }
 
-    /**
-     * @return whether the specified index, data stream or alias exists.
-     *         If the data stream, index or alias contains date math then that is resolved too.
-     */
-    public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) {
-        return hasIndexAbstraction(indexAbstraction, projectResolver.getProjectMetadata(state));
-    }
-
     /**
      * @return whether the specified index, data stream or alias exists.
      *         If the data stream, index or alias contains date math then that is resolved too.
@@ -1041,34 +1000,13 @@ public static void assertExpressionHasNullOrDataSelector(String expression) {
         }
     }
 
-    /**
-     * Resolve an array of expressions to the set of indices and aliases that these expressions match.
-     */
-    public Set<ResolvedExpression> resolveExpressions(ClusterState state, String... expressions) {
-        return resolveExpressions(projectResolver.getProjectMetadata(state), expressions);
-    }
-
     /**
      * Resolve an array of expressions to the set of indices and aliases that these expressions match.
      */
     public Set<ResolvedExpression> resolveExpressions(ProjectMetadata project, String... expressions) {
         return resolveExpressions(project, IndicesOptions.lenientExpandOpen(), false, expressions);
     }
 
-    /**
-     * Resolve the expression to the set of indices, aliases, and, optionally, datastreams that the expression matches.
-     * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the
-     * {@param expressions} are returned as-is, without expanding them further to their respective backing indices.
-     */
-    public Set<ResolvedExpression> resolveExpressions(
-        ClusterState state,
-        IndicesOptions indicesOptions,
-        boolean preserveDataStreams,
-        String... expressions
-    ) {
-        return resolveExpressions(projectResolver.getProjectMetadata(state), indicesOptions, preserveDataStreams, expressions);
-    }
-
     /**
      * Resolve the expression to the set of indices, aliases, and, optionally, datastreams that the expression matches.
      * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the
@@ -1101,17 +1039,6 @@ public Set<ResolvedExpression> resolveExpressions(
         }
     }
 
-    /**
-     * Iterates through the list of indices and selects the effective list of filtering aliases for the
-     * given index.
-     * <p>Only aliases with filters are returned. If the indices list contains a non-filtering reference to
-     * the index itself - null is returned. Returns {@code null} if no filtering is required.
-     * <b>NOTE</b>: The provided expressions must have been resolved already via {@link #resolveExpressionsToResources(Context, String...)}.
-     */
-    public String[] filteringAliases(ClusterState state, String index, Set<ResolvedExpression> resolvedExpressions) {
-        return filteringAliases(projectResolver.getProjectMetadata(state), index, resolvedExpressions);
-    }
-
     /**
      * Iterates through the list of indices and selects the effective list of filtering aliases for the
      * given index.
@@ -1140,31 +1067,6 @@ boolean iterateIndexAliases(int indexAliasesSize, int resolvedExpressionsSize) {
         return indexAliasesSize <= resolvedExpressionsSize;
     }
 
-    /**
-     * Iterates through the list of indices and selects the effective list of required aliases for the given index.
-     * <p>Only aliases where the given predicate tests successfully are returned. If the indices list contains a non-required reference to
-     * the index itself - null is returned. Returns {@code null} if no filtering is required.
-     * <p><b>NOTE</b>: the provided expressions must have been resolved already via
-     * {@link #resolveExpressionsToResources(Context, String...)}.
-     */
-    public String[] indexAliases(
-        ClusterState state,
-        String index,
-        Predicate<AliasMetadata> requiredAlias,
-        Predicate<DataStreamAlias> requiredDataStreamAlias,
-        boolean skipIdentity,
-        Set<ResolvedExpression> resolvedExpressions
-    ) {
-        return indexAliases(
-            projectResolver.getProjectMetadata(state),
-            index,
-            requiredAlias,
-            requiredDataStreamAlias,
-            skipIdentity,
-            resolvedExpressions
-        );
-    }
-
     /**
      * Iterates through the list of indices and selects the effective list of required aliases for the given index.
      * <p>Only aliases where the given predicate tests successfully are returned. If the indices list contains a non-required reference to
@@ -1827,9 +1729,9 @@ private static IndexMetadata.State excludeState(IndicesOptions options) {
             final IndexMetadata.State excludeState;
             if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
                 excludeState = null;
-            } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed() == false) {
+            } else if (options.expandWildcardsOpen()) {
                 excludeState = IndexMetadata.State.CLOSE;
-            } else if (options.expandWildcardsClosed() && options.expandWildcardsOpen() == false) {
+            } else if (options.expandWildcardsClosed()) {
                 excludeState = IndexMetadata.State.OPEN;
             } else {
                 assert false : "this shouldn't get called if wildcards expand to none";

x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json

@@ -36,6 +36,9 @@
         },
         "namespaces": {
           "type": "keyword"
+        },
+        "hidden": {
+          "type": "boolean"
         }
       }
     }

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java

@@ -52,31 +52,48 @@ public void testPushEqualityOnDefaults() throws IOException {
         testPushQuery(value, """
             FROM test
             | WHERE test == "%value"
-            """, "#test.keyword:%value -_ignored:test.keyword", false);
+            """, "*:*", true, true);
     }
 
     public void testPushEqualityOnDefaultsTooBigToPush() throws IOException {
         String value = "a".repeat(between(257, 1000));
         testPushQuery(value, """
             FROM test
             | WHERE test == "%value"
-            """, "*:*", true);
+            """, "*:*", true, true);
+    }
+
+    public void testPushInequalityOnDefaults() throws IOException {
+        String value = "v".repeat(between(0, 256));
+        testPushQuery(value, """
+            FROM test
+            | WHERE test != "%different_value"
+            """, "*:*", true, true);
+    }
+
+    public void testPushInequalityOnDefaultsTooBigToPush() throws IOException {
+        String value = "a".repeat(between(257, 1000));
+        testPushQuery(value, """
+            FROM test
+            | WHERE test != "%value"
+            """, "*:*", true, false);
     }
 
     public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException {
         String value = "a".repeat(between(0, 256));
         testPushQuery(value, """
             FROM test
             | WHERE TO_LOWER(test) == "%value"
-            """, "*:*", true);
+            """, "*:*", true, true);
     }
 
-    private void testPushQuery(String value, String esqlQuery, String luceneQuery, boolean filterInCompute) throws IOException {
+    private void testPushQuery(String value, String esqlQuery, String luceneQuery, boolean filterInCompute, boolean found)
+        throws IOException {
         indexValue(value);
+        String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.length()));
 
-        RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(
-            esqlQuery.replaceAll("%value", value) + "\n| KEEP test"
-        );
+        String replacedQuery = esqlQuery.replaceAll("%value", value).replaceAll("%different_value", differentValue);
+        RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(replacedQuery + "\n| KEEP test");
         builder.profile(true);
         Map<String, Object> result = runEsql(builder, new AssertWarnings.NoWarnings(), RestEsqlTestCase.Mode.SYNC);
         assertResultMap(
@@ -88,7 +105,7 @@ private void testPushQuery(String value, String esqlQuery, String luceneQuery, b
                     .entry("query", matchesMap().extraOk())
             ),
             matchesList().item(matchesMap().entry("name", "test").entry("type", "text")),
-            equalTo(List.of(List.of(value)))
+            equalTo(found ? List.of(List.of(value)) : List.of())
         );
 
         @SuppressWarnings("unchecked")
@@ -100,7 +117,7 @@ private void testPushQuery(String value, String esqlQuery, String luceneQuery, b
             @SuppressWarnings("unchecked")
             List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
             for (Map<String, Object> o : operators) {
-                sig.add(checkOperatorProfile(o, luceneQuery.replaceAll("%value", value)));
+                sig.add(checkOperatorProfile(o, luceneQuery.replaceAll("%value", value).replaceAll("%different_value", differentValue)));
             }
             String description = p.get("description").toString();
             switch (description) {

x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec

@@ -2332,3 +2332,40 @@ warning:Line 2:9: java.lang.IllegalArgumentException: single-value function enco
     @timestamp:date     | message:text
 2023-10-23T13:55:01.546Z|More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
 ;
+
+mvStringNotEquals
+FROM mv_text
+| WHERE message != "Connected to 10.1.0.2"
+| KEEP @timestamp, message
+;
+warning:Line 2:9: evaluation of [message != \"Connected to 10.1.0.2\"] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+    @timestamp:date     | message:text
+2023-10-23T13:55:01.544Z|Connected to 10.1.0.1
+2023-10-23T13:55:01.546Z|More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
+;
+
+mvStringNotEqualsFound
+FROM mv_text
+| WHERE message != "Connected to 10.1.0.1"
+| KEEP @timestamp, message
+;
+warning:Line 2:9: evaluation of [message != \"Connected to 10.1.0.1\"] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+    @timestamp:date     | message:text
+2023-10-23T13:55:01.546Z|More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
+;
+
+mvStringNotEqualsLong
+FROM mv_text
+| WHERE message != "More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100"
+| KEEP @timestamp, message
+;
+warning:Line 2:9: evaluation of [message != \"More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100\"] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+    @timestamp:date     | message:text
+2023-10-23T13:55:01.544Z|Connected to 10.1.0.1
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java

@@ -130,7 +130,7 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) {
     @Override
     public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
         if (right() instanceof Literal lit) {
-            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+            if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
                 if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) {
                     return true;
                 }
@@ -142,7 +142,8 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     @Override
     public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         if (right() instanceof Literal lit) {
-            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+            // Disabled because it cased a bug with !=. Fix incoming shortly.
+            if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
                 String value = ((BytesRef) lit.value()).utf8ToString();
                 if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) {
                     String name = handler.nameOf(fa);

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java

@@ -7793,6 +7793,7 @@ public void testReductionPlanForAggs() {
     }
 
     public void testEqualsPushdownToDelegate() {
+        assumeFalse("disabled from bug", true);
         var optimized = optimizedPlan(physicalPlan("""
             FROM test
             | WHERE job == "v"

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java

@@ -80,7 +80,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
     private static final List<String> ALLOWED_PRODUCTS = List.of("kibana", "fleet");
     private static final int FLEET_ACTIONS_MAPPINGS_VERSION = 2;
     private static final int FLEET_AGENTS_MAPPINGS_VERSION = 2;
-    private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 2;
+    private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 3;
     private static final int FLEET_SECRETS_MAPPINGS_VERSION = 1;
     private static final int FLEET_POLICIES_MAPPINGS_VERSION = 2;
     private static final int FLEET_POLICIES_LEADER_MAPPINGS_VERSION = 1;