Merge branch 'main' into fix_jwt_test_issue

Author: jfreden

docs/changelog/128519.yaml

@@ -0,0 +1,5 @@
+pr: 128519
+summary: Add support for LOOKUP JOIN on aliases
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/128610.yaml

@@ -0,0 +1,6 @@
+pr: 128610
+summary: "Handle the indices pattern `[\"*\", \"-*\"]` when grouping indices by cluster\
+  \ name"
+area: CCS
+type: bug
+issues: []

renovate.json

@@ -19,19 +19,5 @@
     "8.19",
     "8.18",
     "8.17"
-  ],
-  "packageRules": [
-    {
-      "groupName": "wolfi (versioned)",
-      "groupSlug": "wolfi-versioned",
-      "description": "Override the `groupSlug` to create a non-special-character branch name",
-      "matchDatasources": [
-        "docker"
-      ],
-      "matchPackageNames": [
-        "/^docker.elastic.co/wolfi/chainguard-base$/",
-        "/^docker.elastic.co/wolfi/chainguard-base-fips$/"
-      ]
-    }
   ]
 }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -276,6 +276,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00);
     public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES = def(9_087_0_00);
+    public static final TransportVersion JOIN_ON_ALIASES = def(9_088_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -1433,7 +1433,7 @@ static <T> boolean isExplicitAllPattern(Collection<T> aliasesOrIndices, Function
     /**
      * Identifies if this expression list is *,-* which effectively means a request that requests no indices.
      */
-    static boolean isNoneExpression(String[] expressions) {
+    public static boolean isNoneExpression(String[] expressions) {
         return expressions.length == 2 && "*".equals(expressions[0]) && "-*".equals(expressions[1]);
     }
 

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.client.internal.RemoteClusterClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
@@ -201,7 +202,23 @@ public Map<String, OriginalIndices> groupIndices(
         boolean returnLocalAll
     ) {
         final Map<String, OriginalIndices> originalIndicesMap = new HashMap<>();
-        final Map<String, List<String>> groupedIndices = groupClusterIndices(remoteClusterNames, indices);
+        final Map<String, List<String>> groupedIndices;
+        /*
+         * returnLocalAll is used to control whether we'd like to fallback to the local cluster.
+         * While this is acceptable in a few cases, there are cases where we should not fallback to the local
+         * cluster. Consider _resolve/cluster where the specified patterns do not match any remote clusters.
+         * Falling back to the local cluster and returning its details in such cases is not ok. This is why
+         * TransportResolveClusterAction sets returnLocalAll to false wherever it uses groupIndices().
+         *
+         * If such a fallback isn't allowed and the given indices match a pattern whose semantics mean that
+         * it's ok to return an empty result (denoted via ["*", "-*"]), empty groupIndices.
+         */
+        if (returnLocalAll == false && IndexNameExpressionResolver.isNoneExpression(indices)) {
+            groupedIndices = Map.of();
+        } else {
+            groupedIndices = groupClusterIndices(remoteClusterNames, indices);
+        }
+
         if (groupedIndices.isEmpty()) {
             if (returnLocalAll) {
                 // search on _all in the local cluster if neither local indices nor remote indices were specified

x-pack/plugin/build.gradle

@@ -133,6 +133,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.replaceValueInMatch("Size", 49, "Test flamegraph from test-events")
   task.skipTest("esql/90_non_indexed/fetch", "Temporary until backported")
   task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results")
+  task.skipTest("esql/191_lookup_join_on_datastreams/data streams not supported in LOOKUP JOIN", "Added support for aliases in JOINs")
 })
 
 tasks.named('yamlRestCompatTest').configure {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

@@ -484,6 +484,7 @@ static RoleDescriptor kibanaSystem(String name) {
                         "logs-tenable_sc.vulnerability-*",
                         "logs-tenable_io.vulnerability-*",
                         "logs-rapid7_insightvm.vulnerability-*",
+                        "logs-rapid7_insightvm.asset_vulnerability-*",
                         "logs-carbon_black_cloud.asset_vulnerability_summary-*"
                     )
                     .privileges("read", "view_index_metadata")

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -1652,6 +1652,7 @@ public void testKibanaSystemRole() {
             "logs-tenable_sc.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)),
             "logs-tenable_io.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)),
             "logs-rapid7_insightvm.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)),
+            "logs-rapid7_insightvm.asset_vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)),
             "logs-carbon_black_cloud.asset_vulnerability_summary-" + randomAlphaOfLength(randomIntBetween(0, 13))
         ).forEach(indexName -> {
             final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName);

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -46,6 +47,7 @@
  */
 public abstract class QueryList {
     protected final SearchExecutionContext searchExecutionContext;
+    protected final AliasFilter aliasFilter;
     protected final MappedFieldType field;
     protected final Block block;
     @Nullable
@@ -54,10 +56,12 @@ public abstract class QueryList {
     protected QueryList(
         MappedFieldType field,
         SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
         Block block,
         OnlySingleValueParams onlySingleValueParams
     ) {
         this.searchExecutionContext = searchExecutionContext;
+        this.aliasFilter = aliasFilter;
         this.field = field;
         this.block = block;
         this.onlySingleValueParams = onlySingleValueParams;
@@ -74,7 +78,7 @@ int getPositionCount() {
      * Returns a copy of this query list that only returns queries for single-valued positions.
      * That is, it returns `null` queries for either multivalued or null positions.
      * <p>
-     *     Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted.
+     * Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted.
      * </p>
      */
     public abstract QueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage);
@@ -93,6 +97,17 @@ final Query getQuery(int position) {
 
         Query query = doGetQuery(position, firstValueIndex, valueCount);
 
+        if (aliasFilter != null && aliasFilter != AliasFilter.EMPTY) {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(query, BooleanClause.Occur.FILTER);
+            try {
+                builder.add(aliasFilter.getQueryBuilder().toQuery(searchExecutionContext), BooleanClause.Occur.FILTER);
+                query = builder.build();
+            } catch (IOException e) {
+                throw new UncheckedIOException("Error while building query for alias filter", e);
+            }
+        }
+
         if (onlySingleValueParams != null) {
             query = wrapSingleValueQuery(query);
         }
@@ -138,7 +153,12 @@ private Query wrapSingleValueQuery(Query query) {
      * using only the {@link ElementType} of the {@link Block} to determine the
      * query.
      */
-    public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
+    public static QueryList rawTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        Block block
+    ) {
         IntFunction<Object> blockToJavaObject = switch (block.elementType()) {
             case BOOLEAN -> {
                 BooleanBlock booleanBlock = (BooleanBlock) block;
@@ -170,17 +190,22 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC
             case AGGREGATE_METRIC_DOUBLE -> throw new IllegalArgumentException("can't read values from [aggregate metric double] block");
             case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
         };
-        return new TermQueryList(field, searchExecutionContext, block, null, blockToJavaObject);
+        return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, blockToJavaObject);
     }
 
     /**
      * Returns a list of term queries for the given field and the input block of
      * {@code ip} field values.
      */
-    public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) {
+    public static QueryList ipTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        BytesRefBlock block
+    ) {
         BytesRef scratch = new BytesRef();
         byte[] ipBytes = new byte[InetAddressPoint.BYTES];
-        return new TermQueryList(field, searchExecutionContext, block, null, offset -> {
+        return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, offset -> {
             final var bytes = block.getBytesRef(offset, scratch);
             if (ipBytes.length != bytes.length) {
                 // Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes
@@ -195,10 +220,16 @@ public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionCo
      * Returns a list of term queries for the given field and the input block of
      * {@code date} field values.
      */
-    public static QueryList dateTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) {
+    public static QueryList dateTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        LongBlock block
+    ) {
         return new TermQueryList(
             field,
             searchExecutionContext,
+            aliasFilter,
             block,
             null,
             field instanceof RangeFieldMapper.RangeFieldType rangeFieldType
@@ -210,8 +241,13 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution
     /**
      * Returns a list of geo_shape queries for the given field and the input block.
      */
-    public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
-        return new GeoShapeQueryList(field, searchExecutionContext, block, null);
+    public static QueryList geoShapeQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        Block block
+    ) {
+        return new GeoShapeQueryList(field, searchExecutionContext, aliasFilter, block, null);
     }
 
     private static class TermQueryList extends QueryList {
@@ -220,11 +256,12 @@ private static class TermQueryList extends QueryList {
         private TermQueryList(
             MappedFieldType field,
             SearchExecutionContext searchExecutionContext,
+            AliasFilter aliasFilter,
             Block block,
             OnlySingleValueParams onlySingleValueParams,
             IntFunction<Object> blockValueReader
         ) {
-            super(field, searchExecutionContext, block, onlySingleValueParams);
+            super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams);
             this.blockValueReader = blockValueReader;
         }
 
@@ -233,6 +270,7 @@ public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarnin
             return new TermQueryList(
                 field,
                 searchExecutionContext,
+                aliasFilter,
                 block,
                 new OnlySingleValueParams(warnings, multiValueWarningMessage),
                 blockValueReader
@@ -264,10 +302,11 @@ private static class GeoShapeQueryList extends QueryList {
         private GeoShapeQueryList(
             MappedFieldType field,
             SearchExecutionContext searchExecutionContext,
+            AliasFilter aliasFilter,
             Block block,
             OnlySingleValueParams onlySingleValueParams
         ) {
-            super(field, searchExecutionContext, block, onlySingleValueParams);
+            super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams);
 
             this.blockValueReader = blockToGeometry(block);
             this.shapeQuery = shapeQuery();
@@ -278,6 +317,7 @@ public GeoShapeQueryList onlySingleValues(Warnings warnings, String multiValueWa
             return new GeoShapeQueryList(
                 field,
                 searchExecutionContext,
+                aliasFilter,
                 block,
                 new OnlySingleValueParams(warnings, multiValueWarningMessage)
             );