Alias filters

Author: luigidellaquila

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -46,6 +47,7 @@
  */
 public abstract class QueryList {
     protected final SearchExecutionContext searchExecutionContext;
+    protected final AliasFilter aliasFilter;
     protected final MappedFieldType field;
     protected final Block block;
     @Nullable
@@ -54,10 +56,12 @@ public abstract class QueryList {
     protected QueryList(
         MappedFieldType field,
         SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
         Block block,
         OnlySingleValueParams onlySingleValueParams
     ) {
         this.searchExecutionContext = searchExecutionContext;
+        this.aliasFilter = aliasFilter;
         this.field = field;
         this.block = block;
         this.onlySingleValueParams = onlySingleValueParams;
@@ -74,7 +78,7 @@ int getPositionCount() {
      * Returns a copy of this query list that only returns queries for single-valued positions.
      * That is, it returns `null` queries for either multivalued or null positions.
      * <p>
-     *     Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted.
+     * Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted.
      * </p>
      */
     public abstract QueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage);
@@ -93,6 +97,17 @@ final Query getQuery(int position) {
 
         Query query = doGetQuery(position, firstValueIndex, valueCount);
 
+        if (aliasFilter != null && aliasFilter != AliasFilter.EMPTY) {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(query, BooleanClause.Occur.FILTER);
+            try {
+                builder.add(aliasFilter.getQueryBuilder().toQuery(searchExecutionContext), BooleanClause.Occur.FILTER);
+                query = builder.build();
+            } catch (IOException e) {
+                throw new RuntimeException(e);// TODO
+            }
+        }
+
         if (onlySingleValueParams != null) {
             query = wrapSingleValueQuery(query);
         }
@@ -138,7 +153,12 @@ private Query wrapSingleValueQuery(Query query) {
      * using only the {@link ElementType} of the {@link Block} to determine the
      * query.
      */
-    public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
+    public static QueryList rawTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        Block block
+    ) {
         IntFunction<Object> blockToJavaObject = switch (block.elementType()) {
             case BOOLEAN -> {
                 BooleanBlock booleanBlock = (BooleanBlock) block;
@@ -170,17 +190,22 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC
             case AGGREGATE_METRIC_DOUBLE -> throw new IllegalArgumentException("can't read values from [aggregate metric double] block");
             case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
         };
-        return new TermQueryList(field, searchExecutionContext, block, null, blockToJavaObject);
+        return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, blockToJavaObject);
     }
 
     /**
      * Returns a list of term queries for the given field and the input block of
      * {@code ip} field values.
      */
-    public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) {
+    public static QueryList ipTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        BytesRefBlock block
+    ) {
         BytesRef scratch = new BytesRef();
         byte[] ipBytes = new byte[InetAddressPoint.BYTES];
-        return new TermQueryList(field, searchExecutionContext, block, null, offset -> {
+        return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, offset -> {
             final var bytes = block.getBytesRef(offset, scratch);
             if (ipBytes.length != bytes.length) {
                 // Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes
@@ -195,10 +220,16 @@ public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionCo
      * Returns a list of term queries for the given field and the input block of
      * {@code date} field values.
      */
-    public static QueryList dateTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) {
+    public static QueryList dateTermQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        LongBlock block
+    ) {
         return new TermQueryList(
             field,
             searchExecutionContext,
+            aliasFilter,
             block,
             null,
             field instanceof RangeFieldMapper.RangeFieldType rangeFieldType
@@ -210,8 +241,13 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution
     /**
      * Returns a list of geo_shape queries for the given field and the input block.
      */
-    public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
-        return new GeoShapeQueryList(field, searchExecutionContext, block, null);
+    public static QueryList geoShapeQueryList(
+        MappedFieldType field,
+        SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
+        Block block
+    ) {
+        return new GeoShapeQueryList(field, searchExecutionContext, aliasFilter, block, null);
     }
 
     private static class TermQueryList extends QueryList {
@@ -220,11 +256,12 @@ private static class TermQueryList extends QueryList {
         private TermQueryList(
             MappedFieldType field,
             SearchExecutionContext searchExecutionContext,
+            AliasFilter aliasFilter,
             Block block,
             OnlySingleValueParams onlySingleValueParams,
             IntFunction<Object> blockValueReader
         ) {
-            super(field, searchExecutionContext, block, onlySingleValueParams);
+            super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams);
             this.blockValueReader = blockValueReader;
         }
 
@@ -233,6 +270,7 @@ public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarnin
             return new TermQueryList(
                 field,
                 searchExecutionContext,
+                aliasFilter,
                 block,
                 new OnlySingleValueParams(warnings, multiValueWarningMessage),
                 blockValueReader
@@ -264,10 +302,11 @@ private static class GeoShapeQueryList extends QueryList {
         private GeoShapeQueryList(
             MappedFieldType field,
             SearchExecutionContext searchExecutionContext,
+            AliasFilter aliasFilter,
             Block block,
             OnlySingleValueParams onlySingleValueParams
         ) {
-            super(field, searchExecutionContext, block, onlySingleValueParams);
+            super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams);
 
             this.blockValueReader = blockToGeometry(block);
             this.shapeQuery = shapeQuery();
@@ -278,6 +317,7 @@ public GeoShapeQueryList onlySingleValues(Warnings warnings, String multiValueWa
             return new GeoShapeQueryList(
                 field,
                 searchExecutionContext,
+                aliasFilter,
                 block,
                 new OnlySingleValueParams(warnings, multiValueWarningMessage)
             );

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -83,7 +84,7 @@ public void testQueries() throws Exception {
             var inputTerms = makeTermsBlock(List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of()))
         ) {
             MappedFieldType uidField = new KeywordFieldMapper.KeywordFieldType("uid");
-            QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, inputTerms);
+            QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, AliasFilter.EMPTY, inputTerms);
             assertThat(queryList.getPositionCount(), equalTo(6));
             assertThat(queryList.getQuery(0), equalTo(new TermQuery(new Term("uid", new BytesRef("b2")))));
             assertThat(queryList.getQuery(1), equalTo(new TermInSetQuery("uid", List.of(new BytesRef("c1"), new BytesRef("a2")))));
@@ -153,7 +154,12 @@ public void testRandomMatchQueries() throws Exception {
         }).toList();
 
         try (var directoryData = makeDirectoryWith(directoryTermsList); var inputTerms = makeTermsBlock(inputTermsList)) {
-            var queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms);
+            var queryList = QueryList.rawTermQueryList(
+                directoryData.field,
+                directoryData.searchExecutionContext,
+                AliasFilter.EMPTY,
+                inputTerms
+            );
             int maxPageSize = between(1, 256);
             EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator(
                 blockFactory,
@@ -190,8 +196,12 @@ public void testQueries_OnlySingleValues() throws Exception {
                 List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of("a3", "a2", "z2", "xx"))
             )
         ) {
-            QueryList queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms)
-                .onlySingleValues(warnings(), "multi-value found");
+            QueryList queryList = QueryList.rawTermQueryList(
+                directoryData.field,
+                directoryData.searchExecutionContext,
+                AliasFilter.EMPTY,
+                inputTerms
+            ).onlySingleValues(warnings(), "multi-value found");
             // pos -> terms -> docs
             // -----------------------------
             // 0 -> [b2] -> []

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

@@ -637,7 +637,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
         );
         assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales"))));
 
-        // user has permission on the alias, but can't read the key (doc level security)
+        // user has permission on the alias, but can't read the key (doc level security at role level)
         resp = runESQLCommand(
             "metadata1_alias_read2",
             "ROW x = 32.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value | KEEP x, org"
@@ -654,6 +654,31 @@ public void testLookupJoinIndexAllowed() throws Exception {
         assertThat(row.size(), is(2));
         assertThat(row.get(0), is(32.0));
         assertThat(row.get(1), is(nullValue()));
+
+        // user has permission on the alias, the alias has a filter that doesn't allow to see the value
+        resp = runESQLCommand("alias_user1", "ROW x = 12.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
+        assertOK(resp);
+        respMap = entityAsMap(resp);
+        assertThat(
+            respMap.get("columns"),
+            equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword")))
+        );
+        values = (List<?>) respMap.get("values");
+        assertThat(values.size(), is(1));
+        row = (List<?>) values.get(0);
+        assertThat(row.size(), is(2));
+        assertThat(row.get(0), is(12.0));
+        assertThat(row.get(1), is(nullValue()));
+
+        // user has permission on the alias, the alias has a filter that allows to see the value
+        resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
+        assertOK(resp);
+        respMap = entityAsMap(resp);
+        assertThat(
+            respMap.get("columns"),
+            equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword")))
+        );
+        assertThat(respMap.get("values"), equalTo(List.of(List.of(31.0, "sales"))));
     }
 
     @SuppressWarnings("unchecked")

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java

@@ -14,6 +14,8 @@
 import org.elasticsearch.action.support.ChannelActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -52,6 +54,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
@@ -122,8 +125,10 @@
 public abstract class AbstractLookupService<R extends AbstractLookupService.Request, T extends AbstractLookupService.TransportRequest> {
     private final String actionName;
     protected final ClusterService clusterService;
+    protected final IndicesService indicesService;
     private final LookupShardContextFactory lookupShardContextFactory;
     protected final TransportService transportService;
+    IndexNameExpressionResolver indexNameExpressionResolver;
     protected final Executor executor;
     private final BigArrays bigArrays;
     private final BlockFactory blockFactory;
@@ -141,17 +146,21 @@ public abstract class AbstractLookupService<R extends AbstractLookupService.Requ
     AbstractLookupService(
         String actionName,
         ClusterService clusterService,
+        IndicesService indicesService,
         LookupShardContextFactory lookupShardContextFactory,
         TransportService transportService,
+        IndexNameExpressionResolver indexNameExpressionResolver,
         BigArrays bigArrays,
         BlockFactory blockFactory,
         boolean mergePages,
         CheckedBiFunction<StreamInput, BlockFactory, T, IOException> readRequest
     ) {
         this.actionName = actionName;
         this.clusterService = clusterService;
+        this.indicesService = indicesService;
         this.lookupShardContextFactory = lookupShardContextFactory;
         this.transportService = transportService;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.executor = transportService.getThreadPool().executor(ThreadPool.Names.SEARCH);
         this.bigArrays = bigArrays;
         this.blockFactory = blockFactory;
@@ -181,6 +190,7 @@ public ThreadContext getThreadContext() {
     protected abstract QueryList queryList(
         T request,
         SearchExecutionContext context,
+        AliasFilter aliasFilter,
         Block inputBlock,
         @Nullable DataType inputDataType,
         Warnings warnings
@@ -199,13 +209,14 @@ protected abstract QueryList queryList(
     protected static QueryList termQueryList(
         MappedFieldType field,
         SearchExecutionContext searchExecutionContext,
+        AliasFilter aliasFilter,
         Block block,
         @Nullable DataType inputDataType
     ) {
         return switch (inputDataType) {
-            case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, (BytesRefBlock) block);
-            case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, (LongBlock) block);
-            case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, block);
+            case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, aliasFilter, (BytesRefBlock) block);
+            case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, aliasFilter, (LongBlock) block);
+            case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, aliasFilter, block);
         };
     }
 
@@ -265,6 +276,14 @@ private void doLookup(T request, CancellableTask task, ActionListener<List<Page>
         final List<Releasable> releasables = new ArrayList<>(6);
         boolean started = false;
         try {
+
+            ProjectMetadata projMeta = clusterService.state().metadata().getProject();
+            AliasFilter aliasFilter = indicesService.buildAliasFilter(
+                clusterService.state().projectState(),
+                request.shardId.getIndex().getName(),
+                indexNameExpressionResolver.resolveExpressions(projMeta, request.indexPattern)
+            );
+
             LookupShardContext shardContext = lookupShardContextFactory.create(request.shardId);
             releasables.add(shardContext.release);
             final LocalCircuitBreaker localBreaker = new LocalCircuitBreaker(
@@ -310,7 +329,14 @@ private void doLookup(T request, CancellableTask task, ActionListener<List<Page>
                 request.source.source().getColumnNumber(),
                 request.source.text()
             );
-            QueryList queryList = queryList(request, shardContext.executionContext, inputBlock, request.inputDataType, warnings);
+            QueryList queryList = queryList(
+                request,
+                shardContext.executionContext,
+                aliasFilter,
+                inputBlock,
+                request.inputDataType,
+                warnings
+            );
             var queryOperator = new EnrichQuerySourceOperator(
                 driverContext.blockFactory(),
                 EnrichQuerySourceOperator.DEFAULT_MAX_PAGE_SIZE,