From 49d15e6a0ebc70f873ffde503e761d2551f14895 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Sat, 8 Feb 2025 19:42:12 -0500 Subject: [PATCH 01/17] ESQL: Views REST API This is extracted from the Views Prototype at https://github.com/elastic/elasticsearch/pull/134995 and is originally based on the "ESQL: View" prototype by Nik but with the following changes: * No ES|QL language integration, only REST API for adding/removing and managing Views in cluster state * ViewMetadata now inside ProjectMetadata * NodeFeature to ensure metadata only exists in clusters fully upgraded * Wide CRUD support (add, update, delete, list) * View name validation * View count, length and depth restrictions Co-authored-by: Nik Everett --- .../rest-api-spec/api/esql.delete_view.json | 34 +++ .../rest-api-spec/api/esql.get_view.json | 34 +++ .../rest-api-spec/api/esql.list_views.json | 28 +++ .../rest-api-spec/api/esql.put_view.json | 38 +++ .../definitions/referable/esql_views.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../xpack/esql/plugin/EsqlFeatures.java | 8 +- .../xpack/esql/plugin/EsqlPlugin.java | 53 +++- .../xpack/esql/view/ClusterViewService.java | 96 +++++++ .../xpack/esql/view/DeleteViewAction.java | 70 ++++++ .../xpack/esql/view/GetViewAction.java | 117 +++++++++ .../xpack/esql/view/InMemoryViewService.java | 48 ++++ .../xpack/esql/view/ListViewsAction.java | 114 +++++++++ .../xpack/esql/view/PutViewAction.java | 78 ++++++ .../xpack/esql/view/RestDeleteViewAction.java | 39 +++ .../xpack/esql/view/RestGetViewAction.java | 39 +++ .../xpack/esql/view/RestListViewsAction.java | 39 +++ .../xpack/esql/view/RestPutViewAction.java | 47 ++++ .../esql/view/TransportDeleteViewAction.java | 60 +++++ .../esql/view/TransportGetViewAction.java | 38 +++ .../esql/view/TransportListViewsAction.java | 43 ++++ .../esql/view/TransportPutViewAction.java | 60 +++++ .../elasticsearch/xpack/esql/view/View.java | 91 +++++++ .../xpack/esql/view/ViewMetadata.java | 121 +++++++++ .../xpack/esql/view/ViewService.java | 235 ++++++++++++++++++ .../xpack/esql/view/AbstractViewTestCase.java | 72 ++++++ .../esql/view/InMemoryViewServiceTests.java | 172 +++++++++++++ .../xpack/esql/view/LocalStateView.java | 64 +++++ .../xpack/esql/view/ViewCrudTests.java | 154 ++++++++++++ .../xpack/esql/view/ViewMetadataTests.java | 72 ++++++ .../xpack/esql/view/ViewTests.java | 60 +++++ .../xpack/security/operator/Constants.java | 5 + .../rest-api-spec/test/esql/200_view.yml | 91 +++++++ 33 files changed, 2218 insertions(+), 5 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json create mode 100644 server/src/main/resources/transport/definitions/referable/esql_views.csv create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json new file mode 100644 index 0000000000000..3357415dd3d58 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json @@ -0,0 +1,34 @@ +{ + "esql.delete_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete", + "description": "Delete a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "DELETE" + ], + "parts": { + "name": { + "type": "string", + "description": "The name of the view to delete" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json new file mode 100644 index 0000000000000..094302c9b5818 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json @@ -0,0 +1,34 @@ +{ + "esql.get_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-get", + "description": "Get a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "GET" + ], + "parts": { + "name": { + "type": "string", + "description": "The name of the view to get" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json new file mode 100644 index 0000000000000..06dc03774b5d8 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json @@ -0,0 +1,28 @@ +{ + "esql.list_views": { + "documentation": { + "url": null, + "description": "List all defined non-materialized ES|QL views" + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/views", + "methods": [ + "GET" + ] + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json new file mode 100644 index 0000000000000..a4f0a6023acd6 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json @@ -0,0 +1,38 @@ +{ + "esql.put_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put", + "description": "Creates a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "PUT" + ], + "parts": { + "name": { + "type": "string", + "description": "The name of the view to create or update" + } + } + } + ] + }, + "body": { + "description": "Use the `query` element to define the ES|QL query to use as a non-materialized VIEW.", + "required": true + } + } +} diff --git a/server/src/main/resources/transport/definitions/referable/esql_views.csv b/server/src/main/resources/transport/definitions/referable/esql_views.csv new file mode 100644 index 0000000000000..fad15855ad264 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_views.csv @@ -0,0 +1 @@ +9217000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 28e1d47db658d..a30c6bc6cf6d7 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_execution_metadata,9216000 +esql_views,9217000 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java index 89b4231a999d4..bb3bdd75cece9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java @@ -34,9 +34,15 @@ public class EsqlFeatures implements FeatureSpecification { */ public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax"); + /** + * Cluster support for view management (create, get, update, list) + * This marks that the cluster state has support for the ViewMetadata CustomProject + */ + public static final NodeFeature ESQL_VIEWS = new NodeFeature("esql.views"); + private Set snapshotBuildFeatures() { assert Build.current().isSnapshot() : Build.current(); - return Set.of(METRICS_SYNTAX); + return Set.of(METRICS_SYNTAX, ESQL_VIEWS); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 7f98ba0e1ed98..12dfd4bfee35c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.plugin; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -48,6 +49,8 @@ import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -74,6 +77,7 @@ import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.ExpressionWritables; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.plan.PlanWritables; @@ -82,6 +86,21 @@ import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog; import org.elasticsearch.xpack.esql.session.IndexResolver; +import org.elasticsearch.xpack.esql.view.ClusterViewService; +import org.elasticsearch.xpack.esql.view.DeleteViewAction; +import org.elasticsearch.xpack.esql.view.GetViewAction; +import org.elasticsearch.xpack.esql.view.ListViewsAction; +import org.elasticsearch.xpack.esql.view.PutViewAction; +import org.elasticsearch.xpack.esql.view.RestDeleteViewAction; +import org.elasticsearch.xpack.esql.view.RestGetViewAction; +import org.elasticsearch.xpack.esql.view.RestListViewsAction; +import org.elasticsearch.xpack.esql.view.RestPutViewAction; +import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction; +import org.elasticsearch.xpack.esql.view.TransportGetViewAction; +import org.elasticsearch.xpack.esql.view.TransportListViewsAction; +import org.elasticsearch.xpack.esql.view.TransportPutViewAction; +import org.elasticsearch.xpack.esql.view.ViewMetadata; +import org.elasticsearch.xpack.esql.view.ViewService; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -182,6 +201,14 @@ public Collection createComponents(PluginServices services) { ); BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking(); var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize); + EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); + ViewService viewService = new ClusterViewService( + functionRegistry, + services.clusterService(), + services.featureService(), + services.projectResolver(), + ViewService.ViewServiceConfig.fromSettings(settings) + ); setupSharedSecrets(); List> extraCheckers = extraCheckerProviders.stream() .flatMap(p -> p.checkers(services.projectResolver(), services.clusterService()).stream()) @@ -201,7 +228,9 @@ public Collection createComponents(PluginServices services) { ThreadPool.Names.SEARCH, blockFactoryProvider.blockFactory() ), - blockFactoryProvider + blockFactoryProvider, + functionRegistry, + viewService ); } @@ -264,7 +293,11 @@ public List getActions() { new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class), new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class), new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class), - new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class) + new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class), + new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class), + new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class), + new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class), + new ActionHandler(ListViewsAction.INSTANCE, TransportListViewsAction.class) ); } @@ -286,7 +319,11 @@ public List getRestHandlers( new RestEsqlGetAsyncResultAction(), new RestEsqlStopAsyncAction(), new RestEsqlDeleteAsyncResultAction(), - new RestEsqlListQueriesAction() + new RestEsqlListQueriesAction(), + new RestPutViewAction(), + new RestDeleteViewAction(), + new RestGetViewAction(), + new RestListViewsAction() ); } @@ -315,12 +352,22 @@ public List getNamedWriteables() { entries.add(ExpressionQueryBuilder.ENTRY); entries.add(PlanStreamWrapperQueryBuilder.ENTRY); + entries.add(ViewMetadata.ENTRY); entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); return entries; } + @Override + public List getNamedXContent() { + List namedXContent = new ArrayList<>(); + namedXContent.add( + new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent) + ); + return namedXContent; + } + public List> getExecutorBuilders(Settings settings) { final int allocatedProcessors = EsExecutors.allocatedProcessors(settings); return List.of( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java new file mode 100644 index 0000000000000..642a88513d5fa --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.plugin.EsqlFeatures; + +import java.util.Map; +import java.util.function.Function; + +/** + * Implementation of {@link ViewService} that keeps the views in the cluster state. + */ +public class ClusterViewService extends ViewService { + private final ClusterService clusterService; + private final FeatureService featureService; + private final ProjectResolver projectResolver; + + public ClusterViewService( + EsqlFunctionRegistry functionRegistry, + ClusterService clusterService, + FeatureService featureService, + ProjectResolver projectResolver, + ViewServiceConfig config + ) { + super(functionRegistry, config); + this.clusterService = clusterService; + this.featureService = featureService; + this.projectResolver = projectResolver; + } + + @Override + protected ViewMetadata getMetadata() { + return getMetadata(clusterService.state()); + } + + protected ViewMetadata getMetadata(ClusterState clusterState) { + return getProjectMetadata(clusterState).custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); + } + + protected ProjectMetadata getProjectMetadata(ClusterState clusterState) { + return projectResolver.getProjectMetadata(clusterService.state()); + } + + @Override + protected void updateViewMetadata(ActionListener callback, Function> function) { + submitUnbatchedTask("update-esql-view-metadata", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + var project = getProjectMetadata(currentState); + var views = project.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); + Map policies = function.apply(views); + var metadata = ProjectMetadata.builder(project).putCustom(ViewMetadata.TYPE, new ViewMetadata(policies)); + return ClusterState.builder(currentState).putProjectMetadata(metadata).build(); + } + + @Override + public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + callback.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + callback.onFailure(e); + } + }); + } + + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here + private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { + clusterService.submitUnbatchedStateUpdateTask(source, task); + } + + @Override + protected void assertMasterNode() { + assert clusterService.localNode().isMasterNode(); + } + + @Override + protected boolean viewsFeatureEnabled() { + return featureService.clusterHasFeature(clusterService.state(), EsqlFeatures.ESQL_VIEWS); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java new file mode 100644 index 0000000000000..d0776f58120dd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteViewAction extends ActionType { + + public static final DeleteViewAction INSTANCE = new DeleteViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/delete"; + + private DeleteViewAction() { + super(NAME); + } + + public static class Request extends MasterNodeRequest { + private final String name; + + public Request(TimeValue masterNodeTimeout, String name) { + super(masterNodeTimeout); + this.name = Objects.requireNonNull(name, "name cannot be null"); + } + + public Request(StreamInput in) throws IOException { + super(in); + name = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + public String name() { + return name; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java new file mode 100644 index 0000000000000..a5450a1c4b96b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class GetViewAction extends ActionType { + + public static final GetViewAction INSTANCE = new GetViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/get"; + + private GetViewAction() { + super(NAME); + } + + public static class Request extends ActionRequest { + private String name; + + public Request(String name) { + this.name = Objects.requireNonNull(name, "name cannot be null"); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + public String name() { + return name; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final View view; + + public Response(final View view) { + this.view = view; + } + + public View getView() { + return view; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + view.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return view.equals(((Response) o).view); + } + + @Override + public int hashCode() { + return view.hashCode(); + } + + @Override + public String toString() { + return "GetViewAction.Response{view=" + view.toString() + '}'; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java new file mode 100644 index 0000000000000..0a7334a1026e8 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; + +import java.util.Map; +import java.util.function.Function; + +/** + * Simple implementation of {@link ClusterViewService} that keeps the views in memory. + * This is useful for testing. + */ +public class InMemoryViewService extends ViewService { + + private ViewMetadata metadata; + + public InMemoryViewService(EsqlFunctionRegistry functionRegistry) { + this(functionRegistry, ViewServiceConfig.DEFAULT); + } + + public InMemoryViewService(EsqlFunctionRegistry functionRegistry, ViewServiceConfig config) { + super(functionRegistry, config); + this.metadata = ViewMetadata.EMPTY; + } + + @Override + protected ViewMetadata getMetadata() { + return metadata; + } + + @Override + protected void updateViewMetadata(ActionListener callback, Function> function) { + Map updated = function.apply(metadata); + this.metadata = new ViewMetadata(updated); + } + + @Override + protected void assertMasterNode() { + // no-op + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java new file mode 100644 index 0000000000000..fc638f2b1412e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.view.ViewMetadata.VIEWS; + +public class ListViewsAction extends ActionType { + + public static final ListViewsAction INSTANCE = new ListViewsAction(); + public static final String NAME = "cluster:admin/xpack/esql/views"; + + private ListViewsAction() { + super(NAME); + } + + public static class Request extends ActionRequest { + public Request() { + super(); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + return o != null && getClass() == o.getClass(); + } + + @Override + public int hashCode() { + return ListViewsAction.Request.class.hashCode(); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final Map views; + + public Response(final Map views) { + this.views = views; + } + + public Map getViews() { + return views; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + var v = ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); + while (v.hasNext()) { + v.next().toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return views.equals(((Response) o).views); + } + + @Override + public int hashCode() { + return views.hashCode(); + } + + @Override + public String toString() { + return "ListViewsAction.Response{view=" + views.toString() + '}'; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java new file mode 100644 index 0000000000000..d783684df492c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class PutViewAction extends ActionType { + + public static final PutViewAction INSTANCE = new PutViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/put"; + + private PutViewAction() { + super(NAME); + } + + public static class Request extends MasterNodeRequest { + private final String name; + private final View view; + + public Request(TimeValue masterNodeTimeout, String name, View view) { + super(masterNodeTimeout); + this.name = Objects.requireNonNull(name, "name cannot be null"); + this.view = view; + } + + public Request(StreamInput in) throws IOException { + super(in); + name = in.readString(); + view = new View(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + view.writeTo(out); + } + + public String name() { + return name; + } + + public View view() { + return view; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name) && view.equals(request.view); + } + + @Override + public int hashCode() { + return Objects.hash(name, view); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java new file mode 100644 index 0000000000000..f0ad25b2c0db9 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; + +@ServerlessScope(Scope.PUBLIC) +public class RestDeleteViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(DELETE, "/_query/view/{name}")); + } + + @Override + public String getName() { + return "esql_delete_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + DeleteViewAction.Request req = new DeleteViewAction.Request(RestUtils.getMasterNodeTimeout(request), request.param("name")); + return channel -> client.execute(DeleteViewAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java new file mode 100644 index 0000000000000..0e94733822477 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(GET, "/_query/view/{name}")); + } + + @Override + public String getName() { + return "esql_get_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + GetViewAction.Request req = new GetViewAction.Request(request.param("name")); + return channel -> client.execute(TransportGetViewAction.TYPE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java new file mode 100644 index 0000000000000..5e7033ee3c282 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestListViewsAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(GET, "/_query/views")); + } + + @Override + public String getName() { + return "esql_list_views"; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + ListViewsAction.Request req = new ListViewsAction.Request(); + return channel -> client.execute(TransportListViewsAction.TYPE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java new file mode 100644 index 0000000000000..c7250cf8f4786 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestPutViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(PUT, "/_query/view/{name}")); + } + + @Override + public String getName() { + return "esql_put_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentOrSourceParamParser()) { + PutViewAction.Request req = new PutViewAction.Request( + RestUtils.getMasterNodeTimeout(request), + request.param("name"), + View.PARSER.parse(parser, null) + ); + return channel -> client.execute(PutViewAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java new file mode 100644 index 0000000000000..10fe9de0a5037 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportDeleteViewAction extends AcknowledgedTransportMasterNodeAction { + private final ClusterViewService viewService; + + @Inject + public TransportDeleteViewAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + ClusterViewService viewService + ) { + super( + DeleteViewAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + DeleteViewAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.viewService = viewService; + } + + @Override + protected void masterOperation( + Task task, + DeleteViewAction.Request request, + ClusterState state, + ActionListener listener + ) { + viewService.delete(request.name(), listener.map(v -> AcknowledgedResponse.TRUE)); + } + + @Override + protected ClusterBlockException checkBlock(DeleteViewAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java new file mode 100644 index 0000000000000..68471ff5d887d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +public class TransportGetViewAction extends HandledTransportAction { + public static final ActionType TYPE = new ActionType<>(GetViewAction.NAME); + private final ClusterViewService viewService; + + @Inject + public TransportGetViewAction(TransportService transportService, ActionFilters actionFilters, ClusterViewService viewService) { + super(GetViewAction.NAME, transportService, actionFilters, GetViewAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.viewService = viewService; + } + + @Override + protected void doExecute(Task task, GetViewAction.Request request, ActionListener listener) { + View view = viewService.get(request.name()); + if (view == null) { + listener.onFailure(new IllegalArgumentException("View [" + request.name() + "] does not exist")); + } else { + listener.onResponse(new GetViewAction.Response(view)); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java new file mode 100644 index 0000000000000..b08865f846a5a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class TransportListViewsAction extends HandledTransportAction { + public static final ActionType TYPE = new ActionType<>(ListViewsAction.NAME); + private final ClusterViewService viewService; + + @Inject + public TransportListViewsAction(TransportService transportService, ActionFilters actionFilters, ClusterViewService viewService) { + super(ListViewsAction.NAME, transportService, actionFilters, ListViewsAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.viewService = viewService; + } + + @Override + protected void doExecute(Task task, ListViewsAction.Request request, ActionListener listener) { + Map views = new LinkedHashMap<>(); + for (String name : viewService.list()) { + View view = viewService.get(name); + if (view != null) { + views.put(name, viewService.get(name)); + } + } + listener.onResponse(new ListViewsAction.Response(views)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java new file mode 100644 index 0000000000000..f2dde51ae23ef --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportPutViewAction extends AcknowledgedTransportMasterNodeAction { + private final ClusterViewService viewService; + + @Inject + public TransportPutViewAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + ClusterViewService viewService + ) { + super( + PutViewAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + PutViewAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.viewService = viewService; + } + + @Override + protected void masterOperation( + Task task, + PutViewAction.Request request, + ClusterState state, + ActionListener listener + ) { + viewService.put(request.name(), request.view(), listener.map(v -> AcknowledgedResponse.TRUE)); + } + + @Override + protected ClusterBlockException checkBlock(PutViewAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java new file mode 100644 index 0000000000000..9ebe93f25d3d3 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +/** + * Represents an enrich policy including its configuration. + */ +public final class View implements Writeable, ToXContentFragment { + private static final ParseField QUERY = new ParseField("query"); + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view", + false, + (args, ctx) -> new View((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), QUERY); + } + + private final String query; + + public View(String query) { + this.query = query; + } + + public View(StreamInput in) throws IOException { + this.query = in.readString(); + } + + public static View fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token == null) { + token = parser.nextToken(); + } + if (token != XContentParser.Token.START_OBJECT) { + throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "], expected START_OBJECT"); + } + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(query); + } + + public String query() { + return query; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(QUERY.getPreferredName(), query); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + View other = (View) o; + return Objects.equals(query, other.query); + } + + @Override + public int hashCode() { + return Objects.hash(query); + } + + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java new file mode 100644 index 0000000000000..3c36438c13b44 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; + +/** + * Encapsulates view definitions as custom metadata inside ProjectMetadata within cluster state. + */ +public final class ViewMetadata extends AbstractNamedDiffable implements Metadata.ProjectCustom { + public static final String TYPE = "esql_view"; + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(ViewMetadata.class, TYPE, ViewMetadata::new); + private static final TransportVersion ESQL_VIEWS = TransportVersion.fromName("esql_views"); + + static final ParseField VIEWS = new ParseField("views"); + + public static final ViewMetadata EMPTY = new ViewMetadata(Collections.emptyMap()); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view_metadata", + args -> new ViewMetadata((Map) args[0]) + ); + + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + Map patterns = new HashMap<>(); + String fieldName = null; + for (XContentParser.Token token = p.nextToken(); token != XContentParser.Token.END_OBJECT; token = p.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = p.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + patterns.put(fieldName, View.fromXContent(p)); + } else { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + } + return patterns; + }, VIEWS); + } + + public static ViewMetadata fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Map views; + + public ViewMetadata(StreamInput in) throws IOException { + this(in.readMap(View::new)); + } + + public ViewMetadata(Map views) { + this.views = Collections.unmodifiableMap(views); + } + + public Map views() { + return views; + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return ESQL_VIEWS; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(views, StreamOutput::writeWriteable); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params ignored) { + return ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ViewMetadata that = (ViewMetadata) o; + return views.equals(that.views); + } + + @Override + public int hashCode() { + return Objects.hash(views); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java new file mode 100644 index 0000000000000..f6ede5d2fef4f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.parser.EsqlParser; +import org.elasticsearch.xpack.esql.parser.QueryParams; +import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.UnionAll; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public abstract class ViewService { + private final ViewServiceConfig config; + private final EsqlFunctionRegistry functionRegistry; + + public record ViewServiceConfig(int maxViews, int maxViewSize, int maxViewDepth) { + + public static final String MAX_VIEWS_COUNT_SETTING = "esql.views.max_count"; + public static final String MAX_VIEWS_SIZE_SETTING = "esql.views.max_size"; + public static final String MAX_VIEWS_DEPTH_SETTING = "esql.views.max_depth"; + public static final ViewServiceConfig DEFAULT = new ViewServiceConfig(100, 10_000, 10); + + public static ViewServiceConfig fromSettings(Settings settings) { + return new ViewServiceConfig( + settings.getAsInt(MAX_VIEWS_COUNT_SETTING, DEFAULT.maxViews), + settings.getAsInt(MAX_VIEWS_SIZE_SETTING, DEFAULT.maxViewSize), + settings.getAsInt(MAX_VIEWS_DEPTH_SETTING, DEFAULT.maxViewDepth) + ); + } + } + + public ViewService(EsqlFunctionRegistry functionRegistry, ViewServiceConfig config) { + this.functionRegistry = functionRegistry; + this.config = config; + } + + protected abstract ViewMetadata getMetadata(); + + public LogicalPlan replaceViews(LogicalPlan plan, PlanTelemetry telemetry) { + if (viewsFeatureEnabled() == false) { + return plan; + } + ViewMetadata views = getMetadata(); + + List seen = new ArrayList<>(); + while (true) { + LogicalPlan prev = plan; + plan = plan.transformUp(UnresolvedRelation.class, ur -> { + List indexes = new ArrayList<>(); + List subqueries = new ArrayList<>(); + for (String name : ur.indexPattern().indexPattern().split(",")) { + name = name.trim(); + if (views.views().containsKey(name)) { + boolean alreadySeen = seen.contains(name); + seen.add(name); + if (alreadySeen) { + throw viewError("circular view reference ", seen); + } + if (seen.size() > config.maxViewDepth) { + throw viewError("The maximum allowed view depth of " + config.maxViewDepth + " has been exceeded: ", seen); + } + View view = views.views().get(name); + subqueries.add(resolve(view, telemetry)); + } else { + indexes.add(name); + } + } + if (subqueries.isEmpty()) { + // No views defined, just return the original plan + return ur; + } + if (indexes.isEmpty()) { + if (subqueries.size() == 1) { + // only one view, no need for union + return subqueries.getFirst(); + } + } else { + subqueries.add( + 0, + new UnresolvedRelation( + ur.source(), + new IndexPattern(ur.indexPattern().source(), String.join(",", indexes)), + ur.frozen(), + ur.metadataFields(), + ur.indexMode(), + ur.unresolvedMessage() + ) + ); + } + return new UnionAll(ur.source(), subqueries, List.of()); + }); + if (plan.equals(prev)) { + return prev; + } + } + } + + private static LogicalPlan resolve(View view, PlanTelemetry telemetry) { + // TODO don't reparse every time. Store parsed? Or cache parsing? dunno + // this will make super-wrong Source. the _source should be the view. + // if there's a `filter` it applies "under" the view. that's weird. right? + // security to create this + // telemetry + // don't allow circular references + return new EsqlParser().createStatement(view.query(), new QueryParams(), telemetry); + } + + private VerificationException viewError(String type, List seen) { + StringBuilder b = new StringBuilder(); + for (String s : seen) { + if (b.isEmpty()) { + b.append(type); + } else { + b.append(" -> "); + } + b.append(s); + } + throw new VerificationException(b.toString()); + } + + /** + * Adds or modifies a view by name. This method can only be invoked on the master node. + */ + public void put(String name, View view, ActionListener callback) { + assertMasterNode(); + if (viewsFeatureEnabled()) { + validatePutView(name, view); + updateViewMetadata(callback, current -> { + Map original = getMetadata().views(); + Map updated = new HashMap<>(original); + updated.put(name, view); + return updated; + }); + } + } + + private void validatePutView(String name, View view) { + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + // The view name is used in a similar context to an index name and therefore has the same restrictions as an index name + MetadataCreateIndexService.validateIndexOrAliasName( + name, + (viewName, error) -> new IllegalArgumentException("Invalid view name [" + viewName + "], " + error) + ); + if (name.toLowerCase(Locale.ROOT).equals(name) == false) { + throw new IllegalArgumentException("Invalid view name [" + name + "], must be lowercase"); + } + if (view == null) { + throw new IllegalArgumentException("view is missing"); + } + if (Strings.isNullOrEmpty(view.query())) { + throw new IllegalArgumentException("view query is missing or empty"); + } + if (view.query().length() > config.maxViewSize) { + throw new IllegalArgumentException( + "view query is too large: " + view.query().length() + " characters, the maximum allowed is " + config.maxViewSize + ); + } + if (getMetadata().views().containsKey(name) == false && getMetadata().views().size() >= config.maxViews) { + throw new IllegalArgumentException("cannot add view, the maximum number of views is reached: " + config.maxViews); + } + new EsqlParser().createStatement(view.query(), new QueryParams(), new PlanTelemetry(functionRegistry)); + // TODO should we validate this in the transport action and make it async? like plan like a query + // TODO postgresql does. + } + + /** + * Gets the view by name. + */ + public View get(String name) { + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + return viewsFeatureEnabled() ? getMetadata().views().get(name) : null; + } + + /** + * List current view names. + */ + public Set list() { + return viewsFeatureEnabled() ? getMetadata().views().keySet() : Set.of(); + } + + /** + * Removes a view from the cluster state. This method can only be invoked on the master node. + */ + public void delete(String name, ActionListener callback) { + assertMasterNode(); + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + + if (viewsFeatureEnabled()) { + updateViewMetadata(callback, current -> { + Map original = current.views(); + if (original.containsKey(name) == false) { + throw new ResourceNotFoundException("view [{}] not found", name); + } + Map updated = new HashMap<>(original); + updated.remove(name); + return updated; + }); + } + } + + protected abstract void assertMasterNode(); + + protected boolean viewsFeatureEnabled() { + return true; + } + + protected abstract void updateViewMetadata(ActionListener callback, Function> function); +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java new file mode 100644 index 0000000000000..dcbef2ff12036 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.esql.view.ViewService.ViewServiceConfig.DEFAULT; + +public abstract class AbstractViewTestCase extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(LocalStateView.class); + } + + protected ViewService viewService() { + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + FeatureService featureService = getInstanceFromNode(FeatureService.class); + ProjectResolver projectResolver = getInstanceFromNode(ProjectResolver.class); + return new ClusterViewService(new EsqlFunctionRegistry(), clusterService, featureService, projectResolver, DEFAULT); + } + + protected AtomicReference saveView(String name, View policy, ViewService viewService) throws InterruptedException { + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + TestResponseCapture responseCapture = new TestResponseCapture(); + viewService.put(name, policy, responseCapture); + responseCapture.latch.await(); + return responseCapture.error; + } + + protected void deleteView(String name, ViewService viewService) throws Exception { + TestResponseCapture responseCapture = new TestResponseCapture(); + viewService.delete(name, responseCapture); + responseCapture.latch.await(); + if (responseCapture.error.get() != null) { + throw responseCapture.error.get(); + } + } + + protected static class TestResponseCapture implements ActionListener { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference error = new AtomicReference<>(); + + @Override + public void onResponse(Void unused) { + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + error.set(e); + latch.countDown(); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java new file mode 100644 index 0000000000000..230ce1b77e17b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.parser.AbstractStatementParserTests; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; + +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; + +public class InMemoryViewServiceTests extends AbstractStatementParserTests { + EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); + InMemoryViewService viewService = new InMemoryViewService(functionRegistry); + PlanTelemetry telemetry = new PlanTelemetry(functionRegistry); + + public void testPutGet() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + assertThat(viewService.get("view1").query(), equalTo("from emp")); + assertThat(viewService.get("view2").query(), equalTo("from view1")); + assertThat(viewService.get("view3").query(), equalTo("from view2")); + } + + public void testReplaceView() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + LogicalPlan plan = statement("from view3"); + LogicalPlan rewritten = viewService.replaceViews(plan, telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } + + public void testViewDepthExceeded() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + addView("view4", "from view3"); + addView("view5", "from view4"); + addView("view6", "from view5"); + addView("view7", "from view6"); + addView("view8", "from view7"); + addView("view9", "from view8"); + addView("view10", "from view9"); + addView("view11", "from view10"); + + // FROM view11 should fail + Exception e = expectThrows(VerificationException.class, () -> viewService.replaceViews(statement("from view11"), telemetry)); + assertThat(e.getMessage(), startsWith("The maximum allowed view depth of 10 has been exceeded")); + + // But FROM view10 should work + LogicalPlan rewritten = viewService.replaceViews(statement("from view10"), telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } + + public void testModifiedViewDepth() { + var config = new ViewService.ViewServiceConfig(100, 10_000, 1); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from emp", customViewService); + addView("view2", "from view1", customViewService); + addView("view3", "from view2", customViewService); + + // FROM view2 should fail + Exception e = expectThrows( + VerificationException.class, + () -> customViewService.replaceViews(statement("from view2"), telemetry) + ); + assertThat(e.getMessage(), startsWith("The maximum allowed view depth of 1 has been exceeded")); + + // But FROM view1 should work + LogicalPlan rewritten = customViewService.replaceViews(statement("from view1"), telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testViewCountExceeded() throws Exception { + for (int i = 0; i < ViewService.ViewServiceConfig.DEFAULT.maxViews(); i++) { + addView("view" + i, "from emp"); + } + + // FROM view11 should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("viewx", "from emp")); + assertThat(e.getMessage(), startsWith("cannot add view, the maximum number of views is reached: 100")); + } + + public void testModifiedViewCount() { + var config = new ViewService.ViewServiceConfig(1, 10_000, 10); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from emp", customViewService); + + // View2 should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("view2", "from emp", customViewService)); + assertThat(e.getMessage(), startsWith("cannot add view, the maximum number of views is reached: 1")); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testViewLengthExceeded() throws Exception { + addView("view1", "from short"); + + // Long view definition should fail + StringBuilder longView = new StringBuilder("from "); + for (int i = 0; i < ViewService.ViewServiceConfig.DEFAULT.maxViewSize(); i++) { + longView.append("a"); + } + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("viewx", longView.toString())); + assertThat(e.getMessage(), startsWith("view query is too large: 10005 characters, the maximum allowed is 10000")); + } + + public void testModifiedViewLength() { + var config = new ViewService.ViewServiceConfig(100, 6, 10); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from a", customViewService); + + // Just one character longer should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("view2", "from aa", customViewService)); + assertThat(e.getMessage(), startsWith("view query is too large: 7 characters, the maximum allowed is 6")); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testInvalidViewNames() { + var config = ViewService.ViewServiceConfig.DEFAULT; + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + for (var name : Map.of( + "viewX", + "Invalid view name [viewX], must be lowercase", + ".", + "Invalid view name [.], must not be '.' or '..'", + "..", + "Invalid view name [..], must not be '.' or '..'", + "invalid name", + "Invalid view name [invalid name], must not contain the following characters", + "invalid*name", + "Invalid view name [invalid*name], must not contain the following characters" + ).entrySet()) { + Exception e = expectThrows( + "Expected '" + name.getKey() + "' to be an invalid name, but it was not", + IllegalArgumentException.class, + startsWith(name.getValue()), + () -> addView(name.getKey(), "from aa", customViewService) + ); + } + } + + private void addView(String name, String query) { + addView(name, query, viewService); + } + + private void addView(String name, String query, ViewService viewService) { + viewService.put(name, new View(query), ActionListener.noop()); + } + +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java new file mode 100644 index 0000000000000..bf1b5fb7432ea --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.LicenseService; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.action.TransportXPackInfoAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureResponse; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +public class LocalStateView extends LocalStateCompositeXPackPlugin { + + public LocalStateView(final Settings settings, final Path configPath) throws Exception { + super(settings, configPath); + + plugins.add(new EsqlPlugin() { + @Override + protected XPackLicenseState getLicenseState() { + return LocalStateView.this.getLicenseState(); + } + }); + } + + public static class ViewTransportXPackInfoAction extends TransportXPackInfoAction { + @Inject + public ViewTransportXPackInfoAction( + TransportService transportService, + ActionFilters actionFilters, + LicenseService licenseService, + NodeClient client + ) { + super(transportService, actionFilters, licenseService, client); + } + + @Override + protected List> infoActions() { + return Collections.singletonList(XPackInfoFeatureAction.ESQL); + } + } + + @Override + protected Class> getInfoAction() { + return ViewTransportXPackInfoAction.class; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java new file mode 100644 index 0000000000000..e3c60a308f1a1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.parser.ParsingException; +import org.junit.After; +import org.junit.Before; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.esql.view.ViewTests.randomView; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class ViewCrudTests extends AbstractViewTestCase { + + private ViewService viewService; + + @Before + public void setup() throws Exception { + super.setUp(); + this.viewService = viewService(); + } + + @After + public void tearDown() throws Exception { + for (String name : this.viewService.list()) { + deleteView(name, viewService); + } + super.tearDown(); + } + + public void testCrud() throws Exception { + ViewService viewService = viewService(); + View view = randomView(XContentType.JSON); + String name = "my-view"; + + AtomicReference error = saveView(name, view, viewService); + assertThat(error.get(), nullValue()); + + View result = viewService.get(name); + assertThat(result, equalTo(view)); + + Set views = viewService.list(); + assertThat(views.size(), equalTo(1)); + assertThat(views, equalTo(Set.of(name))); + + deleteView(name, viewService); + result = viewService.get(name); + assertThat(result, nullValue()); + } + + public void testUpdate() throws Exception { + ViewService viewService = viewService(); + View view = randomView(XContentType.JSON); + String name = "my-view"; + + AtomicReference error = saveView(name, view, viewService); + assertThat(error.get(), nullValue()); + + view = randomView(XContentType.JSON); + error = saveView(name, view, viewService); + assertThat(error.get(), nullValue()); + View result = viewService.get(name); + assertThat(result, equalTo(view)); + + deleteView(name, viewService); + result = viewService.get(name); + assertThat(result, nullValue()); + } + + public void testPutValidation() throws Exception { + ViewService viewService = viewService(); + View view = randomView(XContentType.JSON); + + { + String nullOrEmptyName = randomBoolean() ? "" : null; + + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> saveView(nullOrEmptyName, view, viewService) + ); + + assertThat(error.getMessage(), equalTo("name is missing or empty")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("my-view", null, viewService)); + + assertThat(error.getMessage(), equalTo("view is missing")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("my#view", view, viewService)); + assertThat(error.getMessage(), equalTo("Invalid view name [my#view], must not contain '#'")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("..", view, viewService)); + assertThat(error.getMessage(), equalTo("Invalid view name [..], must not be '.' or '..'")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("myView", view, viewService)); + assertThat(error.getMessage(), equalTo("Invalid view name [myView], must be lowercase")); + } + { + View invalidView = new View("FROMMM abc"); + ParsingException error = expectThrows(ParsingException.class, () -> saveView("name", invalidView, viewService)); + assertThat(error.getMessage(), containsString("mismatched input 'FROMMM'")); + } + } + + public void testDeleteValidation() { + ViewService viewService = viewService(); + + { + String nullOrEmptyName = randomBoolean() ? "" : null; + + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> deleteView(nullOrEmptyName, viewService)); + + assertThat(error.getMessage(), equalTo("name is missing or empty")); + } + { + ResourceNotFoundException error = expectThrows(ResourceNotFoundException.class, () -> deleteView("my-view", viewService)); + + assertThat(error.getMessage(), equalTo("view [my-view] not found")); + } + } + + public void testGetValidation() { + ViewService viewService = viewService(); + String nullOrEmptyName = randomBoolean() ? "" : null; + + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewService.get(nullOrEmptyName)); + + assertThat(error.getMessage(), equalTo("name is missing or empty")); + + View view = viewService.get("null-view"); + assertNull(view); + } + + public void testListValidation() { + ViewService viewService = viewService(); + Set views = viewService.list(); + assertTrue(views.isEmpty()); + } + +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java new file mode 100644 index 0000000000000..1585022cc7b1a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.test.AbstractChunkedSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.view.ViewTests.randomView; +import static org.hamcrest.Matchers.equalTo; + +public class ViewMetadataTests extends AbstractChunkedSerializingTestCase { + + @Override + protected ViewMetadata doParseInstance(XContentParser parser) throws IOException { + return ViewMetadata.fromXContent(parser); + } + + @Override + protected ViewMetadata createTestInstance() { + return randomViewMetadata(randomFrom(XContentType.values())); + } + + @Override + protected ViewMetadata mutateInstance(ViewMetadata instance) { + HashMap views = new HashMap<>(instance.views()); + views.replaceAll((name, view) -> randomView(randomFrom(XContentType.values()))); + return new ViewMetadata(views); + } + + @Override + protected ViewMetadata createXContextTestInstance(XContentType xContentType) { + return randomViewMetadata(xContentType); + } + + private static ViewMetadata randomViewMetadata(XContentType xContentType) { + int numViews = randomIntBetween(8, 64); + Map views = Maps.newMapWithExpectedSize(numViews); + for (int i = 0; i < numViews; i++) { + View view = randomView(xContentType); + views.put(randomAlphaOfLength(8), view); + } + return new ViewMetadata(views); + } + + @Override + protected Writeable.Reader instanceReader() { + return ViewMetadata::new; + } + + @Override + protected void assertEqualInstances(ViewMetadata expectedInstance, ViewMetadata newInstance) { + assertNotSame(expectedInstance, newInstance); + assertThat(newInstance.views().size(), equalTo(expectedInstance.views().size())); + for (Map.Entry entry : newInstance.views().entrySet()) { + View actual = entry.getValue(); + View expected = expectedInstance.views().get(entry.getKey()); + ViewTests.assertEqualViews(expected, actual); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java new file mode 100644 index 0000000000000..732745c4d64be --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class ViewTests extends AbstractXContentSerializingTestCase { + + @Override + protected View doParseInstance(XContentParser parser) throws IOException { + return View.fromXContent(parser); + } + + @Override + protected View createTestInstance() { + return randomView(randomFrom(XContentType.values())); + } + + @Override + protected View mutateInstance(View instance) { + return randomView(randomFrom(XContentType.values())); + } + + @Override + protected View createXContextTestInstance(XContentType xContentType) { + return randomView(xContentType); + } + + public static View randomView(XContentType xContentType) { + String query = "FROM " + randomAlphaOfLength(10); + return new View(query); + } + + @Override + protected Writeable.Reader instanceReader() { + return View::new; + } + + @Override + protected void assertEqualInstances(View expectedInstance, View newInstance) { + assertNotSame(expectedInstance, newInstance); + assertEqualViews(expectedInstance, newInstance); + } + + public static void assertEqualViews(View expectedInstance, View newInstance) { + assertThat(newInstance.query(), equalTo(expectedInstance.query())); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 0957ec55e882a..b6e4796700fc0 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -174,6 +174,11 @@ public class Constants { "cluster:admin/xpack/enrich/get", "cluster:admin/xpack/enrich/put", "cluster:admin/xpack/enrich/reindex", + "cluster:admin/xpack/esql/view/put", + "cluster:admin/xpack/esql/view/delete", + "cluster:admin/xpack/esql/view/get", + "cluster:admin/xpack/esql/views", + "cluster:internal/xpack/inference/clear_inference_endpoint_cache", "cluster:admin/xpack/inference/ccm/delete", "cluster:admin/xpack/inference/ccm/put", "cluster:admin/xpack/inference/delete", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml new file mode 100644 index 0000000000000..ab9ba547e8d10 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml @@ -0,0 +1,91 @@ +--- +setup: + - requires: + cluster_features: [ "esql.views" ] + reason: "Views not yet supported in cluster state" + - do: + indices.create: + index: test + body: + mappings: + properties: + content: + type: keyword + color: + type: keyword + animal: + type: keyword + id: + type: integer + - do: + bulk: + index: "test" + refresh: true + body: + - { "index": { } } + - { "content": "This is a brown fox", "color": "brown", "animal": "fox", "id": 1 } + - { "index": { } } + - { "content": "This is a brown dog", "color": "brown", "animal": "dog", "id": 2 } + - { "index": { } } + - { "content": "This dog is really white", "color": "white", "animal": "dog", "id": 3 } + - { "index": { } } + - { "content": "The dog is brown but this document is very very long", "color": "brown", "animal": "dog", "id": 4 } + - { "index": { } } + - { "content": "There is also a white cat", "color": "white", "animal": "cat", "id": 5 } + - { "index": { } } + - { "content": "The quick brown fox jumps over the lazy dog", "color": "brown", "animal": "dog", "id": 6 } + +--- +crud: + - do: + esql.put_view: + name: dogs + body: + query: 'FROM test | WHERE animal == "dog"' + + - do: + esql.get_view: + name: dogs + + - match: { query: 'FROM test | WHERE animal == "dog"' } + + - do: + esql.list_views: { } + + - match: { views: { dogs: { query: 'FROM test | WHERE animal == "dog"'} } } + + - do: + esql.delete_view: + name: dogs + + - do: + esql.list_views: { } + + - match: { views: { } } + +--- +basic: + - requires: + capabilities: + - method: PUT + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ views_v1 ] + reason: "Views not yet supported" + test_runner_features: [ capabilities, allowed_warnings_regex, warnings_regex ] + - do: + esql.put_view: + name: dogs + body: + query: 'FROM test | WHERE animal == "dog"' + + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM dogs | WHERE color == "white" | KEEP id | SORT id' + - match: { columns.0.name: "id" } + - match: { columns.0.type: "integer" } + - length: { values: 1 } + - match: { values.0.0: 3 } From 507bf1c2f6cfc96f881fa4f795522c70826c1395 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 11 Nov 2025 10:39:47 +0100 Subject: [PATCH 02/17] Update docs/changelog/137818.yaml --- docs/changelog/137818.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/137818.yaml diff --git a/docs/changelog/137818.yaml b/docs/changelog/137818.yaml new file mode 100644 index 0000000000000..e3c1932ba58f4 --- /dev/null +++ b/docs/changelog/137818.yaml @@ -0,0 +1,5 @@ +pr: 137818 +summary: Views REST API +area: "ES|QL, Indices APIs" +type: feature +issues: [] From b0ae25cf850b22756766e9139c9b7aec4a7b2c90 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 11 Nov 2025 10:43:55 +0100 Subject: [PATCH 03/17] Refine changelog --- docs/changelog/137818.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/137818.yaml b/docs/changelog/137818.yaml index e3c1932ba58f4..685df5a102665 100644 --- a/docs/changelog/137818.yaml +++ b/docs/changelog/137818.yaml @@ -1,5 +1,5 @@ pr: 137818 -summary: Views REST API -area: "ES|QL, Indices APIs" +summary: ES|QL Views REST API +area: "ES|QL" type: feature issues: [] From 73da7cda9232f0f9a89a002bef609de99a996229 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 12 Nov 2025 12:45:20 +0100 Subject: [PATCH 04/17] Expand get-views to cover use cases for listing views --- .../rest-api-spec/api/esql.get_view.json | 10 +- .../xpack/esql/view/GetViewAction.java | 61 +++++----- .../xpack/esql/view/RestGetViewAction.java | 16 ++- .../esql/view/TransportGetViewAction.java | 25 ++++- .../xpack/esql/view/AbstractViewTestCase.java | 49 +++++--- .../xpack/esql/view/ViewCrudTests.java | 106 +++++++++--------- 6 files changed, 159 insertions(+), 108 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json index 094302c9b5818..5bfef0bc75867 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json @@ -23,10 +23,16 @@ ], "parts": { "name": { - "type": "string", - "description": "The name of the view to get" + "type": "list", + "description": "A comma-separated list of view names" } } + }, + { + "path": "/_query/view", + "methods": [ + "GET" + ] } ] } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java index a5450a1c4b96b..f215ed13bfedd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java @@ -6,18 +6,24 @@ */ package org.elasticsearch.xpack.esql.view; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.TreeMap; + +import static org.elasticsearch.xpack.esql.view.ViewMetadata.VIEWS; public class GetViewAction extends ActionType { @@ -28,30 +34,20 @@ private GetViewAction() { super(NAME); } - public static class Request extends ActionRequest { - private String name; + public static class Request extends LocalClusterStateRequest { + private List names; - public Request(String name) { - this.name = Objects.requireNonNull(name, "name cannot be null"); + public Request(TimeValue masterNodeTimeout, String... names) { + super(masterNodeTimeout); + this.names = List.of(names); } public Request(StreamInput in) throws IOException { super(in); } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(name); - } - - public String name() { - return name; - } - - @Override - public ActionRequestValidationException validate() { - return null; + public List names() { + return names; } @Override @@ -59,25 +55,27 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return name.equals(request.name); + return Objects.equals(names, request.names); } @Override public int hashCode() { - return name.hashCode(); + return Objects.hash(names); } } public static class Response extends ActionResponse implements ToXContentObject { - private final View view; + private final Map views; - public Response(final View view) { - this.view = view; + public Response(Map views) { + Objects.requireNonNull(views, "views cannot be null"); + // use a treemap to guarantee ordering in the set, then transform it to the list of named policies + this.views = new TreeMap<>(views); } - public View getView() { - return view; + public Map getViews() { + return views; } @Override @@ -88,7 +86,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - view.toXContent(builder, params); + var v = ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); + while (v.hasNext()) { + v.next().toXContent(builder, params); + } builder.endObject(); return builder; } @@ -101,17 +102,17 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - return view.equals(((Response) o).view); + return views.equals(((Response) o).views); } @Override public int hashCode() { - return view.hashCode(); + return views.hashCode(); } @Override public String toString() { - return "GetViewAction.Response{view=" + view.toString() + '}'; + return "GetViewAction.Response{" + views.toString() + '}'; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java index 0e94733822477..12254a74a5dd0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java @@ -8,10 +8,13 @@ package org.elasticsearch.xpack.esql.view; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -23,7 +26,7 @@ public class RestGetViewAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(GET, "/_query/view/{name}")); + return List.of(new Route(GET, "/_query/view/{name}"), new Route(GET, "/_query/view")); } @Override @@ -33,7 +36,14 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - GetViewAction.Request req = new GetViewAction.Request(request.param("name")); - return channel -> client.execute(TransportGetViewAction.TYPE, req, new RestToXContentListener<>(channel)); + GetViewAction.Request req = new GetViewAction.Request( + RestUtils.getMasterNodeTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")) + ); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + GetViewAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java index 68471ff5d887d..5b43eb1fa381d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java @@ -16,6 +16,10 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; + public class TransportGetViewAction extends HandledTransportAction { public static final ActionType TYPE = new ActionType<>(GetViewAction.NAME); private final ClusterViewService viewService; @@ -28,11 +32,24 @@ public TransportGetViewAction(TransportService transportService, ActionFilters a @Override protected void doExecute(Task task, GetViewAction.Request request, ActionListener listener) { - View view = viewService.get(request.name()); - if (view == null) { - listener.onFailure(new IllegalArgumentException("View [" + request.name() + "] does not exist")); + TreeMap views = new TreeMap<>(); + List missing = new ArrayList<>(); + var names = request.names(); + if (names.isEmpty()) { + names = new ArrayList<>(viewService.list()); + } + for (String name : names) { + View view = viewService.get(name); + if (view == null) { + missing.add(name); + } else { + views.put(name, view); + } + } + if (missing.isEmpty() == false) { + listener.onFailure(new IllegalArgumentException("Views do not exist: " + String.join(", ", missing))); } else { - listener.onResponse(new GetViewAction.Response(view)); + listener.onResponse(new GetViewAction.Response(views)); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java index dcbef2ff12036..530f03b4a6d70 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.plugins.Plugin; @@ -18,6 +19,7 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -37,30 +39,47 @@ protected ViewService viewService() { return new ClusterViewService(new EsqlFunctionRegistry(), clusterService, featureService, projectResolver, DEFAULT); } - protected AtomicReference saveView(String name, View policy, ViewService viewService) throws InterruptedException { - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - TestResponseCapture responseCapture = new TestResponseCapture(); - viewService.put(name, policy, responseCapture); - responseCapture.latch.await(); - return responseCapture.error; - } + protected class TestViewsApi { + protected ViewService viewService = viewService(); + + protected AtomicReference save(String name, View policy) throws InterruptedException { + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + TestResponseCapture responseCapture = new TestResponseCapture<>(); + viewService.put(name, policy, responseCapture); + responseCapture.latch.await(); + return responseCapture.error; + } + + protected void delete(String name) throws Exception { + TestResponseCapture responseCapture = new TestResponseCapture<>(); + viewService.delete(name, responseCapture); + responseCapture.latch.await(); + if (responseCapture.error.get() != null) { + throw responseCapture.error.get(); + } + } - protected void deleteView(String name, ViewService viewService) throws Exception { - TestResponseCapture responseCapture = new TestResponseCapture(); - viewService.delete(name, responseCapture); - responseCapture.latch.await(); - if (responseCapture.error.get() != null) { - throw responseCapture.error.get(); + public Map get(String... names) throws Exception { + TestResponseCapture responseCapture = new TestResponseCapture<>(); + TransportGetViewAction getViewAction = getInstanceFromNode(TransportGetViewAction.class); + GetViewAction.Request request = new GetViewAction.Request(TimeValue.THIRTY_SECONDS, names); + getViewAction.doExecute(null, request, responseCapture); + if (responseCapture.error.get() != null) { + throw responseCapture.error.get(); + } + return responseCapture.response.getViews(); } } - protected static class TestResponseCapture implements ActionListener { + protected static class TestResponseCapture implements ActionListener { CountDownLatch latch = new CountDownLatch(1); AtomicReference error = new AtomicReference<>(); + T response; @Override - public void onResponse(Void unused) { + public void onResponse(T response) { latch.countDown(); + this.response = response; } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java index e3c60a308f1a1..7b62a446d812c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -13,142 +13,140 @@ import org.junit.After; import org.junit.Before; -import java.util.Set; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.esql.view.ViewTests.randomView; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; public class ViewCrudTests extends AbstractViewTestCase { - private ViewService viewService; + private TestViewsApi viewsApi; @Before public void setup() throws Exception { super.setUp(); - this.viewService = viewService(); + this.viewsApi = new TestViewsApi(); } @After public void tearDown() throws Exception { - for (String name : this.viewService.list()) { - deleteView(name, viewService); + for (String name : this.viewsApi.viewService.list()) { + viewsApi.delete(name); } super.tearDown(); } public void testCrud() throws Exception { - ViewService viewService = viewService(); View view = randomView(XContentType.JSON); String name = "my-view"; - AtomicReference error = saveView(name, view, viewService); + AtomicReference error = viewsApi.save(name, view); assertThat(error.get(), nullValue()); + assertView(viewsApi.get(name), name, view); - View result = viewService.get(name); - assertThat(result, equalTo(view)); + viewsApi.delete(name); + assertThat(viewsApi.get(name).size(), equalTo(0)); + } - Set views = viewService.list(); - assertThat(views.size(), equalTo(1)); - assertThat(views, equalTo(Set.of(name))); + public void testList() throws Exception { + for (int i = 0; i < 10; i++) { + View view = randomView(XContentType.JSON); + String name = "my-view-" + i; - deleteView(name, viewService); - result = viewService.get(name); - assertThat(result, nullValue()); + AtomicReference error = viewsApi.save(name, view); + assertThat(error.get(), nullValue()); + assertView(viewsApi.get(name), name, view); + assertThat(viewsApi.get().size(), equalTo(1 + i)); + } + for (int i = 0; i < 10; i++) { + String name = "my-view-" + i; + assertThat(viewsApi.get(name).size(), equalTo(1)); + viewsApi.delete(name); + expectThrows(name, IllegalArgumentException.class, equalTo("Views do not exist: " + name), () -> viewsApi.get(name)); + assertThat(viewsApi.get().size(), equalTo(9 - i)); + } } public void testUpdate() throws Exception { - ViewService viewService = viewService(); View view = randomView(XContentType.JSON); String name = "my-view"; - AtomicReference error = saveView(name, view, viewService); + AtomicReference error = viewsApi.save(name, view); assertThat(error.get(), nullValue()); view = randomView(XContentType.JSON); - error = saveView(name, view, viewService); + error = viewsApi.save(name, view); assertThat(error.get(), nullValue()); - View result = viewService.get(name); - assertThat(result, equalTo(view)); + assertView(viewsApi.get(name), name, view); - deleteView(name, viewService); - result = viewService.get(name); - assertThat(result, nullValue()); + viewsApi.delete(name); + assertThat(viewsApi.get(name).size(), equalTo(0)); } public void testPutValidation() throws Exception { - ViewService viewService = viewService(); View view = randomView(XContentType.JSON); { String nullOrEmptyName = randomBoolean() ? "" : null; - - IllegalArgumentException error = expectThrows( - IllegalArgumentException.class, - () -> saveView(nullOrEmptyName, view, viewService) - ); - + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save(nullOrEmptyName, view)); assertThat(error.getMessage(), equalTo("name is missing or empty")); } { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("my-view", null, viewService)); - + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("my-view", null)); assertThat(error.getMessage(), equalTo("view is missing")); } { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("my#view", view, viewService)); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("my#view", view)); assertThat(error.getMessage(), equalTo("Invalid view name [my#view], must not contain '#'")); } { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("..", view, viewService)); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("..", view)); assertThat(error.getMessage(), equalTo("Invalid view name [..], must not be '.' or '..'")); } { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> saveView("myView", view, viewService)); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("myView", view)); assertThat(error.getMessage(), equalTo("Invalid view name [myView], must be lowercase")); } { View invalidView = new View("FROMMM abc"); - ParsingException error = expectThrows(ParsingException.class, () -> saveView("name", invalidView, viewService)); + ParsingException error = expectThrows(ParsingException.class, () -> viewsApi.save("name", invalidView)); assertThat(error.getMessage(), containsString("mismatched input 'FROMMM'")); } } public void testDeleteValidation() { - ViewService viewService = viewService(); - { String nullOrEmptyName = randomBoolean() ? "" : null; - - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> deleteView(nullOrEmptyName, viewService)); - + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.delete(nullOrEmptyName)); assertThat(error.getMessage(), equalTo("name is missing or empty")); } { - ResourceNotFoundException error = expectThrows(ResourceNotFoundException.class, () -> deleteView("my-view", viewService)); - + ResourceNotFoundException error = expectThrows(ResourceNotFoundException.class, () -> viewsApi.delete("my-view")); assertThat(error.getMessage(), equalTo("view [my-view] not found")); } } - public void testGetValidation() { - ViewService viewService = viewService(); + public void testGetValidation() throws Exception { String nullOrEmptyName = randomBoolean() ? "" : null; - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewService.get(nullOrEmptyName)); - + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.get(nullOrEmptyName)); assertThat(error.getMessage(), equalTo("name is missing or empty")); - - View view = viewService.get("null-view"); - assertNull(view); + assertThat(viewsApi.get("null-view").size(), equalTo(0)); } - public void testListValidation() { - ViewService viewService = viewService(); - Set views = viewService.list(); - assertTrue(views.isEmpty()); + public void testListValidation() throws Exception { + Map result = viewsApi.get("null-view"); + assertTrue(result.isEmpty()); } + private void assertView(Map result, String name, View view) { + assertThat(result.size(), equalTo(1)); + View found = result.get(name); + assertThat(found, not(nullValue())); + assertThat(found, equalTo(view)); + } } From 21ca8f00649df05a7229b4d875d8c3175478e62c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 12 Nov 2025 15:57:21 +0100 Subject: [PATCH 05/17] Fixed failing tests after changing behaviour of get to throw exception on missing view --- .../xpack/esql/view/AbstractViewTestCase.java | 4 ++ .../xpack/esql/view/ViewCrudTests.java | 38 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java index 530f03b4a6d70..525c51610655d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -60,6 +60,10 @@ protected void delete(String name) throws Exception { } public Map get(String... names) throws Exception { + if (names == null || names.length == 1 && names[0] == null) { + // This is only for consistent testing, in production this is already checked in the REST API + throw new IllegalArgumentException("name is missing or empty"); + } TestResponseCapture responseCapture = new TestResponseCapture<>(); TransportGetViewAction getViewAction = getInstanceFromNode(TransportGetViewAction.class); GetViewAction.Request request = new GetViewAction.Request(TimeValue.THIRTY_SECONDS, names); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java index 7b62a446d812c..c0ca18559541e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -49,7 +49,7 @@ public void testCrud() throws Exception { assertView(viewsApi.get(name), name, view); viewsApi.delete(name); - assertThat(viewsApi.get(name).size(), equalTo(0)); + assertViewMissing(viewsApi, name, 0); } public void testList() throws Exception { @@ -66,8 +66,7 @@ public void testList() throws Exception { String name = "my-view-" + i; assertThat(viewsApi.get(name).size(), equalTo(1)); viewsApi.delete(name); - expectThrows(name, IllegalArgumentException.class, equalTo("Views do not exist: " + name), () -> viewsApi.get(name)); - assertThat(viewsApi.get().size(), equalTo(9 - i)); + assertViewMissing(viewsApi, name, 9 - i); } } @@ -84,7 +83,7 @@ public void testUpdate() throws Exception { assertView(viewsApi.get(name), name, view); viewsApi.delete(name); - assertThat(viewsApi.get(name).size(), equalTo(0)); + assertViewMissing(viewsApi, name, 0); } public void testPutValidation() throws Exception { @@ -131,16 +130,22 @@ public void testDeleteValidation() { } public void testGetValidation() throws Exception { - String nullOrEmptyName = randomBoolean() ? "" : null; - - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.get(nullOrEmptyName)); - assertThat(error.getMessage(), equalTo("name is missing or empty")); - assertThat(viewsApi.get("null-view").size(), equalTo(0)); - } - - public void testListValidation() throws Exception { - Map result = viewsApi.get("null-view"); - assertTrue(result.isEmpty()); + expectThrows("null name", IllegalArgumentException.class, equalTo("name is missing or empty"), () -> viewsApi.get((String) null)); + expectThrows("empty name", IllegalArgumentException.class, equalTo("name is missing or empty"), () -> viewsApi.get("")); + expectThrows("missing view", IllegalArgumentException.class, equalTo("Views do not exist: name"), () -> viewsApi.get("name")); + expectThrows( + "missing views", + IllegalArgumentException.class, + equalTo("Views do not exist: v1, v2"), + () -> viewsApi.get("v1", "v2") + ); + viewsApi.save("v2", randomView(XContentType.JSON)); + expectThrows( + "partially missing views", + IllegalArgumentException.class, + equalTo("Views do not exist: v1, v3"), + () -> viewsApi.get("v1", "v2", "v3") + ); } private void assertView(Map result, String name, View view) { @@ -149,4 +154,9 @@ private void assertView(Map result, String name, View view) { assertThat(found, not(nullValue())); assertThat(found, equalTo(view)); } + + private void assertViewMissing(TestViewsApi viewsApi, String name, int viewCount) throws Exception { + expectThrows(name, IllegalArgumentException.class, equalTo("Views do not exist: " + name), () -> viewsApi.get(name)); + assertThat(viewsApi.get().size(), equalTo(viewCount)); + } } From 050d89c8bf54f9d442c179b584a34ee101bfc219 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 12 Nov 2025 16:16:46 +0100 Subject: [PATCH 06/17] Fixed failing yaml-tests after changing behaviour of get to return older list structures --- .../org/elasticsearch/xpack/esql/view/GetViewAction.java | 8 ++++++++ .../resources/rest-api-spec/test/esql/200_view.yml | 9 ++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java index f215ed13bfedd..57592c3d07342 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java @@ -14,6 +14,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -46,6 +49,11 @@ public Request(StreamInput in) throws IOException { super(in); } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + public List names() { return names; } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml index ab9ba547e8d10..0bd58a8a74411 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml @@ -47,19 +47,14 @@ crud: esql.get_view: name: dogs - - match: { query: 'FROM test | WHERE animal == "dog"' } - - - do: - esql.list_views: { } - - - match: { views: { dogs: { query: 'FROM test | WHERE animal == "dog"'} } } + - match: { views: { dogs: { query: 'FROM test | WHERE animal == "dog"' } } } - do: esql.delete_view: name: dogs - do: - esql.list_views: { } + esql.get_view: { } - match: { views: { } } From f529f11b850967611bd31329bcbb4ca75390643c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 12 Nov 2025 16:25:28 +0100 Subject: [PATCH 07/17] Remove list_views API (covered by get_view now) --- .../rest-api-spec/api/esql.list_views.json | 28 ----- .../xpack/esql/plugin/EsqlPlugin.java | 9 +- .../xpack/esql/view/ListViewsAction.java | 114 ------------------ .../xpack/esql/view/RestListViewsAction.java | 39 ------ .../esql/view/TransportListViewsAction.java | 43 ------- .../xpack/security/operator/Constants.java | 1 - 6 files changed, 2 insertions(+), 232 deletions(-) delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json deleted file mode 100644 index 06dc03774b5d8..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "esql.list_views": { - "documentation": { - "url": null, - "description": "List all defined non-materialized ES|QL views" - }, - "stability": "experimental", - "visibility": "public", - "headers": { - "accept": [ - "application/json" - ], - "content_type": [ - "application/json" - ] - }, - "url": { - "paths": [ - { - "path": "/_query/views", - "methods": [ - "GET" - ] - } - ] - } - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 12dfd4bfee35c..d1ece541d7689 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -89,15 +89,12 @@ import org.elasticsearch.xpack.esql.view.ClusterViewService; import org.elasticsearch.xpack.esql.view.DeleteViewAction; import org.elasticsearch.xpack.esql.view.GetViewAction; -import org.elasticsearch.xpack.esql.view.ListViewsAction; import org.elasticsearch.xpack.esql.view.PutViewAction; import org.elasticsearch.xpack.esql.view.RestDeleteViewAction; import org.elasticsearch.xpack.esql.view.RestGetViewAction; -import org.elasticsearch.xpack.esql.view.RestListViewsAction; import org.elasticsearch.xpack.esql.view.RestPutViewAction; import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction; import org.elasticsearch.xpack.esql.view.TransportGetViewAction; -import org.elasticsearch.xpack.esql.view.TransportListViewsAction; import org.elasticsearch.xpack.esql.view.TransportPutViewAction; import org.elasticsearch.xpack.esql.view.ViewMetadata; import org.elasticsearch.xpack.esql.view.ViewService; @@ -296,8 +293,7 @@ public List getActions() { new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class), new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class), new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class), - new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class), - new ActionHandler(ListViewsAction.INSTANCE, TransportListViewsAction.class) + new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class) ); } @@ -322,8 +318,7 @@ public List getRestHandlers( new RestEsqlListQueriesAction(), new RestPutViewAction(), new RestDeleteViewAction(), - new RestGetViewAction(), - new RestListViewsAction() + new RestGetViewAction() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java deleted file mode 100644 index fc638f2b1412e..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ListViewsAction.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.view; - -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.xpack.esql.view.ViewMetadata.VIEWS; - -public class ListViewsAction extends ActionType { - - public static final ListViewsAction INSTANCE = new ListViewsAction(); - public static final String NAME = "cluster:admin/xpack/esql/views"; - - private ListViewsAction() { - super(NAME); - } - - public static class Request extends ActionRequest { - public Request() { - super(); - } - - public Request(StreamInput in) throws IOException { - super(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - return o != null && getClass() == o.getClass(); - } - - @Override - public int hashCode() { - return ListViewsAction.Request.class.hashCode(); - } - } - - public static class Response extends ActionResponse implements ToXContentObject { - - private final Map views; - - public Response(final Map views) { - this.views = views; - } - - public Map getViews() { - return views; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - var v = ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); - while (v.hasNext()) { - v.next().toXContent(builder, params); - } - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - return views.equals(((Response) o).views); - } - - @Override - public int hashCode() { - return views.hashCode(); - } - - @Override - public String toString() { - return "ListViewsAction.Response{view=" + views.toString() + '}'; - } - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java deleted file mode 100644 index 5e7033ee3c282..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestListViewsAction.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.view; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.Scope; -import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.GET; - -@ServerlessScope(Scope.PUBLIC) -public class RestListViewsAction extends BaseRestHandler { - @Override - public List routes() { - return List.of(new Route(GET, "/_query/views")); - } - - @Override - public String getName() { - return "esql_list_views"; - } - - @Override - protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - ListViewsAction.Request req = new ListViewsAction.Request(); - return channel -> client.execute(TransportListViewsAction.TYPE, req, new RestToXContentListener<>(channel)); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java deleted file mode 100644 index b08865f846a5a..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportListViewsAction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.view; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.injection.guice.Inject; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.transport.TransportService; - -import java.util.LinkedHashMap; -import java.util.Map; - -public class TransportListViewsAction extends HandledTransportAction { - public static final ActionType TYPE = new ActionType<>(ListViewsAction.NAME); - private final ClusterViewService viewService; - - @Inject - public TransportListViewsAction(TransportService transportService, ActionFilters actionFilters, ClusterViewService viewService) { - super(ListViewsAction.NAME, transportService, actionFilters, ListViewsAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); - this.viewService = viewService; - } - - @Override - protected void doExecute(Task task, ListViewsAction.Request request, ActionListener listener) { - Map views = new LinkedHashMap<>(); - for (String name : viewService.list()) { - View view = viewService.get(name); - if (view != null) { - views.put(name, viewService.get(name)); - } - } - listener.onResponse(new ListViewsAction.Response(views)); - } -} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b6e4796700fc0..e6f4c397adffb 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -177,7 +177,6 @@ public class Constants { "cluster:admin/xpack/esql/view/put", "cluster:admin/xpack/esql/view/delete", "cluster:admin/xpack/esql/view/get", - "cluster:admin/xpack/esql/views", "cluster:internal/xpack/inference/clear_inference_endpoint_cache", "cluster:admin/xpack/inference/ccm/delete", "cluster:admin/xpack/inference/ccm/put", From 2694cef10a9bd881b0375122b4e87e15de72608b Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 14:46:28 +0100 Subject: [PATCH 08/17] Simply fixes based on code review --- .../xpack/esql/view/TransportGetViewAction.java | 6 ++++-- .../main/java/org/elasticsearch/xpack/esql/view/View.java | 8 -------- .../xpack/esql/view/InMemoryViewService.java | 0 .../elasticsearch/xpack/security/operator/Constants.java | 1 - 4 files changed, 4 insertions(+), 11 deletions(-) rename x-pack/plugin/esql/src/{main => test}/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java (100%) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java index 5b43eb1fa381d..d9987d74b3d6c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java @@ -17,6 +17,8 @@ import org.elasticsearch.transport.TransportService; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.TreeMap; @@ -34,9 +36,9 @@ public TransportGetViewAction(TransportService transportService, ActionFilters a protected void doExecute(Task task, GetViewAction.Request request, ActionListener listener) { TreeMap views = new TreeMap<>(); List missing = new ArrayList<>(); - var names = request.names(); + Collection names = request.names(); if (names.isEmpty()) { - names = new ArrayList<>(viewService.list()); + names = Collections.unmodifiableSet(viewService.list()); } for (String name : names) { View view = viewService.get(name); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java index 9ebe93f25d3d3..05f0081818c0e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.esql.view; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -47,13 +46,6 @@ public View(StreamInput in) throws IOException { } public static View fromXContent(XContentParser parser) throws IOException { - XContentParser.Token token = parser.currentToken(); - if (token == null) { - token = parser.nextToken(); - } - if (token != XContentParser.Token.START_OBJECT) { - throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "], expected START_OBJECT"); - } return PARSER.parse(parser, null); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java similarity index 100% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index e6f4c397adffb..aa51a5f1978b2 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -177,7 +177,6 @@ public class Constants { "cluster:admin/xpack/esql/view/put", "cluster:admin/xpack/esql/view/delete", "cluster:admin/xpack/esql/view/get", - "cluster:internal/xpack/inference/clear_inference_endpoint_cache", "cluster:admin/xpack/inference/ccm/delete", "cluster:admin/xpack/inference/ccm/put", "cluster:admin/xpack/inference/delete", From fe44dbbbb240e13f3c2fe1a7854ddf230523d1f9 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 14:58:46 +0100 Subject: [PATCH 09/17] Fix transport version numbers --- .../resources/transport/definitions/referable/esql_views.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.3.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/esql_views.csv b/server/src/main/resources/transport/definitions/referable/esql_views.csv index fad15855ad264..023d490c87211 100644 --- a/server/src/main/resources/transport/definitions/referable/esql_views.csv +++ b/server/src/main/resources/transport/definitions/referable/esql_views.csv @@ -1 +1 @@ -9217000 +9218000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index b29f7625613b5..cc96fbc37f368 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -resharding_shard_summary_in_esql,9217000 +esql_views,9218000 From 507c7816f8c57e3f090b9a839ae678633cbdd4ad Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 19:05:42 +0100 Subject: [PATCH 10/17] Update REST API to use ProjectID where possible --- .../xpack/esql/view/ClusterViewService.java | 28 +++++++++++---- .../esql/view/TransportDeleteViewAction.java | 15 ++++---- .../esql/view/TransportGetViewAction.java | 6 ++-- .../esql/view/TransportPutViewAction.java | 2 +- .../xpack/esql/view/ViewService.java | 34 ++++++++++++------- .../xpack/esql/view/AbstractViewTestCase.java | 17 +++++++--- .../xpack/esql/view/InMemoryViewService.java | 12 ++++++- .../esql/view/InMemoryViewServiceTests.java | 10 +++--- .../xpack/esql/view/ViewCrudTests.java | 2 +- 9 files changed, 86 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java index 642a88513d5fa..1a307447c45ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -42,25 +43,38 @@ public ClusterViewService( this.projectResolver = projectResolver; } + public ProjectId getProjectId() { + return projectResolver.getProjectId(); + } + @Override protected ViewMetadata getMetadata() { - return getMetadata(clusterService.state()); + return getMetadata(getProjectId()); + } + + @Override + protected ViewMetadata getMetadata(ProjectId projectId) { + return getMetadata(clusterService.state().metadata().getProject(projectId)); } - protected ViewMetadata getMetadata(ClusterState clusterState) { - return getProjectMetadata(clusterState).custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); + protected ViewMetadata getMetadata(ProjectMetadata projectMetadata) { + return projectMetadata.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); } - protected ProjectMetadata getProjectMetadata(ClusterState clusterState) { - return projectResolver.getProjectMetadata(clusterService.state()); + protected ProjectMetadata getProjectMetadata(ProjectId projectId) { + return clusterService.state().metadata().getProject(projectId); } @Override - protected void updateViewMetadata(ActionListener callback, Function> function) { + protected void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ) { submitUnbatchedTask("update-esql-view-metadata", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - var project = getProjectMetadata(currentState); + var project = getProjectMetadata(projectId); var views = project.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); Map policies = function.apply(views); var metadata = ProjectMetadata.builder(project).putCustom(ViewMetadata.TYPE, new ViewMetadata(policies)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java index 10fe9de0a5037..7099172e1dc4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java @@ -9,10 +9,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeProjectAction; +import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -20,7 +21,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -public class TransportDeleteViewAction extends AcknowledgedTransportMasterNodeAction { +public class TransportDeleteViewAction extends AcknowledgedTransportMasterNodeProjectAction { private final ClusterViewService viewService; @Inject @@ -29,6 +30,7 @@ public TransportDeleteViewAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, + ProjectResolver projectResolver, ClusterViewService viewService ) { super( @@ -38,6 +40,7 @@ public TransportDeleteViewAction( threadPool, actionFilters, DeleteViewAction.Request::new, + projectResolver, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.viewService = viewService; @@ -47,14 +50,14 @@ public TransportDeleteViewAction( protected void masterOperation( Task task, DeleteViewAction.Request request, - ClusterState state, + ProjectState state, ActionListener listener ) { - viewService.delete(request.name(), listener.map(v -> AcknowledgedResponse.TRUE)); + viewService.delete(state.projectId(), request.name(), listener.map(v -> AcknowledgedResponse.TRUE)); } @Override - protected ClusterBlockException checkBlock(DeleteViewAction.Request request, ClusterState state) { + protected ClusterBlockException checkBlock(DeleteViewAction.Request request, ProjectState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java index d9987d74b3d6c..9b77e70973aa4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -34,14 +35,15 @@ public TransportGetViewAction(TransportService transportService, ActionFilters a @Override protected void doExecute(Task task, GetViewAction.Request request, ActionListener listener) { + ProjectId projectId = viewService.getProjectId(); TreeMap views = new TreeMap<>(); List missing = new ArrayList<>(); Collection names = request.names(); if (names.isEmpty()) { - names = Collections.unmodifiableSet(viewService.list()); + names = Collections.unmodifiableSet(viewService.list(projectId)); } for (String name : names) { - View view = viewService.get(name); + View view = viewService.get(projectId, name); if (view == null) { missing.add(name); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java index f2dde51ae23ef..bfc932381ecfa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java @@ -50,7 +50,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) { - viewService.put(request.name(), request.view(), listener.map(v -> AcknowledgedResponse.TRUE)); + viewService.put(state.projectId(), request.name(), request.view(), listener.map(v -> AcknowledgedResponse.TRUE)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java index f6ede5d2fef4f..5d3d161074106 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java @@ -10,6 +10,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.VerificationException; @@ -57,6 +58,8 @@ public ViewService(EsqlFunctionRegistry functionRegistry, ViewServiceConfig conf protected abstract ViewMetadata getMetadata(); + protected abstract ViewMetadata getMetadata(ProjectId projectId); + public LogicalPlan replaceViews(LogicalPlan plan, PlanTelemetry telemetry) { if (viewsFeatureEnabled() == false) { return plan; @@ -142,12 +145,12 @@ private VerificationException viewError(String type, List seen) { /** * Adds or modifies a view by name. This method can only be invoked on the master node. */ - public void put(String name, View view, ActionListener callback) { + public void put(ProjectId projectId, String name, View view, ActionListener callback) { assertMasterNode(); if (viewsFeatureEnabled()) { - validatePutView(name, view); - updateViewMetadata(callback, current -> { - Map original = getMetadata().views(); + validatePutView(projectId, name, view); + updateViewMetadata(projectId, callback, current -> { + Map original = getMetadata(projectId).views(); Map updated = new HashMap<>(original); updated.put(name, view); return updated; @@ -155,7 +158,7 @@ public void put(String name, View view, ActionListener callback) { } } - private void validatePutView(String name, View view) { + private void validatePutView(ProjectId projectId, String name, View view) { if (Strings.isNullOrEmpty(name)) { throw new IllegalArgumentException("name is missing or empty"); } @@ -178,7 +181,8 @@ private void validatePutView(String name, View view) { "view query is too large: " + view.query().length() + " characters, the maximum allowed is " + config.maxViewSize ); } - if (getMetadata().views().containsKey(name) == false && getMetadata().views().size() >= config.maxViews) { + Map views = getMetadata(projectId).views(); + if (views.containsKey(name) == false && views.size() >= config.maxViews) { throw new IllegalArgumentException("cannot add view, the maximum number of views is reached: " + config.maxViews); } new EsqlParser().createStatement(view.query(), new QueryParams(), new PlanTelemetry(functionRegistry)); @@ -189,31 +193,31 @@ private void validatePutView(String name, View view) { /** * Gets the view by name. */ - public View get(String name) { + public View get(ProjectId projectId, String name) { if (Strings.isNullOrEmpty(name)) { throw new IllegalArgumentException("name is missing or empty"); } - return viewsFeatureEnabled() ? getMetadata().views().get(name) : null; + return viewsFeatureEnabled() ? getMetadata(projectId).views().get(name) : null; } /** * List current view names. */ - public Set list() { - return viewsFeatureEnabled() ? getMetadata().views().keySet() : Set.of(); + public Set list(ProjectId projectId) { + return viewsFeatureEnabled() ? getMetadata(projectId).views().keySet() : Set.of(); } /** * Removes a view from the cluster state. This method can only be invoked on the master node. */ - public void delete(String name, ActionListener callback) { + public void delete(ProjectId projectId, String name, ActionListener callback) { assertMasterNode(); if (Strings.isNullOrEmpty(name)) { throw new IllegalArgumentException("name is missing or empty"); } if (viewsFeatureEnabled()) { - updateViewMetadata(callback, current -> { + updateViewMetadata(projectId, callback, current -> { Map original = current.views(); if (original.containsKey(name) == false) { throw new ResourceNotFoundException("view [{}] not found", name); @@ -231,5 +235,9 @@ protected boolean viewsFeatureEnabled() { return true; } - protected abstract void updateViewMetadata(ActionListener callback, Function> function); + protected abstract void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java index 525c51610655d..9a0bf3d38e43e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; @@ -32,27 +33,33 @@ protected Collection> getPlugins() { return List.of(LocalStateView.class); } - protected ViewService viewService() { + protected ViewService viewService(ProjectResolver projectResolver) { ClusterService clusterService = getInstanceFromNode(ClusterService.class); FeatureService featureService = getInstanceFromNode(FeatureService.class); - ProjectResolver projectResolver = getInstanceFromNode(ProjectResolver.class); return new ClusterViewService(new EsqlFunctionRegistry(), clusterService, featureService, projectResolver, DEFAULT); } protected class TestViewsApi { - protected ViewService viewService = viewService(); + protected final ViewService viewService; + protected final ProjectId projectId; + + public TestViewsApi() { + ProjectResolver projectResolver = getInstanceFromNode(ProjectResolver.class); + this.viewService = viewService(projectResolver); + this.projectId = projectResolver.getProjectId(); + } protected AtomicReference save(String name, View policy) throws InterruptedException { IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); TestResponseCapture responseCapture = new TestResponseCapture<>(); - viewService.put(name, policy, responseCapture); + viewService.put(projectId, name, policy, responseCapture); responseCapture.latch.await(); return responseCapture.error; } protected void delete(String name) throws Exception { TestResponseCapture responseCapture = new TestResponseCapture<>(); - viewService.delete(name, responseCapture); + viewService.delete(projectId, name, responseCapture); responseCapture.latch.await(); if (responseCapture.error.get() != null) { throw responseCapture.error.get(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java index 0a7334a1026e8..966a278600494 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.view; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import java.util.Map; @@ -36,7 +37,16 @@ protected ViewMetadata getMetadata() { } @Override - protected void updateViewMetadata(ActionListener callback, Function> function) { + protected ViewMetadata getMetadata(ProjectId projectId) { + return metadata; + } + + @Override + protected void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ) { Map updated = function.apply(metadata); this.metadata = new ViewMetadata(updated); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java index 230ce1b77e17b..101cd8c6ca865 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.view; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.parser.AbstractStatementParserTests; @@ -23,14 +24,15 @@ public class InMemoryViewServiceTests extends AbstractStatementParserTests { EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); InMemoryViewService viewService = new InMemoryViewService(functionRegistry); PlanTelemetry telemetry = new PlanTelemetry(functionRegistry); + ProjectId projectId = ProjectId.fromId("1"); public void testPutGet() throws Exception { addView("view1", "from emp"); addView("view2", "from view1"); addView("view3", "from view2"); - assertThat(viewService.get("view1").query(), equalTo("from emp")); - assertThat(viewService.get("view2").query(), equalTo("from view1")); - assertThat(viewService.get("view3").query(), equalTo("from view2")); + assertThat(viewService.get(projectId, "view1").query(), equalTo("from emp")); + assertThat(viewService.get(projectId, "view2").query(), equalTo("from view1")); + assertThat(viewService.get(projectId, "view3").query(), equalTo("from view2")); } public void testReplaceView() throws Exception { @@ -166,7 +168,7 @@ private void addView(String name, String query) { } private void addView(String name, String query, ViewService viewService) { - viewService.put(name, new View(query), ActionListener.noop()); + viewService.put(projectId, name, new View(query), ActionListener.noop()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java index c0ca18559541e..98eba02fab614 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -34,7 +34,7 @@ public void setup() throws Exception { @After public void tearDown() throws Exception { - for (String name : this.viewsApi.viewService.list()) { + for (String name : this.viewsApi.viewService.list(viewsApi.projectId)) { viewsApi.delete(name); } super.tearDown(); From ef4e0bfff0d633709deb15f7b54da4bb51f737aa Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 19:22:35 +0100 Subject: [PATCH 11/17] Fixed things that did not get backported correctly --- .../xpack/esql/view/TransportPutViewAction.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java index bfc932381ecfa..9d3e753ff764e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java @@ -9,10 +9,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeProjectAction; +import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -20,7 +21,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -public class TransportPutViewAction extends AcknowledgedTransportMasterNodeAction { +public class TransportPutViewAction extends AcknowledgedTransportMasterNodeProjectAction { private final ClusterViewService viewService; @Inject @@ -29,7 +30,8 @@ public TransportPutViewAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - ClusterViewService viewService + ClusterViewService viewService, + ProjectResolver projectResolver ) { super( PutViewAction.NAME, @@ -38,6 +40,7 @@ public TransportPutViewAction( threadPool, actionFilters, PutViewAction.Request::new, + projectResolver, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.viewService = viewService; @@ -47,14 +50,14 @@ public TransportPutViewAction( protected void masterOperation( Task task, PutViewAction.Request request, - ClusterState state, + ProjectState state, ActionListener listener ) { viewService.put(state.projectId(), request.name(), request.view(), listener.map(v -> AcknowledgedResponse.TRUE)); } @Override - protected ClusterBlockException checkBlock(PutViewAction.Request request, ClusterState state) { + protected ClusterBlockException checkBlock(PutViewAction.Request request, ProjectState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } } From 2065a220dd4fe513dac9742a1955486bd3807e06 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 19:23:44 +0100 Subject: [PATCH 12/17] Fixed named writable --- .../org/elasticsearch/xpack/esql/view/ViewMetadata.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java index 3c36438c13b44..1a17b1f9f3c28 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java @@ -32,7 +32,11 @@ */ public final class ViewMetadata extends AbstractNamedDiffable implements Metadata.ProjectCustom { public static final String TYPE = "esql_view"; - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(ViewMetadata.class, TYPE, ViewMetadata::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Metadata.ProjectCustom.class, + TYPE, + ViewMetadata::new + ); private static final TransportVersion ESQL_VIEWS = TransportVersion.fromName("esql_views"); static final ParseField VIEWS = new ParseField("views"); From 3cd972dceca2621d541cf4362120b31608e5e05f Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 13 Nov 2025 19:25:58 +0100 Subject: [PATCH 13/17] Missing project ID --- .../elasticsearch/xpack/esql/view/TransportPutViewAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java index 9d3e753ff764e..4adf1a7b12a73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java @@ -58,6 +58,6 @@ protected void masterOperation( @Override protected ClusterBlockException checkBlock(PutViewAction.Request request, ProjectState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + return state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE); } } From adbee24d8a3db0739941fe22b7287af1f27bbcbd Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 14 Nov 2025 12:20:07 +0100 Subject: [PATCH 14/17] Register NamedDiff --- .../org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java | 2 +- .../org/elasticsearch/xpack/esql/view/ViewMetadata.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index d1ece541d7689..9f7708c6f1bd2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -347,7 +347,7 @@ public List getNamedWriteables() { entries.add(ExpressionQueryBuilder.ENTRY); entries.add(PlanStreamWrapperQueryBuilder.ENTRY); - entries.add(ViewMetadata.ENTRY); + entries.addAll(ViewMetadata.ENTRIES); entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java index 1a17b1f9f3c28..61c462fbdfe7f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -24,6 +25,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,10 +34,9 @@ */ public final class ViewMetadata extends AbstractNamedDiffable implements Metadata.ProjectCustom { public static final String TYPE = "esql_view"; - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( - Metadata.ProjectCustom.class, - TYPE, - ViewMetadata::new + public static final List ENTRIES = List.of( + new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, TYPE, ViewMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, TYPE, in -> ViewMetadata.readDiffFrom(Metadata.ProjectCustom.class, TYPE, in)) ); private static final TransportVersion ESQL_VIEWS = TransportVersion.fromName("esql_views"); From 4ddbebf302330135e5e585a18795b1cb0245b053 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 14 Nov 2025 12:36:26 +0100 Subject: [PATCH 15/17] Make views tests not fail in release mode --- .../xpack/esql/view/AbstractViewTestCase.java | 8 +++++--- .../xpack/esql/view/ViewCrudTests.java | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java index 9a0bf3d38e43e..85a4e5b6ce1c0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -6,14 +6,13 @@ */ package org.elasticsearch.xpack.esql.view; +import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; @@ -44,13 +43,16 @@ protected class TestViewsApi { protected final ProjectId projectId; public TestViewsApi() { + if (Build.current().isSnapshot() == false) { + // The TestResponseCapture implementation waits forever if views are not enabled, so lets rather fail early + throw new IllegalStateException("Views tests cannot run in release mode yet"); + } ProjectResolver projectResolver = getInstanceFromNode(ProjectResolver.class); this.viewService = viewService(projectResolver); this.projectId = projectResolver.getProjectId(); } protected AtomicReference save(String name, View policy) throws InterruptedException { - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); TestResponseCapture responseCapture = new TestResponseCapture<>(); viewService.put(projectId, name, policy, responseCapture); responseCapture.latch.await(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java index 98eba02fab614..47cd2a9f02f0d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -7,11 +7,15 @@ package org.elasticsearch.xpack.esql.view; +import org.elasticsearch.Build; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.junit.After; import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -26,6 +30,16 @@ public class ViewCrudTests extends AbstractViewTestCase { private TestViewsApi viewsApi; + @Rule + // TODO: Remove this once we make ViewMetadata no longer snapshot-only + public TestRule skipIfNotSnapshot = (base, description) -> new Statement() { + @Override + public void evaluate() throws Throwable { + assumeTrue("These tests only work in SNAPSHOT builds", Build.current().isSnapshot()); + base.evaluate(); + } + }; + @Before public void setup() throws Exception { super.setUp(); From 88216ddcd3bf988d612babbae978f636129901db Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 14 Nov 2025 17:47:43 +0100 Subject: [PATCH 16/17] Disable for release build and have tests that assert this --- .../xpack/esql/plugin/EsqlPlugin.java | 45 +++++++++++++------ .../xpack/esql/view/ViewRestTests.java | 40 +++++++++++++++++ 2 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewRestTests.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 9f7708c6f1bd2..c0901fbe31445 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.plugin; +import org.elasticsearch.Build; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -280,7 +281,7 @@ public List> getSettings() { @Override public List getActions() { - return List.of( + List releasedActions = List.of( new ActionHandler(EsqlQueryAction.INSTANCE, TransportEsqlQueryAction.class), new ActionHandler(EsqlAsyncGetResultAction.INSTANCE, TransportEsqlAsyncGetResultsAction.class), new ActionHandler(EsqlStatsAction.INSTANCE, TransportEsqlStatsAction.class), @@ -290,11 +291,20 @@ public List getActions() { new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class), new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class), new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class), - new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class), - new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class), - new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class), - new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class) + new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class) ); + if (Build.current().isSnapshot()) { + List actions = new ArrayList<>(releasedActions); + actions.addAll( + List.of( + new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class), + new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class), + new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class) + ) + ); + return actions; + } + return releasedActions; } @Override @@ -309,17 +319,20 @@ public List getRestHandlers( Supplier nodesInCluster, Predicate clusterSupportsFeature ) { - return List.of( + List releasedRestHandlers = List.of( new RestEsqlQueryAction(), new RestEsqlAsyncQueryAction(), new RestEsqlGetAsyncResultAction(), new RestEsqlStopAsyncAction(), new RestEsqlDeleteAsyncResultAction(), - new RestEsqlListQueriesAction(), - new RestPutViewAction(), - new RestDeleteViewAction(), - new RestGetViewAction() + new RestEsqlListQueriesAction() ); + if (Build.current().isSnapshot()) { + List restHandlers = new ArrayList<>(releasedRestHandlers); + restHandlers.addAll(List.of(new RestPutViewAction(), new RestDeleteViewAction(), new RestGetViewAction())); + return restHandlers; + } + return releasedRestHandlers; } @Override @@ -347,7 +360,9 @@ public List getNamedWriteables() { entries.add(ExpressionQueryBuilder.ENTRY); entries.add(PlanStreamWrapperQueryBuilder.ENTRY); - entries.addAll(ViewMetadata.ENTRIES); + if (Build.current().isSnapshot()) { + entries.addAll(ViewMetadata.ENTRIES); + } entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); @@ -357,9 +372,11 @@ public List getNamedWriteables() { @Override public List getNamedXContent() { List namedXContent = new ArrayList<>(); - namedXContent.add( - new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent) - ); + if (Build.current().isSnapshot()) { + namedXContent.add( + new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent) + ); + } return namedXContent; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewRestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewRestTests.java new file mode 100644 index 0000000000000..5d2121e7a1153 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewRestTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.Build; +import org.elasticsearch.core.TimeValue; + +public class ViewRestTests extends AbstractViewTestCase { + + public void testSnapshot() throws Exception { + assumeTrue("Skipping test because we're not in a SNAPSHOT build", Build.current().isSnapshot()); + GetViewAction.Request request = new GetViewAction.Request(TimeValue.timeValueMinutes(1)); + TestResponseCapture responseCapture = new TestResponseCapture<>(); + client().admin().cluster().execute(GetViewAction.INSTANCE, request, responseCapture); + if (responseCapture.error.get() != null) { + fail(responseCapture.error.get(), "Failed to get views in SNAPSHOT build"); + } + if (responseCapture.response == null) { + fail("Response is null"); + } + } + + public void testReleased() { + assumeFalse("Skipping test because we're in a SNAPSHOT build", Build.current().isSnapshot()); + GetViewAction.Request request = new GetViewAction.Request(TimeValue.timeValueMinutes(1)); + TestResponseCapture responseCapture = new TestResponseCapture<>(); + client().admin().cluster().execute(GetViewAction.INSTANCE, request, responseCapture); + if (responseCapture.error.get() == null) { + fail("Expected to fail to get views in release build, but no error was returned"); + } + if (responseCapture.response != null) { + fail("Received unexpected response in release build: " + responseCapture.response); + } + } +} From d55fe6e0b23ad12872f10e1bb236282fe05e6f73 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 14 Nov 2025 17:50:58 +0100 Subject: [PATCH 17/17] Update transport versions --- .../resources/transport/definitions/referable/esql_views.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.3.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/esql_views.csv b/server/src/main/resources/transport/definitions/referable/esql_views.csv index 023d490c87211..711a96c6acee7 100644 --- a/server/src/main/resources/transport/definitions/referable/esql_views.csv +++ b/server/src/main/resources/transport/definitions/referable/esql_views.csv @@ -1 +1 @@ -9218000 +9220000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index afc3bb444e49d..fcdb137b3079b 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -search_project_routing,9219000 +esql_views,9220000