diff --git a/docs/changelog/137818.yaml b/docs/changelog/137818.yaml new file mode 100644 index 0000000000000..685df5a102665 --- /dev/null +++ b/docs/changelog/137818.yaml @@ -0,0 +1,5 @@ +pr: 137818 +summary: ES|QL Views REST API +area: "ES|QL" +type: feature +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json new file mode 100644 index 0000000000000..3357415dd3d58 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json @@ -0,0 +1,34 @@ +{ + "esql.delete_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete", + "description": "Delete a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "DELETE" + ], + "parts": { + "name": { + "type": "string", + "description": "The name of the view to delete" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json new file mode 100644 index 0000000000000..5bfef0bc75867 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json @@ -0,0 +1,40 @@ +{ + "esql.get_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-get", + "description": "Get a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "GET" + ], + "parts": { + "name": { + "type": "list", + "description": "A comma-separated list of view names" + } + } + }, + { + "path": "/_query/view", + "methods": [ + "GET" + ] + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json new file mode 100644 index 0000000000000..a4f0a6023acd6 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json @@ -0,0 +1,38 @@ +{ + "esql.put_view": { + "documentation": { + "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put", + "description": "Creates a non-materialized VIEW for ESQL." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query/view/{name}", + "methods": [ + "PUT" + ], + "parts": { + "name": { + "type": "string", + "description": "The name of the view to create or update" + } + } + } + ] + }, + "body": { + "description": "Use the `query` element to define the ES|QL query to use as a non-materialized VIEW.", + "required": true + } + } +} diff --git a/server/src/main/resources/transport/definitions/referable/esql_views.csv b/server/src/main/resources/transport/definitions/referable/esql_views.csv new file mode 100644 index 0000000000000..023d490c87211 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_views.csv @@ -0,0 +1 @@ +9218000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index b29f7625613b5..cc96fbc37f368 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -resharding_shard_summary_in_esql,9217000 +esql_views,9218000 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java index 89b4231a999d4..bb3bdd75cece9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java @@ -34,9 +34,15 @@ public class EsqlFeatures implements FeatureSpecification { */ public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax"); + /** + * Cluster support for view management (create, get, update, list) + * This marks that the cluster state has support for the ViewMetadata CustomProject + */ + public static final NodeFeature ESQL_VIEWS = new NodeFeature("esql.views"); + private Set snapshotBuildFeatures() { assert Build.current().isSnapshot() : Build.current(); - return Set.of(METRICS_SYNTAX); + return Set.of(METRICS_SYNTAX, ESQL_VIEWS); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 7f98ba0e1ed98..9f7708c6f1bd2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.plugin; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -48,6 +49,8 @@ import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -74,6 +77,7 @@ import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.ExpressionWritables; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.plan.PlanWritables; @@ -82,6 +86,18 @@ import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog; import org.elasticsearch.xpack.esql.session.IndexResolver; +import org.elasticsearch.xpack.esql.view.ClusterViewService; +import org.elasticsearch.xpack.esql.view.DeleteViewAction; +import org.elasticsearch.xpack.esql.view.GetViewAction; +import org.elasticsearch.xpack.esql.view.PutViewAction; +import org.elasticsearch.xpack.esql.view.RestDeleteViewAction; +import org.elasticsearch.xpack.esql.view.RestGetViewAction; +import org.elasticsearch.xpack.esql.view.RestPutViewAction; +import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction; +import org.elasticsearch.xpack.esql.view.TransportGetViewAction; +import org.elasticsearch.xpack.esql.view.TransportPutViewAction; +import org.elasticsearch.xpack.esql.view.ViewMetadata; +import org.elasticsearch.xpack.esql.view.ViewService; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -182,6 +198,14 @@ public Collection createComponents(PluginServices services) { ); BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking(); var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize); + EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); + ViewService viewService = new ClusterViewService( + functionRegistry, + services.clusterService(), + services.featureService(), + services.projectResolver(), + ViewService.ViewServiceConfig.fromSettings(settings) + ); setupSharedSecrets(); List> extraCheckers = extraCheckerProviders.stream() .flatMap(p -> p.checkers(services.projectResolver(), services.clusterService()).stream()) @@ -201,7 +225,9 @@ public Collection createComponents(PluginServices services) { ThreadPool.Names.SEARCH, blockFactoryProvider.blockFactory() ), - blockFactoryProvider + blockFactoryProvider, + functionRegistry, + viewService ); } @@ -264,7 +290,10 @@ public List getActions() { new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class), new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class), new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class), - new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class) + new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class), + new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class), + new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class), + new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class) ); } @@ -286,7 +315,10 @@ public List getRestHandlers( new RestEsqlGetAsyncResultAction(), new RestEsqlStopAsyncAction(), new RestEsqlDeleteAsyncResultAction(), - new RestEsqlListQueriesAction() + new RestEsqlListQueriesAction(), + new RestPutViewAction(), + new RestDeleteViewAction(), + new RestGetViewAction() ); } @@ -315,12 +347,22 @@ public List getNamedWriteables() { entries.add(ExpressionQueryBuilder.ENTRY); entries.add(PlanStreamWrapperQueryBuilder.ENTRY); + entries.addAll(ViewMetadata.ENTRIES); entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); return entries; } + @Override + public List getNamedXContent() { + List namedXContent = new ArrayList<>(); + namedXContent.add( + new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent) + ); + return namedXContent; + } + public List> getExecutorBuilders(Settings settings) { final int allocatedProcessors = EsExecutors.allocatedProcessors(settings); return List.of( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java new file mode 100644 index 0000000000000..1a307447c45ce --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.plugin.EsqlFeatures; + +import java.util.Map; +import java.util.function.Function; + +/** + * Implementation of {@link ViewService} that keeps the views in the cluster state. + */ +public class ClusterViewService extends ViewService { + private final ClusterService clusterService; + private final FeatureService featureService; + private final ProjectResolver projectResolver; + + public ClusterViewService( + EsqlFunctionRegistry functionRegistry, + ClusterService clusterService, + FeatureService featureService, + ProjectResolver projectResolver, + ViewServiceConfig config + ) { + super(functionRegistry, config); + this.clusterService = clusterService; + this.featureService = featureService; + this.projectResolver = projectResolver; + } + + public ProjectId getProjectId() { + return projectResolver.getProjectId(); + } + + @Override + protected ViewMetadata getMetadata() { + return getMetadata(getProjectId()); + } + + @Override + protected ViewMetadata getMetadata(ProjectId projectId) { + return getMetadata(clusterService.state().metadata().getProject(projectId)); + } + + protected ViewMetadata getMetadata(ProjectMetadata projectMetadata) { + return projectMetadata.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); + } + + protected ProjectMetadata getProjectMetadata(ProjectId projectId) { + return clusterService.state().metadata().getProject(projectId); + } + + @Override + protected void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ) { + submitUnbatchedTask("update-esql-view-metadata", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + var project = getProjectMetadata(projectId); + var views = project.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY); + Map policies = function.apply(views); + var metadata = ProjectMetadata.builder(project).putCustom(ViewMetadata.TYPE, new ViewMetadata(policies)); + return ClusterState.builder(currentState).putProjectMetadata(metadata).build(); + } + + @Override + public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + callback.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + callback.onFailure(e); + } + }); + } + + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here + private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { + clusterService.submitUnbatchedStateUpdateTask(source, task); + } + + @Override + protected void assertMasterNode() { + assert clusterService.localNode().isMasterNode(); + } + + @Override + protected boolean viewsFeatureEnabled() { + return featureService.clusterHasFeature(clusterService.state(), EsqlFeatures.ESQL_VIEWS); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java new file mode 100644 index 0000000000000..d0776f58120dd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/DeleteViewAction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteViewAction extends ActionType { + + public static final DeleteViewAction INSTANCE = new DeleteViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/delete"; + + private DeleteViewAction() { + super(NAME); + } + + public static class Request extends MasterNodeRequest { + private final String name; + + public Request(TimeValue masterNodeTimeout, String name) { + super(masterNodeTimeout); + this.name = Objects.requireNonNull(name, "name cannot be null"); + } + + public Request(StreamInput in) throws IOException { + super(in); + name = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + public String name() { + return name; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java new file mode 100644 index 0000000000000..57592c3d07342 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/GetViewAction.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static org.elasticsearch.xpack.esql.view.ViewMetadata.VIEWS; + +public class GetViewAction extends ActionType { + + public static final GetViewAction INSTANCE = new GetViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/get"; + + private GetViewAction() { + super(NAME); + } + + public static class Request extends LocalClusterStateRequest { + private List names; + + public Request(TimeValue masterNodeTimeout, String... names) { + super(masterNodeTimeout); + this.names = List.of(names); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + public List names() { + return names; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(names, request.names); + } + + @Override + public int hashCode() { + return Objects.hash(names); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final Map views; + + public Response(Map views) { + Objects.requireNonNull(views, "views cannot be null"); + // use a treemap to guarantee ordering in the set, then transform it to the list of named policies + this.views = new TreeMap<>(views); + } + + public Map getViews() { + return views; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + var v = ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); + while (v.hasNext()) { + v.next().toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return views.equals(((Response) o).views); + } + + @Override + public int hashCode() { + return views.hashCode(); + } + + @Override + public String toString() { + return "GetViewAction.Response{" + views.toString() + '}'; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java new file mode 100644 index 0000000000000..d783684df492c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/PutViewAction.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class PutViewAction extends ActionType { + + public static final PutViewAction INSTANCE = new PutViewAction(); + public static final String NAME = "cluster:admin/xpack/esql/view/put"; + + private PutViewAction() { + super(NAME); + } + + public static class Request extends MasterNodeRequest { + private final String name; + private final View view; + + public Request(TimeValue masterNodeTimeout, String name, View view) { + super(masterNodeTimeout); + this.name = Objects.requireNonNull(name, "name cannot be null"); + this.view = view; + } + + public Request(StreamInput in) throws IOException { + super(in); + name = in.readString(); + view = new View(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + view.writeTo(out); + } + + public String name() { + return name; + } + + public View view() { + return view; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name) && view.equals(request.view); + } + + @Override + public int hashCode() { + return Objects.hash(name, view); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java new file mode 100644 index 0000000000000..f0ad25b2c0db9 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestDeleteViewAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; + +@ServerlessScope(Scope.PUBLIC) +public class RestDeleteViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(DELETE, "/_query/view/{name}")); + } + + @Override + public String getName() { + return "esql_delete_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + DeleteViewAction.Request req = new DeleteViewAction.Request(RestUtils.getMasterNodeTimeout(request), request.param("name")); + return channel -> client.execute(DeleteViewAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java new file mode 100644 index 0000000000000..12254a74a5dd0 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestGetViewAction.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(GET, "/_query/view/{name}"), new Route(GET, "/_query/view")); + } + + @Override + public String getName() { + return "esql_get_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + GetViewAction.Request req = new GetViewAction.Request( + RestUtils.getMasterNodeTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")) + ); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + GetViewAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java new file mode 100644 index 0000000000000..c7250cf8f4786 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/RestPutViewAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestPutViewAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(PUT, "/_query/view/{name}")); + } + + @Override + public String getName() { + return "esql_put_view"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentOrSourceParamParser()) { + PutViewAction.Request req = new PutViewAction.Request( + RestUtils.getMasterNodeTimeout(request), + request.param("name"), + View.PARSER.parse(parser, null) + ); + return channel -> client.execute(PutViewAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java new file mode 100644 index 0000000000000..7099172e1dc4f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportDeleteViewAction.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeProjectAction; +import org.elasticsearch.cluster.ProjectState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportDeleteViewAction extends AcknowledgedTransportMasterNodeProjectAction { + private final ClusterViewService viewService; + + @Inject + public TransportDeleteViewAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + ProjectResolver projectResolver, + ClusterViewService viewService + ) { + super( + DeleteViewAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + DeleteViewAction.Request::new, + projectResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.viewService = viewService; + } + + @Override + protected void masterOperation( + Task task, + DeleteViewAction.Request request, + ProjectState state, + ActionListener listener + ) { + viewService.delete(state.projectId(), request.name(), listener.map(v -> AcknowledgedResponse.TRUE)); + } + + @Override + protected ClusterBlockException checkBlock(DeleteViewAction.Request request, ProjectState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java new file mode 100644 index 0000000000000..9b77e70973aa4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportGetViewAction.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; + +public class TransportGetViewAction extends HandledTransportAction { + public static final ActionType TYPE = new ActionType<>(GetViewAction.NAME); + private final ClusterViewService viewService; + + @Inject + public TransportGetViewAction(TransportService transportService, ActionFilters actionFilters, ClusterViewService viewService) { + super(GetViewAction.NAME, transportService, actionFilters, GetViewAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.viewService = viewService; + } + + @Override + protected void doExecute(Task task, GetViewAction.Request request, ActionListener listener) { + ProjectId projectId = viewService.getProjectId(); + TreeMap views = new TreeMap<>(); + List missing = new ArrayList<>(); + Collection names = request.names(); + if (names.isEmpty()) { + names = Collections.unmodifiableSet(viewService.list(projectId)); + } + for (String name : names) { + View view = viewService.get(projectId, name); + if (view == null) { + missing.add(name); + } else { + views.put(name, view); + } + } + if (missing.isEmpty() == false) { + listener.onFailure(new IllegalArgumentException("Views do not exist: " + String.join(", ", missing))); + } else { + listener.onResponse(new GetViewAction.Response(views)); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java new file mode 100644 index 0000000000000..4adf1a7b12a73 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/TransportPutViewAction.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeProjectAction; +import org.elasticsearch.cluster.ProjectState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportPutViewAction extends AcknowledgedTransportMasterNodeProjectAction { + private final ClusterViewService viewService; + + @Inject + public TransportPutViewAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + ClusterViewService viewService, + ProjectResolver projectResolver + ) { + super( + PutViewAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + PutViewAction.Request::new, + projectResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.viewService = viewService; + } + + @Override + protected void masterOperation( + Task task, + PutViewAction.Request request, + ProjectState state, + ActionListener listener + ) { + viewService.put(state.projectId(), request.name(), request.view(), listener.map(v -> AcknowledgedResponse.TRUE)); + } + + @Override + protected ClusterBlockException checkBlock(PutViewAction.Request request, ProjectState state) { + return state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java new file mode 100644 index 0000000000000..05f0081818c0e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/View.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +/** + * Represents an enrich policy including its configuration. + */ +public final class View implements Writeable, ToXContentFragment { + private static final ParseField QUERY = new ParseField("query"); + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view", + false, + (args, ctx) -> new View((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), QUERY); + } + + private final String query; + + public View(String query) { + this.query = query; + } + + public View(StreamInput in) throws IOException { + this.query = in.readString(); + } + + public static View fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(query); + } + + public String query() { + return query; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(QUERY.getPreferredName(), query); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + View other = (View) o; + return Objects.equals(query, other.query); + } + + @Override + public int hashCode() { + return Objects.hash(query); + } + + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java new file mode 100644 index 0000000000000..61c462fbdfe7f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewMetadata.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Encapsulates view definitions as custom metadata inside ProjectMetadata within cluster state. + */ +public final class ViewMetadata extends AbstractNamedDiffable implements Metadata.ProjectCustom { + public static final String TYPE = "esql_view"; + public static final List ENTRIES = List.of( + new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, TYPE, ViewMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, TYPE, in -> ViewMetadata.readDiffFrom(Metadata.ProjectCustom.class, TYPE, in)) + ); + private static final TransportVersion ESQL_VIEWS = TransportVersion.fromName("esql_views"); + + static final ParseField VIEWS = new ParseField("views"); + + public static final ViewMetadata EMPTY = new ViewMetadata(Collections.emptyMap()); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view_metadata", + args -> new ViewMetadata((Map) args[0]) + ); + + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + Map patterns = new HashMap<>(); + String fieldName = null; + for (XContentParser.Token token = p.nextToken(); token != XContentParser.Token.END_OBJECT; token = p.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = p.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + patterns.put(fieldName, View.fromXContent(p)); + } else { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + } + return patterns; + }, VIEWS); + } + + public static ViewMetadata fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Map views; + + public ViewMetadata(StreamInput in) throws IOException { + this(in.readMap(View::new)); + } + + public ViewMetadata(Map views) { + this.views = Collections.unmodifiableMap(views); + } + + public Map views() { + return views; + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return ESQL_VIEWS; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(views, StreamOutput::writeWriteable); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params ignored) { + return ChunkedToXContentHelper.xContentObjectFieldObjects(VIEWS.getPreferredName(), views); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ViewMetadata that = (ViewMetadata) o; + return views.equals(that.views); + } + + @Override + public int hashCode() { + return Objects.hash(views); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java new file mode 100644 index 0000000000000..5d3d161074106 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewService.java @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.parser.EsqlParser; +import org.elasticsearch.xpack.esql.parser.QueryParams; +import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.UnionAll; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public abstract class ViewService { + private final ViewServiceConfig config; + private final EsqlFunctionRegistry functionRegistry; + + public record ViewServiceConfig(int maxViews, int maxViewSize, int maxViewDepth) { + + public static final String MAX_VIEWS_COUNT_SETTING = "esql.views.max_count"; + public static final String MAX_VIEWS_SIZE_SETTING = "esql.views.max_size"; + public static final String MAX_VIEWS_DEPTH_SETTING = "esql.views.max_depth"; + public static final ViewServiceConfig DEFAULT = new ViewServiceConfig(100, 10_000, 10); + + public static ViewServiceConfig fromSettings(Settings settings) { + return new ViewServiceConfig( + settings.getAsInt(MAX_VIEWS_COUNT_SETTING, DEFAULT.maxViews), + settings.getAsInt(MAX_VIEWS_SIZE_SETTING, DEFAULT.maxViewSize), + settings.getAsInt(MAX_VIEWS_DEPTH_SETTING, DEFAULT.maxViewDepth) + ); + } + } + + public ViewService(EsqlFunctionRegistry functionRegistry, ViewServiceConfig config) { + this.functionRegistry = functionRegistry; + this.config = config; + } + + protected abstract ViewMetadata getMetadata(); + + protected abstract ViewMetadata getMetadata(ProjectId projectId); + + public LogicalPlan replaceViews(LogicalPlan plan, PlanTelemetry telemetry) { + if (viewsFeatureEnabled() == false) { + return plan; + } + ViewMetadata views = getMetadata(); + + List seen = new ArrayList<>(); + while (true) { + LogicalPlan prev = plan; + plan = plan.transformUp(UnresolvedRelation.class, ur -> { + List indexes = new ArrayList<>(); + List subqueries = new ArrayList<>(); + for (String name : ur.indexPattern().indexPattern().split(",")) { + name = name.trim(); + if (views.views().containsKey(name)) { + boolean alreadySeen = seen.contains(name); + seen.add(name); + if (alreadySeen) { + throw viewError("circular view reference ", seen); + } + if (seen.size() > config.maxViewDepth) { + throw viewError("The maximum allowed view depth of " + config.maxViewDepth + " has been exceeded: ", seen); + } + View view = views.views().get(name); + subqueries.add(resolve(view, telemetry)); + } else { + indexes.add(name); + } + } + if (subqueries.isEmpty()) { + // No views defined, just return the original plan + return ur; + } + if (indexes.isEmpty()) { + if (subqueries.size() == 1) { + // only one view, no need for union + return subqueries.getFirst(); + } + } else { + subqueries.add( + 0, + new UnresolvedRelation( + ur.source(), + new IndexPattern(ur.indexPattern().source(), String.join(",", indexes)), + ur.frozen(), + ur.metadataFields(), + ur.indexMode(), + ur.unresolvedMessage() + ) + ); + } + return new UnionAll(ur.source(), subqueries, List.of()); + }); + if (plan.equals(prev)) { + return prev; + } + } + } + + private static LogicalPlan resolve(View view, PlanTelemetry telemetry) { + // TODO don't reparse every time. Store parsed? Or cache parsing? dunno + // this will make super-wrong Source. the _source should be the view. + // if there's a `filter` it applies "under" the view. that's weird. right? + // security to create this + // telemetry + // don't allow circular references + return new EsqlParser().createStatement(view.query(), new QueryParams(), telemetry); + } + + private VerificationException viewError(String type, List seen) { + StringBuilder b = new StringBuilder(); + for (String s : seen) { + if (b.isEmpty()) { + b.append(type); + } else { + b.append(" -> "); + } + b.append(s); + } + throw new VerificationException(b.toString()); + } + + /** + * Adds or modifies a view by name. This method can only be invoked on the master node. + */ + public void put(ProjectId projectId, String name, View view, ActionListener callback) { + assertMasterNode(); + if (viewsFeatureEnabled()) { + validatePutView(projectId, name, view); + updateViewMetadata(projectId, callback, current -> { + Map original = getMetadata(projectId).views(); + Map updated = new HashMap<>(original); + updated.put(name, view); + return updated; + }); + } + } + + private void validatePutView(ProjectId projectId, String name, View view) { + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + // The view name is used in a similar context to an index name and therefore has the same restrictions as an index name + MetadataCreateIndexService.validateIndexOrAliasName( + name, + (viewName, error) -> new IllegalArgumentException("Invalid view name [" + viewName + "], " + error) + ); + if (name.toLowerCase(Locale.ROOT).equals(name) == false) { + throw new IllegalArgumentException("Invalid view name [" + name + "], must be lowercase"); + } + if (view == null) { + throw new IllegalArgumentException("view is missing"); + } + if (Strings.isNullOrEmpty(view.query())) { + throw new IllegalArgumentException("view query is missing or empty"); + } + if (view.query().length() > config.maxViewSize) { + throw new IllegalArgumentException( + "view query is too large: " + view.query().length() + " characters, the maximum allowed is " + config.maxViewSize + ); + } + Map views = getMetadata(projectId).views(); + if (views.containsKey(name) == false && views.size() >= config.maxViews) { + throw new IllegalArgumentException("cannot add view, the maximum number of views is reached: " + config.maxViews); + } + new EsqlParser().createStatement(view.query(), new QueryParams(), new PlanTelemetry(functionRegistry)); + // TODO should we validate this in the transport action and make it async? like plan like a query + // TODO postgresql does. + } + + /** + * Gets the view by name. + */ + public View get(ProjectId projectId, String name) { + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + return viewsFeatureEnabled() ? getMetadata(projectId).views().get(name) : null; + } + + /** + * List current view names. + */ + public Set list(ProjectId projectId) { + return viewsFeatureEnabled() ? getMetadata(projectId).views().keySet() : Set.of(); + } + + /** + * Removes a view from the cluster state. This method can only be invoked on the master node. + */ + public void delete(ProjectId projectId, String name, ActionListener callback) { + assertMasterNode(); + if (Strings.isNullOrEmpty(name)) { + throw new IllegalArgumentException("name is missing or empty"); + } + + if (viewsFeatureEnabled()) { + updateViewMetadata(projectId, callback, current -> { + Map original = current.views(); + if (original.containsKey(name) == false) { + throw new ResourceNotFoundException("view [{}] not found", name); + } + Map updated = new HashMap<>(original); + updated.remove(name); + return updated; + }); + } + } + + protected abstract void assertMasterNode(); + + protected boolean viewsFeatureEnabled() { + return true; + } + + protected abstract void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ); +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java new file mode 100644 index 0000000000000..85a4e5b6ce1c0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/AbstractViewTestCase.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.Build; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.esql.view.ViewService.ViewServiceConfig.DEFAULT; + +public abstract class AbstractViewTestCase extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(LocalStateView.class); + } + + protected ViewService viewService(ProjectResolver projectResolver) { + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + FeatureService featureService = getInstanceFromNode(FeatureService.class); + return new ClusterViewService(new EsqlFunctionRegistry(), clusterService, featureService, projectResolver, DEFAULT); + } + + protected class TestViewsApi { + protected final ViewService viewService; + protected final ProjectId projectId; + + public TestViewsApi() { + if (Build.current().isSnapshot() == false) { + // The TestResponseCapture implementation waits forever if views are not enabled, so lets rather fail early + throw new IllegalStateException("Views tests cannot run in release mode yet"); + } + ProjectResolver projectResolver = getInstanceFromNode(ProjectResolver.class); + this.viewService = viewService(projectResolver); + this.projectId = projectResolver.getProjectId(); + } + + protected AtomicReference save(String name, View policy) throws InterruptedException { + TestResponseCapture responseCapture = new TestResponseCapture<>(); + viewService.put(projectId, name, policy, responseCapture); + responseCapture.latch.await(); + return responseCapture.error; + } + + protected void delete(String name) throws Exception { + TestResponseCapture responseCapture = new TestResponseCapture<>(); + viewService.delete(projectId, name, responseCapture); + responseCapture.latch.await(); + if (responseCapture.error.get() != null) { + throw responseCapture.error.get(); + } + } + + public Map get(String... names) throws Exception { + if (names == null || names.length == 1 && names[0] == null) { + // This is only for consistent testing, in production this is already checked in the REST API + throw new IllegalArgumentException("name is missing or empty"); + } + TestResponseCapture responseCapture = new TestResponseCapture<>(); + TransportGetViewAction getViewAction = getInstanceFromNode(TransportGetViewAction.class); + GetViewAction.Request request = new GetViewAction.Request(TimeValue.THIRTY_SECONDS, names); + getViewAction.doExecute(null, request, responseCapture); + if (responseCapture.error.get() != null) { + throw responseCapture.error.get(); + } + return responseCapture.response.getViews(); + } + } + + protected static class TestResponseCapture implements ActionListener { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference error = new AtomicReference<>(); + T response; + + @Override + public void onResponse(T response) { + latch.countDown(); + this.response = response; + } + + @Override + public void onFailure(Exception e) { + error.set(e); + latch.countDown(); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java new file mode 100644 index 0000000000000..966a278600494 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewService.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; + +import java.util.Map; +import java.util.function.Function; + +/** + * Simple implementation of {@link ClusterViewService} that keeps the views in memory. + * This is useful for testing. + */ +public class InMemoryViewService extends ViewService { + + private ViewMetadata metadata; + + public InMemoryViewService(EsqlFunctionRegistry functionRegistry) { + this(functionRegistry, ViewServiceConfig.DEFAULT); + } + + public InMemoryViewService(EsqlFunctionRegistry functionRegistry, ViewServiceConfig config) { + super(functionRegistry, config); + this.metadata = ViewMetadata.EMPTY; + } + + @Override + protected ViewMetadata getMetadata() { + return metadata; + } + + @Override + protected ViewMetadata getMetadata(ProjectId projectId) { + return metadata; + } + + @Override + protected void updateViewMetadata( + ProjectId projectId, + ActionListener callback, + Function> function + ) { + Map updated = function.apply(metadata); + this.metadata = new ViewMetadata(updated); + } + + @Override + protected void assertMasterNode() { + // no-op + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java new file mode 100644 index 0000000000000..101cd8c6ca865 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.parser.AbstractStatementParserTests; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; + +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; + +public class InMemoryViewServiceTests extends AbstractStatementParserTests { + EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); + InMemoryViewService viewService = new InMemoryViewService(functionRegistry); + PlanTelemetry telemetry = new PlanTelemetry(functionRegistry); + ProjectId projectId = ProjectId.fromId("1"); + + public void testPutGet() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + assertThat(viewService.get(projectId, "view1").query(), equalTo("from emp")); + assertThat(viewService.get(projectId, "view2").query(), equalTo("from view1")); + assertThat(viewService.get(projectId, "view3").query(), equalTo("from view2")); + } + + public void testReplaceView() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + LogicalPlan plan = statement("from view3"); + LogicalPlan rewritten = viewService.replaceViews(plan, telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } + + public void testViewDepthExceeded() throws Exception { + addView("view1", "from emp"); + addView("view2", "from view1"); + addView("view3", "from view2"); + addView("view4", "from view3"); + addView("view5", "from view4"); + addView("view6", "from view5"); + addView("view7", "from view6"); + addView("view8", "from view7"); + addView("view9", "from view8"); + addView("view10", "from view9"); + addView("view11", "from view10"); + + // FROM view11 should fail + Exception e = expectThrows(VerificationException.class, () -> viewService.replaceViews(statement("from view11"), telemetry)); + assertThat(e.getMessage(), startsWith("The maximum allowed view depth of 10 has been exceeded")); + + // But FROM view10 should work + LogicalPlan rewritten = viewService.replaceViews(statement("from view10"), telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } + + public void testModifiedViewDepth() { + var config = new ViewService.ViewServiceConfig(100, 10_000, 1); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from emp", customViewService); + addView("view2", "from view1", customViewService); + addView("view3", "from view2", customViewService); + + // FROM view2 should fail + Exception e = expectThrows( + VerificationException.class, + () -> customViewService.replaceViews(statement("from view2"), telemetry) + ); + assertThat(e.getMessage(), startsWith("The maximum allowed view depth of 1 has been exceeded")); + + // But FROM view1 should work + LogicalPlan rewritten = customViewService.replaceViews(statement("from view1"), telemetry); + assertThat(rewritten, equalTo(statement("from emp"))); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testViewCountExceeded() throws Exception { + for (int i = 0; i < ViewService.ViewServiceConfig.DEFAULT.maxViews(); i++) { + addView("view" + i, "from emp"); + } + + // FROM view11 should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("viewx", "from emp")); + assertThat(e.getMessage(), startsWith("cannot add view, the maximum number of views is reached: 100")); + } + + public void testModifiedViewCount() { + var config = new ViewService.ViewServiceConfig(1, 10_000, 10); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from emp", customViewService); + + // View2 should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("view2", "from emp", customViewService)); + assertThat(e.getMessage(), startsWith("cannot add view, the maximum number of views is reached: 1")); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testViewLengthExceeded() throws Exception { + addView("view1", "from short"); + + // Long view definition should fail + StringBuilder longView = new StringBuilder("from "); + for (int i = 0; i < ViewService.ViewServiceConfig.DEFAULT.maxViewSize(); i++) { + longView.append("a"); + } + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("viewx", longView.toString())); + assertThat(e.getMessage(), startsWith("view query is too large: 10005 characters, the maximum allowed is 10000")); + } + + public void testModifiedViewLength() { + var config = new ViewService.ViewServiceConfig(100, 6, 10); + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + try { + addView("view1", "from a", customViewService); + + // Just one character longer should fail + Exception e = expectThrows(IllegalArgumentException.class, () -> addView("view2", "from aa", customViewService)); + assertThat(e.getMessage(), startsWith("view query is too large: 7 characters, the maximum allowed is 6")); + } catch (Exception e) { + throw new AssertionError("unexpected exception", e); + } + } + + public void testInvalidViewNames() { + var config = ViewService.ViewServiceConfig.DEFAULT; + InMemoryViewService customViewService = new InMemoryViewService(functionRegistry, config); + for (var name : Map.of( + "viewX", + "Invalid view name [viewX], must be lowercase", + ".", + "Invalid view name [.], must not be '.' or '..'", + "..", + "Invalid view name [..], must not be '.' or '..'", + "invalid name", + "Invalid view name [invalid name], must not contain the following characters", + "invalid*name", + "Invalid view name [invalid*name], must not contain the following characters" + ).entrySet()) { + Exception e = expectThrows( + "Expected '" + name.getKey() + "' to be an invalid name, but it was not", + IllegalArgumentException.class, + startsWith(name.getValue()), + () -> addView(name.getKey(), "from aa", customViewService) + ); + } + } + + private void addView(String name, String query) { + addView(name, query, viewService); + } + + private void addView(String name, String query, ViewService viewService) { + viewService.put(projectId, name, new View(query), ActionListener.noop()); + } + +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java new file mode 100644 index 0000000000000..bf1b5fb7432ea --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/LocalStateView.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.LicenseService; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.action.TransportXPackInfoAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureResponse; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +public class LocalStateView extends LocalStateCompositeXPackPlugin { + + public LocalStateView(final Settings settings, final Path configPath) throws Exception { + super(settings, configPath); + + plugins.add(new EsqlPlugin() { + @Override + protected XPackLicenseState getLicenseState() { + return LocalStateView.this.getLicenseState(); + } + }); + } + + public static class ViewTransportXPackInfoAction extends TransportXPackInfoAction { + @Inject + public ViewTransportXPackInfoAction( + TransportService transportService, + ActionFilters actionFilters, + LicenseService licenseService, + NodeClient client + ) { + super(transportService, actionFilters, licenseService, client); + } + + @Override + protected List> infoActions() { + return Collections.singletonList(XPackInfoFeatureAction.ESQL); + } + } + + @Override + protected Class> getInfoAction() { + return ViewTransportXPackInfoAction.class; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java new file mode 100644 index 0000000000000..47cd2a9f02f0d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewCrudTests.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.Build; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.parser.ParsingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.esql.view.ViewTests.randomView; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class ViewCrudTests extends AbstractViewTestCase { + + private TestViewsApi viewsApi; + + @Rule + // TODO: Remove this once we make ViewMetadata no longer snapshot-only + public TestRule skipIfNotSnapshot = (base, description) -> new Statement() { + @Override + public void evaluate() throws Throwable { + assumeTrue("These tests only work in SNAPSHOT builds", Build.current().isSnapshot()); + base.evaluate(); + } + }; + + @Before + public void setup() throws Exception { + super.setUp(); + this.viewsApi = new TestViewsApi(); + } + + @After + public void tearDown() throws Exception { + for (String name : this.viewsApi.viewService.list(viewsApi.projectId)) { + viewsApi.delete(name); + } + super.tearDown(); + } + + public void testCrud() throws Exception { + View view = randomView(XContentType.JSON); + String name = "my-view"; + + AtomicReference error = viewsApi.save(name, view); + assertThat(error.get(), nullValue()); + assertView(viewsApi.get(name), name, view); + + viewsApi.delete(name); + assertViewMissing(viewsApi, name, 0); + } + + public void testList() throws Exception { + for (int i = 0; i < 10; i++) { + View view = randomView(XContentType.JSON); + String name = "my-view-" + i; + + AtomicReference error = viewsApi.save(name, view); + assertThat(error.get(), nullValue()); + assertView(viewsApi.get(name), name, view); + assertThat(viewsApi.get().size(), equalTo(1 + i)); + } + for (int i = 0; i < 10; i++) { + String name = "my-view-" + i; + assertThat(viewsApi.get(name).size(), equalTo(1)); + viewsApi.delete(name); + assertViewMissing(viewsApi, name, 9 - i); + } + } + + public void testUpdate() throws Exception { + View view = randomView(XContentType.JSON); + String name = "my-view"; + + AtomicReference error = viewsApi.save(name, view); + assertThat(error.get(), nullValue()); + + view = randomView(XContentType.JSON); + error = viewsApi.save(name, view); + assertThat(error.get(), nullValue()); + assertView(viewsApi.get(name), name, view); + + viewsApi.delete(name); + assertViewMissing(viewsApi, name, 0); + } + + public void testPutValidation() throws Exception { + View view = randomView(XContentType.JSON); + + { + String nullOrEmptyName = randomBoolean() ? "" : null; + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save(nullOrEmptyName, view)); + assertThat(error.getMessage(), equalTo("name is missing or empty")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("my-view", null)); + assertThat(error.getMessage(), equalTo("view is missing")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("my#view", view)); + assertThat(error.getMessage(), equalTo("Invalid view name [my#view], must not contain '#'")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("..", view)); + assertThat(error.getMessage(), equalTo("Invalid view name [..], must not be '.' or '..'")); + } + { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.save("myView", view)); + assertThat(error.getMessage(), equalTo("Invalid view name [myView], must be lowercase")); + } + { + View invalidView = new View("FROMMM abc"); + ParsingException error = expectThrows(ParsingException.class, () -> viewsApi.save("name", invalidView)); + assertThat(error.getMessage(), containsString("mismatched input 'FROMMM'")); + } + } + + public void testDeleteValidation() { + { + String nullOrEmptyName = randomBoolean() ? "" : null; + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> viewsApi.delete(nullOrEmptyName)); + assertThat(error.getMessage(), equalTo("name is missing or empty")); + } + { + ResourceNotFoundException error = expectThrows(ResourceNotFoundException.class, () -> viewsApi.delete("my-view")); + assertThat(error.getMessage(), equalTo("view [my-view] not found")); + } + } + + public void testGetValidation() throws Exception { + expectThrows("null name", IllegalArgumentException.class, equalTo("name is missing or empty"), () -> viewsApi.get((String) null)); + expectThrows("empty name", IllegalArgumentException.class, equalTo("name is missing or empty"), () -> viewsApi.get("")); + expectThrows("missing view", IllegalArgumentException.class, equalTo("Views do not exist: name"), () -> viewsApi.get("name")); + expectThrows( + "missing views", + IllegalArgumentException.class, + equalTo("Views do not exist: v1, v2"), + () -> viewsApi.get("v1", "v2") + ); + viewsApi.save("v2", randomView(XContentType.JSON)); + expectThrows( + "partially missing views", + IllegalArgumentException.class, + equalTo("Views do not exist: v1, v3"), + () -> viewsApi.get("v1", "v2", "v3") + ); + } + + private void assertView(Map result, String name, View view) { + assertThat(result.size(), equalTo(1)); + View found = result.get(name); + assertThat(found, not(nullValue())); + assertThat(found, equalTo(view)); + } + + private void assertViewMissing(TestViewsApi viewsApi, String name, int viewCount) throws Exception { + expectThrows(name, IllegalArgumentException.class, equalTo("Views do not exist: " + name), () -> viewsApi.get(name)); + assertThat(viewsApi.get().size(), equalTo(viewCount)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java new file mode 100644 index 0000000000000..1585022cc7b1a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewMetadataTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.test.AbstractChunkedSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.view.ViewTests.randomView; +import static org.hamcrest.Matchers.equalTo; + +public class ViewMetadataTests extends AbstractChunkedSerializingTestCase { + + @Override + protected ViewMetadata doParseInstance(XContentParser parser) throws IOException { + return ViewMetadata.fromXContent(parser); + } + + @Override + protected ViewMetadata createTestInstance() { + return randomViewMetadata(randomFrom(XContentType.values())); + } + + @Override + protected ViewMetadata mutateInstance(ViewMetadata instance) { + HashMap views = new HashMap<>(instance.views()); + views.replaceAll((name, view) -> randomView(randomFrom(XContentType.values()))); + return new ViewMetadata(views); + } + + @Override + protected ViewMetadata createXContextTestInstance(XContentType xContentType) { + return randomViewMetadata(xContentType); + } + + private static ViewMetadata randomViewMetadata(XContentType xContentType) { + int numViews = randomIntBetween(8, 64); + Map views = Maps.newMapWithExpectedSize(numViews); + for (int i = 0; i < numViews; i++) { + View view = randomView(xContentType); + views.put(randomAlphaOfLength(8), view); + } + return new ViewMetadata(views); + } + + @Override + protected Writeable.Reader instanceReader() { + return ViewMetadata::new; + } + + @Override + protected void assertEqualInstances(ViewMetadata expectedInstance, ViewMetadata newInstance) { + assertNotSame(expectedInstance, newInstance); + assertThat(newInstance.views().size(), equalTo(expectedInstance.views().size())); + for (Map.Entry entry : newInstance.views().entrySet()) { + View actual = entry.getValue(); + View expected = expectedInstance.views().get(entry.getKey()); + ViewTests.assertEqualViews(expected, actual); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java new file mode 100644 index 0000000000000..732745c4d64be --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/ViewTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.view; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class ViewTests extends AbstractXContentSerializingTestCase { + + @Override + protected View doParseInstance(XContentParser parser) throws IOException { + return View.fromXContent(parser); + } + + @Override + protected View createTestInstance() { + return randomView(randomFrom(XContentType.values())); + } + + @Override + protected View mutateInstance(View instance) { + return randomView(randomFrom(XContentType.values())); + } + + @Override + protected View createXContextTestInstance(XContentType xContentType) { + return randomView(xContentType); + } + + public static View randomView(XContentType xContentType) { + String query = "FROM " + randomAlphaOfLength(10); + return new View(query); + } + + @Override + protected Writeable.Reader instanceReader() { + return View::new; + } + + @Override + protected void assertEqualInstances(View expectedInstance, View newInstance) { + assertNotSame(expectedInstance, newInstance); + assertEqualViews(expectedInstance, newInstance); + } + + public static void assertEqualViews(View expectedInstance, View newInstance) { + assertThat(newInstance.query(), equalTo(expectedInstance.query())); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 0957ec55e882a..aa51a5f1978b2 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -174,6 +174,9 @@ public class Constants { "cluster:admin/xpack/enrich/get", "cluster:admin/xpack/enrich/put", "cluster:admin/xpack/enrich/reindex", + "cluster:admin/xpack/esql/view/put", + "cluster:admin/xpack/esql/view/delete", + "cluster:admin/xpack/esql/view/get", "cluster:admin/xpack/inference/ccm/delete", "cluster:admin/xpack/inference/ccm/put", "cluster:admin/xpack/inference/delete", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml new file mode 100644 index 0000000000000..0bd58a8a74411 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/200_view.yml @@ -0,0 +1,86 @@ +--- +setup: + - requires: + cluster_features: [ "esql.views" ] + reason: "Views not yet supported in cluster state" + - do: + indices.create: + index: test + body: + mappings: + properties: + content: + type: keyword + color: + type: keyword + animal: + type: keyword + id: + type: integer + - do: + bulk: + index: "test" + refresh: true + body: + - { "index": { } } + - { "content": "This is a brown fox", "color": "brown", "animal": "fox", "id": 1 } + - { "index": { } } + - { "content": "This is a brown dog", "color": "brown", "animal": "dog", "id": 2 } + - { "index": { } } + - { "content": "This dog is really white", "color": "white", "animal": "dog", "id": 3 } + - { "index": { } } + - { "content": "The dog is brown but this document is very very long", "color": "brown", "animal": "dog", "id": 4 } + - { "index": { } } + - { "content": "There is also a white cat", "color": "white", "animal": "cat", "id": 5 } + - { "index": { } } + - { "content": "The quick brown fox jumps over the lazy dog", "color": "brown", "animal": "dog", "id": 6 } + +--- +crud: + - do: + esql.put_view: + name: dogs + body: + query: 'FROM test | WHERE animal == "dog"' + + - do: + esql.get_view: + name: dogs + + - match: { views: { dogs: { query: 'FROM test | WHERE animal == "dog"' } } } + + - do: + esql.delete_view: + name: dogs + + - do: + esql.get_view: { } + + - match: { views: { } } + +--- +basic: + - requires: + capabilities: + - method: PUT + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ views_v1 ] + reason: "Views not yet supported" + test_runner_features: [ capabilities, allowed_warnings_regex, warnings_regex ] + - do: + esql.put_view: + name: dogs + body: + query: 'FROM test | WHERE animal == "dog"' + + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM dogs | WHERE color == "white" | KEEP id | SORT id' + - match: { columns.0.name: "id" } + - match: { columns.0.type: "integer" } + - length: { values: 1 } + - match: { values.0.0: 3 }