diff --git a/docs/changelog/118282.yaml b/docs/changelog/118282.yaml new file mode 100644 index 0000000000000..cf5746ab7b725 --- /dev/null +++ b/docs/changelog/118282.yaml @@ -0,0 +1,5 @@ +pr: 118282 +summary: "[Connector API] Support soft deletes of connectors" +area: Extract&Transform +type: feature +issues: [] diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc index 76621d7f1843b..50faef5261113 100644 --- a/docs/reference/connector/apis/delete-connector-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -6,8 +6,7 @@ beta::[] -Removes a connector and associated sync jobs. -This is a destructive action that is not recoverable. +Soft-deletes a connector and removes associated sync jobs. Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually. diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 302773e0af831..d36e26c73278e 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -27,6 +27,9 @@ To get started with Connector APIs, check out <`:: (Required, string) +`deleted`:: +(Optional, boolean) A flag indicating whether to return connectors that have been soft-deleted. Defaults to `false`. + [[get-connector-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index 4a93ecf2b0109..7e619c7a369b3 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -41,6 +41,9 @@ To get started with Connector APIs, check out < createComponents(PluginServices services) { @Override public Collection getSystemIndexDescriptors(Settings settings) { Collection systemIndices = new ArrayList<>( - List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor()) + List.of( + SearchApplicationIndexService.getSystemIndexDescriptor(), + QueryRulesIndexService.getSystemIndexDescriptor(), + ConnectorIndexService.getConnectorsDeletedSystemIndexDescriptor() + ) ); if (ConnectorSecretsFeature.isEnabled()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java index ba121f2cf865e..2f7b3769666ff 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java @@ -9,6 +9,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction; import org.elasticsearch.xpack.application.rules.retriever.QueryRuleRetrieverBuilder; @@ -23,7 +24,8 @@ public Set getFeatures() { return Set.of( QUERY_RULES_TEST_API, QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED, - ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES + ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES, + ConnectorIndexService.CONNECTOR_SOFT_DELETES_FEATURE ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 5e1fde0dfb942..192ce69ba0c90 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -10,10 +10,12 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; @@ -24,7 +26,9 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -33,6 +37,8 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -59,6 +65,7 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; +import org.elasticsearch.xpack.core.template.TemplateUtils; import java.time.Instant; import java.util.ArrayList; @@ -76,21 +83,62 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { + // The client to perform any operations on user indices (alias, ...). private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; + public static String DELETED_CONNECTORS_INDEX_NAME = ".connectors-deleted"; + private static final int DELETED_CONNECTORS_INDEX_VERSION = 1; + private static final String DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE = "connectors-deleted.version"; + private static final String DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "connectors-deleted.managed.index.version"; + + public static final NodeFeature CONNECTOR_SOFT_DELETES_FEATURE = new NodeFeature("connector_soft_deletes"); + + /** + * Returns the {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. + * + * @return The {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. + */ + public static SystemIndexDescriptor getConnectorsDeletedSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + + String templateSource = TemplateUtils.loadTemplate( + "/connectors-deleted.json", + Version.CURRENT.toString(), + DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE, + Map.of(DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(DELETED_CONNECTORS_INDEX_VERSION)) + ); + request.source(templateSource, XContentType.JSON); + + return SystemIndexDescriptor.builder() + .setIndexPattern(DELETED_CONNECTORS_INDEX_NAME + "*") + .setPrimaryIndex(DELETED_CONNECTORS_INDEX_NAME + "-" + DELETED_CONNECTORS_INDEX_VERSION) + .setDescription("Index storing deleted connectors") + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setAliasName(DELETED_CONNECTORS_INDEX_NAME) + .setOrigin(CONNECTORS_ORIGIN) + .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) + .setThreadPools(ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS) + .build(); + } + /** * @param client A client for executing actions on the connector index */ public ConnectorIndexService(Client client) { this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -194,13 +242,19 @@ private Connector createConnectorWithDefaultValues( * Gets the {@link Connector} from the underlying index. * * @param connectorId The id of the connector object. + * @param isDeleted If set to true, it returns only soft-deleted connector; otherwise, it returns non-deleted connector. * @param listener The action listener to invoke on response/failure. */ - public void getConnector(String connectorId, ActionListener listener) { + public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { + final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; + + // Only .connectors-delete is system index now + Client requestCli = isDeleted ? clientWithOrigin : client; + try { - final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); + final GetRequest getRequest = new GetRequest(indexName).id(connectorId).realtime(true); - client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { + requestCli.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -229,21 +283,35 @@ public void getConnector(String connectorId, ActionListener listener) { + try { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { - final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexSoftDeletedConnectorRequest = new IndexRequest(DELETED_CONNECTORS_INDEX_NAME).opType( + DocWriteRequest.OpType.INDEX + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .id(connectorId) + .source(connector.getSourceRef(), XContentType.JSON); + + clientWithOrigin.index(indexSoftDeletedConnectorRequest, l.delegateFailure((ll, indexResponse) -> { + final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, ll, (lll, deleteResponse) -> { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + lll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId( + connectorId, + lll.map(r -> deleteResponse) + ); + } else { + lll.onResponse(deleteResponse); + } + })); + })); - try { - client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse)); - } else { - l.onResponse(deleteResponse); - } })); } catch (Exception e) { listener.onFailure(e); @@ -260,6 +328,7 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac * @param connectorNames Filter connectors by connector names, if provided. * @param serviceTypes Filter connectors by service types, if provided. * @param searchQuery Apply a wildcard search on index name, connector name, and description, if provided. + * @param isDeleted If set to true, it returns only soft-deleted connectors; otherwise, it returns non-deleted connectors. * @param listener Invoked with search results or upon failure. */ public void listConnectors( @@ -269,16 +338,21 @@ public void listConnectors( List connectorNames, List serviceTypes, String searchQuery, + Boolean isDeleted, ActionListener listener ) { try { + final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; + // Only .connectors-delete is system index now + Client requestCli = isDeleted ? clientWithOrigin : client; final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); - final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - client.search(req, new ActionListener<>() { + final SearchRequest req = new SearchRequest(indexName).source(source); + + requestCli.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -367,7 +441,7 @@ public void updateConnectorConfiguration(UpdateConnectorConfigurationAction.Requ Map configurationValues = request.getConfigurationValues(); String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy( WriteRequest.RefreshPolicy.IMMEDIATE @@ -599,7 +673,7 @@ public void updateConnectorFilteringDraft( ActionListener listener ) { try { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), XContentType.JSON @@ -659,7 +733,7 @@ public void updateConnectorDraftFilteringValidation( FilteringValidationInfo validation, ActionListener listener ) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -703,7 +777,7 @@ public void updateConnectorDraftFilteringValidation( * @param listener Listener to respond to a successful response or an error. */ public void activateConnectorDraftFiltering(String connectorId, ActionListener listener) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -956,7 +1030,7 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request request, ActionListener listener) { try { String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED @@ -1003,7 +1077,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A try { String connectorId = request.getConnectorId(); ConnectorStatus newStatus = request.getStatus(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 2edd47b1fce3a..120d00f176a0e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class GetConnectorAction { @@ -36,22 +37,31 @@ private GetConnectorAction() {/* no instances */} public static class Request extends ConnectorActionRequest implements ToXContentObject { private final String connectorId; + private final Boolean isDeleted; private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); - public Request(String connectorId) { + private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); + + public Request(String connectorId, Boolean isDeleted) { this.connectorId = connectorId; + this.isDeleted = isDeleted; } public Request(StreamInput in) throws IOException { super(in); this.connectorId = in.readString(); + this.isDeleted = in.readBoolean(); } public String getConnectorId() { return connectorId; } + public Boolean getDeleted() { + return isDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -67,6 +77,7 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(connectorId); + out.writeBoolean(isDeleted); } @Override @@ -74,12 +85,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(connectorId, request.connectorId); + return Objects.equals(connectorId, request.connectorId) && Objects.equals(isDeleted, request.isDeleted); } @Override public int hashCode() { - return Objects.hash(connectorId); + return Objects.hash(connectorId, isDeleted); } @Override @@ -87,6 +98,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } builder.endObject(); return builder; @@ -95,11 +107,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_connector_request", false, - (p) -> new Request((String) p[0]) + (p) -> new Request((String) p[0], (Boolean) p[1]) ); static { PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static Request parse(XContentParser parser) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index e543d805b7099..44a52cd87e014 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -46,11 +46,13 @@ public static class Request extends ConnectorActionRequest implements ToXContent private final List connectorNames; private final List connectorServiceTypes; private final String connectorSearchQuery; + private final Boolean isDeleted; private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams"); private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names"); private static final ParseField NAMES_FIELD = new ParseField("names"); private static final ParseField SEARCH_QUERY_FIELD = new ParseField("query"); + private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); public Request(StreamInput in) throws IOException { super(in); @@ -59,6 +61,7 @@ public Request(StreamInput in) throws IOException { this.connectorNames = in.readOptionalStringCollectionAsList(); this.connectorServiceTypes = in.readOptionalStringCollectionAsList(); this.connectorSearchQuery = in.readOptionalString(); + this.isDeleted = in.readOptionalBoolean(); } public Request( @@ -66,13 +69,15 @@ public Request( List indexNames, List connectorNames, List serviceTypes, - String connectorSearchQuery + String connectorSearchQuery, + Boolean isDeleted ) { this.pageParams = pageParams; this.indexNames = indexNames; this.connectorNames = connectorNames; this.connectorServiceTypes = serviceTypes; this.connectorSearchQuery = connectorSearchQuery; + this.isDeleted = isDeleted; } public PageParams getPageParams() { @@ -95,6 +100,10 @@ public String getConnectorSearchQuery() { return connectorSearchQuery; } + public Boolean getDeleted() { + return isDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -120,6 +129,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringCollection(connectorNames); out.writeOptionalStringCollection(connectorServiceTypes); out.writeOptionalString(connectorSearchQuery); + out.writeOptionalBoolean(isDeleted); } @Override @@ -131,7 +141,8 @@ public boolean equals(Object o) { && Objects.equals(indexNames, request.indexNames) && Objects.equals(connectorNames, request.connectorNames) && Objects.equals(connectorServiceTypes, request.connectorServiceTypes) - && Objects.equals(connectorSearchQuery, request.connectorSearchQuery); + && Objects.equals(connectorSearchQuery, request.connectorSearchQuery) + && Objects.equals(isDeleted, request.isDeleted); } @Override @@ -147,7 +158,8 @@ public int hashCode() { (List) p[1], (List) p[2], (List) p[3], - (String) p[4] + (String) p[4], + (Boolean) p[5] ) ); @@ -157,6 +169,7 @@ public int hashCode() { PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD); PARSER.declareStringArray(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD); PARSER.declareString(optionalConstructorArg(), SEARCH_QUERY_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static ListConnectorAction.Request parse(XContentParser parser) { @@ -172,6 +185,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NAMES_FIELD.getPreferredName(), connectorNames); builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connectorServiceTypes); builder.field(SEARCH_QUERY_FIELD.getPreferredName(), connectorSearchQuery); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } builder.endObject(); return builder; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java index 8d3d5914ca695..8aab02deed5a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java @@ -36,7 +36,8 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM)); + Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); + GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), isDeleted); return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 765f6eca12290..5bb8db98fc11b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -43,13 +43,15 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); List serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0])); String searchQuery = restRequest.param("query"); + Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); ListConnectorAction.Request request = new ListConnectorAction.Request( new PageParams(from, size), indexNames, connectorNames, serviceTypes, - searchQuery + searchQuery, + isDeleted ); return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java index e0beeb9b19515..209c326eb8044 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java @@ -34,6 +34,6 @@ public TransportGetConnectorAction(TransportService transportService, ActionFilt @Override protected void doExecute(Task task, GetConnectorAction.Request request, ActionListener listener) { - connectorIndexService.getConnector(request.getConnectorId(), listener.map(GetConnectorAction.Response::new)); + connectorIndexService.getConnector(request.getConnectorId(), request.getDeleted(), listener.map(GetConnectorAction.Response::new)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java index 8ff180aa6189b..f7a3f9a374b44 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java @@ -44,6 +44,8 @@ protected void doExecute(Task task, ListConnectorAction.Request request, ActionL request.getConnectorNames(), request.getConnectorServiceTypes(), request.getConnectorSearchQuery(), + request.getDeleted(), + listener.map(r -> new ListConnectorAction.Response(r.connectors(), r.totalResults())) ); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 12abca3a78591..d8bf916fbb5d2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -118,6 +118,59 @@ public void testDeleteConnector() throws Exception { expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); } + public void testDeleteConnector_expectSoftDeletion() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + List connectors = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + connectors.add(connector); + } + + String connectorIdToDelete = connectorIds.get(0); + DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); + + expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); + + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorIdToDelete); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorIdToDelete)); + assertThat(softDeletedConnector.getServiceType(), equalTo(connectors.get(0).getServiceType())); + } + + public void testDeleteConnector_expectSoftDeletionMultipleConnectors() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + } + + // Delete all of them + for (int i = 0; i < numConnectors; i++) { + String connectorIdToDelete = connectorIds.get(i); + DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + } + + // Connectors were deleted from main index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId)); + } + + // Soft deleted connectors available in system index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorId); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorId)); + } + } + public void testUpdateConnectorConfiguration_FullConfiguration() throws Exception { Connector connector = ConnectorTestUtils.getRandomConnector(); String connectorId = randomUUID(); @@ -886,11 +939,19 @@ public void onFailure(Exception e) { return resp.get(); } + private Connector awaitGetSoftDeletedConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, true); + } + private Connector awaitGetConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, false); + } + + private Connector awaitGetConnector(String connectorId, Boolean isDeleted) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.getConnector(connectorId, new ActionListener<>() { + connectorIndexService.getConnector(connectorId, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorSearchResult connectorResult) { // Serialize the sourceRef to Connector class for unit tests @@ -924,11 +985,23 @@ private ConnectorIndexService.ConnectorResult awaitListConnector( List names, List serviceTypes, String searchQuery + ) throws Exception { + return awaitListConnector(from, size, indexNames, names, serviceTypes, searchQuery, false); + } + + private ConnectorIndexService.ConnectorResult awaitListConnector( + int from, + int size, + List indexNames, + List names, + List serviceTypes, + String searchQuery, + Boolean isDeleted ) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, new ActionListener<>() { + connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorIndexService.ConnectorResult result) { resp.set(result); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java index 2b8e8735fa2cc..849bb1eaaefd9 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java @@ -23,7 +23,7 @@ protected Writeable.Reader instanceReader() { @Override protected GetConnectorAction.Request createTestInstance() { - return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10)); + return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10), randomBoolean()); } @Override @@ -38,6 +38,6 @@ protected GetConnectorAction.Request doParseInstance(XContentParser parser) thro @Override protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) { - return new GetConnectorAction.Request(instance.getConnectorId()); + return new GetConnectorAction.Request(instance.getConnectorId(), instance.getDeleted()); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java index 3390ece073e5f..a815432a7f8ca 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java @@ -31,7 +31,8 @@ protected ListConnectorAction.Request createTestInstance() { List.of(generateRandomStringArray(10, 10, false)), List.of(generateRandomStringArray(10, 10, false)), List.of(generateRandomStringArray(10, 10, false)), - randomAlphaOfLengthBetween(3, 10) + randomAlphaOfLengthBetween(3, 10), + randomBoolean() ); } @@ -52,7 +53,8 @@ protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorActi instance.getIndexNames(), instance.getConnectorNames(), instance.getConnectorServiceTypes(), - instance.getConnectorSearchQuery() + instance.getConnectorSearchQuery(), + instance.getDeleted() ); } }