From 93ac07ce27a3841058a7b9710efa9cc54698b357 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Mon, 9 Dec 2024 17:02:50 +0100 Subject: [PATCH 01/26] [Connector API] Add interface for soft-deletes --- .../connector/ConnectorIndexService.java | 17 ++++++++------- .../connector/action/GetConnectorAction.java | 21 +++++++++++++++---- .../connector/action/ListConnectorAction.java | 20 +++++++++++++++--- .../action/RestGetConnectorAction.java | 3 ++- .../action/RestListConnectorAction.java | 4 +++- .../action/TransportGetConnectorAction.java | 2 +- .../action/TransportListConnectorAction.java | 2 ++ .../connector/ConnectorIndexServiceTests.java | 20 ++++++++++++++++-- ...ectorActionRequestBWCSerializingTests.java | 4 ++-- ...ectorActionRequestBWCSerializingTests.java | 6 ++++-- 10 files changed, 76 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 5e1fde0dfb942..59ce2006b5046 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -194,9 +194,10 @@ private Connector createConnectorWithDefaultValues( * Gets the {@link Connector} from the underlying index. * * @param connectorId The id of the connector object. + * @param isDeleted If set to true, it returns only soft-deleted connector; otherwise, it returns non-deleted connector. * @param listener The action listener to invoke on response/failure. */ - public void getConnector(String connectorId, ActionListener listener) { + public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { try { final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); @@ -260,6 +261,7 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac * @param connectorNames Filter connectors by connector names, if provided. * @param serviceTypes Filter connectors by service types, if provided. * @param searchQuery Apply a wildcard search on index name, connector name, and description, if provided. + * @param isDeleted If set to true, it returns only soft-deleted connectors; otherwise, it returns non-deleted connectors. * @param listener Invoked with search results or upon failure. */ public void listConnectors( @@ -269,6 +271,7 @@ public void listConnectors( List connectorNames, List serviceTypes, String searchQuery, + Boolean isDeleted, ActionListener listener ) { try { @@ -367,7 +370,7 @@ public void updateConnectorConfiguration(UpdateConnectorConfigurationAction.Requ Map configurationValues = request.getConfigurationValues(); String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy( WriteRequest.RefreshPolicy.IMMEDIATE @@ -599,7 +602,7 @@ public void updateConnectorFilteringDraft( ActionListener listener ) { try { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), XContentType.JSON @@ -659,7 +662,7 @@ public void updateConnectorDraftFilteringValidation( FilteringValidationInfo validation, ActionListener listener ) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -703,7 +706,7 @@ public void updateConnectorDraftFilteringValidation( * @param listener Listener to respond to a successful response or an error. */ public void activateConnectorDraftFiltering(String connectorId, ActionListener listener) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -956,7 +959,7 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request request, ActionListener listener) { try { String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED @@ -1003,7 +1006,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A try { String connectorId = request.getConnectorId(); ConnectorStatus newStatus = request.getStatus(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 2edd47b1fce3a..120d00f176a0e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class GetConnectorAction { @@ -36,22 +37,31 @@ private GetConnectorAction() {/* no instances */} public static class Request extends ConnectorActionRequest implements ToXContentObject { private final String connectorId; + private final Boolean isDeleted; private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); - public Request(String connectorId) { + private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); + + public Request(String connectorId, Boolean isDeleted) { this.connectorId = connectorId; + this.isDeleted = isDeleted; } public Request(StreamInput in) throws IOException { super(in); this.connectorId = in.readString(); + this.isDeleted = in.readBoolean(); } public String getConnectorId() { return connectorId; } + public Boolean getDeleted() { + return isDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -67,6 +77,7 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(connectorId); + out.writeBoolean(isDeleted); } @Override @@ -74,12 +85,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(connectorId, request.connectorId); + return Objects.equals(connectorId, request.connectorId) && Objects.equals(isDeleted, request.isDeleted); } @Override public int hashCode() { - return Objects.hash(connectorId); + return Objects.hash(connectorId, isDeleted); } @Override @@ -87,6 +98,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } builder.endObject(); return builder; @@ -95,11 +107,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_connector_request", false, - (p) -> new Request((String) p[0]) + (p) -> new Request((String) p[0], (Boolean) p[1]) ); static { PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static Request parse(XContentParser parser) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index e543d805b7099..44a52cd87e014 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -46,11 +46,13 @@ public static class Request extends ConnectorActionRequest implements ToXContent private final List connectorNames; private final List connectorServiceTypes; private final String connectorSearchQuery; + private final Boolean isDeleted; private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams"); private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names"); private static final ParseField NAMES_FIELD = new ParseField("names"); private static final ParseField SEARCH_QUERY_FIELD = new ParseField("query"); + private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); public Request(StreamInput in) throws IOException { super(in); @@ -59,6 +61,7 @@ public Request(StreamInput in) throws IOException { this.connectorNames = in.readOptionalStringCollectionAsList(); this.connectorServiceTypes = in.readOptionalStringCollectionAsList(); this.connectorSearchQuery = in.readOptionalString(); + this.isDeleted = in.readOptionalBoolean(); } public Request( @@ -66,13 +69,15 @@ public Request( List indexNames, List connectorNames, List serviceTypes, - String connectorSearchQuery + String connectorSearchQuery, + Boolean isDeleted ) { this.pageParams = pageParams; this.indexNames = indexNames; this.connectorNames = connectorNames; this.connectorServiceTypes = serviceTypes; this.connectorSearchQuery = connectorSearchQuery; + this.isDeleted = isDeleted; } public PageParams getPageParams() { @@ -95,6 +100,10 @@ public String getConnectorSearchQuery() { return connectorSearchQuery; } + public Boolean getDeleted() { + return isDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -120,6 +129,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringCollection(connectorNames); out.writeOptionalStringCollection(connectorServiceTypes); out.writeOptionalString(connectorSearchQuery); + out.writeOptionalBoolean(isDeleted); } @Override @@ -131,7 +141,8 @@ public boolean equals(Object o) { && Objects.equals(indexNames, request.indexNames) && Objects.equals(connectorNames, request.connectorNames) && Objects.equals(connectorServiceTypes, request.connectorServiceTypes) - && Objects.equals(connectorSearchQuery, request.connectorSearchQuery); + && Objects.equals(connectorSearchQuery, request.connectorSearchQuery) + && Objects.equals(isDeleted, request.isDeleted); } @Override @@ -147,7 +158,8 @@ public int hashCode() { (List) p[1], (List) p[2], (List) p[3], - (String) p[4] + (String) p[4], + (Boolean) p[5] ) ); @@ -157,6 +169,7 @@ public int hashCode() { PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD); PARSER.declareStringArray(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD); PARSER.declareString(optionalConstructorArg(), SEARCH_QUERY_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static ListConnectorAction.Request parse(XContentParser parser) { @@ -172,6 +185,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NAMES_FIELD.getPreferredName(), connectorNames); builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connectorServiceTypes); builder.field(SEARCH_QUERY_FIELD.getPreferredName(), connectorSearchQuery); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } builder.endObject(); return builder; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java index 8d3d5914ca695..8aab02deed5a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java @@ -36,7 +36,8 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM)); + Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); + GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), isDeleted); return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 765f6eca12290..5bb8db98fc11b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -43,13 +43,15 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); List serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0])); String searchQuery = restRequest.param("query"); + Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); ListConnectorAction.Request request = new ListConnectorAction.Request( new PageParams(from, size), indexNames, connectorNames, serviceTypes, - searchQuery + searchQuery, + isDeleted ); return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java index e0beeb9b19515..209c326eb8044 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java @@ -34,6 +34,6 @@ public TransportGetConnectorAction(TransportService transportService, ActionFilt @Override protected void doExecute(Task task, GetConnectorAction.Request request, ActionListener listener) { - connectorIndexService.getConnector(request.getConnectorId(), listener.map(GetConnectorAction.Response::new)); + connectorIndexService.getConnector(request.getConnectorId(), request.getDeleted(), listener.map(GetConnectorAction.Response::new)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java index 8ff180aa6189b..f7a3f9a374b44 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java @@ -44,6 +44,8 @@ protected void doExecute(Task task, ListConnectorAction.Request request, ActionL request.getConnectorNames(), request.getConnectorServiceTypes(), request.getConnectorSearchQuery(), + request.getDeleted(), + listener.map(r -> new ListConnectorAction.Response(r.connectors(), r.totalResults())) ); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 12abca3a78591..f37600e9d076c 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -887,10 +887,14 @@ public void onFailure(Exception e) { } private Connector awaitGetConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, false); + } + + private Connector awaitGetConnector(String connectorId, Boolean isDeleted) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.getConnector(connectorId, new ActionListener<>() { + connectorIndexService.getConnector(connectorId, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorSearchResult connectorResult) { // Serialize the sourceRef to Connector class for unit tests @@ -924,11 +928,23 @@ private ConnectorIndexService.ConnectorResult awaitListConnector( List names, List serviceTypes, String searchQuery + ) throws Exception { + return awaitListConnector(from, size, indexNames, names, serviceTypes, searchQuery, false); + } + + private ConnectorIndexService.ConnectorResult awaitListConnector( + int from, + int size, + List indexNames, + List names, + List serviceTypes, + String searchQuery, + Boolean isDeleted ) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, new ActionListener<>() { + connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorIndexService.ConnectorResult result) { resp.set(result); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java index 2b8e8735fa2cc..849bb1eaaefd9 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java @@ -23,7 +23,7 @@ protected Writeable.Reader instanceReader() { @Override protected GetConnectorAction.Request createTestInstance() { - return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10)); + return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10), randomBoolean()); } @Override @@ -38,6 +38,6 @@ protected GetConnectorAction.Request doParseInstance(XContentParser parser) thro @Override protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) { - return new GetConnectorAction.Request(instance.getConnectorId()); + return new GetConnectorAction.Request(instance.getConnectorId(), instance.getDeleted()); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java index 3390ece073e5f..a815432a7f8ca 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java @@ -31,7 +31,8 @@ protected ListConnectorAction.Request createTestInstance() { List.of(generateRandomStringArray(10, 10, false)), List.of(generateRandomStringArray(10, 10, false)), List.of(generateRandomStringArray(10, 10, false)), - randomAlphaOfLengthBetween(3, 10) + randomAlphaOfLengthBetween(3, 10), + randomBoolean() ); } @@ -52,7 +53,8 @@ protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorActi instance.getIndexNames(), instance.getConnectorNames(), instance.getConnectorServiceTypes(), - instance.getConnectorSearchQuery() + instance.getConnectorSearchQuery(), + instance.getDeleted() ); } } From 12567b24e91fda90a877f9c9e641c7388168d4c5 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 10 Dec 2024 10:21:31 +0100 Subject: [PATCH 02/26] Define connector deleted system index --- .../main/resources/connectors-deleted.json | 307 ++++++++++++++++++ .../xpack/application/EnterpriseSearch.java | 7 +- .../connector/ConnectorIndexService.java | 40 +++ 3 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json diff --git a/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json b/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json new file mode 100644 index 0000000000000..9e37b31f5672f --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json @@ -0,0 +1,307 @@ +{ + "settings": { + "index": { + "number_of_shards": "1", + "auto_expand_replicas": "0-1" + } + }, + "mappings":{ + "_doc":{ + "dynamic":"false", + "_meta": { + "version": "${connectors-deleted.version}", + "managed_index_mappings_version": ${connectors-deleted.managed.index.version} + }, + "properties":{ + "api_key_id":{ + "type":"keyword" + }, + "configuration":{ + "type":"object" + }, + "custom_scheduling":{ + "type":"object" + }, + "description":{ + "type":"text" + }, + "error":{ + "type":"keyword" + }, + "features":{ + "properties":{ + "filtering_advanced_config":{ + "type":"boolean" + }, + "filtering_rules":{ + "type":"boolean" + }, + "incremental_sync":{ + "properties":{ + "enabled":{ + "type":"boolean" + } + } + }, + "sync_rules":{ + "properties":{ + "advanced":{ + "properties":{ + "enabled":{ + "type":"boolean" + } + } + }, + "basic":{ + "properties":{ + "enabled":{ + "type":"boolean" + } + } + } + } + } + } + }, + "filtering":{ + "properties":{ + "active":{ + "properties":{ + "advanced_snippet":{ + "properties":{ + "created_at":{ + "type":"date" + }, + "updated_at":{ + "type":"date" + }, + "value":{ + "type":"object" + } + } + }, + "rules":{ + "properties":{ + "created_at":{ + "type":"date" + }, + "field":{ + "type":"keyword" + }, + "id":{ + "type":"keyword" + }, + "order":{ + "type":"short" + }, + "policy":{ + "type":"keyword" + }, + "rule":{ + "type":"keyword" + }, + "updated_at":{ + "type":"date" + }, + "value":{ + "type":"keyword" + } + } + }, + "validation":{ + "properties":{ + "errors":{ + "properties":{ + "ids":{ + "type":"keyword" + }, + "messages":{ + "type":"text" + } + } + }, + "state":{ + "type":"keyword" + } + } + } + } + }, + "domain":{ + "type":"keyword" + }, + "draft":{ + "properties":{ + "advanced_snippet":{ + "properties":{ + "created_at":{ + "type":"date" + }, + "updated_at":{ + "type":"date" + }, + "value":{ + "type":"object" + } + } + }, + "rules":{ + "properties":{ + "created_at":{ + "type":"date" + }, + "field":{ + "type":"keyword" + }, + "id":{ + "type":"keyword" + }, + "order":{ + "type":"short" + }, + "policy":{ + "type":"keyword" + }, + "rule":{ + "type":"keyword" + }, + "updated_at":{ + "type":"date" + }, + "value":{ + "type":"keyword" + } + } + }, + "validation":{ + "properties":{ + "errors":{ + "properties":{ + "ids":{ + "type":"keyword" + }, + "messages":{ + "type":"text" + } + } + }, + "state":{ + "type":"keyword" + } + } + } + } + } + } + }, + "index_name":{ + "type":"keyword" + }, + "is_native":{ + "type":"boolean" + }, + "language":{ + "type":"keyword" + }, + "last_access_control_sync_error":{ + "type":"keyword" + }, + "last_access_control_sync_scheduled_at":{ + "type":"date" + }, + "last_access_control_sync_status":{ + "type":"keyword" + }, + "last_deleted_document_count":{ + "type":"long" + }, + "last_incremental_sync_scheduled_at":{ + "type":"date" + }, + "last_indexed_document_count":{ + "type":"long" + }, + "last_seen":{ + "type":"date" + }, + "last_sync_error":{ + "type":"keyword" + }, + "last_sync_scheduled_at":{ + "type":"date" + }, + "last_sync_status":{ + "type":"keyword" + }, + "last_synced":{ + "type":"date" + }, + "name":{ + "type":"keyword" + }, + "pipeline":{ + "properties":{ + "extract_binary_content":{ + "type":"boolean" + }, + "name":{ + "type":"keyword" + }, + "reduce_whitespace":{ + "type":"boolean" + }, + "run_ml_inference":{ + "type":"boolean" + } + } + }, + "scheduling":{ + "properties":{ + "access_control":{ + "properties":{ + "enabled":{ + "type":"boolean" + }, + "interval":{ + "type":"text" + } + } + }, + "full":{ + "properties":{ + "enabled":{ + "type":"boolean" + }, + "interval":{ + "type":"text" + } + } + }, + "incremental":{ + "properties":{ + "enabled":{ + "type":"boolean" + }, + "interval":{ + "type":"text" + } + } + } + } + }, + "service_type":{ + "type":"keyword" + }, + "status":{ + "type":"keyword" + }, + "sync_cursor":{ + "type":"object" + }, + "sync_now":{ + "type":"boolean" + } + } + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index df1c76ccf770f..ee7349ead43f7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction; import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig; import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction; import org.elasticsearch.xpack.application.connector.action.GetConnectorAction; @@ -477,7 +478,11 @@ public Collection createComponents(PluginServices services) { @Override public Collection getSystemIndexDescriptors(Settings settings) { Collection systemIndices = new ArrayList<>( - List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor()) + List.of( + SearchApplicationIndexService.getSystemIndexDescriptor(), + QueryRulesIndexService.getSystemIndexDescriptor(), + ConnectorIndexService.getConnectorsDeletedSystemIndexDescriptor() + ) ); if (ConnectorSecretsFeature.isEnabled()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 59ce2006b5046..79639010b004f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -10,10 +10,12 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; @@ -33,6 +35,8 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -59,6 +63,7 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; +import org.elasticsearch.xpack.core.template.TemplateUtils; import java.time.Instant; import java.util.ArrayList; @@ -76,6 +81,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. @@ -86,6 +92,40 @@ public class ConnectorIndexService { public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; + public static String DELETED_CONNECTORS_INDEX_NAME = ".connectors-deleted"; + private static final int DELETED_CONNECTORS_INDEX_VERSION = 1; + private static final String DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE = "connectors-deleted.version"; + private static final String DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "connectors-deleted.managed.index.version"; + + /** + * Returns the {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. + * + * @return The {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. + */ + public static SystemIndexDescriptor getConnectorsDeletedSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + + String templateSource = TemplateUtils.loadTemplate( + "/connectors-deleted.json", + Version.CURRENT.toString(), + DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE, + Map.of(DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(DELETED_CONNECTORS_INDEX_VERSION)) + ); + request.source(templateSource, XContentType.JSON); + + return SystemIndexDescriptor.builder() + .setIndexPattern(DELETED_CONNECTORS_INDEX_NAME + "*") + .setPrimaryIndex(DELETED_CONNECTORS_INDEX_NAME + "-" + DELETED_CONNECTORS_INDEX_VERSION) + .setDescription("Index storing deleted connectors") + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setAliasName(DELETED_CONNECTORS_INDEX_NAME) + .setOrigin(CONNECTORS_ORIGIN) + .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) + .setThreadPools(ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS) + .build(); + } + /** * @param client A client for executing actions on the connector index */ From 4fdc2caf7cd591b1acfe9cc69a2d3319a1448446 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 10 Dec 2024 11:13:36 +0100 Subject: [PATCH 03/26] Got soft-delete logic working --- .../connector/ConnectorIndexService.java | 52 +++++++++++++------ .../action/DeleteConnectorAction.java | 2 +- .../xpack/security/operator/Constants.java | 2 +- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 79639010b004f..20c924bc2f50b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -82,13 +83,17 @@ import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { + // The client to perform any operations on user indices (alias, ...). private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; @@ -131,6 +136,7 @@ public static SystemIndexDescriptor getConnectorsDeletedSystemIndexDescriptor() */ public ConnectorIndexService(Client client) { this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, ENT_SEARCH_ORIGIN); } /** @@ -238,8 +244,9 @@ private Connector createConnectorWithDefaultValues( * @param listener The action listener to invoke on response/failure. */ public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { + final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; try { - final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); + final GetRequest getRequest = new GetRequest(indexName).id(connectorId).realtime(true); client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { @@ -270,21 +277,35 @@ public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { + try { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { - final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + IndexRequest indexSoftDeletedConnectorRequest = new IndexRequest(DELETED_CONNECTORS_INDEX_NAME).opType( + DocWriteRequest.OpType.INDEX + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .id(connectorId) + .source(connector.getSourceRef(), XContentType.JSON); + + clientWithOrigin.index(indexSoftDeletedConnectorRequest, l.delegateFailureAndWrap((ll, indexResponse) -> { + final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, ll, (lll, deleteResponse) -> { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + lll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId( + connectorId, + lll.map(r -> deleteResponse) + ); + } else { + lll.onResponse(deleteResponse); + } + })); + })); - try { - client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse)); - } else { - l.onResponse(deleteResponse); - } })); } catch (Exception e) { listener.onFailure(e); @@ -315,12 +336,13 @@ public void listConnectors( ActionListener listener ) { try { + final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); - final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); + final SearchRequest req = new SearchRequest(indexName).source(source); client.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java index 930068a2a46ef..83efccd49577f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java @@ -28,7 +28,7 @@ public class DeleteConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/delete"; + public static final String NAME = "cluster:admin/xpack/connector/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index c91314716cf9e..46288f364da09 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -132,7 +132,7 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", - "indices:data/write/xpack/connector/delete", + "cluster:admin/xpack/connector/delete", "indices:data/read/xpack/connector/get", "indices:data/read/xpack/connector/list", "indices:data/write/xpack/connector/post", From d9be2162331b1953ceae1dddb964da639a4fc412 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 10 Dec 2024 14:04:32 +0100 Subject: [PATCH 04/26] Add unit tests --- muted-tests.yml | 2 - .../connector/ConnectorIndexService.java | 2 +- .../connector/ConnectorIndexServiceTests.java | 57 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index eecb7ac3d7e59..f64e537bdb99c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -97,8 +97,6 @@ tests: - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 -- class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests - issue: https://github.com/elastic/elasticsearch/issues/116087 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=cat.shards/10_basic/Help} issue: https://github.com/elastic/elasticsearch/issues/116110 diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 20c924bc2f50b..f9d63f41e3da2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -287,7 +287,7 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac .id(connectorId) .source(connector.getSourceRef(), XContentType.JSON); - clientWithOrigin.index(indexSoftDeletedConnectorRequest, l.delegateFailureAndWrap((ll, indexResponse) -> { + clientWithOrigin.index(indexSoftDeletedConnectorRequest, l.delegateFailure((ll, indexResponse) -> { final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, ll, (lll, deleteResponse) -> { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index f37600e9d076c..d8bf916fbb5d2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -118,6 +118,59 @@ public void testDeleteConnector() throws Exception { expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); } + public void testDeleteConnector_expectSoftDeletion() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + List connectors = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + connectors.add(connector); + } + + String connectorIdToDelete = connectorIds.get(0); + DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); + + expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); + + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorIdToDelete); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorIdToDelete)); + assertThat(softDeletedConnector.getServiceType(), equalTo(connectors.get(0).getServiceType())); + } + + public void testDeleteConnector_expectSoftDeletionMultipleConnectors() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + } + + // Delete all of them + for (int i = 0; i < numConnectors; i++) { + String connectorIdToDelete = connectorIds.get(i); + DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + } + + // Connectors were deleted from main index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId)); + } + + // Soft deleted connectors available in system index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorId); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorId)); + } + } + public void testUpdateConnectorConfiguration_FullConfiguration() throws Exception { Connector connector = ConnectorTestUtils.getRandomConnector(); String connectorId = randomUUID(); @@ -886,6 +939,10 @@ public void onFailure(Exception e) { return resp.get(); } + private Connector awaitGetSoftDeletedConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, true); + } + private Connector awaitGetConnector(String connectorId) throws Exception { return awaitGetConnector(connectorId, false); } From 183ed7ae2a0c4a3a386017480a15c2e93bc8e3f4 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 10 Dec 2024 16:43:51 +0100 Subject: [PATCH 05/26] Add yaml e2e test and attempt to update permissions --- .../rest-api-spec/api/connector.get.json | 7 + .../rest-api-spec/api/connector.list.json | 5 + .../privilege/ClusterPrivilegeResolver.java | 10 +- x-pack/plugin/ent-search/qa/rest/roles.yml | 2 + .../entsearch/connector/20_connector_list.yml | 134 ++++++++++++++++++ .../connector/30_connector_delete.yml | 27 ++++ .../application/EnterpriseSearchFeatures.java | 4 +- .../connector/ConnectorIndexService.java | 3 + 8 files changed, 189 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json index 2645df28c5d1e..c6c1178f177e5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json @@ -26,6 +26,13 @@ } } ] + }, + "params": { + "deleted": { + "type": "boolean", + "default": false, + "description": "A flag indicating whether to return connectors that have been soft-deleted." + } } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json index 67d2250d3c661..17992d412a567 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json @@ -47,6 +47,11 @@ "query": { "type": "string", "description": "A search string for querying connectors, filtering results by matching against connector names, descriptions, and index names" + }, + "deleted": { + "type": "boolean", + "default": false, + "description": "A flag indicating whether to list connectors that have been soft-deleted." } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index 00d45fb135fb2..f3897a0619694 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -177,12 +177,18 @@ public class ClusterPrivilegeResolver { ); private static final Set MANAGE_SEARCH_APPLICATION_PATTERN = Set.of("cluster:admin/xpack/application/search_application/*"); - private static final Set MANAGE_CONNECTOR_PATTERN = Set.of("cluster:admin/xpack/connector/*"); + private static final Set MANAGE_CONNECTOR_PATTERN = Set.of( + "cluster:admin/xpack/connector/*", + "indices:data/read/xpack/connector/*", + "indices:data/write/xpack/connector/*" + ); private static final Set MONITOR_CONNECTOR_PATTERN = Set.of( "cluster:admin/xpack/connector/get", "cluster:admin/xpack/connector/list", "cluster:admin/xpack/connector/sync_job/get", - "cluster:admin/xpack/connector/sync_job/list" + "cluster:admin/xpack/connector/sync_job/list", + "indices:data/read/xpack/connector/get", + "indices:data/read/xpack/connector/list" ); private static final Set READ_CONNECTOR_SECRETS_PATTERN = Set.of("cluster:admin/xpack/connector/secret/get"); private static final Set WRITE_CONNECTOR_SECRETS_PATTERN = Set.of( diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml index d32f05b7b749e..911404c3c6c5f 100644 --- a/x-pack/plugin/ent-search/qa/rest/roles.yml +++ b/x-pack/plugin/ent-search/qa/rest/roles.yml @@ -2,6 +2,7 @@ admin: cluster: - manage_search_application - manage_behavioral_analytics + - manage_connector - manage - monitor indices: @@ -18,6 +19,7 @@ user: cluster: - post_behavioral_analytics_event - manage_api_key + - monitor_connector - read_connector_secrets - write_connector_secrets indices: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml index 10e4620ca5603..6c3b7ae1b68c6 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml @@ -280,3 +280,137 @@ setup: +--- +"List Connectors - Soft deleted connectors initial empty state": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.list: + deleted: true + + - match: { count: 0 } + + +--- +"List Connectors - Single soft deleted connector": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.delete: + connector_id: connector-a + + - do: + connector.list: {} + + - match: { count: 2 } + + - do: + connector.list: + deleted: true + + - match: { count: 1 } + +--- +"List Connectors - Soft deleted connectors": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.delete: + connector_id: connector-a + - do: + connector.delete: + connector_id: connector-b + - do: + connector.delete: + connector_id: connector-c + + - do: + connector.list: + deleted: true + + - match: { count: 3 } + + # Alphabetical order by index_name for results + - match: { results.0.id: "connector-a" } + - match: { results.0.index_name: "search-1-test" } + - match: { results.0.language: "pl" } + + - match: { results.1.id: "connector-b" } + - match: { results.1.index_name: "search-2-test" } + - match: { results.1.language: "en" } + + - match: { results.2.id: "connector-c" } + - match: { results.2.index_name: "search-3-test" } + - match: { results.2.language: "nl" } + + +--- +"List Connectors - Soft deleted with from": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.delete: + connector_id: connector-a + - do: + connector.delete: + connector_id: connector-b + - do: + connector.delete: + connector_id: connector-c + + - do: + connector.list: + from: 1 + deleted: true + + - match: { count: 3 } + + # Alphabetical order by index_name for results + - match: { results.0.id: "connector-b" } + - match: { results.0.index_name: "search-2-test" } + - match: { results.0.language: "en" } + + - match: { results.1.id: "connector-c" } + - match: { results.1.index_name: "search-3-test" } + - match: { results.1.language: "nl" } + +--- +"List Connector- Soft deleted with size": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.delete: + connector_id: connector-a + - do: + connector.delete: + connector_id: connector-b + - do: + connector.delete: + connector_id: connector-c + + - do: + connector.list: + size: 2 + deleted: true + + - match: { count: 3 } + + # Alphabetical order by index_name for results + - match: { results.0.id: "connector-a" } + - match: { results.0.index_name: "search-1-test" } + - match: { results.0.language: "pl" } + + - match: { results.1.id: "connector-b" } + - match: { results.1.index_name: "search-2-test" } + - match: { results.1.language: "en" } + diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml index 484c5264227f9..6567c4934ef46 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml @@ -108,6 +108,33 @@ setup: connector_id: test-nonexistent-connector +--- +"Delete Connector - Supports soft deletes": + - requires: + cluster_features: ["connector_soft_deletes"] + reason: Soft deletes were introduced in 9.0 release + + - do: + connector.delete: + connector_id: test-connector-to-delete + + - match: { acknowledged: true } + + - do: + catch: "missing" + connector.get: + connector_id: test-connector-to-delete + + - do: + connector.get: + connector_id: test-connector-to-delete + deleted: true + + - match: { id: test-connector-to-delete } + - match: { index_name: search-1-test } + - match: { service_type: super-connector } + - match: { name: my-connector } + --- "Delete connector fails for unprivileged user": - skip: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java index ba121f2cf865e..2f7b3769666ff 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java @@ -9,6 +9,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction; import org.elasticsearch.xpack.application.rules.retriever.QueryRuleRetrieverBuilder; @@ -23,7 +24,8 @@ public Set getFeatures() { return Set.of( QUERY_RULES_TEST_API, QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED, - ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES + ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES, + ConnectorIndexService.CONNECTOR_SOFT_DELETES_FEATURE ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index f9d63f41e3da2..ef2015dba3477 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -102,6 +103,8 @@ public class ConnectorIndexService { private static final String DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE = "connectors-deleted.version"; private static final String DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "connectors-deleted.managed.index.version"; + public static final NodeFeature CONNECTOR_SOFT_DELETES_FEATURE = new NodeFeature("connector_soft_deletes"); + /** * Returns the {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. * From 397032baaa801995fcf9abc92d4002aaf583b792 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 11 Dec 2024 10:36:51 +0100 Subject: [PATCH 06/26] Fix permissions --- .../authz/privilege/ClusterPrivilegeResolver.java | 10 ++-------- .../connector/ConnectorIndexService.java | 14 ++++++++++---- .../connector/action/DeleteConnectorAction.java | 2 +- .../xpack/security/operator/Constants.java | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index f3897a0619694..00d45fb135fb2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -177,18 +177,12 @@ public class ClusterPrivilegeResolver { ); private static final Set MANAGE_SEARCH_APPLICATION_PATTERN = Set.of("cluster:admin/xpack/application/search_application/*"); - private static final Set MANAGE_CONNECTOR_PATTERN = Set.of( - "cluster:admin/xpack/connector/*", - "indices:data/read/xpack/connector/*", - "indices:data/write/xpack/connector/*" - ); + private static final Set MANAGE_CONNECTOR_PATTERN = Set.of("cluster:admin/xpack/connector/*"); private static final Set MONITOR_CONNECTOR_PATTERN = Set.of( "cluster:admin/xpack/connector/get", "cluster:admin/xpack/connector/list", "cluster:admin/xpack/connector/sync_job/get", - "cluster:admin/xpack/connector/sync_job/list", - "indices:data/read/xpack/connector/get", - "indices:data/read/xpack/connector/list" + "cluster:admin/xpack/connector/sync_job/list" ); private static final Set READ_CONNECTOR_SECRETS_PATTERN = Set.of("cluster:admin/xpack/connector/secret/get"); private static final Set WRITE_CONNECTOR_SECRETS_PATTERN = Set.of( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index ef2015dba3477..192ce69ba0c90 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -84,7 +84,6 @@ import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. @@ -139,7 +138,7 @@ public static SystemIndexDescriptor getConnectorsDeletedSystemIndexDescriptor() */ public ConnectorIndexService(Client client) { this.client = client; - this.clientWithOrigin = new OriginSettingClient(client, ENT_SEARCH_ORIGIN); + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -248,10 +247,14 @@ private Connector createConnectorWithDefaultValues( */ public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; + + // Only .connectors-delete is system index now + Client requestCli = isDeleted ? clientWithOrigin : client; + try { final GetRequest getRequest = new GetRequest(indexName).id(connectorId).realtime(true); - client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { + requestCli.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -340,13 +343,16 @@ public void listConnectors( ) { try { final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; + // Only .connectors-delete is system index now + Client requestCli = isDeleted ? clientWithOrigin : client; final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(indexName).source(source); - client.search(req, new ActionListener<>() { + + requestCli.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java index 83efccd49577f..930068a2a46ef 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java @@ -28,7 +28,7 @@ public class DeleteConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/delete"; + public static final String NAME = "indices:data/write/xpack/connector/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 46288f364da09..c91314716cf9e 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -132,7 +132,7 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", - "cluster:admin/xpack/connector/delete", + "indices:data/write/xpack/connector/delete", "indices:data/read/xpack/connector/get", "indices:data/read/xpack/connector/list", "indices:data/write/xpack/connector/post", From d8523e0cdc904b8bc393dfe86760bd270f851278 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 11 Dec 2024 10:42:03 +0100 Subject: [PATCH 07/26] Update docs --- docs/reference/connector/apis/delete-connector-api.asciidoc | 3 +-- docs/reference/connector/apis/get-connector-api.asciidoc | 3 +++ docs/reference/connector/apis/list-connectors-api.asciidoc | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc index 76621d7f1843b..50faef5261113 100644 --- a/docs/reference/connector/apis/delete-connector-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -6,8 +6,7 @@ beta::[] -Removes a connector and associated sync jobs. -This is a destructive action that is not recoverable. +Soft-deletes a connector and removes associated sync jobs. Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually. diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 302773e0af831..185004a57585d 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -27,6 +27,9 @@ To get started with Connector APIs, check out <`:: (Required, string) +`deleted`:: +(Optional, string) A flag indicating whether to return connectors that have been soft-deleted. Defaults to `false`. + [[get-connector-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index 4a93ecf2b0109..4ed03b7e99038 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -41,6 +41,9 @@ To get started with Connector APIs, check out < Date: Wed, 11 Dec 2024 11:09:28 +0100 Subject: [PATCH 08/26] Fix docs --- docs/reference/connector/apis/get-connector-api.asciidoc | 2 +- docs/reference/connector/apis/list-connectors-api.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 185004a57585d..d36e26c73278e 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -28,7 +28,7 @@ To get started with Connector APIs, check out < Date: Wed, 11 Dec 2024 11:13:06 +0100 Subject: [PATCH 09/26] Update docs/changelog/118282.yaml --- docs/changelog/118282.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/118282.yaml diff --git a/docs/changelog/118282.yaml b/docs/changelog/118282.yaml new file mode 100644 index 0000000000000..cf5746ab7b725 --- /dev/null +++ b/docs/changelog/118282.yaml @@ -0,0 +1,5 @@ +pr: 118282 +summary: "[Connector API] Support soft deletes of connectors" +area: Extract&Transform +type: feature +issues: [] From bf98ea86fa4b00776a5ad49fbd837326d418ebb4 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 12 Dec 2024 16:33:57 +0100 Subject: [PATCH 10/26] Change logic --- .../main/resources/connectors-deleted.json | 307 ------------------ .../elastic-connectors-mappings.json | 3 + .../xpack/application/EnterpriseSearch.java | 7 +- .../application/connector/Connector.java | 32 +- .../connector/ConnectorIndexService.java | 136 +++----- .../connector/ConnectorTemplateRegistry.java | 2 +- .../connector/ConnectorIndexServiceTests.java | 13 +- 7 files changed, 81 insertions(+), 419 deletions(-) delete mode 100644 x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json diff --git a/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json b/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json deleted file mode 100644 index 9e37b31f5672f..0000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/connectors-deleted.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "1", - "auto_expand_replicas": "0-1" - } - }, - "mappings":{ - "_doc":{ - "dynamic":"false", - "_meta": { - "version": "${connectors-deleted.version}", - "managed_index_mappings_version": ${connectors-deleted.managed.index.version} - }, - "properties":{ - "api_key_id":{ - "type":"keyword" - }, - "configuration":{ - "type":"object" - }, - "custom_scheduling":{ - "type":"object" - }, - "description":{ - "type":"text" - }, - "error":{ - "type":"keyword" - }, - "features":{ - "properties":{ - "filtering_advanced_config":{ - "type":"boolean" - }, - "filtering_rules":{ - "type":"boolean" - }, - "incremental_sync":{ - "properties":{ - "enabled":{ - "type":"boolean" - } - } - }, - "sync_rules":{ - "properties":{ - "advanced":{ - "properties":{ - "enabled":{ - "type":"boolean" - } - } - }, - "basic":{ - "properties":{ - "enabled":{ - "type":"boolean" - } - } - } - } - } - } - }, - "filtering":{ - "properties":{ - "active":{ - "properties":{ - "advanced_snippet":{ - "properties":{ - "created_at":{ - "type":"date" - }, - "updated_at":{ - "type":"date" - }, - "value":{ - "type":"object" - } - } - }, - "rules":{ - "properties":{ - "created_at":{ - "type":"date" - }, - "field":{ - "type":"keyword" - }, - "id":{ - "type":"keyword" - }, - "order":{ - "type":"short" - }, - "policy":{ - "type":"keyword" - }, - "rule":{ - "type":"keyword" - }, - "updated_at":{ - "type":"date" - }, - "value":{ - "type":"keyword" - } - } - }, - "validation":{ - "properties":{ - "errors":{ - "properties":{ - "ids":{ - "type":"keyword" - }, - "messages":{ - "type":"text" - } - } - }, - "state":{ - "type":"keyword" - } - } - } - } - }, - "domain":{ - "type":"keyword" - }, - "draft":{ - "properties":{ - "advanced_snippet":{ - "properties":{ - "created_at":{ - "type":"date" - }, - "updated_at":{ - "type":"date" - }, - "value":{ - "type":"object" - } - } - }, - "rules":{ - "properties":{ - "created_at":{ - "type":"date" - }, - "field":{ - "type":"keyword" - }, - "id":{ - "type":"keyword" - }, - "order":{ - "type":"short" - }, - "policy":{ - "type":"keyword" - }, - "rule":{ - "type":"keyword" - }, - "updated_at":{ - "type":"date" - }, - "value":{ - "type":"keyword" - } - } - }, - "validation":{ - "properties":{ - "errors":{ - "properties":{ - "ids":{ - "type":"keyword" - }, - "messages":{ - "type":"text" - } - } - }, - "state":{ - "type":"keyword" - } - } - } - } - } - } - }, - "index_name":{ - "type":"keyword" - }, - "is_native":{ - "type":"boolean" - }, - "language":{ - "type":"keyword" - }, - "last_access_control_sync_error":{ - "type":"keyword" - }, - "last_access_control_sync_scheduled_at":{ - "type":"date" - }, - "last_access_control_sync_status":{ - "type":"keyword" - }, - "last_deleted_document_count":{ - "type":"long" - }, - "last_incremental_sync_scheduled_at":{ - "type":"date" - }, - "last_indexed_document_count":{ - "type":"long" - }, - "last_seen":{ - "type":"date" - }, - "last_sync_error":{ - "type":"keyword" - }, - "last_sync_scheduled_at":{ - "type":"date" - }, - "last_sync_status":{ - "type":"keyword" - }, - "last_synced":{ - "type":"date" - }, - "name":{ - "type":"keyword" - }, - "pipeline":{ - "properties":{ - "extract_binary_content":{ - "type":"boolean" - }, - "name":{ - "type":"keyword" - }, - "reduce_whitespace":{ - "type":"boolean" - }, - "run_ml_inference":{ - "type":"boolean" - } - } - }, - "scheduling":{ - "properties":{ - "access_control":{ - "properties":{ - "enabled":{ - "type":"boolean" - }, - "interval":{ - "type":"text" - } - } - }, - "full":{ - "properties":{ - "enabled":{ - "type":"boolean" - }, - "interval":{ - "type":"text" - } - } - }, - "incremental":{ - "properties":{ - "enabled":{ - "type":"boolean" - }, - "interval":{ - "type":"text" - } - } - } - } - }, - "service_type":{ - "type":"keyword" - }, - "status":{ - "type":"keyword" - }, - "sync_cursor":{ - "type":"object" - }, - "sync_now":{ - "type":"boolean" - } - } - } - } -} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json index 651e1c84da73a..1940cc83f5e71 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json @@ -302,6 +302,9 @@ }, "sync_now": { "type": "boolean" + }, + "deleted": { + "type": "boolean" } } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index ee7349ead43f7..df1c76ccf770f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -46,7 +46,6 @@ import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction; import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig; import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature; -import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction; import org.elasticsearch.xpack.application.connector.action.GetConnectorAction; @@ -478,11 +477,7 @@ public Collection createComponents(PluginServices services) { @Override public Collection getSystemIndexDescriptors(Settings settings) { Collection systemIndices = new ArrayList<>( - List.of( - SearchApplicationIndexService.getSystemIndexDescriptor(), - QueryRulesIndexService.getSystemIndexDescriptor(), - ConnectorIndexService.getConnectorsDeletedSystemIndexDescriptor() - ) + List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor()) ); if (ConnectorSecretsFeature.isEnabled()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index 46275bb623b7a..5fc5eae0936ee 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -63,6 +63,7 @@ *
  • A {@link ConnectorStatus} indicating the current status of the connector.
    • *
  • A sync cursor, used for incremental syncs.
    • *
  • A boolean flag 'syncNow', which, when set, triggers an immediate synchronization operation.
    • + *
  • A boolean flag 'isDeleted', when set indicates that connector has been soft-deleted.
    • * */ public class Connector implements NamedWriteable, ToXContentObject { @@ -106,6 +107,7 @@ public class Connector implements NamedWriteable, ToXContentObject { @Nullable private final Object syncCursor; private final boolean syncNow; + private final boolean isDeleted; /** * Constructor for Connector. @@ -132,6 +134,7 @@ public class Connector implements NamedWriteable, ToXContentObject { * @param status Current status of the connector. * @param syncCursor Position or state indicating the current point of synchronization. * @param syncNow Flag indicating whether an immediate synchronization is requested. + * @param isDeleted Flag indicating whether connector has been soft-deleted. */ private Connector( String connectorId, @@ -155,7 +158,8 @@ private Connector( String serviceType, ConnectorStatus status, Object syncCursor, - boolean syncNow + boolean syncNow, + Boolean isDeleted ) { this.connectorId = connectorId; this.apiKeyId = apiKeyId; @@ -179,6 +183,7 @@ private Connector( this.status = status; this.syncCursor = syncCursor; this.syncNow = syncNow; + this.isDeleted = isDeleted; } public Connector(StreamInput in) throws IOException { @@ -204,6 +209,7 @@ public Connector(StreamInput in) throws IOException { this.status = in.readEnum(ConnectorStatus.class); this.syncCursor = in.readGenericValue(); this.syncNow = in.readBoolean(); + this.isDeleted = in.readBoolean(); } public static final ParseField ID_FIELD = new ParseField("id"); @@ -226,6 +232,7 @@ public Connector(StreamInput in) throws IOException { public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField SYNC_CURSOR_FIELD = new ParseField("sync_cursor"); static final ParseField SYNC_NOW_FIELD = new ParseField("sync_now"); + public static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -265,7 +272,8 @@ public Connector(StreamInput in) throws IOException { .setServiceType((String) args[i++]) .setStatus((ConnectorStatus) args[i++]) .setSyncCursor(args[i++]) - .setSyncNow((Boolean) args[i]) + .setSyncNow((Boolean) args[i++]) + .setIsDeleted((Boolean) args[i]) .build(); } ); @@ -357,6 +365,7 @@ public String getAccessControlIndexName() { ); PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> p.map(), null, SYNC_CURSOR_FIELD); PARSER.declareBoolean(optionalConstructorArg(), SYNC_NOW_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static Connector fromXContentBytes(BytesReference source, String docId, XContentType xContentType) { @@ -433,6 +442,7 @@ public void toInnerXContent(XContentBuilder builder, Params params) throws IOExc builder.field(STATUS_FIELD.getPreferredName(), status.toString()); } builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } @Override @@ -469,6 +479,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(status); out.writeGenericValue(syncCursor); out.writeBoolean(syncNow); + out.writeBoolean(isDeleted); } public String getConnectorId() { @@ -559,6 +570,10 @@ public boolean isSyncNow() { return syncNow; } + public boolean isDeleted() { + return isDeleted; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -566,6 +581,7 @@ public boolean equals(Object o) { Connector connector = (Connector) o; return isNative == connector.isNative && syncNow == connector.syncNow + && isDeleted == connector.isDeleted && Objects.equals(connectorId, connector.connectorId) && Objects.equals(apiKeyId, connector.apiKeyId) && Objects.equals(apiKeySecretId, connector.apiKeySecretId) @@ -612,7 +628,8 @@ public int hashCode() { serviceType, status, syncCursor, - syncNow + syncNow, + isDeleted ); } @@ -645,6 +662,7 @@ public static class Builder { private ConnectorStatus status = ConnectorStatus.CREATED; private Object syncCursor; private boolean syncNow; + private boolean isDeleted; public Builder setConnectorId(String connectorId) { this.connectorId = connectorId; @@ -756,6 +774,11 @@ public Builder setSyncNow(Boolean syncNow) { return this; } + public Builder setIsDeleted(Boolean isDeleted) { + this.isDeleted = Objects.requireNonNullElse(isDeleted, false); + return this; + } + public Connector build() { return new Connector( connectorId, @@ -779,7 +802,8 @@ public Connector build() { serviceType, status, syncCursor, - syncNow + syncNow, + isDeleted ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 192ce69ba0c90..8aa1bd740aaa6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -10,14 +10,9 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -26,7 +21,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexNotFoundException; @@ -37,8 +31,6 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; -import org.elasticsearch.indices.ExecutorNames; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -65,7 +57,6 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; -import org.elasticsearch.xpack.core.template.TemplateUtils; import java.time.Instant; import java.util.ArrayList; @@ -83,62 +74,23 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; -import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { - // The client to perform any operations on user indices (alias, ...). private final Client client; - // The client to interact with the system index (internal user). - private final Client clientWithOrigin; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; - public static String DELETED_CONNECTORS_INDEX_NAME = ".connectors-deleted"; - private static final int DELETED_CONNECTORS_INDEX_VERSION = 1; - private static final String DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE = "connectors-deleted.version"; - private static final String DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "connectors-deleted.managed.index.version"; - public static final NodeFeature CONNECTOR_SOFT_DELETES_FEATURE = new NodeFeature("connector_soft_deletes"); - /** - * Returns the {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. - * - * @return The {@link SystemIndexDescriptor} for the Deleted {@link Connector} system index. - */ - public static SystemIndexDescriptor getConnectorsDeletedSystemIndexDescriptor() { - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - - String templateSource = TemplateUtils.loadTemplate( - "/connectors-deleted.json", - Version.CURRENT.toString(), - DELETED_CONNECTORS_MAPPING_VERSION_VARIABLE, - Map.of(DELETED_CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(DELETED_CONNECTORS_INDEX_VERSION)) - ); - request.source(templateSource, XContentType.JSON); - - return SystemIndexDescriptor.builder() - .setIndexPattern(DELETED_CONNECTORS_INDEX_NAME + "*") - .setPrimaryIndex(DELETED_CONNECTORS_INDEX_NAME + "-" + DELETED_CONNECTORS_INDEX_VERSION) - .setDescription("Index storing deleted connectors") - .setMappings(request.mappings()) - .setSettings(request.settings()) - .setAliasName(DELETED_CONNECTORS_INDEX_NAME) - .setOrigin(CONNECTORS_ORIGIN) - .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) - .setThreadPools(ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS) - .build(); - } - /** * @param client A client for executing actions on the connector index */ public ConnectorIndexService(Client client) { this.client = client; - this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -245,16 +197,12 @@ private Connector createConnectorWithDefaultValues( * @param isDeleted If set to true, it returns only soft-deleted connector; otherwise, it returns non-deleted connector. * @param listener The action listener to invoke on response/failure. */ - public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { - final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; - - // Only .connectors-delete is system index now - Client requestCli = isDeleted ? clientWithOrigin : client; + public void getConnector(String connectorId, boolean isDeleted, ActionListener listener) { try { - final GetRequest getRequest = new GetRequest(indexName).id(connectorId).realtime(true); + final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); - requestCli.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { + client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -265,6 +213,10 @@ public void getConnector(String connectorId, Boolean isDeleted, ActionListener listener) { - try { - getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { + public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener listener) { - IndexRequest indexSoftDeletedConnectorRequest = new IndexRequest(DELETED_CONNECTORS_INDEX_NAME).opType( - DocWriteRequest.OpType.INDEX - ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + try { + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) .id(connectorId) - .source(connector.getSourceRef(), XContentType.JSON); - - clientWithOrigin.index(indexSoftDeletedConnectorRequest, l.delegateFailure((ll, indexResponse) -> { - final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, ll, (lll, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - lll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId( - connectorId, - lll.map(r -> deleteResponse) - ); - } else { - lll.onResponse(deleteResponse); - } - })); - })); - + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true)) + ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> updateResponse)); + } else { + l.onResponse(updateResponse); + } })); } catch (Exception e) { listener.onFailure(e); } - } /** @@ -338,21 +278,18 @@ public void listConnectors( List connectorNames, List serviceTypes, String searchQuery, - Boolean isDeleted, + boolean isDeleted, ActionListener listener ) { try { - final String indexName = isDeleted ? DELETED_CONNECTORS_INDEX_NAME : CONNECTOR_INDEX_NAME; - // Only .connectors-delete is system index now - Client requestCli = isDeleted ? clientWithOrigin : client; final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) - .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery)) + .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery, isDeleted)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); - final SearchRequest req = new SearchRequest(indexName).source(source); + final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - requestCli.search(req, new ActionListener<>() { + client.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -390,7 +327,8 @@ private QueryBuilder buildListQuery( List indexNames, List connectorNames, List serviceTypes, - String searchQuery + String searchQuery, + boolean isDeleted ) { boolean filterByIndexNames = indexNames != null && indexNames.isEmpty() == false; boolean filterByConnectorNames = indexNames != null && connectorNames.isEmpty() == false; @@ -422,7 +360,12 @@ private QueryBuilder buildListQuery( ); } } - return usesFilter ? boolFilterQueryBuilder : new MatchAllQueryBuilder(); + + if (isDeleted == false) { + boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true)); + } + + return boolFilterQueryBuilder; } /** @@ -1143,6 +1086,11 @@ private Map getConnectorConfigurationFromSearchResult(ConnectorS return (Map) searchResult.getResultMap().get(Connector.CONFIGURATION_FIELD.getPreferredName()); } + private boolean getIsDeletedFromSearchResult(ConnectorSearchResult searchResult) { + Boolean isDeletedFlag = (Boolean) searchResult.getResultMap().get(Connector.IS_DELETED_FIELD.getPreferredName()); + return Boolean.TRUE.equals(isDeletedFlag); + } + private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) { final List connectorResults = Arrays.stream(response.getHits().getHits()) .map(ConnectorIndexService::hitToConnector) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index 9b8cc7cfdbe4f..a0f0621dbf205 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -31,7 +31,7 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { // This number must be incremented when we make changes to built-in templates. - static final int REGISTRY_VERSION = 3; + static final int REGISTRY_VERSION = 4; // Connector indices constants diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index d8bf916fbb5d2..57ba5f725ad03 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; @@ -111,7 +110,7 @@ public void testDeleteConnector() throws Exception { } String connectorIdToDelete = connectorIds.get(0); - DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); assertThat(resp.status(), equalTo(RestStatus.OK)); expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); @@ -130,7 +129,7 @@ public void testDeleteConnector_expectSoftDeletion() throws Exception { } String connectorIdToDelete = connectorIds.get(0); - DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); assertThat(resp.status(), equalTo(RestStatus.OK)); expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); @@ -153,7 +152,7 @@ public void testDeleteConnector_expectSoftDeletionMultipleConnectors() throws Ex // Delete all of them for (int i = 0; i < numConnectors; i++) { String connectorIdToDelete = connectorIds.get(i); - DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); assertThat(resp.status(), equalTo(RestStatus.OK)); } @@ -880,13 +879,13 @@ public void testUpdateConnectorApiKeyIdOrApiKeySecretId() throws Exception { assertThat(updateApiKeyIdRequest.getApiKeySecretId(), equalTo(indexedConnector.getApiKeySecretId())); } - private DeleteResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception { + private UpdateResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception { CountDownLatch latch = new CountDownLatch(1); - final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); connectorIndexService.deleteConnector(connectorId, deleteConnectorSyncJobs, new ActionListener<>() { @Override - public void onResponse(DeleteResponse deleteResponse) { + public void onResponse(UpdateResponse deleteResponse) { resp.set(deleteResponse); latch.countDown(); } From b4eb41159c2c031fc955ee98eb1f5f930ef8fb8c Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Fri, 13 Dec 2024 15:22:52 +0100 Subject: [PATCH 11/26] Fix tests --- .../connector/apis/get-connector-api.asciidoc | 2 +- .../apis/list-connectors-api.asciidoc | 2 +- .../entsearch/connector/20_connector_list.yml | 15 +++++--- .../connector/ConnectorIndexService.java | 38 ++++++++++--------- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index d36e26c73278e..51dab7979cbbb 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -28,7 +28,7 @@ To get started with Connector APIs, check out < listener) { try { - final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( - new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true)) - ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> updateResponse)); - } else { - l.onResponse(updateResponse); - } + // ensure that if connector is soft-deleted, deleting it again results in 404 + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy( + WriteRequest.RefreshPolicy.IMMEDIATE + ) + .doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true)) + ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, ll.map(r -> updateResponse)); + } else { + ll.onResponse(updateResponse); + } + })); })); } catch (Exception e) { listener.onFailure(e); @@ -288,7 +293,6 @@ public void listConnectors( .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - client.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { From 1b0845f5c4315f1e370e37c5e9742c11c4892b24 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Fri, 13 Dec 2024 15:24:46 +0100 Subject: [PATCH 12/26] Remove unnecessary privilege from yaml rest test --- x-pack/plugin/ent-search/qa/rest/roles.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml index 911404c3c6c5f..d32f05b7b749e 100644 --- a/x-pack/plugin/ent-search/qa/rest/roles.yml +++ b/x-pack/plugin/ent-search/qa/rest/roles.yml @@ -2,7 +2,6 @@ admin: cluster: - manage_search_application - manage_behavioral_analytics - - manage_connector - manage - monitor indices: @@ -19,7 +18,6 @@ user: cluster: - post_behavioral_analytics_event - manage_api_key - - monitor_connector - read_connector_secrets - write_connector_secrets indices: From b51848ecf2f4729481315cddbf411cf5dbd3b9e8 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Fri, 13 Dec 2024 15:39:46 +0100 Subject: [PATCH 13/26] Update changelog --- docs/changelog/118282.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/118282.yaml diff --git a/docs/changelog/118282.yaml b/docs/changelog/118282.yaml deleted file mode 100644 index cf5746ab7b725..0000000000000 --- a/docs/changelog/118282.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 118282 -summary: "[Connector API] Support soft deletes of connectors" -area: Extract&Transform -type: feature -issues: [] From b206844018e3319608bdad9902db617038d1eec7 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Fri, 13 Dec 2024 15:40:30 +0100 Subject: [PATCH 14/26] Update docs/changelog/118669.yaml --- docs/changelog/118669.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/118669.yaml diff --git a/docs/changelog/118669.yaml b/docs/changelog/118669.yaml new file mode 100644 index 0000000000000..4e0d10aaac816 --- /dev/null +++ b/docs/changelog/118669.yaml @@ -0,0 +1,5 @@ +pr: 118669 +summary: "[Connector API] Support soft-deletes of connectors" +area: Extract&Transform +type: feature +issues: [] From 6ff5a8dfe8c08ae96ddebd2c6013837e6275b80c Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Mon, 16 Dec 2024 12:11:21 +0100 Subject: [PATCH 15/26] Adapt yaml tests --- .../entsearch/connector/20_connector_list.yml | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml index f18cab8c52079..f35aa3ee9da6a 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml @@ -71,7 +71,7 @@ setup: - match: { results.1.language: "nl" } --- -"List Connector- with size": +"List Connector - with size": - do: connector.list: size: 2 @@ -335,14 +335,14 @@ setup: - match: { count: 3 } # Alphabetical order by index_name for results - - match: { results.0.id: "connector-a" } - - match: { results.0.index_name: "search-1-test" } - - match: { results.0.language: "pl" } + - match: { results.0.id: "connector-b" } + - match: { results.0.index_name: "content-search-2-test" } + - match: { results.0.language: "en" } - match: { results.0.deleted: true } - - match: { results.1.id: "connector-b" } - - match: { results.1.index_name: "search-2-test" } - - match: { results.1.language: "en" } + - match: { results.1.id: "connector-a" } + - match: { results.1.index_name: "search-1-test" } + - match: { results.1.language: "pl" } - match: { results.1.deleted: true } - match: { results.2.id: "connector-c" } @@ -375,18 +375,18 @@ setup: - match: { count: 3 } # Alphabetical order by index_name for results - - match: { results.0.id: "connector-b" } - - match: { results.0.index_name: "search-2-test" } - - match: { results.0.language: "en" } + - match: { results.0.id: "connector-a" } + - match: { results.0.index_name: "search-1-test" } + - match: { results.0.language: "pl" } - match: { results.0.deleted: true } - match: { results.1.id: "connector-c" } - match: { results.1.index_name: "search-3-test" } - match: { results.1.language: "nl" } - - match: { results.1.deleted: true } + - match: { results.0.deleted: true } --- -"List Connector- Soft deleted with size": +"List Connector - Soft deleted with size": - requires: cluster_features: ["connector_soft_deletes"] reason: Soft deletes were introduced in 9.0 release @@ -409,13 +409,14 @@ setup: - match: { count: 3 } # Alphabetical order by index_name for results - - match: { results.0.id: "connector-a" } - - match: { results.0.index_name: "search-1-test" } - - match: { results.0.language: "pl" } + - match: { results.0.id: "connector-b" } + - match: { results.0.index_name: "content-search-2-test" } + - match: { results.0.language: "en" } - match: { results.0.deleted: true } - - match: { results.1.id: "connector-b" } - - match: { results.1.index_name: "search-2-test" } - - match: { results.1.language: "en" } + - match: { results.1.id: "connector-a" } + - match: { results.1.index_name: "search-1-test" } + - match: { results.1.language: "pl" } - match: { results.1.deleted: true } + From e697134d75d9236c8ddf9c0fbadc6990cefbf17b Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 17 Dec 2024 17:00:48 +0100 Subject: [PATCH 16/26] Undo changes to muted-tests.yml --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e2d1dcb3c6f38..42845fda82180 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -85,6 +85,8 @@ tests: - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 +- class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests + issue: https://github.com/elastic/elasticsearch/issues/116087 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=cat.shards/10_basic/Help} issue: https://github.com/elastic/elasticsearch/issues/116110 From 4eb2a88ca9464187a39894a540409e2c9b6d0b19 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 18 Dec 2024 14:20:20 +0100 Subject: [PATCH 17/26] Fix compilation issue after other PR got merged --- .../xpack/application/connector/ConnectorIndexService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 7414bda680021..f8b5fe864193d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -840,7 +840,7 @@ public void updateConnectorNative(UpdateConnectorNativeAction.Request request, A String connectorId = request.getConnectorId(); boolean isNative = request.isNative(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { String indexName = getConnectorIndexNameFromSearchResult(connector); @@ -951,7 +951,7 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ return; } - getConnector(connectorId, l.delegateFailure((ll, connector) -> { + getConnector(connectorId, false, l.delegateFailure((ll, connector) -> { Boolean isNativeConnector = getConnectorIsNativeFlagFromSearchResult(connector); Boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false; From 7525c99092450d3f714159cc7c20c855e9b99e41 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 19 Dec 2024 17:02:00 +0100 Subject: [PATCH 18/26] Exclude soft-deleted connector from checks about index_name already in use --- .../entsearch/connector/10_connector_put.yml | 24 +++++++++++++++++++ .../connector/ConnectorIndexService.java | 7 ++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml index 4240467ea4ff3..a4d9176cf01e3 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml @@ -218,6 +218,30 @@ setup: body: index_name: search-test +--- +'Create Connector - Index name used by deleted connector': + - do: + connector.put: + connector_id: test-connector-1 + body: + index_name: search-test + + - match: { result: 'created' } + + - do: + connector.delete: + connector_id: test-connector-1 + + - match: { acknowledged: true } + + - do: + connector.put: + connector_id: test-connector-2 + body: + index_name: search-test + + - match: { result: 'created' } + --- 'Create Connector - Without index attached': - do: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index f8b5fe864193d..15f60c65b3a6a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -1182,8 +1182,8 @@ private static ConnectorSearchResult hitToConnector(SearchHit searchHit) { } /** - * This method determines if any documents in the connector index have the same index name as the one specified, - * excluding the document with the given _id if it is provided. + * This method determines if any records in the connector index have the same index name as the one specified, + * excluding the docs marked as deleted (soft-deleted) and document with the given _id if it is provided. * * @param indexName The name of the index to check for existence in the connector index. * @param connectorId The ID of the {@link Connector} to exclude from the search. Can be null if no document should be excluded. @@ -1199,6 +1199,9 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A boolFilterQueryBuilder.must().add(new TermQueryBuilder(Connector.INDEX_NAME_FIELD.getPreferredName(), indexName)); + // exclude soft-deleted connectors + boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true)); + // If we know the connector _id, exclude this from search query if (connectorId != null) { boolFilterQueryBuilder.mustNot(new IdsQueryBuilder().addIds(connectorId)); From af7ad48a2270da7404feb8c08f8d336a766ba523 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Mon, 30 Dec 2024 13:19:55 +0100 Subject: [PATCH 19/26] Update docs/reference/connector/apis/get-connector-api.asciidoc Co-authored-by: Tim Grein --- docs/reference/connector/apis/get-connector-api.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 51dab7979cbbb..620ed48a21e28 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -28,7 +28,7 @@ To get started with Connector APIs, check out < Date: Mon, 30 Dec 2024 13:20:11 +0100 Subject: [PATCH 20/26] Update rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json Co-authored-by: Tim Grein --- .../src/main/resources/rest-api-spec/api/connector.list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json index 17992d412a567..5082ef61f6642 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json @@ -51,7 +51,7 @@ "deleted": { "type": "boolean", "default": false, - "description": "A flag indicating whether to list connectors that have been soft-deleted." + "description": "A flag indicating whether to return connectors that have been soft-deleted." } } } From 7a9a90d3f9a65076fc8740579ead1bffb6feb228 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Mon, 30 Dec 2024 14:20:53 +0100 Subject: [PATCH 21/26] Adapt comments, add connector wire serializing test --- .../connector/ConnectorIndexService.java | 14 ++++--- .../connector/ConnectorSerializingTests.java | 37 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 15f60c65b3a6a..0742884f4a192 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -195,7 +195,8 @@ private Connector createConnectorWithDefaultValues( * Gets the {@link Connector} from the underlying index. * * @param connectorId The id of the connector object. - * @param isDeleted If set to true, it returns only soft-deleted connector; otherwise, it returns non-deleted connector. + * @param isDeleted If false, returns only the non-deleted connector with the matching ID; + * otherwise, returns the connector with the matching ID. * @param listener The action listener to invoke on response/failure. */ public void getConnector(String connectorId, boolean isDeleted, ActionListener listener) { @@ -214,7 +215,9 @@ public void getConnector(String connectorId, boolean isDeleted, ActionListener getConnectorConfigurationFromSearchResult(ConnectorS return (Map) searchResult.getResultMap().get(Connector.CONFIGURATION_FIELD.getPreferredName()); } - private boolean getIsDeletedFromSearchResult(ConnectorSearchResult searchResult) { + private boolean isConnectorDeleted(ConnectorSearchResult searchResult) { Boolean isDeletedFlag = (Boolean) searchResult.getResultMap().get(Connector.IS_DELETED_FIELD.getPreferredName()); return Boolean.TRUE.equals(isDeletedFlag); } @@ -1182,7 +1186,7 @@ private static ConnectorSearchResult hitToConnector(SearchHit searchHit) { } /** - * This method determines if any records in the connector index have the same index name as the one specified, + * This method determines if any documents in the connector index have the same index name as the one specified, * excluding the docs marked as deleted (soft-deleted) and document with the given _id if it is provided. * * @param indexName The name of the index to check for existence in the connector index. diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java new file mode 100644 index 0000000000000..57c1e3e30b0c0 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +public class ConnectorSerializingTests extends AbstractBWCWireSerializationTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return Connector::new; + } + + @Override + protected Connector createTestInstance() { + return ConnectorTestUtils.getRandomConnector(); + } + + @Override + protected Connector mutateInstance(Connector instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected Connector mutateInstanceForVersion(Connector instance, TransportVersion version) { + return instance; + } +} From 8d4ec5380bbd905683e0f40928f8018c9aa620de Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Mon, 30 Dec 2024 14:41:09 +0100 Subject: [PATCH 22/26] Introduce new transport versions for passing the delete flag --- .../java/org/elasticsearch/TransportVersions.java | 1 + .../xpack/application/connector/Connector.java | 11 +++++++++-- .../connector/action/GetConnectorAction.java | 11 +++++++++-- .../connector/action/ListConnectorAction.java | 11 +++++++++-- ...GetConnectorActionRequestBWCSerializingTests.java | 12 ++++++++++++ ...istConnectorActionRequestBWCSerializingTests.java | 11 +++++++++++ 6 files changed, 51 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index d3e235f1cd82a..3370aee8d9ea2 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -140,6 +140,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0); public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_809_00_0); public static final TransportVersion NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION = def(8_810_00_0); + public static final TransportVersion CONNECTOR_API_SUPPORT_SOFT_DELETES = def(8_811_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index 5fc5eae0936ee..0ea0a9a8440e5 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; @@ -209,7 +210,11 @@ public Connector(StreamInput in) throws IOException { this.status = in.readEnum(ConnectorStatus.class); this.syncCursor = in.readGenericValue(); this.syncNow = in.readBoolean(); - this.isDeleted = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + this.isDeleted = in.readBoolean(); + } else { + this.isDeleted = false; + } } public static final ParseField ID_FIELD = new ParseField("id"); @@ -479,7 +484,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(status); out.writeGenericValue(syncCursor); out.writeBoolean(syncNow); - out.writeBoolean(isDeleted); + if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + out.writeBoolean(isDeleted); + } } public String getConnectorId() { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 120d00f176a0e..afb4ec2b81629 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.connector.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -51,7 +52,11 @@ public Request(String connectorId, Boolean isDeleted) { public Request(StreamInput in) throws IOException { super(in); this.connectorId = in.readString(); - this.isDeleted = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + this.isDeleted = in.readBoolean(); + } else { + this.isDeleted = false; + } } public String getConnectorId() { @@ -77,7 +82,9 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(connectorId); - out.writeBoolean(isDeleted); + if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + out.writeBoolean(isDeleted); + } } @Override diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index 44a52cd87e014..c609d122c5cd8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.connector.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -61,7 +62,11 @@ public Request(StreamInput in) throws IOException { this.connectorNames = in.readOptionalStringCollectionAsList(); this.connectorServiceTypes = in.readOptionalStringCollectionAsList(); this.connectorSearchQuery = in.readOptionalString(); - this.isDeleted = in.readOptionalBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + this.isDeleted = in.readBoolean(); + } else { + this.isDeleted = false; + } } public Request( @@ -129,7 +134,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringCollection(connectorNames); out.writeOptionalStringCollection(connectorServiceTypes); out.writeOptionalString(connectorSearchQuery); - out.writeOptionalBoolean(isDeleted); + if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { + out.writeBoolean(isDeleted); + } } @Override diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java index 849bb1eaaefd9..8d169026a714f 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java @@ -8,11 +8,16 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; public class GetConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { @@ -40,4 +45,11 @@ protected GetConnectorAction.Request doParseInstance(XContentParser parser) thro protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) { return new GetConnectorAction.Request(instance.getConnectorId(), instance.getDeleted()); } + + @Override + protected List bwcVersions() { + return getAllBWCVersions().stream() + .filter(v -> v.onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) + .collect(Collectors.toList()); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java index a815432a7f8ca..1528dcecfa9a2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -16,6 +17,9 @@ import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; public class ListConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { @Override @@ -57,4 +61,11 @@ protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorActi instance.getDeleted() ); } + + @Override + protected List bwcVersions() { + return getAllBWCVersions().stream() + .filter(v -> v.onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) + .collect(Collectors.toList()); + } } From ea9394e32ae18771008973b8e19f2114bd44223f Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 8 Jan 2025 15:35:54 +0100 Subject: [PATCH 23/26] Get rid of wire serialisation, use include_deleted instead of deleted flag --- .../connector/apis/get-connector-api.asciidoc | 2 +- .../apis/list-connectors-api.asciidoc | 2 +- .../rest-api-spec/api/connector.get.json | 2 +- .../rest-api-spec/api/connector.list.json | 2 +- .../org/elasticsearch/TransportVersions.java | 1 - .../entsearch/connector/20_connector_list.yml | 10 +-- .../connector/30_connector_delete.yml | 2 +- .../application/connector/Connector.java | 70 +----------------- .../connector/ConnectorIndexService.java | 12 ++-- .../connector/action/GetConnectorAction.java | 38 ++++------ .../connector/action/ListConnectorAction.java | 44 +++--------- .../action/RestGetConnectorAction.java | 2 +- .../action/RestListConnectorAction.java | 2 +- .../action/TransportGetConnectorAction.java | 18 +++-- .../action/TransportListConnectorAction.java | 14 ++-- .../connector/syncjob/ConnectorSyncJob.java | 48 +------------ .../connector/ConnectorSerializingTests.java | 37 ---------- .../application/connector/ConnectorTests.java | 33 --------- ...ectorActionRequestBWCSerializingTests.java | 55 -------------- ...ectorActionRequestBWCSerializingTests.java | 71 ------------------- ...ctorSecretResponseBWCSerializingTests.java | 8 --- ...ctorSecretResponseBWCSerializingTests.java | 8 --- .../syncjob/ConnectorSyncJobTests.java | 29 -------- ...cJobActionResponseBWCSerializingTests.java | 8 --- ...JobsActionResponseBWCSerializingTests.java | 8 --- 25 files changed, 55 insertions(+), 471 deletions(-) delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 620ed48a21e28..804fc93c34a8b 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -27,7 +27,7 @@ To get started with Connector APIs, check out <`:: (Required, string) -`deleted`:: +`include_deleted`:: (Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`. [[get-connector-api-response-codes]] diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index 8f3708789dfca..821f2e7ab0694 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -41,7 +41,7 @@ To get started with Connector APIs, check out <A boolean flag 'isDeleted', when set indicates that connector has been soft-deleted. * */ -public class Connector implements NamedWriteable, ToXContentObject { +public class Connector implements ToXContentObject { public static final String NAME = Connector.class.getName().toUpperCase(Locale.ROOT); @@ -187,36 +183,6 @@ private Connector( this.isDeleted = isDeleted; } - public Connector(StreamInput in) throws IOException { - this.connectorId = in.readOptionalString(); - this.apiKeyId = in.readOptionalString(); - this.apiKeySecretId = in.readOptionalString(); - this.configuration = in.readMap(ConnectorConfiguration::new); - this.customScheduling = in.readMap(ConnectorCustomSchedule::new); - this.description = in.readOptionalString(); - this.error = in.readOptionalString(); - this.features = in.readOptionalWriteable(ConnectorFeatures::new); - this.filtering = in.readOptionalCollectionAsList(ConnectorFiltering::new); - this.syncJobFiltering = in.readOptionalWriteable(FilteringRules::new); - this.indexName = in.readOptionalString(); - this.isNative = in.readBoolean(); - this.language = in.readOptionalString(); - this.lastSeen = in.readOptionalInstant(); - this.syncInfo = in.readOptionalWriteable(ConnectorSyncInfo::new); - this.name = in.readOptionalString(); - this.pipeline = in.readOptionalWriteable(ConnectorIngestPipeline::new); - this.scheduling = in.readOptionalWriteable(ConnectorScheduling::new); - this.serviceType = in.readOptionalString(); - this.status = in.readEnum(ConnectorStatus.class); - this.syncCursor = in.readGenericValue(); - this.syncNow = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - this.isDeleted = in.readBoolean(); - } else { - this.isDeleted = false; - } - } - public static final ParseField ID_FIELD = new ParseField("id"); public static final ParseField API_KEY_ID_FIELD = new ParseField("api_key_id"); public static final ParseField API_KEY_SECRET_ID_FIELD = new ParseField("api_key_secret_id"); @@ -460,35 +426,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalString(connectorId); - out.writeOptionalString(apiKeyId); - out.writeOptionalString(apiKeySecretId); - out.writeMap(configuration, StreamOutput::writeWriteable); - out.writeMap(customScheduling, StreamOutput::writeWriteable); - out.writeOptionalString(description); - out.writeOptionalString(error); - out.writeOptionalWriteable(features); - out.writeOptionalCollection(filtering); - out.writeOptionalWriteable(syncJobFiltering); - out.writeOptionalString(indexName); - out.writeBoolean(isNative); - out.writeOptionalString(language); - out.writeOptionalInstant(lastSeen); - out.writeOptionalWriteable(syncInfo); - out.writeOptionalString(name); - out.writeOptionalWriteable(pipeline); - out.writeOptionalWriteable(scheduling); - out.writeOptionalString(serviceType); - out.writeEnum(status); - out.writeGenericValue(syncCursor); - out.writeBoolean(syncNow); - if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - out.writeBoolean(isDeleted); - } - } - public String getConnectorId() { return connectorId; } @@ -640,11 +577,6 @@ public int hashCode() { ); } - @Override - public String getWriteableName() { - return NAME; - } - public static class Builder { private String connectorId; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 0742884f4a192..aed8cbcf2485b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -277,7 +277,7 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac * @param connectorNames Filter connectors by connector names, if provided. * @param serviceTypes Filter connectors by service types, if provided. * @param searchQuery Apply a wildcard search on index name, connector name, and description, if provided. - * @param isDeleted If false, filters to include only non-deleted connectors; otherwise, no filter is applied. + * @param includeDeleted If false, filters to include only non-deleted connectors; otherwise, no filter is applied. * @param listener Invoked with search results or upon failure. */ public void listConnectors( @@ -287,13 +287,13 @@ public void listConnectors( List connectorNames, List serviceTypes, String searchQuery, - boolean isDeleted, + boolean includeDeleted, ActionListener listener ) { try { final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) - .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery, isDeleted)) + .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery, includeDeleted)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); @@ -329,7 +329,7 @@ public void onFailure(Exception e) { * @param connectorNames List of connector names for filtering, or null/empty to skip. * @param serviceTypes List of connector service types for filtering, or null/empty to skip. * @param searchQuery Search query for wildcard filtering on index name, connector name, and description, or null/empty to skip. - * @param isDeleted If false, filters to include only non-deleted connectors; otherwise, no filter is applied. + * @param includeDeleted If false, filters to include only non-deleted connectors; otherwise, no filter is applied. * @return A {@link QueryBuilder} customized based on provided filters. */ private QueryBuilder buildListQuery( @@ -337,7 +337,7 @@ private QueryBuilder buildListQuery( List connectorNames, List serviceTypes, String searchQuery, - boolean isDeleted + boolean includeDeleted ) { boolean filterByIndexNames = indexNames != null && indexNames.isEmpty() == false; boolean filterByConnectorNames = indexNames != null && connectorNames.isEmpty() == false; @@ -370,7 +370,7 @@ private QueryBuilder buildListQuery( } } - if (isDeleted == false) { + if (includeDeleted == false) { boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true)); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index afb4ec2b81629..82f6cf77def58 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -7,10 +7,10 @@ package org.elasticsearch.xpack.application.connector.action; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -38,33 +38,23 @@ private GetConnectorAction() {/* no instances */} public static class Request extends ConnectorActionRequest implements ToXContentObject { private final String connectorId; - private final Boolean isDeleted; + private final Boolean includeDeleted; private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); - private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); + private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted"); - public Request(String connectorId, Boolean isDeleted) { + public Request(String connectorId, Boolean includeDeleted) { this.connectorId = connectorId; - this.isDeleted = isDeleted; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.connectorId = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - this.isDeleted = in.readBoolean(); - } else { - this.isDeleted = false; - } + this.includeDeleted = includeDeleted; } public String getConnectorId() { return connectorId; } - public Boolean getDeleted() { - return isDeleted; + public Boolean getIncludeDeleted() { + return includeDeleted; } @Override @@ -80,11 +70,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(connectorId); - if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - out.writeBoolean(isDeleted); - } + TransportAction.localOnly(); } @Override @@ -92,12 +78,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(connectorId, request.connectorId) && Objects.equals(isDeleted, request.isDeleted); + return Objects.equals(connectorId, request.connectorId) && Objects.equals(includeDeleted, request.includeDeleted); } @Override public int hashCode() { - return Objects.hash(connectorId, isDeleted); + return Objects.hash(connectorId, includeDeleted); } @Override @@ -105,7 +91,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId); - builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); + builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted); } builder.endObject(); return builder; @@ -119,7 +105,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws ); static { PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD); - PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD); } public static Request parse(XContentParser parser) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index c609d122c5cd8..1220c3c58fbb8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -7,10 +7,10 @@ package org.elasticsearch.xpack.application.connector.action; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -47,27 +47,13 @@ public static class Request extends ConnectorActionRequest implements ToXContent private final List connectorNames; private final List connectorServiceTypes; private final String connectorSearchQuery; - private final Boolean isDeleted; + private final Boolean includeDeleted; private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams"); private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names"); private static final ParseField NAMES_FIELD = new ParseField("names"); private static final ParseField SEARCH_QUERY_FIELD = new ParseField("query"); - private static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); - - public Request(StreamInput in) throws IOException { - super(in); - this.pageParams = new PageParams(in); - this.indexNames = in.readOptionalStringCollectionAsList(); - this.connectorNames = in.readOptionalStringCollectionAsList(); - this.connectorServiceTypes = in.readOptionalStringCollectionAsList(); - this.connectorSearchQuery = in.readOptionalString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - this.isDeleted = in.readBoolean(); - } else { - this.isDeleted = false; - } - } + private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted"); public Request( PageParams pageParams, @@ -75,14 +61,14 @@ public Request( List connectorNames, List serviceTypes, String connectorSearchQuery, - Boolean isDeleted + Boolean includeDeleted ) { this.pageParams = pageParams; this.indexNames = indexNames; this.connectorNames = connectorNames; this.connectorServiceTypes = serviceTypes; this.connectorSearchQuery = connectorSearchQuery; - this.isDeleted = isDeleted; + this.includeDeleted = includeDeleted; } public PageParams getPageParams() { @@ -105,8 +91,8 @@ public String getConnectorSearchQuery() { return connectorSearchQuery; } - public Boolean getDeleted() { - return isDeleted; + public Boolean getIncludeDeleted() { + return includeDeleted; } @Override @@ -128,15 +114,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - pageParams.writeTo(out); - out.writeOptionalStringCollection(indexNames); - out.writeOptionalStringCollection(connectorNames); - out.writeOptionalStringCollection(connectorServiceTypes); - out.writeOptionalString(connectorSearchQuery); - if (out.getTransportVersion().onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) { - out.writeBoolean(isDeleted); - } + TransportAction.localOnly(); } @Override @@ -149,7 +127,7 @@ public boolean equals(Object o) { && Objects.equals(connectorNames, request.connectorNames) && Objects.equals(connectorServiceTypes, request.connectorServiceTypes) && Objects.equals(connectorSearchQuery, request.connectorSearchQuery) - && Objects.equals(isDeleted, request.isDeleted); + && Objects.equals(includeDeleted, request.includeDeleted); } @Override @@ -176,7 +154,7 @@ public int hashCode() { PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD); PARSER.declareStringArray(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD); PARSER.declareString(optionalConstructorArg(), SEARCH_QUERY_FIELD); - PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD); } public static ListConnectorAction.Request parse(XContentParser parser) { @@ -192,7 +170,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NAMES_FIELD.getPreferredName(), connectorNames); builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connectorServiceTypes); builder.field(SEARCH_QUERY_FIELD.getPreferredName(), connectorSearchQuery); - builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); + builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted); } builder.endObject(); return builder; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java index 8aab02deed5a0..b6c812cfb1bdd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java @@ -36,7 +36,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); + Boolean isDeleted = restRequest.paramAsBoolean("include_deleted", false); GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), isDeleted); return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 5bb8db98fc11b..9fd859607e458 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -43,7 +43,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); List serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0])); String searchQuery = restRequest.param("query"); - Boolean isDeleted = restRequest.paramAsBoolean("deleted", false); + Boolean isDeleted = restRequest.paramAsBoolean("include_deleted", false); ListConnectorAction.Request request = new ListConnectorAction.Request( new PageParams(from, size), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java index 209c326eb8044..98891427ab848 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -17,23 +17,21 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; -public class TransportGetConnectorAction extends HandledTransportAction { +public class TransportGetConnectorAction extends TransportAction { protected final ConnectorIndexService connectorIndexService; @Inject public TransportGetConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) { - super( - GetConnectorAction.NAME, - transportService, - actionFilters, - GetConnectorAction.Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE - ); + super(GetConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.connectorIndexService = new ConnectorIndexService(client); } @Override protected void doExecute(Task task, GetConnectorAction.Request request, ActionListener listener) { - connectorIndexService.getConnector(request.getConnectorId(), request.getDeleted(), listener.map(GetConnectorAction.Response::new)); + connectorIndexService.getConnector( + request.getConnectorId(), + request.getIncludeDeleted(), + listener.map(GetConnectorAction.Response::new) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java index f7a3f9a374b44..408aa6fc12405 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -18,18 +18,12 @@ import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.core.action.util.PageParams; -public class TransportListConnectorAction extends HandledTransportAction { +public class TransportListConnectorAction extends TransportAction { protected final ConnectorIndexService connectorIndexService; @Inject public TransportListConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) { - super( - ListConnectorAction.NAME, - transportService, - actionFilters, - ListConnectorAction.Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE - ); + super(ListConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.connectorIndexService = new ConnectorIndexService(client); } @@ -44,7 +38,7 @@ protected void doExecute(Task task, ListConnectorAction.Request request, ActionL request.getConnectorNames(), request.getConnectorServiceTypes(), request.getConnectorSearchQuery(), - request.getDeleted(), + request.getIncludeDeleted(), listener.map(r -> new ListConnectorAction.Response(r.connectors(), r.totalResults())) ); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index 4aabb9e1af663..5ba45ea4757bf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -10,9 +10,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -65,7 +62,7 @@ *
  • The hostname of the worker to run the sync job.
    • * */ -public class ConnectorSyncJob implements Writeable, ToXContentObject { +public class ConnectorSyncJob implements ToXContentObject { static final ParseField CANCELATION_REQUESTED_AT_FIELD = new ParseField("cancelation_requested_at"); @@ -213,27 +210,6 @@ private ConnectorSyncJob( this.workerHostname = workerHostname; } - public ConnectorSyncJob(StreamInput in) throws IOException { - this.cancelationRequestedAt = in.readOptionalInstant(); - this.canceledAt = in.readOptionalInstant(); - this.completedAt = in.readOptionalInstant(); - this.connector = in.readNamedWriteable(Connector.class); - this.createdAt = in.readInstant(); - this.deletedDocumentCount = in.readLong(); - this.error = in.readOptionalString(); - this.id = in.readString(); - this.indexedDocumentCount = in.readLong(); - this.indexedDocumentVolume = in.readLong(); - this.jobType = in.readEnum(ConnectorSyncJobType.class); - this.lastSeen = in.readOptionalInstant(); - this.metadata = in.readMap(StreamInput::readString, StreamInput::readGenericValue); - this.startedAt = in.readOptionalInstant(); - this.status = in.readEnum(ConnectorSyncStatus.class); - this.totalDocumentCount = in.readOptionalLong(); - this.triggerMethod = in.readEnum(ConnectorSyncJobTriggerMethod.class); - this.workerHostname = in.readOptionalString(); - } - @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "connector_sync_job", @@ -548,28 +524,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalInstant(cancelationRequestedAt); - out.writeOptionalInstant(canceledAt); - out.writeOptionalInstant(completedAt); - out.writeNamedWriteable(connector); - out.writeInstant(createdAt); - out.writeLong(deletedDocumentCount); - out.writeOptionalString(error); - out.writeString(id); - out.writeLong(indexedDocumentCount); - out.writeLong(indexedDocumentVolume); - out.writeEnum(jobType); - out.writeOptionalInstant(lastSeen); - out.writeMap(metadata, StreamOutput::writeString, StreamOutput::writeGenericValue); - out.writeOptionalInstant(startedAt); - out.writeEnum(status); - out.writeOptionalLong(totalDocumentCount); - out.writeEnum(triggerMethod); - out.writeOptionalString(workerHostname); - } - public boolean equals(Object other) { if (this == other) return true; if (other == null || getClass() != other.getClass()) return false; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java deleted file mode 100644 index 57c1e3e30b0c0..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSerializingTests.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class ConnectorSerializingTests extends AbstractBWCWireSerializationTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return Connector::new; - } - - @Override - protected Connector createTestInstance() { - return ConnectorTestUtils.getRandomConnector(); - } - - @Override - protected Connector mutateInstance(Connector instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected Connector mutateInstanceForVersion(Connector instance, TransportVersion version) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java index bcb647d978abb..bb37a2b1e80b4 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -9,43 +9,20 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; import java.io.IOException; -import java.util.List; -import static java.util.Collections.emptyList; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.CoreMatchers.equalTo; public class ConnectorTests extends ESTestCase { - private NamedWriteableRegistry namedWriteableRegistry; - - @Before - public void registerNamedObjects() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - } - - public final void testRandomSerialization() throws IOException { - for (int runs = 0; runs < 10; runs++) { - Connector testInstance = ConnectorTestUtils.getRandomConnector(); - assertTransportSerialization(testInstance); - } - } - public void testToXContent() throws IOException { String connectorId = "test-connector"; String content = XContentHelper.stripWhitespace(""" @@ -383,14 +360,4 @@ public void testToXContent_withOptionalFieldsMissing() throws IOException { assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); assertThat(parsed.getApiKeySecretId(), equalTo(null)); } - - private void assertTransportSerialization(Connector testInstance) throws IOException { - Connector deserializedInstance = copyInstance(testInstance); - assertNotSame(testInstance, deserializedInstance); - assertThat(testInstance, equalTo(deserializedInstance)); - } - - private Connector copyInstance(Connector instance) throws IOException { - return copyWriteable(instance, namedWriteableRegistry, Connector::new); - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java deleted file mode 100644 index 8d169026a714f..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractBWCSerializationTestCase; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; - -import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; - -public class GetConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return GetConnectorAction.Request::new; - } - - @Override - protected GetConnectorAction.Request createTestInstance() { - return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10), randomBoolean()); - } - - @Override - protected GetConnectorAction.Request mutateInstance(GetConnectorAction.Request instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected GetConnectorAction.Request doParseInstance(XContentParser parser) throws IOException { - return GetConnectorAction.Request.parse(parser); - } - - @Override - protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) { - return new GetConnectorAction.Request(instance.getConnectorId(), instance.getDeleted()); - } - - @Override - protected List bwcVersions() { - return getAllBWCVersions().stream() - .filter(v -> v.onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) - .collect(Collectors.toList()); - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java deleted file mode 100644 index 1528dcecfa9a2..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractBWCSerializationTestCase; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; -import org.elasticsearch.xpack.core.action.util.PageParams; - -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; - -import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; - -public class ListConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { - @Override - protected Writeable.Reader instanceReader() { - return ListConnectorAction.Request::new; - } - - @Override - protected ListConnectorAction.Request createTestInstance() { - PageParams pageParams = EnterpriseSearchModuleTestUtils.randomPageParams(); - return new ListConnectorAction.Request( - pageParams, - List.of(generateRandomStringArray(10, 10, false)), - List.of(generateRandomStringArray(10, 10, false)), - List.of(generateRandomStringArray(10, 10, false)), - randomAlphaOfLengthBetween(3, 10), - randomBoolean() - ); - } - - @Override - protected ListConnectorAction.Request mutateInstance(ListConnectorAction.Request instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected ListConnectorAction.Request doParseInstance(XContentParser parser) throws IOException { - return ListConnectorAction.Request.parse(parser); - } - - @Override - protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorAction.Request instance, TransportVersion version) { - return new ListConnectorAction.Request( - instance.getPageParams(), - instance.getIndexNames(), - instance.getConnectorNames(), - instance.getConnectorServiceTypes(), - instance.getConnectorSearchQuery(), - instance.getDeleted() - ); - } - - @Override - protected List bwcVersions() { - return getAllBWCVersions().stream() - .filter(v -> v.onOrAfter(TransportVersions.CONNECTOR_API_SUPPORT_SOFT_DELETES)) - .collect(Collectors.toList()); - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java index 964c5e15d845d..5523ad4f50fe2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java @@ -8,22 +8,14 @@ package org.elasticsearch.xpack.application.connector.secrets.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class DeleteConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return DeleteConnectorSecretResponse::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java index 4448024814df3..c5ea4cbc8e550 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java @@ -8,22 +8,14 @@ package org.elasticsearch.xpack.application.connector.secrets.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class GetConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return GetConnectorSecretResponse::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java index ed3338c715bdf..449dd258359c4 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java @@ -8,17 +8,14 @@ package org.elasticsearch.xpack.application.connector.syncjob; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; -import org.junit.Before; import java.io.IOException; import java.time.Instant; -import java.util.List; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -26,22 +23,6 @@ public class ConnectorSyncJobTests extends ESTestCase { - private NamedWriteableRegistry namedWriteableRegistry; - - @Before - public void registerNamedObjects() { - namedWriteableRegistry = new NamedWriteableRegistry( - List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)) - ); - } - - public final void testRandomSerialization() throws IOException { - for (int run = 0; run < 10; run++) { - ConnectorSyncJob syncJob = ConnectorSyncJobTestUtils.getRandomConnectorSyncJob(); - assertTransportSerialization(syncJob); - } - } - public void testFromXContent_WithAllFields_AllSet() throws IOException { String content = XContentHelper.stripWhitespace(""" { @@ -332,14 +313,4 @@ public void testSyncJobConnectorFromXContent_WithAllNonOptionalFieldsSet_DoesNot ConnectorSyncJob.syncJobConnectorFromXContentBytes(new BytesArray(content), null, XContentType.JSON); } - - private void assertTransportSerialization(ConnectorSyncJob testInstance) throws IOException { - ConnectorSyncJob deserializedInstance = copyInstance(testInstance); - assertNotSame(testInstance, deserializedInstance); - assertThat(testInstance, equalTo(deserializedInstance)); - } - - private ConnectorSyncJob copyInstance(ConnectorSyncJob instance) throws IOException { - return copyWriteable(instance, namedWriteableRegistry, ConnectorSyncJob::new); - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java index 00f6e7cf57fc1..ff915090f3c20 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java @@ -8,23 +8,15 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class GetConnectorSyncJobActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< GetConnectorSyncJobAction.Response> { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return GetConnectorSyncJobAction.Response::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java index bc7b6320dddbe..ead5975d590a6 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java @@ -8,23 +8,15 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class ListConnectorSyncJobsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< ListConnectorSyncJobsAction.Response> { - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return ListConnectorSyncJobsAction.Response::new; From 90862f282f0be477df1e92d4b8fb892e9520a005 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 8 Jan 2025 16:19:51 +0100 Subject: [PATCH 24/26] Remove unused import --- .../xpack/application/EnterpriseSearchFeatures.java | 4 +--- .../xpack/application/connector/ConnectorIndexService.java | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java index 2f7b3769666ff..ba121f2cf865e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java @@ -9,7 +9,6 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction; import org.elasticsearch.xpack.application.rules.retriever.QueryRuleRetrieverBuilder; @@ -24,8 +23,7 @@ public Set getFeatures() { return Set.of( QUERY_RULES_TEST_API, QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED, - ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES, - ConnectorIndexService.CONNECTOR_SOFT_DELETES_FEATURE + ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 2a1406f0e1975..207ead649a379 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -23,7 +23,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; @@ -88,8 +87,6 @@ public class ConnectorIndexService { public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; - public static final NodeFeature CONNECTOR_SOFT_DELETES_FEATURE = new NodeFeature("connector_soft_deletes"); - /** * @param client A client for executing actions on the connector index */ From 4d848cb48a64aa95287a7cfb79d2e10d34f9329e Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 9 Jan 2025 07:55:15 +0100 Subject: [PATCH 25/26] Final tweaks --- .../connector/ConnectorIndexService.java | 16 ++++++++-------- .../connector/ConnectorTemplateRegistry.java | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 207ead649a379..34948a70d2f75 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -194,12 +194,12 @@ private Connector createConnectorWithDefaultValues( /** * Gets the {@link Connector} from the underlying index. * - * @param connectorId The id of the connector object. - * @param isDeleted If false, returns only the non-deleted connector with the matching ID; - * otherwise, returns the connector with the matching ID. - * @param listener The action listener to invoke on response/failure. + * @param connectorId The id of the connector object. + * @param includeDeleted If false, returns only the non-deleted connector with the matching ID; + * if true, returns the connector with the matching ID. + * @param listener The action listener to invoke on response/failure. */ - public void getConnector(String connectorId, boolean isDeleted, ActionListener listener) { + public void getConnector(String connectorId, boolean includeDeleted, ActionListener listener) { try { final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); @@ -215,7 +215,7 @@ public void getConnector(String connectorId, boolean isDeleted, ActionListener Date: Thu, 9 Jan 2025 12:16:52 +0100 Subject: [PATCH 26/26] Adapt variable name in rest layer --- .../application/connector/action/RestGetConnectorAction.java | 4 ++-- .../application/connector/action/RestListConnectorAction.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java index b6c812cfb1bdd..81960db46ad62 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java @@ -36,8 +36,8 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - Boolean isDeleted = restRequest.paramAsBoolean("include_deleted", false); - GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), isDeleted); + Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false); + GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), includeDeleted); return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 9fd859607e458..df33718462911 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -43,7 +43,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); List serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0])); String searchQuery = restRequest.param("query"); - Boolean isDeleted = restRequest.paramAsBoolean("include_deleted", false); + Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false); ListConnectorAction.Request request = new ListConnectorAction.Request( new PageParams(from, size), @@ -51,7 +51,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient connectorNames, serviceTypes, searchQuery, - isDeleted + includeDeleted ); return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel));