[Connector API] Support hard deletes with new URL param in delete endpoint (#120200)

* [Connector API] Add hard delete support

* Undo accidental change

* undo accidental build gradle change

* Tweak typos

* Update docs/changelog/120200.yaml

* [CI] Auto commit changes from spotless

* Fix yaml test

* Actually skip the feature check since we don't have the feature anyway

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: jedrazb

docs/changelog/120200.yaml

@@ -0,0 +1,5 @@
+pr: 120200
+summary: "[Connector API] Support hard deletes with new URL param in delete endpoint"
+area: Extract&Transform
+type: feature
+issues: []

docs/reference/connector/apis/delete-connector-api.asciidoc

@@ -13,7 +13,7 @@ beta::[]
 For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
 --
 
-Soft-deletes a connector and removes associated sync jobs.
+Deletes a connector and optionally removes associated sync jobs.
 
 Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
 
@@ -37,6 +37,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `<connector_id>`::
 (Required, string)
 
+`<hard>`::
+(Optional, boolean) If `true`, the connector doc is deleted. If `false`, connector doc is marked as deleted (soft deletion). Defaults to `false`.
+
 `delete_sync_jobs`::
 (Optional, boolean) A flag indicating if associated sync jobs should be also removed. Defaults to `false`.
 

rest-api-spec/src/main/resources/rest-api-spec/api/connector.delete.json

@@ -28,6 +28,11 @@
       ]
     },
     "params": {
+      "hard": {
+        "type": "boolean",
+        "default": false,
+        "description":  "If true, the connector doc is deleted. If false, connector doc is marked as deleted (soft-deleted)."
+      },
       "delete_sync_jobs": {
         "type": "boolean",
         "default": false,

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml

@@ -280,9 +280,7 @@ setup:
 
 ---
 "List Connectors - Soft deleted connectors / no deleted":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.list:
@@ -293,9 +291,7 @@ setup:
 
 ---
 "List Connectors - Single soft deleted connector":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.delete:
@@ -312,11 +308,91 @@ setup:
 
   - match: { count: 3 }
 
+
+---
+"List Connectors - Single hard deleted connector":
+
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+        hard: true
+
+  - do:
+      connector.list: {}
+
+  - match: { count: 2 }
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 2 }
+
+
+---
+"List Connectors - All hard deleted connectors":
+
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+        hard: true
+
+  - do:
+      connector.delete:
+        connector_id: connector-b
+        hard: true
+
+  - do:
+      connector.delete:
+        connector_id: connector-c
+        hard: true
+
+  - do:
+      connector.list: {}
+
+  - match: { count: 0 }
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 0 }
+
+---
+"List Connectors - 2 hard deleted connectors, 1 soft deleted":
+
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+        hard: false
+
+  - do:
+      connector.delete:
+        connector_id: connector-b
+        hard: true
+
+  - do:
+      connector.delete:
+        connector_id: connector-c
+        hard: true
+
+  - do:
+      connector.list: {}
+
+  - match: { count: 0 }
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 1 }
+
 ---
 "List Connectors - Soft deleted connectors":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.delete:
@@ -353,9 +429,7 @@ setup:
 
 ---
 "List Connectors - Soft deleted with from":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.delete:
@@ -387,9 +461,7 @@ setup:
 
 ---
 "List Connector - Soft deleted with size":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.delete:

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml

@@ -27,6 +27,31 @@ setup:
         connector_id: test-connector-to-delete
 
 
+---
+"Delete Connector - Hard Delete":
+  - do:
+      connector.put:
+        connector_id: test-connector-hard-delete
+        body:
+          index_name: search-2-test
+          name: my-hard-delete-connector
+          language: en
+          is_native: false
+          service_type: super-connector
+
+  - do:
+      connector.delete:
+        connector_id: test-connector-hard-delete
+        hard: true
+
+  - match: { acknowledged: true }
+
+  - do:
+      catch: "missing"
+      connector.get:
+        connector_id: test-connector-hard-delete
+        include_deleted: true
+
 ---
 "Delete Connector - deletes associated sync jobs":
 
@@ -107,12 +132,9 @@ setup:
       connector.delete:
         connector_id: test-nonexistent-connector
 
-
 ---
 "Delete Connector - Supports soft deletes":
-  - requires:
-      cluster_features: ["connector_soft_deletes"]
-      reason: Soft deletes were introduced in 9.0 release
+
 
   - do:
       connector.delete:

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -232,40 +234,71 @@ public void getConnector(String connectorId, boolean includeDeleted, ActionListe
     }
 
     /**
-     * Soft deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
+     * Deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
      *
      * @param connectorId          The id of the {@link Connector}.
+     * @param hardDelete           If set to true, the {@link Connector} is permanently deleted; otherwise, it is soft-deleted.
      * @param shouldDeleteSyncJobs The flag indicating if {@link ConnectorSyncJob} should also be deleted.
      * @param listener             The action listener to invoke on response/failure.
      */
-    public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener<UpdateResponse> listener) {
+    public void deleteConnector(
+        String connectorId,
+        boolean hardDelete,
+        boolean shouldDeleteSyncJobs,
+        ActionListener<DocWriteResponse> listener
+    ) {
 
         try {
-            // ensure that if connector is soft-deleted, deleting it again results in 404
-            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
-                final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
-                    WriteRequest.RefreshPolicy.IMMEDIATE
-                )
-                    .doc(
-                        new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
-                            .id(connectorId)
-                            .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
-                    );
-                clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
-                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                        ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                        return;
-                    }
-                    if (shouldDeleteSyncJobs) {
-                        new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
-                            connectorId,
-                            ll.map(r -> updateResponse)
+            if (hardDelete) {
+                final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+                clientWithOrigin.delete(
+                    deleteRequest,
+                    new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> {
+                        if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
+                            l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                            return;
+                        }
+                        if (shouldDeleteSyncJobs) {
+                            new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
+                                connectorId,
+                                l.map(r -> deleteResponse)
+                            );
+                        } else {
+                            l.onResponse(deleteResponse);
+                        }
+                    })
+                );
+            } else {
+                getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
+                    final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
+                        WriteRequest.RefreshPolicy.IMMEDIATE
+                    )
+                        .doc(
+                            new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+                                .id(connectorId)
+                                .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
                         );
-                    } else {
-                        ll.onResponse(updateResponse);
-                    }
+                    clientWithOrigin.update(
+                        updateRequest,
+                        new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+                            if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                                ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                                return;
+                            }
+                            if (shouldDeleteSyncJobs) {
+                                new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
+                                    connectorId,
+                                    ll.map(r -> updateResponse)
+                                );
+                            } else {
+                                ll.onResponse(updateResponse);
+                            }
+                        })
+                    );
                 }));
-            }));
+            }
         } catch (Exception e) {
             listener.onFailure(e);
         }