[Connector API] Support soft-deletes of connectors (#118669)

* [Connector API] Add interface for soft-deletes

* Define connector deleted system index

* Got soft-delete logic working

* Add unit tests

* Add yaml e2e test and attempt to update permissions

* Fix permissions

* Update docs

* Fix docs

* Update docs/changelog/118282.yaml

* Change logic

* Fix tests

* Remove unnecessary privilege from yaml rest test

* Update changelog

* Update docs/changelog/118669.yaml

* Adapt yaml tests

* Undo changes to muted-tests.yml

* Fix compilation issue after other PR got merged

* Exclude soft-deleted connector from checks about index_name already in use

* Update docs/reference/connector/apis/get-connector-api.asciidoc

Co-authored-by: Tim Grein <[email protected]>

* Update rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json

Co-authored-by: Tim Grein <[email protected]>

* Adapt comments, add connector wire serializing test

* Introduce new transport versions for passing the delete flag

* Get rid of wire serialisation, use include_deleted instead of deleted flag

* Remove unused import

* Final tweaks

* Adapt variable name in rest layer

---------

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Tim Grein <[email protected]>

Author: 3 people

docs/changelog/118669.yaml

@@ -0,0 +1,5 @@
+pr: 118669
+summary: "[Connector API] Support soft-deletes of connectors"
+area: Extract&Transform
+type: feature
+issues: []

docs/reference/connector/apis/delete-connector-api.asciidoc

@@ -6,14 +6,14 @@
 
 beta::[]
 
+
 .New API reference
 [sidebar]
 --
 For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
 --
 
-Removes a connector and associated sync jobs.
-This is a destructive action that is not recoverable.
+Soft-deletes a connector and removes associated sync jobs.
 
 Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
 

docs/reference/connector/apis/get-connector-api.asciidoc

@@ -33,6 +33,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `<connector_id>`::
 (Required, string)
 
+`include_deleted`::
+(Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`.
+
 [[get-connector-api-response-codes]]
 ==== {api-response-codes-title}
 

docs/reference/connector/apis/list-connectors-api.asciidoc

@@ -47,6 +47,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `service_type`::
 (Optional, string) A comma-separated list of connector service types, used to filter search results.
 
+`include_deleted`::
+(Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`.
+
 [[list-connector-api-example]]
 ==== {api-examples-title}
 

rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json

@@ -26,6 +26,13 @@
           }
         }
       ]
+    },
+    "params": {
+      "include_deleted": {
+        "type": "boolean",
+        "default": false,
+        "description": "A flag indicating whether to return connectors that have been soft-deleted."
+      }
     }
   }
 }

rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json

@@ -47,6 +47,11 @@
       "query": {
         "type": "string",
         "description": "A search string for querying connectors, filtering results by matching against connector names, descriptions, and index names"
+      },
+      "include_deleted": {
+        "type": "boolean",
+        "default": false,
+        "description": "A flag indicating whether to return connectors that have been soft-deleted."
       }
     }
   }

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json

@@ -302,6 +302,9 @@
         },
         "sync_now": {
           "type": "boolean"
+        },
+        "deleted": {
+          "type": "boolean"
         }
       }
     }

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml

@@ -218,6 +218,30 @@ setup:
         body:
           index_name: search-test
 
+---
+'Create Connector - Index name used by deleted connector':
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          index_name: search-test
+
+  - match: { result: 'created' }
+
+  - do:
+      connector.delete:
+        connector_id: test-connector-1
+
+  - match: { acknowledged: true }
+
+  - do:
+      connector.put:
+        connector_id: test-connector-2
+        body:
+          index_name: search-test
+
+  - match: { result: 'created' }
+
 ---
 'Create Connector - Without index attached':
   - do:

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml

@@ -71,7 +71,7 @@ setup:
   - match: { results.1.language: "nl" }
 
 ---
-"List Connector- with size":
+"List Connector - with size":
   - do:
       connector.list:
         size: 2
@@ -106,7 +106,6 @@ setup:
 
   - match: { count: 0 }
 
-
 ---
 "List Connector - filter by index names":
   - do:
@@ -279,4 +278,145 @@ setup:
       connector.list: { }
 
 
+---
+"List Connectors - Soft deleted connectors / no deleted":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+
+---
+"List Connectors - Single soft deleted connector":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+
+  - do:
+      connector.list: {}
+
+  - match: { count: 2 }
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+---
+"List Connectors - Soft deleted connectors":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-b" }
+  - match: { results.0.index_name: "content-search-2-test" }
+  - match: { results.0.language: "en" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-a" }
+  - match: { results.1.index_name: "search-1-test" }
+  - match: { results.1.language: "pl" }
+  - match: { results.1.deleted: true }
+
+  - match: { results.2.id: "connector-c" }
+  - match: { results.2.index_name: "search-3-test" }
+  - match: { results.2.language: "nl" }
+  - match: { results.2.deleted: true }
+
+
+---
+"List Connectors - Soft deleted with from":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        from: 1
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-a" }
+  - match: { results.0.index_name: "search-1-test" }
+  - match: { results.0.language: "pl" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-c" }
+  - match: { results.1.index_name: "search-3-test" }
+  - match: { results.1.language: "nl" }
+  - match: { results.0.deleted: true }
+
+---
+"List Connector - Soft deleted with size":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        size: 2
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-b" }
+  - match: { results.0.index_name: "content-search-2-test" }
+  - match: { results.0.language: "en" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-a" }
+  - match: { results.1.index_name: "search-1-test" }
+  - match: { results.1.language: "pl" }
+  - match: { results.1.deleted: true }
+
 

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml

@@ -108,6 +108,33 @@ setup:
         connector_id: test-nonexistent-connector
 
 
+---
+"Delete Connector - Supports soft deletes":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: test-connector-to-delete
+
+  - match: { acknowledged: true }
+
+  - do:
+      catch: "missing"
+      connector.get:
+        connector_id: test-connector-to-delete
+
+  - do:
+      connector.get:
+        connector_id: test-connector-to-delete
+        include_deleted: true
+
+  - match: { id: test-connector-to-delete }
+  - match: { index_name: search-1-test }
+  - match: { service_type: super-connector }
+  - match: { name: my-connector }
+
 ---
 "Delete connector fails for unprivileged user":
   - skip: