feat: handle secrets for data connectors (#413)

Add support for saving and managing secrets for data connectors.

Details:
* Add API endpoints to list, update and delete saved secrets for a given data connector.

Author: leafty

bases/renku_data_services/data_api/app.py

@@ -140,6 +140,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         url_prefix=url_prefix,
         data_connector_repo=config.data_connector_repo,
         data_connector_to_project_link_repo=config.data_connector_to_project_link_repo,
+        data_connector_secret_repo=config.data_connector_secret_repo,
         authenticator=config.authenticator,
     )
     app.blueprint(

components/renku_data_services/app_config/config.py

@@ -44,7 +44,11 @@
     ServerOptionsDefaults,
     generate_default_resource_pool,
 )
-from renku_data_services.data_connectors.db import DataConnectorProjectLinkRepository, DataConnectorRepository
+from renku_data_services.data_connectors.db import (
+    DataConnectorProjectLinkRepository,
+    DataConnectorRepository,
+    DataConnectorSecretRepository,
+)
 from renku_data_services.db_config import DBConfig
 from renku_data_services.git.gitlab import DummyGitlabAPI, GitlabAPI
 from renku_data_services.k8s.clients import DummyCoreClient, DummySchedulingClient, K8sCoreClient, K8sSchedulingClient
@@ -180,6 +184,7 @@ class Config:
     _data_connector_to_project_link_repo: DataConnectorProjectLinkRepository | None = field(
         default=None, repr=False, init=False
     )
+    _data_connector_secret_repo: DataConnectorSecretRepository | None = field(default=None, repr=False, init=False)
 
     def __post_init__(self) -> None:
         spec_file = Path(renku_data_services.crc.__file__).resolve().parent / "api.spec.yaml"
@@ -427,6 +432,18 @@ def data_connector_to_project_link_repo(self) -> DataConnectorProjectLinkReposit
             )
         return self._data_connector_to_project_link_repo
 
+    @property
+    def data_connector_secret_repo(self) -> DataConnectorSecretRepository:
+        """The DB adapter for data connector secrets."""
+        if not self._data_connector_secret_repo:
+            self._data_connector_secret_repo = DataConnectorSecretRepository(
+                session_maker=self.db.async_session_maker,
+                data_connector_repo=self.data_connector_repo,
+                user_repo=self.kc_user_repo,
+                secret_service_public_key=self.secrets_service_public_key,
+            )
+        return self._data_connector_secret_repo
+
     @classmethod
     def from_env(cls, prefix: str = "") -> "Config":
         """Create a config from environment variables."""

components/renku_data_services/data_connectors/api.spec.yaml

@@ -244,7 +244,7 @@ paths:
           content:
             "application/json":
               schema:
-                $ref: "#/components/schemas/CloudStorageSecretGetList"
+                $ref: "#/components/schemas/DataConnectorSecretsList"
         "404":
           description: Storage was not found
           content:
@@ -255,21 +255,22 @@ paths:
           $ref: "#/components/responses/Error"
       tags:
         - data_connectors
-    post:
+    patch:
       summary: Save secrets for a data connector
+      description: New secrets will be added and existing secrets will have their value updated. Using `null` as a value will remove the corresponding secret.
       requestBody:
         required: true
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/CloudStorageSecretPostList"
+              $ref: "#/components/schemas/DataConnectorSecretPatchList"
       responses:
         "201":
           description: The secrets for cloud storage were saved
           content:
             "application/json":
               schema:
-                $ref: "#/components/schemas/CloudStorageSecretGetList"
+                $ref: "#/components/schemas/DataConnectorSecretsList"
         default:
           $ref: "#/components/responses/Error"
       tags:
@@ -306,10 +307,6 @@ components:
           $ref: "#/components/schemas/Slug"
         storage:
           $ref: "#/components/schemas/CloudStorageCore"
-        secrets:
-          type: array
-          items:
-            $ref: "#/components/schemas/CloudStorageSecretGet"
         creation_date:
           $ref: "#/components/schemas/CreationDate"
         created_by:
@@ -498,67 +495,49 @@ components:
           $ref: "#/components/schemas/Ulid"
       required:
         - project_id
-    CloudStorageSecretPost:
+    DataConnectorSecretsList:
+      description: A list of data connectors
+      type: array
+      items:
+        $ref: "#/components/schemas/DataConnectorSecret"
+    DataConnectorSecret:
+      description: Information about a credential saved for a data connector
       type: object
-      description: Data for storing secret for a storage field
       properties:
         name:
-          type: string
-          description: Name of the field to store credential for
-          minLength: 1
-          maxLength: 99
-        value:
-          $ref: "#/components/schemas/SecretValue"
+          $ref: "#/components/schemas/DataConnectorSecretFieldName"
+        secret_id:
+          $ref: "#/components/schemas/Ulid"
       required:
         - name
-        - value
-    CloudStorageSecretPostList:
-      description: List of storage secrets that are saved
-      type: array
-      items:
-        $ref: "#/components/schemas/CloudStorageSecretPost"
-    CloudStorageSecretGetList:
-      description: List of storage secrets that are saved
+        - secret_id
+    DataConnectorSecretPatchList:
+      description: List of secrets to be saved for a data connector
       type: array
       items:
-        $ref: "#/components/schemas/CloudStorageSecretGet"
-    CloudStorageSecretGet:
-      type: object
-      description: Data for saved storage secrets
+        $ref: "#/components/schemas/DataConnectorSecretPatch"
+    DataConnectorSecretPatch:
+      description: Information about a credential to save for a data connector
       properties:
         name:
-          type: string
-          description: Name of the field to store credential for
-          minLength: 1
-          maxLength: 99
-        secret_id:
-          $ref: "#/components/schemas/Ulid"
+          $ref: "#/components/schemas/DataConnectorSecretFieldName"
+        value:
+          $ref: "#/components/schemas/SecretValueNullable"
       required:
         - name
-        - secret_id
-    SecretValue:
+        - value
+    DataConnectorSecretFieldName:
+      description: Name of the credential field
+      type: string
+      minLength: 1
+      maxLength: 99
+      example: "secret_key"
+    SecretValueNullable:
       description: Secret value that can be any text
       type: string
       minLength: 1
       maxLength: 5000
-    RCloneEntry:
-      type: object
-      description: Schema for a storage type in rclone, like S3 or Azure Blob Storage. Contains fields for that storage type.
-      properties:
-        name:
-          type: string
-          description: Human readable name of the provider
-        description:
-          type: string
-          description: description of the provider
-        prefix:
-          type: string
-          description: Machine readable name of the provider
-        options:
-          description: Fields/properties used for this storage.
-          type: array
-          items:
-            $ref: "#/components/schemas/RCloneOption"
+      nullable: true
     RCloneOption:
       type: object
       description: Single field on an RClone storage, like "remote" or "access_key_id"

components/renku_data_services/data_connectors/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-09-17T13:48:49+00:00
+#   timestamp: 2024-10-03T13:29:06+00:00
 
 from __future__ import annotations
 
@@ -246,31 +246,11 @@ class DataConnectorToProjectLinkPost(BaseAPISpec):
     )
 
 
-class CloudStorageSecretPost(BaseAPISpec):
+class DataConnectorSecret(BaseAPISpec):
     name: str = Field(
         ...,
-        description="Name of the field to store credential for",
-        max_length=99,
-        min_length=1,
-    )
-    value: str = Field(
-        ...,
-        description="Secret value that can be any text",
-        max_length=5000,
-        min_length=1,
-    )
-
-
-class CloudStorageSecretPostList(RootModel[List[CloudStorageSecretPost]]):
-    root: List[CloudStorageSecretPost] = Field(
-        ..., description="List of storage secrets that are saved"
-    )
-
-
-class CloudStorageSecretGet(BaseAPISpec):
-    name: str = Field(
-        ...,
-        description="Name of the field to store credential for",
+        description="Name of the credential field",
+        example="secret_key",
         max_length=99,
         min_length=1,
     )
@@ -283,14 +263,19 @@ class CloudStorageSecretGet(BaseAPISpec):
     )
 
 
-class RCloneEntry(BaseAPISpec):
-    name: Optional[str] = Field(None, description="Human readable name of the provider")
-    description: Optional[str] = Field(None, description="description of the provider")
-    prefix: Optional[str] = Field(
-        None, description="Machine readable name of the provider"
+class DataConnectorSecretPatch(BaseAPISpec):
+    name: str = Field(
+        ...,
+        description="Name of the credential field",
+        example="secret_key",
+        max_length=99,
+        min_length=1,
     )
-    options: Optional[List[RCloneOption]] = Field(
-        None, description="Fields/properties used for this storage."
+    value: Optional[str] = Field(
+        ...,
+        description="Secret value that can be any text",
+        max_length=5000,
+        min_length=1,
     )
 
 
@@ -337,7 +322,6 @@ class DataConnector(BaseAPISpec):
         pattern="^(?!.*\\.git$|.*\\.atom$|.*[\\-._][\\-._].*)[a-zA-Z0-9][a-zA-Z0-9\\-_.]*$",
     )
     storage: CloudStorageCore
-    secrets: Optional[List[CloudStorageSecretGet]] = None
     creation_date: datetime = Field(
         ...,
         description="The date and time the resource was created (in UTC and ISO-8601 format)",
@@ -450,9 +434,15 @@ class DataConnectorToProjectLinksList(RootModel[List[DataConnectorToProjectLink]
     )
 
 
-class CloudStorageSecretGetList(RootModel[List[CloudStorageSecretGet]]):
-    root: List[CloudStorageSecretGet] = Field(
-        ..., description="List of storage secrets that are saved"
+class DataConnectorSecretsList(RootModel[List[DataConnectorSecret]]):
+    root: List[DataConnectorSecret] = Field(
+        ..., description="A list of data connectors"
+    )
+
+
+class DataConnectorSecretPatchList(RootModel[List[DataConnectorSecretPatch]]):
+    root: List[DataConnectorSecretPatch] = Field(
+        ..., description="List of secrets to be saved for a data connector"
     )
 
 

components/renku_data_services/data_connectors/blueprints.py

@@ -15,16 +15,21 @@
 )
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
 from renku_data_services.base_api.etag import extract_if_none_match, if_match_required
-from renku_data_services.base_api.misc import validate_query
+from renku_data_services.base_api.misc import validate_body_root_model, validate_query
 from renku_data_services.base_api.pagination import PaginationRequest, paginate
 from renku_data_services.base_models.validation import validate_and_dump, validated_json
 from renku_data_services.data_connectors import apispec, models
 from renku_data_services.data_connectors.core import (
     dump_storage_with_sensitive_fields,
     validate_data_connector_patch,
+    validate_data_connector_secrets_patch,
     validate_unsaved_data_connector,
 )
-from renku_data_services.data_connectors.db import DataConnectorProjectLinkRepository, DataConnectorRepository
+from renku_data_services.data_connectors.db import (
+    DataConnectorProjectLinkRepository,
+    DataConnectorRepository,
+    DataConnectorSecretRepository,
+)
 from renku_data_services.storage.rclone import RCloneValidator
 
 
@@ -34,6 +39,7 @@ class DataConnectorsBP(CustomBlueprint):
 
     data_connector_repo: DataConnectorRepository
     data_connector_to_project_link_repo: DataConnectorProjectLinkRepository
+    data_connector_secret_repo: DataConnectorSecretRepository
     authenticator: base_models.Authenticator
 
     def get_all(self) -> BlueprintFactoryResponse:
@@ -262,6 +268,64 @@ async def _get_all_data_connectors_links_to_project(
 
         return "/projects/<project_id:ulid>/data_connector_links", ["GET"], _get_all_data_connectors_links_to_project
 
+    def get_secrets(self) -> BlueprintFactoryResponse:
+        """List all saved secrets for a data connector."""
+
+        @authenticate(self.authenticator)
+        @only_authenticated
+        async def _get_secrets(
+            _: Request,
+            user: base_models.APIUser,
+            data_connector_id: ULID,
+        ) -> JSONResponse:
+            secrets = await self.data_connector_secret_repo.get_data_connector_secrets(
+                user=user, data_connector_id=data_connector_id
+            )
+            return validated_json(
+                apispec.DataConnectorSecretsList, [self._dump_data_connector_secret(secret) for secret in secrets]
+            )
+
+        return "/data_connectors/<data_connector_id:ulid>/secrets", ["GET"], _get_secrets
+
+    def patch_secrets(self) -> BlueprintFactoryResponse:
+        """Create, update or delete saved secrets for a data connector."""
+
+        @authenticate(self.authenticator)
+        @only_authenticated
+        @validate_body_root_model(json=apispec.DataConnectorSecretPatchList)
+        async def _patch_secrets(
+            _: Request,
+            user: base_models.APIUser,
+            data_connector_id: ULID,
+            body: apispec.DataConnectorSecretPatchList,
+        ) -> JSONResponse:
+            unsaved_secrets = validate_data_connector_secrets_patch(put=body)
+            secrets = await self.data_connector_secret_repo.patch_data_connector_secrets(
+                user=user, data_connector_id=data_connector_id, secrets=unsaved_secrets
+            )
+            return validated_json(
+                apispec.DataConnectorSecretsList, [self._dump_data_connector_secret(secret) for secret in secrets]
+            )
+
+        return "/data_connectors/<data_connector_id:ulid>/secrets", ["PATCH"], _patch_secrets
+
+    def delete_secrets(self) -> BlueprintFactoryResponse:
+        """Delete all saved secrets for a data connector."""
+
+        @authenticate(self.authenticator)
+        @only_authenticated
+        async def _delete_secrets(
+            _: Request,
+            user: base_models.APIUser,
+            data_connector_id: ULID,
+        ) -> HTTPResponse:
+            await self.data_connector_secret_repo.delete_data_connector_secrets(
+                user=user, data_connector_id=data_connector_id
+            )
+            return HTTPResponse(status=204)
+
+        return "/data_connectors/<data_connector_id:ulid>/secrets", ["DELETE"], _delete_secrets
+
     @staticmethod
     def _dump_data_connector(data_connector: models.DataConnector, validator: RCloneValidator) -> dict[str, Any]:
         """Dumps a data connector for API responses."""
@@ -291,3 +355,11 @@ def _dump_data_connector_to_project_link(link: models.DataConnectorToProjectLink
             creation_date=link.creation_date,
             created_by=link.created_by,
         )
+
+    @staticmethod
+    def _dump_data_connector_secret(secret: models.DataConnectorSecret) -> dict[str, Any]:
+        """Dumps a data connector secret for API responses."""
+        return dict(
+            name=secret.name,
+            secret_id=str(secret.secret_id),
+        )