Skip to content

Commit 7ecf26e

Browse files
leaftyleafty
committed
feat: link data connectors to projects (#410)
Adds the possibility to link a data connector to a project. Details: * Add API to link a data connector to a project * Add API to list projects connected to a data connector * Add API to list data connectors connected to a project Note that `read` permissions are cascaded to direct members of the connected project. * Creating a link from a private data connector to a project requires the `editor` role on both sides of the link. * Creating a link from a public data connector to a project requires the `editor` role on the project. * Removing a link requires either the `editor` role on the project or the `owner` role on the data connector.
1 parent 41945bf commit 7ecf26e

File tree

14 files changed

+1158
-20
lines changed

14 files changed

+1158
-20
lines changed

‎bases/renku_data_services/data_api/app.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
139139
name="data_connectors",
140140
url_prefix=url_prefix,
141141
data_connector_repo=config.data_connector_repo,
142+
data_connector_to_project_link_repo=config.data_connector_to_project_link_repo,
142143
authenticator=config.authenticator,
143144
)
144145
app.blueprint(

‎components/renku_data_services/app_config/config.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
ServerOptionsDefaults,
4545
generate_default_resource_pool,
4646
)
47-
from renku_data_services.data_connectors.db import DataConnectorRepository
47+
from renku_data_services.data_connectors.db import DataConnectorProjectLinkRepository, DataConnectorRepository
4848
from renku_data_services.db_config import DBConfig
4949
from renku_data_services.git.gitlab import DummyGitlabAPI, GitlabAPI
5050
from renku_data_services.k8s.clients import DummyCoreClient, DummySchedulingClient, K8sCoreClient, K8sSchedulingClient
@@ -177,6 +177,9 @@ class Config:
177177
_git_repositories_repo: GitRepositoriesRepository | None = field(default=None, repr=False, init=False)
178178
_platform_repo: PlatformRepository | None = field(default=None, repr=False, init=False)
179179
_data_connector_repo: DataConnectorRepository | None = field(default=None, repr=False, init=False)
180+
_data_connector_to_project_link_repo: DataConnectorProjectLinkRepository | None = field(
181+
default=None, repr=False, init=False
182+
)
180183

181184
def __post_init__(self) -> None:
182185
spec_file = Path(renku_data_services.crc.__file__).resolve().parent / "api.spec.yaml"
@@ -415,6 +418,15 @@ def data_connector_repo(self) -> DataConnectorRepository:
415418
)
416419
return self._data_connector_repo
417420

421+
@property
422+
def data_connector_to_project_link_repo(self) -> DataConnectorProjectLinkRepository:
423+
"""The DB adapter for data connector to project links."""
424+
if not self._data_connector_to_project_link_repo:
425+
self._data_connector_to_project_link_repo = DataConnectorProjectLinkRepository(
426+
session_maker=self.db.async_session_maker, authz=self.authz
427+
)
428+
return self._data_connector_to_project_link_repo
429+
418430
@classmethod
419431
def from_env(cls, prefix: str = "") -> "Config":
420432
"""Create a config from environment variables."""

‎components/renku_data_services/authz/authz.py‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from renku_data_services.authz.config import AuthzConfig
3333
from renku_data_services.authz.models import Change, Member, MembershipChange, Role, Scope, Visibility
3434
from renku_data_services.base_models.core import InternalServiceAdmin
35-
from renku_data_services.data_connectors.models import DataConnector, DataConnectorUpdate
35+
from renku_data_services.data_connectors.models import DataConnector, DataConnectorToProjectLink, DataConnectorUpdate
3636
from renku_data_services.errors import errors
3737
from renku_data_services.namespace.models import Group, GroupUpdate, Namespace, NamespaceKind, NamespaceUpdate
3838
from renku_data_services.project.models import Project, ProjectUpdate
@@ -59,6 +59,7 @@ def authz(self) -> "Authz":
5959
| list[UserInfo]
6060
| DataConnector
6161
| DataConnectorUpdate
62+
| DataConnectorToProjectLink
6263
| None,
6364
)
6465
_T = TypeVar("_T")
@@ -97,6 +98,7 @@ class _Relation(StrEnum):
9798
project_namespace: str = "project_namespace"
9899
data_connector_platform: str = "data_connector_platform"
99100
data_connector_namespace: str = "data_connector_namespace"
101+
linked_to: str = "linked_to"
100102

101103
@classmethod
102104
def from_role(cls, role: Role) -> "_Relation":
@@ -140,6 +142,8 @@ class AuthzOperation(StrEnum):
140142
update: str = "update"
141143
update_or_insert: str = "update_or_insert"
142144
insert_many: str = "insert_many"
145+
create_link: str = "create_link"
146+
delete_link: str = "delete_link"
143147

144148

145149
class _AuthzConverter:
@@ -539,6 +543,19 @@ async def _get_authz_change(
539543
if result.old.namespace.id != result.new.namespace.id:
540544
user = _extract_user_from_args(*func_args, **func_kwargs)
541545
authz_change.extend(await db_repo.authz._update_data_connector_namespace(user, result.new))
546+
case AuthzOperation.create_link, ResourceType.data_connector if isinstance(
547+
result, DataConnectorToProjectLink
548+
):
549+
user = _extract_user_from_args(*func_args, **func_kwargs)
550+
authz_change = await db_repo.authz._add_data_connector_to_project_link(user, result)
551+
case AuthzOperation.delete_link, ResourceType.data_connector if result is None:
552+
# NOTE: This means that the link does not exist in the first place so nothing was deleted
553+
pass
554+
case AuthzOperation.delete_link, ResourceType.data_connector if isinstance(
555+
result, DataConnectorToProjectLink
556+
):
557+
user = _extract_user_from_args(*func_args, **func_kwargs)
558+
authz_change = await db_repo.authz._remove_data_connector_to_project_link(user, result)
542559
case _:
543560
resource_id: str | ULID | None = "unknown"
544561
if isinstance(result, (Project, Namespace, Group, DataConnector)):
@@ -664,6 +681,17 @@ async def _remove_project(
664681
ReadRelationshipsRequest(consistency=consistency, relationship_filter=rel_filter)
665682
)
666683
rels: list[Relationship] = []
684+
async for response in responses:
685+
rels.append(response.relationship)
686+
# Project is also a subject for "linked_to" relations
687+
rel_filter = RelationshipFilter(
688+
optional_subject_filter=SubjectFilter(
689+
subject_type=ResourceType.project.value, optional_subject_id=str(project.id)
690+
)
691+
)
692+
responses: AsyncIterable[ReadRelationshipsResponse] = self.client.ReadRelationships(
693+
ReadRelationshipsRequest(consistency=consistency, relationship_filter=rel_filter)
694+
)
667695
async for response in responses:
668696
rels.append(response.relationship)
669697
apply = WriteRelationshipsRequest(
@@ -1617,3 +1645,82 @@ async def _update_data_connector_namespace(
16171645
]
16181646
)
16191647
return _AuthzChange(apply=apply_change, undo=undo_change)
1648+
1649+
async def _add_data_connector_to_project_link(
1650+
self, user: base_models.APIUser, link: DataConnectorToProjectLink
1651+
) -> _AuthzChange:
1652+
"""Links a data connector to a project."""
1653+
# NOTE: we manually check for permissions here since it is not trivially expressed through decorators
1654+
allowed_from = await self.has_permission(
1655+
user, ResourceType.data_connector, link.data_connector_id, Scope.ADD_LINK
1656+
)
1657+
if not allowed_from:
1658+
raise errors.MissingResourceError(
1659+
message=f"The user with ID {user.id} cannot perform operation {Scope.ADD_LINK} "
1660+
f"on {ResourceType.data_connector.value} "
1661+
f"with ID {link.data_connector_id} or the resource does not exist."
1662+
)
1663+
allowed_to = await self.has_permission(user, ResourceType.project, link.project_id, Scope.WRITE)
1664+
if not allowed_to:
1665+
raise errors.MissingResourceError(
1666+
message=f"The user with ID {user.id} cannot perform operation {Scope.WRITE} "
1667+
f"on {ResourceType.project.value} "
1668+
f"with ID {link.project_id} or the resource does not exist."
1669+
)
1670+
1671+
data_connector_res = _AuthzConverter.data_connector(link.data_connector_id)
1672+
project_subject = SubjectReference(object=_AuthzConverter.project(link.project_id))
1673+
relationship = Relationship(
1674+
resource=data_connector_res,
1675+
relation=_Relation.linked_to.value,
1676+
subject=project_subject,
1677+
)
1678+
apply = WriteRelationshipsRequest(
1679+
updates=[RelationshipUpdate(operation=RelationshipUpdate.OPERATION_TOUCH, relationship=relationship)]
1680+
)
1681+
undo = WriteRelationshipsRequest(
1682+
updates=[RelationshipUpdate(operation=RelationshipUpdate.OPERATION_DELETE, relationship=relationship)]
1683+
)
1684+
change = _AuthzChange(
1685+
apply=apply,
1686+
undo=undo,
1687+
)
1688+
return change
1689+
1690+
async def _remove_data_connector_to_project_link(
1691+
self, user: base_models.APIUser, link: DataConnectorToProjectLink
1692+
) -> _AuthzChange:
1693+
"""Remove the relationships associated with the link from a data connector to a project."""
1694+
# NOTE: we manually check for permissions here since it is not trivially expressed through decorators
1695+
allowed_from = await self.has_permission(
1696+
user, ResourceType.data_connector, link.data_connector_id, Scope.DELETE
1697+
)
1698+
allowed_to, zed_token = await self._has_permission(user, ResourceType.project, link.project_id, Scope.WRITE)
1699+
allowed = allowed_from or allowed_to
1700+
if not allowed:
1701+
raise errors.MissingResourceError(
1702+
message=f"The user with ID {user.id} cannot perform operation {AuthzOperation.delete_link}"
1703+
f"on the data connector to project link with ID {link.id} or the resource does not exist."
1704+
)
1705+
consistency = Consistency(at_least_as_fresh=zed_token) if zed_token else Consistency(fully_consistent=True)
1706+
rel_filter = RelationshipFilter(
1707+
resource_type=ResourceType.data_connector.value,
1708+
optional_resource_id=str(link.data_connector_id),
1709+
optional_relation=_Relation.linked_to.value,
1710+
optional_subject_filter=SubjectFilter(
1711+
subject_type=ResourceType.project.value, optional_subject_id=str(link.project_id)
1712+
),
1713+
)
1714+
responses: AsyncIterable[ReadRelationshipsResponse] = self.client.ReadRelationships(
1715+
ReadRelationshipsRequest(consistency=consistency, relationship_filter=rel_filter)
1716+
)
1717+
rels: list[Relationship] = []
1718+
async for response in responses:
1719+
rels.append(response.relationship)
1720+
apply = WriteRelationshipsRequest(
1721+
updates=[RelationshipUpdate(operation=RelationshipUpdate.OPERATION_DELETE, relationship=i) for i in rels]
1722+
)
1723+
undo = WriteRelationshipsRequest(
1724+
updates=[RelationshipUpdate(operation=RelationshipUpdate.OPERATION_TOUCH, relationship=i) for i in rels]
1725+
)
1726+
return _AuthzChange(apply=apply, undo=undo)

‎components/renku_data_services/authz/models.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class Scope(Enum):
5050
DELETE: str = "delete"
5151
CHANGE_MEMBERSHIP: str = "change_membership"
5252
READ_CHILDREN: str = "read_children"
53+
ADD_LINK: str = "add_link"
5354

5455

5556
@dataclass

‎components/renku_data_services/data_connectors/api.spec.yaml‎

Lines changed: 107 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ paths:
5151
default:
5252
$ref: "#/components/responses/Error"
5353
tags:
54-
- data connectors
54+
- data_connectors
5555
post:
5656
summary: Create a new data connector
5757
requestBody:
@@ -70,7 +70,7 @@ paths:
7070
default:
7171
$ref: "#/components/responses/Error"
7272
tags:
73-
- data connectors
73+
- data_connectors
7474
/data_connectors/{data_connector_id}:
7575
parameters:
7676
- in: path
@@ -97,7 +97,7 @@ paths:
9797
default:
9898
$ref: "#/components/responses/Error"
9999
tags:
100-
- data connectors
100+
- data_connectors
101101
patch:
102102
summary: Update specific fields of an existing data connector
103103
parameters:
@@ -124,7 +124,7 @@ paths:
124124
default:
125125
$ref: "#/components/responses/Error"
126126
tags:
127-
- data connectors
127+
- data_connectors
128128
delete:
129129
summary: Remove a data connector
130130
responses:
@@ -133,7 +133,7 @@ paths:
133133
default:
134134
$ref: "#/components/responses/Error"
135135
tags:
136-
- data connectors
136+
- data_connectors
137137
/namespaces/{namespace}/data_connectors/{slug}:
138138
parameters:
139139
- in: path
@@ -164,8 +164,70 @@ paths:
164164
default:
165165
$ref: "#/components/responses/Error"
166166
tags:
167-
- data connectors
168-
167+
- data_connectors
168+
/data_connectors/{data_connector_id}/project_links:
169+
parameters:
170+
- in: path
171+
name: data_connector_id
172+
required: true
173+
schema:
174+
$ref: "#/components/schemas/Ulid"
175+
description: the ID of the data connector
176+
get:
177+
summary: Get all links from a given data connector to projects
178+
responses:
179+
"200":
180+
description: List of data connector to project links
181+
content:
182+
application/json:
183+
schema:
184+
$ref: "#/components/schemas/DataConnectorToProjectLinksList"
185+
default:
186+
$ref: "#/components/responses/Error"
187+
tags:
188+
- data_connectors
189+
post:
190+
summary: Create a new link from a data connector to a project
191+
requestBody:
192+
required: true
193+
content:
194+
application/json:
195+
schema:
196+
$ref: "#/components/schemas/DataConnectorToProjectLinkPost"
197+
responses:
198+
"201":
199+
description: The data connector was connected to a project
200+
content:
201+
application/json:
202+
schema:
203+
$ref: "#/components/schemas/DataConnectorToProjectLink"
204+
default:
205+
$ref: "#/components/responses/Error"
206+
tags:
207+
- data_connectors
208+
/data_connectors/{data_connector_id}/project_links/{link_id}:
209+
parameters:
210+
- in: path
211+
name: data_connector_id
212+
required: true
213+
schema:
214+
$ref: "#/components/schemas/Ulid"
215+
description: the ID of the data connector
216+
- in: path
217+
name: link_id
218+
required: true
219+
schema:
220+
$ref: "#/components/schemas/Ulid"
221+
description: the ID of the link between a data connector and a project
222+
delete:
223+
summary: Remove a link from a data connector to a project
224+
responses:
225+
"204":
226+
description: The data connector was removed or did not exist in the first place
227+
default:
228+
$ref: "#/components/responses/Error"
229+
tags:
230+
- data_connectors
169231
/data_connectors/{data_connector_id}/secrets:
170232
parameters:
171233
- in: path
@@ -192,7 +254,7 @@ paths:
192254
default:
193255
$ref: "#/components/responses/Error"
194256
tags:
195-
- data connectors
257+
- data_connectors
196258
post:
197259
summary: Save secrets for a data connector
198260
requestBody:
@@ -211,7 +273,7 @@ paths:
211273
default:
212274
$ref: "#/components/responses/Error"
213275
tags:
214-
- data connectors
276+
- data_connectors
215277
delete:
216278
summary: Remove all saved secrets for a data connector
217279
responses:
@@ -220,7 +282,7 @@ paths:
220282
default:
221283
$ref: "#/components/responses/Error"
222284
tags:
223-
- data connectors
285+
- data_connectors
224286
components:
225287
schemas:
226288
DataConnectorsList:
@@ -401,6 +463,41 @@ components:
401463
- target_path
402464
example:
403465
storage_url: s3://giab
466+
DataConnectorToProjectLinksList:
467+
description: A list of links from a data connector to a project
468+
type: array
469+
items:
470+
$ref: "#/components/schemas/DataConnectorToProjectLink"
471+
DataConnectorToProjectLink:
472+
description: A link from a data connector to a project in Renku 2.0
473+
type: object
474+
additionalProperties: false
475+
properties:
476+
id:
477+
$ref: "#/components/schemas/Ulid"
478+
data_connector_id:
479+
$ref: "#/components/schemas/Ulid"
480+
project_id:
481+
$ref: "#/components/schemas/Ulid"
482+
creation_date:
483+
$ref: "#/components/schemas/CreationDate"
484+
created_by:
485+
$ref: "#/components/schemas/UserId"
486+
required:
487+
- id
488+
- data_connector_id
489+
- project_id
490+
- creation_date
491+
- created_by
492+
DataConnectorToProjectLinkPost:
493+
description: A link to be created from a data connector to a project in Renku 2.0
494+
type: object
495+
additionalProperties: false
496+
properties:
497+
project_id:
498+
$ref: "#/components/schemas/Ulid"
499+
required:
500+
- project_id
404501
CloudStorageSecretPost:
405502
type: object
406503
description: Data for storing secret for a storage field

0 commit comments

Comments
 (0)