Skip to content

Commit ae87e85

Browse files
olevskisgaist
olevski
and
sgaist
committed
feat: use cloud storage from amalthea (#387)
Co-authored-by: Samuel Gaist <[email protected]>
1 parent d2b1dcf commit ae87e85

File tree

19 files changed

+225
-126
lines changed

19 files changed

+225
-126
lines changed

‎bases/renku_data_services/data_api/app.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
143143
nb_config=config.nb_config,
144144
internal_gitlab_authenticator=config.gitlab_authenticator,
145145
git_repo=config.git_repositories_repo,
146+
rp_repo=config.rp_repo,
146147
)
147148
notebooks_new = NotebooksNewBP(
148149
name="notebooks",
@@ -151,6 +152,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
151152
nb_config=config.nb_config,
152153
project_repo=config.project_repo,
153154
session_repo=config.session_repo,
155+
storage_repo=config.storage_v2_repo,
154156
rp_repo=config.rp_repo,
155157
internal_gitlab_authenticator=config.gitlab_authenticator,
156158
)

‎components/renku_data_services/notebooks/api.spec.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1022,11 +1022,12 @@ components:
10221022
storage_id:
10231023
allOf:
10241024
- "$ref": "#/components/schemas/Ulid"
1025-
- description: If the storage_id is provided then this config must replace an existing storage config in the session
1025+
- description: The storage ID is used to know which storage config from the DB should be overriden
10261026
required:
10271027
- configuration
10281028
- source_path
10291029
- target_path
1030+
- storage_id
10301031
ServerName:
10311032
type: string
10321033
minLength: 5

‎components/renku_data_services/notebooks/api/amalthea_patches/git_sidecar.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ async def main(server: "UserServer") -> list[dict[str, Any]]:
2222
commit_sha = getattr(server, "commit_sha", None)
2323

2424
volume_mount = {
25-
"mountPath": server.work_dir.absolute().as_posix(),
25+
"mountPath": server.work_dir.as_posix(),
2626
"name": "workspace",
2727
}
2828
if gl_project_path:
@@ -51,7 +51,7 @@ async def main(server: "UserServer") -> list[dict[str, Any]]:
5151
"env": [
5252
{
5353
"name": "GIT_RPC_MOUNT_PATH",
54-
"value": server.work_dir.absolute().as_posix(),
54+
"value": server.work_dir.as_posix(),
5555
},
5656
{
5757
"name": "GIT_RPC_PORT",

‎components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None:
3535
env = [
3636
{
3737
"name": f"{prefix}WORKSPACE_MOUNT_PATH",
38-
"value": server.workspace_mount_path.absolute().as_posix(),
38+
"value": server.workspace_mount_path.as_posix(),
3939
},
4040
{
4141
"name": f"{prefix}MOUNT_PATH",
42-
"value": server.work_dir.absolute().as_posix(),
42+
"value": server.work_dir.as_posix(),
4343
},
4444
{
4545
"name": f"{prefix}LFS_AUTO_FETCH",
@@ -134,7 +134,7 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None:
134134
},
135135
"volumeMounts": [
136136
{
137-
"mountPath": server.workspace_mount_path.absolute().as_posix(),
137+
"mountPath": server.workspace_mount_path.as_posix(),
138138
"name": amalthea_session_work_volume,
139139
},
140140
*etc_cert_volume_mount,
@@ -161,11 +161,11 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None:
161161
env = [
162162
{
163163
"name": f"{prefix}WORKSPACE_MOUNT_PATH",
164-
"value": server.workspace_mount_path.absolute().as_posix(),
164+
"value": server.workspace_mount_path.as_posix(),
165165
},
166166
{
167167
"name": f"{prefix}MOUNT_PATH",
168-
"value": server.work_dir.absolute().as_posix(),
168+
"value": server.work_dir.as_posix(),
169169
},
170170
{
171171
"name": f"{prefix}LFS_AUTO_FETCH",
@@ -260,7 +260,7 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None:
260260
},
261261
"volumeMounts": [
262262
{
263-
"mountPath": server.workspace_mount_path.absolute().as_posix(),
263+
"mountPath": server.workspace_mount_path.as_posix(),
264264
"name": "workspace",
265265
},
266266
*etc_cert_volume_mount,

‎components/renku_data_services/notebooks/api/amalthea_patches/jupyter_server.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def env(server: "UserServer") -> list[dict[str, Any]]:
4343
"path": "/statefulset/spec/template/spec/containers/0/env/-",
4444
"value": {
4545
"name": "NOTEBOOK_DIR",
46-
"value": server.work_dir.absolute().as_posix(),
46+
"value": server.work_dir.as_posix(),
4747
},
4848
},
4949
{
@@ -53,7 +53,7 @@ def env(server: "UserServer") -> list[dict[str, Any]]:
5353
# relative to $HOME.
5454
"value": {
5555
"name": "MOUNT_PATH",
56-
"value": server.work_dir.absolute().as_posix(),
56+
"value": server.work_dir.as_posix(),
5757
},
5858
},
5959
{
@@ -223,7 +223,7 @@ def rstudio_env_variables(server: "UserServer") -> list[dict[str, Any]]:
223223
"path": "/statefulset/spec/template/spec/containers/0/volumeMounts/-",
224224
"value": {
225225
"name": secret_name,
226-
"mountPath": mount_location.absolute().as_posix(),
226+
"mountPath": mount_location.as_posix(),
227227
"subPath": mount_location.name,
228228
"readOnly": True,
229229
},

‎components/renku_data_services/notebooks/api/classes/cloud_storage/__init__.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,15 @@
66
class ICloudStorageRequest(Protocol):
77
"""The abstract class for cloud storage."""
88

9-
exists: bool
109
mount_folder: str
11-
source_folder: str
12-
bucket: str
10+
source_path: str
1311

1412
def get_manifest_patch(
1513
self,
1614
base_name: str,
1715
namespace: str,
18-
labels: dict[str, str] = {},
19-
annotations: dict[str, str] = {},
16+
labels: dict[str, str] | None = None,
17+
annotations: dict[str, str] | None = None,
2018
) -> list[dict[str, Any]]:
2119
"""The patches applied to a jupyter server to insert the storage in the session."""
2220
...

‎components/renku_data_services/notebooks/api/classes/image.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import re
55
from dataclasses import dataclass, field
66
from enum import Enum
7-
from pathlib import Path
7+
from pathlib import PurePosixPath
88
from typing import Any, Optional, Self, cast
99

1010
import requests
@@ -101,7 +101,7 @@ def get_image_config(self, image: "Image") -> Optional[dict[str, Any]]:
101101
return None
102102
return cast(dict[str, Any], res.json())
103103

104-
def image_workdir(self, image: "Image") -> Optional[Path]:
104+
def image_workdir(self, image: "Image") -> Optional[PurePosixPath]:
105105
"""Query the docker API to get the workdir of an image."""
106106
config = self.get_image_config(image)
107107
if config is None:
@@ -112,7 +112,7 @@ def image_workdir(self, image: "Image") -> Optional[Path]:
112112
workdir = nested_config.get("WorkingDir", "/")
113113
if workdir == "":
114114
workdir = "/"
115-
return Path(workdir)
115+
return PurePosixPath(workdir)
116116

117117
def with_oauth2_token(self, oauth2_token: str) -> "ImageRepoDockerAPI":
118118
"""Return a docker API instance with the token as authentication."""

‎components/renku_data_services/notebooks/api/classes/k8s_client.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -353,10 +353,13 @@ def __init__(self, url: str, server_type: type[_SessionType]):
353353
self.url = url
354354
self.client = httpx.AsyncClient()
355355
self.server_type: type[_SessionType] = server_type
356+
self.url_path_name = "servers"
357+
if server_type == AmaltheaSessionV1Alpha1:
358+
self.url_path_name = "sessions"
356359

357360
async def list_servers(self, safe_username: str) -> list[_SessionType]:
358361
"""List the jupyter servers."""
359-
url = urljoin(self.url, f"/users/{safe_username}/servers")
362+
url = urljoin(self.url, f"/users/{safe_username}/{self.url_path_name}")
360363
try:
361364
res = await self.client.get(url, timeout=10)
362365
except httpx.RequestError as err:
@@ -374,7 +377,7 @@ async def list_servers(self, safe_username: str) -> list[_SessionType]:
374377

375378
async def get_server(self, name: str) -> _SessionType | None:
376379
"""Get a specific jupyter server."""
377-
url = urljoin(self.url, f"/servers/{name}")
380+
url = urljoin(self.url, f"/{self.url_path_name}/{name}")
378381
try:
379382
res = await self.client.get(url, timeout=10)
380383
except httpx.RequestError as err:
@@ -487,10 +490,9 @@ async def delete_server(self, server_name: str, safe_username: str) -> None:
487490
"""Delete the server."""
488491
server = await self.get_server(server_name, safe_username)
489492
if not server:
490-
raise MissingResourceError(
491-
f"Cannot find server {server_name} for user " f"{safe_username} in order to delete it."
492-
)
493-
return await self.renku_ns_client.delete_server(server_name)
493+
return None
494+
await self.renku_ns_client.delete_server(server_name)
495+
return None
494496

495497
async def patch_tokens(self, server_name: str, renku_tokens: RenkuTokens, gitlab_token: GitlabToken) -> None:
496498
"""Patch the Renku and Gitlab access tokens used in a session."""

‎components/renku_data_services/notebooks/api/classes/server.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from abc import ABC
44
from collections.abc import Sequence
55
from itertools import chain
6-
from pathlib import Path
6+
from pathlib import PurePosixPath
77
from typing import Any
88
from urllib.parse import urljoin, urlparse
99

@@ -44,8 +44,8 @@ def __init__(
4444
user_secrets: K8sUserSecrets | None,
4545
cloudstorage: Sequence[ICloudStorageRequest],
4646
k8s_client: K8sClient,
47-
workspace_mount_path: Path,
48-
work_dir: Path,
47+
workspace_mount_path: PurePosixPath,
48+
work_dir: PurePosixPath,
4949
config: _NotebooksConfig,
5050
internal_gitlab_user: APIUser,
5151
using_default_image: bool = False,
@@ -205,7 +205,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
205205
"pvc": {
206206
"enabled": True,
207207
"storageClassName": self.config.sessions.storage.pvs_storage_class,
208-
"mountPath": self.workspace_mount_path.absolute().as_posix(),
208+
"mountPath": self.workspace_mount_path.as_posix(),
209209
},
210210
}
211211
else:
@@ -214,7 +214,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
214214
"size": storage_size,
215215
"pvc": {
216216
"enabled": False,
217-
"mountPath": self.workspace_mount_path.absolute().as_posix(),
217+
"mountPath": self.workspace_mount_path.as_posix(),
218218
},
219219
}
220220
# Authentication
@@ -257,7 +257,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
257257
"jupyterServer": {
258258
"defaultUrl": self.server_options.default_url,
259259
"image": self.image,
260-
"rootDir": self.work_dir.absolute().as_posix(),
260+
"rootDir": self.work_dir.as_posix(),
261261
"resources": self.server_options.to_k8s_resources(
262262
enforce_cpu_limits=self.config.sessions.enforce_cpu_limits
263263
),
@@ -378,8 +378,8 @@ def __init__(
378378
user_secrets: K8sUserSecrets | None,
379379
cloudstorage: Sequence[ICloudStorageRequest],
380380
k8s_client: K8sClient,
381-
workspace_mount_path: Path,
382-
work_dir: Path,
381+
workspace_mount_path: PurePosixPath,
382+
work_dir: PurePosixPath,
383383
config: _NotebooksConfig,
384384
gitlab_project: Project | None,
385385
internal_gitlab_user: APIUser,
@@ -503,8 +503,8 @@ def __init__(
503503
user_secrets: K8sUserSecrets | None,
504504
cloudstorage: Sequence[ICloudStorageRequest],
505505
k8s_client: K8sClient,
506-
workspace_mount_path: Path,
507-
work_dir: Path,
506+
workspace_mount_path: PurePosixPath,
507+
work_dir: PurePosixPath,
508508
repositories: list[Repository],
509509
config: _NotebooksConfig,
510510
internal_gitlab_user: APIUser,

0 commit comments

Comments
 (0)