Merge pull request #23 from DenisaCG/contentsManager

Set up backend content manager

Author: DenisaCG

conftest.py

@@ -17,19 +17,19 @@ def drives_base_config():
 
 
 @pytest.fixture
-def drives_s3_config(drives_base_config):
+def drives_config(drives_base_config):
     return drives_base_config()
 
 
 @pytest.fixture
-def drives_s3_manager(drives_base_config):
-    from .jupyter_drives.managers.s3 import S3Manager
+def drives_manager(drives_base_config):
+    from .jupyter_drives.manager import JupyterDrivesManager
 
-    return S3Manager(drives_base_config)
+    return JupyterDrivesManager(drives_base_config)
 
 
 @pytest.fixture
-def drives_valid_s3_manager(drives_s3_manager):
-    drives_s3_manager._config.access_key_id = "valid"
-    drives_s3_manager._config.secret_access = "valid"
-    return drives_s3_manager
+def drives_valid_manager(drives_manager):
+    drives_manager._config.access_key_id = "valid"
+    drives_manager._config.secret_access = "valid"
+    return drives_manager

jupyter_drives/__init__.py

@@ -33,15 +33,14 @@ def _load_jupyter_server_extension(server_app):
         JupyterLab application instance
     """
     from .handlers import setup_handlers
-    from .base import DrivesConfig
 
     setup_handlers(server_app.web_app, server_app.config)
     name = "jupyter_drives"
     server_app.log.info(f"Registered {name} server extension")
 
 # Entry points
-def get_s3_manager(config: "traitlets.config.Config") -> "jupyter_drives.managers.JupyterDrivesManager":
-    """S3 Manager factory"""
-    from .managers.s3 import S3Manager
+def get_manager(config: "traitlets.config.Config") -> "jupyter_drives.managers.JupyterDrivesManager":
+    """Drives Manager factory"""
+    from .manager import JupyterDrivesManager
 
-    return S3Manager(config)
+    return JupyterDrivesManager(config)

jupyter_drives/base.py

@@ -7,9 +7,14 @@
 # Supported third-party services
 MANAGERS = {}
 
+# Moved to the architecture of having one provider independent manager.
+# Keeping the loop in case of future developments that need this feature.
 for entry in entrypoints.get_group_all("jupyter_drives.manager_v1"):
     MANAGERS[entry.name] = entry
 
+# Supported providers
+PROVIDERS = ['s3', 'gcs', 'http']
+
 class DrivesConfig(Configurable):
     """
     Allows configuration of supported drives via jupyter_notebook_config.py
@@ -65,7 +70,7 @@ def set_default_api_base_url(self):
             return "https://www.googleapis.com/"   
 
     provider = Enum(
-        MANAGERS.keys(),
+        PROVIDERS,
         default_value="s3",
         config=True,
         help="The source control provider.",

jupyter_drives/handlers.py

@@ -12,7 +12,7 @@
 import traitlets
 
 from .base import MANAGERS, DrivesConfig
-from .managers.manager import JupyterDrivesManager
+from .manager import JupyterDrivesManager
 
 NAMESPACE = "jupyter-drives"
 
@@ -59,7 +59,7 @@ async def get(self):
     async def post(self):
         body = self.get_json_body()
         result = await self._manager.mount_drive(**body)
-        self.finish(result["message"])
+        self.finish(result)
 
 class ContentsJupyterDrivesHandler(JupyterDrivesAPIHandler):
     """
@@ -99,7 +99,7 @@ def setup_handlers(web_app: tornado.web.Application, config: traitlets.config.Co
     log = log or logging.getLogger(__name__)
 
     provider = DrivesConfig(config=config).provider
-    entry_point = MANAGERS.get(provider)
+    entry_point = MANAGERS.get('drives_manager')
     if entry_point is None:
         log.error(f"JupyterDrives Manager: No manager defined for provider '{provider}'.")
         raise NotImplementedError()

jupyter_drives/managers/manager.py jupyter_drives/manager.pyjupyter_drives/managers/manager.py renamed to jupyter_drives/manager.py

@@ -1,23 +1,25 @@
-import abc
 import http 
 import json
 import logging
 from typing import Dict, List, Optional, Tuple, Union, Any
 
-import nbformat
 import tornado
-import traitlets
 import httpx
+import traitlets
 from jupyter_server.utils import url_path_join
 
-from ..log import get_logger
-from ..base import DrivesConfig
+import obstore as obs
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+from .log import get_logger
+from .base import DrivesConfig
 
 import re
 
-class JupyterDrivesManager(abc.ABC):
+class JupyterDrivesManager():
     """
-    Abstract base class for jupyter-drives manager.
+    Jupyter-drives manager class.
 
     Args:
         config: Server extension configuration object
@@ -26,12 +28,12 @@ class JupyterDrivesManager(abc.ABC):
 
     The manager will receive the global server configuration object;
     so it can add configuration parameters if needed.
-    It needs them to extract the ``DrivesConfig`` from it to pass it to this
-    parent class (see ``S3Manager`` for an example).
+    It needs them to extract the ``DrivesConfig``.
     """
-    def __init__(self, config: DrivesConfig) -> None:
-        self._config = config
+    def __init__(self, config: traitlets.config.Config) -> None:
+        self._config = DrivesConfig(config=config)
         self._client = httpx.AsyncClient()
+        self._content_managers = {}
 
     @property
     def base_api_url(self) -> str:
@@ -50,19 +52,56 @@ def per_page_argument(self) -> Optional[Tuple[str, int]]:
             [str, int]: (query argument name, value)
             None: the provider does not support pagination
         """
-        return None
+        return ("per_page", 100)
 
-    @abc.abstractclassmethod
     async def list_drives(self): 
         """Get list of available drives.
 
         Returns: 
             List of available drives and their properties.
         """
-        raise NotImplementedError()
+        data = []
+        if self._config.access_key_id and self._config.secret_access_key:
+            if self._config.provider == "s3":
+                S3Drive = get_driver(Provider.S3)
+                drives = [S3Drive(self._config.access_key_id, self._config.secret_access_key)]
+
+            elif self._config.provider == 'gcs':
+                GCSDrive = get_driver(Provider.GOOGLE_STORAGE)
+                drives = [GCSDrive(self._config.access_key_id, self._config.secret_access_key)] # verfiy credentials needed
+            
+            else: 
+               raise tornado.web.HTTPError(
+                status_code= httpx.codes.NOT_IMPLEMENTED,
+                reason="Listing drives not supported for given provider.",
+                )
+
+            results = []
+            for drive in drives:
+                results += drive.list_containers()
+            
+            for result in results:
+                data.append(
+                    {
+                        "name": result.name,
+                        "region": self._config.region_name if self._config.region_name is not None else "eu-north-1",
+                        "creation_date": result.extra["creation_date"],
+                        "mounted": "true" if result.name not in self._content_managers else "false",
+                        "provider": self._config.provider
+                    }
+                )
+        else:
+            raise tornado.web.HTTPError(
+            status_code= httpx.codes.BAD_REQUEST,
+            reason="No credentials specified. Please set them in your user jupyter_server_config file.",
+            )
+        
+        response = {
+            "data": data
+        }
+        return response
 
-    @abc.abstractclassmethod
-    async def mount_drive(self, drive_name, **kwargs):
+    async def mount_drive(self, drive_name, provider, region):
         """Mount a drive.
 
         Args:
@@ -71,46 +110,75 @@ async def mount_drive(self, drive_name, **kwargs):
         Returns:
             The content manager for the drive.
         """
-        raise NotImplementedError()
+        try: 
+            # check if content manager doesn't already exist
+            if drive_name not in self._content_managers or self._content_managers[drive_name] is None:
+                if provider == 's3':
+                    store = obs.store.S3Store.from_url("s3://" + drive_name + "/", config = {"aws_access_key_id": self._config.access_key_id, "aws_secret_access_key": self._config.secret_access_key, "aws_region": region})
+                elif provider == 'gcs':
+                    store = obs.store.GCSStore.from_url("gs://" + drive_name + "/", config = {}) # add gcs config
+                elif provider == 'http':
+                    store = obs.store.HTTPStore.from_url(drive_name, client_options = {}) # add http client config
+                
+                self._content_managers[drive_name] = store
+
+            else:
+                raise tornado.web.HTTPError(
+                status_code= httpx.codes.CONFLICT,
+                reason= "Drive already mounted."
+                )
+                
+        except Exception as e:
+            raise tornado.web.HTTPError(
+            status_code= httpx.codes.BAD_REQUEST,
+            reason= f"The following error occured when mouting the drive: {e}"
+            )
+
+        return 
 
-    @abc.abstractclassmethod
-    async def unmount_drive(self, drive_name: str, **kwargs):
+    async def unmount_drive(self, drive_name: str):
         """Unmount a drive.
 
         Args:
             drive_name: name of drive to unmount
         """
-        raise NotImplementedError()
+        if drive_name in self._content_managers:
+            self._content_managers.pop(drive_name, None)
+
+        else:
+            raise tornado.web.HTTPError(
+            status_code= httpx.codes.NOT_FOUND,
+            reason="Drive is not mounted or doesn't exist.",
+            )
+        
+        return
 
-    @abc.abstractclassmethod
     async def get_contents(self, drive_name, path, **kwargs):
         """Get contents of a file or directory.
 
         Args:
             drive_name: name of drive to get the contents of
             path: path to file or directory
         """
-        raise NotImplementedError()
+        print('Get contents function called.')
 
-    @abc.abstractclassmethod
     async def new_file(self, drive_name, path, **kwargs):
         """Create a new file or directory at the given path.
         
         Args:
             drive_name: name of drive where the new content is created
             path: path where new content should be created
         """
-        raise NotImplementedError()
+        print('New file function called.')
 
-    @abc.abstractclassmethod
     async def rename_file(self, drive_name, path, **kwargs):
         """Rename a file.
         
         Args:
             drive_name: name of drive where file is located
             path: path of file
         """
-        raise NotImplementedError()
+        print('Rename file function called.')
 
     async def _call_provider(
         self,