diff --git a/cterasdk/asynchronous/core/files/browser.py b/cterasdk/asynchronous/core/files/browser.py index 1f33b1cb..75af9ad2 100644 --- a/cterasdk/asynchronous/core/files/browser.py +++ b/cterasdk/asynchronous/core/files/browser.py @@ -1,275 +1,337 @@ from .. import query -from ....cio.core import CorePath, Open, OpenMany, Upload, UploadFile, Download, \ - DownloadMany, UnShare, CreateDirectory, GetMetadata, ListVersions, RecursiveIterator, \ +from ....cio.core.commands import Open, OpenMany, Upload, Download, EnsureDirectory, \ + DownloadMany, UnShare, CreateDirectory, GetMetadata, GetProperties, ListVersions, RecursiveIterator, \ Delete, Recover, Rename, GetShareMetadata, Link, Copy, Move, ResourceIterator, GetPermalink from ..base_command import BaseCommand +from ....lib.storage import commonfs from . import io class FileBrowser(BaseCommand): - - def __init__(self, core): - super().__init__(core) - self._scope = f'/{self._core.context}/webdav' + """Async File Browser API.""" async def handle(self, path): """ - Get File Handle. + Get a file handle. - :param str path: Path to a file + :param str path: Path to a file. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.core.OpenError: Raised on error to obtain file handle. """ - return await Open(io.handle, self._core, self.normalize(path)).a_execute() + return await Open(io.handle, self._core, path).a_execute() async def handle_many(self, directory, *objects): """ - Get a Zip Archive File Handle. + Get a ZIP archive file handle. - :param str directory: Path to a folder - :param args objects: List of files and folders + :param str directory: Path to a folder. + :param args objects: List of files and folders to include. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.core.GetMetadataError: If directory not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If target path is not a directory. """ - return await OpenMany(io.handle_many, self._core, self.normalize(directory), *objects).a_execute() + async with EnsureDirectory(io.listdir, self._core, directory) as (_, resource): + return await OpenMany(io.handle_many, self._core, resource, directory, *objects).a_execute() async def download(self, path, destination=None): """ - Download a file + Download a file. - :param str path: Path - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str path: Path. + :param str, optional destination: File destination. If directory, original filename preserved. Defaults to default directory. + :returns: Path to local file. + :rtype: str + :raises cterasdk.exceptions.io.core.OpenError: Raised on error to obtain file handle. """ - return await Download(io.handle, self._core, self.normalize(path), destination).a_execute() + return await Download(io.handle, self._core, path, destination).a_execute() - async def download_many(self, target, objects, destination=None): + async def download_many(self, directory, objects, destination=None): """ Download selected files and/or directories as a ZIP archive. .. warning:: - The provided list of objects is not validated. Only existing files and directories - will be included in the resulting ZIP file. + Only existing files and directories will be included in the resulting ZIP file. + + :param str directory: Path to a folder. + :param list[str] objects: List of files and / or directory names to download. + :param str destination: Optional path to destination file or directory. Defaults to default download directory. + :returns: Path to local file. + :rtype: str + :raises cterasdk.exceptions.io.core.GetMetadataError: If directory not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If target path is not a directory. + """ + async with EnsureDirectory(io.listdir, self._core, directory) as (_, resource): + return await DownloadMany(io.handle_many, self._core, resource, directory, objects, destination).a_execute() - :param str target: - Path to the cloud folder containing the files and directories to download. - :param list[str] objects: - List of file and/or directory names to include in the download. - :param str destination: - Optional. Path to the destination file or directory. If a directory is provided, - the original filename will be preserved. Defaults to the default download directory. + async def listdir(self, path=None, include_deleted=False): """ - return await DownloadMany(io.handle_many, self._core, self.normalize(target), objects, destination).a_execute() + List directory contents. - async def listdir(self, path=None, depth=None, include_deleted=False): + :param str, optional path: Path, defaults to Cloud Drive root. + :param bool, optional include_deleted: Include deleted files. Defaults to False. + :returns: Directory contents. + :rtype: AsyncIterator[cterasdk.cio.core.types.PortalResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If directory not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If target path is not a directory. + :raises cterasdk.exceptions.io.core.ListDirectoryError: Raised on error fetching directory contents. """ - List Directory + async with EnsureDirectory(io.listdir, self._core, path): + async for o in ResourceIterator(query.iterator, self._core, path, None, include_deleted, None, None).a_execute(): + yield o - :param str,optional path: Path, defaults to the Cloud Drive root - :param bool,optional include_deleted: Include deleted files, defaults to False + async def properties(self, path): + """ + Get object properties. + + :param str path: Path. + :returns: Object properties. + :rtype: cterasdk.cio.core.types.PortalResource + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error retrieving object metadata. """ - async for o in ResourceIterator(query.iterator, self._core, self.normalize(path), depth, include_deleted, None, None).a_execute(): - yield o + return await GetProperties(io.listdir, self._core, path, False).a_execute() async def exists(self, path): """ - Check if item exists + Check if item exists. - :param str path: Path + :param str path: Path. + :returns: True if exists, False otherwise. + :rtype: bool """ - async with GetMetadata(io.listdir, self._core, self.normalize(path), True) as (exists, *_): + async with GetMetadata(io.listdir, self._core, path, True) as (exists, *_): return exists async def versions(self, path): """ - List snapshots of a file or directory + List snapshots of a file or directory. - :param str path: Path + :param str path: Path. + :returns: List of versions. + :rtype: list[cterasdk.cio.core.types.PreviousVersion] + :raises cterasdk.exceptions.io.core.GetVersionsError: Raised on error retrieving versions. """ - return await ListVersions(io.versions, self._core, self.normalize(path)).a_execute() + return await ListVersions(io.versions, self._core, path).a_execute() async def walk(self, path=None, include_deleted=False): """ - Walk Directory Contents + Walk directory contents. - :param str,optional path: Path to walk, defaults to the root directory - :param bool,optional include_deleted: Include deleted files, defaults to False + :param str, optional path: Path to walk, defaults to root directory. + :param bool, optional include_deleted: Include deleted files. Defaults to False. + :returns: Async generator of file-system objects. + :rtype: AsyncIterator[cterasdk.cio.edge.types.PortalResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If directory not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If target path is not a directory. """ - async for o in RecursiveIterator(query.iterator, self._core, self.normalize(path), include_deleted).a_generate(): - yield o + async with EnsureDirectory(io.listdir, self._core, path): + async for o in RecursiveIterator(query.iterator, self._core, path, include_deleted).a_generate(): + yield o async def public_link(self, path, access='RO', expire_in=30): """ - Create a public link to a file or a folder + Create a public link to a file or folder. - :param str path: The path of the file to create a link to - :param str,optional access: Access policy of the link, defaults to 'RO' - :param int,optional expire_in: Number of days until the link expires, defaults to 30 + :param str path: Path of file/folder. + :param str, optional access: Access policy. Defaults 'RO'. + :param int, optional expire_in: Days until link expires. Defaults 30. + :returns: Public link. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateLinkError: Raised on failure generating public link. """ - return await Link(io.public_link, self._core, self.normalize(path), access, expire_in).a_execute() + return await Link(io.public_link, self._core, path, access, expire_in).a_execute() async def copy(self, *paths, destination=None, resolver=None, cursor=None, wait=False): """ - Copy one or more files or folders + Copy one or more files or folders. - :param list[str] paths: List of paths - :param str destination: Destination - :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` - :param cterasdk.common.object.Object cursor: Resume copy from cursor - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to copy. + :param str destination: Destination path. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults None. + :param cterasdk.common.object.Object, optional cursor: Resume copy from cursor. + :param bool, optional wait: Wait for task completion. Defaults False. + :returns: Task status object or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.CopyError: Raised on failure copying resources. """ try: - return await Copy(io.copy, self._core, wait, *[self.normalize(path) for path in paths], - destination=self.normalize(destination), resolver=resolver, cursor=cursor).a_execute() + return await Copy(io.copy, self._core, wait, *paths, destination=destination, resolver=resolver, cursor=cursor).a_execute() except ValueError: raise ValueError('Copy destination was not specified.') async def permalink(self, path): """ - Get Permalink for Path. + Get permalink for a path. :param str path: Path. + :returns: Permalink. + :rtype: str + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error retrieving object metadata. """ - return await GetPermalink(io.listdir, self._core, self.normalize(path)).a_execute() - - def normalize(self, entries): - return CorePath.instance(self._scope, entries) + return await GetPermalink(io.listdir, self._core, path).a_execute() class CloudDrive(FileBrowser): + """Async CloudDrive API with upload and sharing functionality.""" - async def upload(self, name, destination, handle, size=None): + async def upload(self, destination, handle, name=None, size=None): """ Upload from file handle. - :param str name: File name. - :param str destination: Path to remote directory. - :param object handle: Handle. - :param str,optional size: File size, defaults to content length + :param str destination: Remote path. + :param object handle: File-like handle. + :param str, optional size: File size. Defaults to content length. + :param str, optional name: Filename to use if it cannot be derived from ``destination`` + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.core.UploadError: Raised on upload failure. """ - return await Upload(io.upload, self._core, io.listdir, name, self.normalize(destination), size, handle).a_execute() + return await Upload(io.upload, self._core, io.listdir, destination, handle, name, size).a_execute() async def upload_file(self, path, destination): """ Upload a file. - :param str path: Local path - :param str destination: Remote path + :param str path: Local path. + :param str destination: Remote path. + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.core.UploadError: Raised on upload failure. """ - return await UploadFile(io.upload, self._core, io.listdir, path, self.normalize(destination)).a_execute() + _, name = commonfs.split_file_directory(path) + with open(path, 'rb') as handle: + return await self.upload(destination, handle, name, commonfs.properties(path)['size']) async def mkdir(self, path): """ - Create a new directory + Create a directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateDirectoryError: Raised on error creating directory. """ - return await CreateDirectory(io.mkdir, self._core, self.normalize(path)).a_execute() + return await CreateDirectory(io.mkdir, self._core, path).a_execute() async def makedirs(self, path): """ - Create a directory recursively + Recursively create a directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateDirectoryError: Raised on error creating directory. """ - return await CreateDirectory(io.mkdir, self._core, self.normalize(path), True).a_execute() + return await CreateDirectory(io.mkdir, self._core, path, True).a_execute() - async def rename(self, path, name, *, wait=False): + async def rename(self, path, name, *, resolver=None, wait=False): """ - Rename a file + Rename a file or folder. - :param str path: Path of the file or directory to rename - :param str name: The name to rename to - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param str path: Path to rename. + :param str name: New name. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults None. + :param bool, optional wait: Wait for task completion. Defaults False. + :returns: Task status object or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.RenameError: Raised on error renaming object. """ - return await Rename(io.move, self._core, self.normalize(path), name, wait).a_execute() + return await Rename(io.move, self._core, wait, path, name, resolver).a_execute() async def delete(self, *paths, wait=False): """ - Delete one or more files or folders + Delete one or more files or folders. - :param str path: Path - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to delete. + :param bool, optional wait: Wait for task completion. Defaults False. + :returns: Task status object or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.DeleteError: Raised on error deleting resources. """ - return await Delete(io.delete, self._core, wait, *[self.normalize(path) for path in paths]).a_execute() + return await Delete(io.delete, self._core, wait, *paths).a_execute() async def undelete(self, *paths, wait=False): """ - Recover one or more files or folders + Recover one or more files or folders. - :param str path: Path - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to recover. + :param bool, optional wait: Wait for task completion. Defaults False. + :returns: Task status object or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.RecoverError: Raised on error recovering resources. """ - return await Recover(io.undelete, self._core, wait, *[self.normalize(path) for path in paths]).a_execute() + return await Recover(io.undelete, self._core, wait, *paths).a_execute() async def move(self, *paths, destination=None, resolver=None, cursor=None, wait=False): """ - Move one or more files or folders + Move one or more files or folders. - :param list[str] paths: List of paths - :param str destination: Destination - :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` - :param cterasdk.common.object.Object cursor: Resume copy from cursor - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to move. + :param str destination: Destination path. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults None. + :param cterasdk.common.object.Object, optional cursor: Resume move from cursor. + :param bool, optional wait: Wait for task completion. Defaults False. + :returns: Task status object or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.MoveError: Raised on error moving resources. """ try: - return await Move(io.move, self._core, wait, *[self.normalize(path) for path in paths], - destination=self.normalize(destination), resolver=resolver, cursor=cursor).a_execute() + return await Move(io.move, self._core, wait, *paths, destination=destination, resolver=resolver, cursor=cursor).a_execute() except ValueError: raise ValueError('Move destination was not specified.') async def get_share_info(self, path): """ - Get share settings and recipients + Get share settings and recipients. - :param str path: Path + :param str path: Path. + :returns: Share metadata. + :rtype: object + :raises cterasdk.exceptions.io.core.GetShareMetadataError: Raised on error obtaining share metadata. """ - return await GetShareMetadata(io.list_shares, self._core, self.normalize(path)).a_execute() + return await GetShareMetadata(io.list_shares, self._core, path).a_execute() async def share(self, path, recipients, as_project=True, allow_reshare=True, allow_sync=True): """ - Share a file or a folder + Share a file or folder. - :param str path: The path of the file or folder to share - :param list[cterasdk.core.types.Collaborator] recipients: A list of share recipients - :param bool,optional as_project: Share as a team project, defaults to True when the item is a cloud folder else False - :param bool,optional allow_reshare: Allow recipients to re-share this item, defaults to True - :param bool,optional allow_sync: Allow recipients to sync this item, defaults to True when the item is a cloud folder else False - :return: A list of all recipients added to the collaboration share + :param str path: Path to file/folder. + :param list[cterasdk.core.types.Collaborator] recipients: Recipients to share with. + :param bool, optional as_project: Share as team project. Defaults True if cloud folder. + :param bool, optional allow_reshare: Allow re-share. Defaults True. + :param bool, optional allow_sync: Allow sync. Defaults True if cloud folder. + :returns: Current list of share members. :rtype: list[cterasdk.core.types.Collaborator] """ - return await io.share(self._core, self.normalize(path), recipients, as_project, allow_reshare, allow_sync) + return await io.share(self._core, path, recipients, as_project, allow_reshare, allow_sync) async def add_share_recipients(self, path, recipients): """ - Add share recipients + Add share recipients. - :param str path: The path of the file or folder - :param list[cterasdk.core.types.Collaborator] recipients: A list of share recipients - :return: A list of all recipients added + :param str path: Path of file/folder. + :param list[cterasdk.core.types.Collaborator] recipients: Recipients to add. + :returns: Current list of share members. :rtype: list[cterasdk.core.types.Collaborator] """ - return await io.add_share_recipients(self._core, self.normalize(path), recipients) + return await io.add_share_recipients(self._core, path, recipients) async def remove_share_recipients(self, path, accounts): """ - Remove share recipients + Remove share recipients. - :param str path: The path of the file or folder - :param list[cterasdk.core.types.PortalAccount] accounts: A list of portal user or group accounts - :return: A list of all share recipients removed - :rtype: list[cterasdk.core.types.PortalAccount] + :param str path: Path of file/folder. + :param list[cterasdk.core.types.PortalAccount] accounts: Accounts to remove. + :returns: Current list of share members. + :rtype: list[cterasdk.core.types.Collaborator] """ - return await io.remove_share_recipients(self._core, self.normalize(path), accounts) + return await io.remove_share_recipients(self._core, path, accounts) async def unshare(self, path): """ - Unshare a file or a folder + Unshare a file or folder. + + :param str path: Path of file/folder. """ - return await UnShare(io.update_share, self._core, self.normalize(path)).a_execute() + return await UnShare(io.update_share, self._core, path).a_execute() diff --git a/cterasdk/asynchronous/core/files/io.py b/cterasdk/asynchronous/core/files/io.py index 23dfc18c..adead1a5 100644 --- a/cterasdk/asynchronous/core/files/io.py +++ b/cterasdk/asynchronous/core/files/io.py @@ -1,4 +1,4 @@ -from ....cio import core as fs +from ....cio.core import commands as fs async def listdir(core, param): @@ -33,9 +33,8 @@ async def handle(core, param): return await core.io.download(param) -async def handle_many(core, param, directory): - async with fs.EnsureDirectory(listdir, core, directory) as (_, resource): - return await core.io.download_zip(str(resource.cloudFolderInfo.uid), param) +async def handle_many(core, cloudfolder, param): + return await core.io.download_zip(cloudfolder, param) async def upload(core, cloudfolder, param): diff --git a/cterasdk/asynchronous/edge/files/browser.py b/cterasdk/asynchronous/edge/files/browser.py index 9e372051..58450c2d 100644 --- a/cterasdk/asynchronous/edge/files/browser.py +++ b/cterasdk/asynchronous/edge/files/browser.py @@ -1,65 +1,100 @@ from ..base_command import BaseCommand -from ....cio.edge import EdgePath, ListDirectory, RecursiveIterator, GetMetadata, Open, OpenMany, Upload, \ - UploadFile, CreateDirectory, Copy, Move, Delete, Download, DownloadMany +from ....cio.edge.commands import ListDirectory, RecursiveIterator, GetMetadata, Open, OpenMany, Upload, \ + CreateDirectory, Copy, Move, Delete, Download, DownloadMany, Rename, EnsureDirectory +from ....lib.storage import commonfs from . import io class FileBrowser(BaseCommand): - """ Edge Filer File Browser APIs """ + """Edge Filer File Browser API.""" async def listdir(self, path): """ - List Directory + List directory contents. - :param str path: Path + :param str path: Path. + :returns: Directory contents. + :rtype: AsyncIterator[cterasdk.cio.edge.types.EdgeResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. + :raises cterasdk.exceptions.io.edge.ListDirectoryError: Raised on error to fetch directory contents. """ - for o in await ListDirectory(io.listdir, self._edge, self.normalize(path)).a_execute(): - yield o + async with EnsureDirectory(io.listdir, self._edge, path): + for o in await ListDirectory(io.listdir, self._edge, path).a_execute(): + yield o async def walk(self, path=None): """ - Walk Directory Contents + Walk directory contents. - :param str, defaults to the root directory path: Path to walk + :param str, optional path: Path to walk. Defaults to the root directory. + :returns: A generator of file-system objects. + :rtype: AsyncIterator[cterasdk.cio.edge.types.EdgeResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. """ - async for o in RecursiveIterator(io.listdir, self._edge, self.normalize(path)).a_generate(): - yield o + async with EnsureDirectory(io.listdir, self._edge, path): + async for o in RecursiveIterator(io.listdir, self._edge, path).a_generate(): + yield o + + async def properties(self, path): + """ + Get object properties. + + :param str path: Path. + :returns: Object properties. + :rtype: cterasdk.cio.edge.types.EdgeResource + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error to obtain object metadata. + """ + async with GetMetadata(io.listdir, self._edge, path, False) as (_, metadata): + return metadata async def exists(self, path): """ - Check if item exists + Check whether an item exists. - :param str path: Path + :param str path: Path. + :returns: ``True`` if the item exists, ``False`` otherwise. + :rtype: bool """ - async with GetMetadata(io.listdir, self._edge, self.normalize(path), True) as (exists, *_): + async with GetMetadata(io.listdir, self._edge, path, True) as (exists, *_): return exists async def handle(self, path): """ - Get File Handle. + Get file handle. - :param str path: Path to a file + :param str path: Path to a file. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.edge.OpenError: Raised on error to obtain a file handle. """ - return await Open(io.handle, self._edge, self.normalize(path)).a_execute() + return await Open(io.handle, self._edge, path).a_execute() async def handle_many(self, directory, *objects): """ - Get a Zip Archive File Handle. + Get a ZIP archive file handle. - :param str directory: Path to a folder - :param args objects: List of files and folders + :param str directory: Path to a folder. + :param args objects: Files and folders to include. + :returns: File handle. + :rtype: object """ return await OpenMany(io.handle_many, self._edge, directory, *objects).a_execute() async def download(self, path, destination=None): """ - Download a file + Download a file. - :param str path: The file path on the Edge Filer - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str path: The file path on the Edge Filer. + :param str, optional destination: + File destination. If a directory is provided, the original filename is preserved. + Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str + :raises cterasdk.exceptions.io.edge.OpenError: Raised on error to obtain a file handle. """ - return await Download(io.handle, self._edge, self.normalize(path), destination).a_execute() + return await Download(io.handle, self._edge, path, destination).a_execute() async def download_many(self, target, objects, destination=None): """ @@ -75,77 +110,105 @@ async def download_many(self, target, objects, destination=None): List of file and/or directory names to include in the download. :param str destination: Optional. Path to the destination file or directory. If a directory is provided, - the original filename will be preserved. Defaults to the default download directory. + the original filename is preserved. Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str """ - return await DownloadMany(io.handle_many, self._edge, self.normalize(target), objects, destination).a_execute() + return await DownloadMany(io.handle_many, self._edge, target, objects, destination).a_execute() - async def upload(self, name, destination, handle): + async def upload(self, destination, handle, name=None): """ - Upload from file handle. + Upload from a file handle. - :param str name: File name. - :param str destination: Path to remote directory. - :param object handle: Handle. + :param str destination: Remote path. + :param object handle: File-like handle. + :param str, optional name: Filename to use if it cannot be derived from ``destination`` + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.edge.UploadError: Raised on upload failure. """ - return await Upload(io.upload, self._edge, io.listdir, name, self.normalize(destination), handle).a_execute() + return await Upload(io.upload, self._edge, io.listdir, destination, handle, name).a_execute() async def upload_file(self, path, destination): """ Upload a file. - :param str path: Local path - :param str destination: Remote path + :param str path: Local path. + :param str destination: Remote path. + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.edge.UploadError: Raised on upload failure. """ - return await UploadFile(io.upload, self._edge, io.listdir, path, self.normalize(destination)).a_execute() + _, name = commonfs.split_file_directory(path) + with open(path, 'rb') as handle: + return await self.upload(destination, handle, name) async def mkdir(self, path): """ - Create a new directory + Create a new directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.edge.CreateDirectoryError: Raised on error to create a directory. """ - return await CreateDirectory(io.mkdir, self._edge, self.normalize(path)).a_execute() + return await CreateDirectory(io.mkdir, self._edge, path).a_execute() async def makedirs(self, path): """ - Create a directory recursively + Create a directory recursively. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.edge.CreateDirectoryError: Raised on error to create a directory. """ - return await CreateDirectory(io.mkdir, self._edge, self.normalize(path), True).a_execute() + return await CreateDirectory(io.mkdir, self._edge, path, True).a_execute() async def copy(self, path, destination=None, overwrite=False): """ - Copy a file or a folder + Copy a file or directory. - :param str path: Source file or folder path - :param str destination: Destination folder path - :param bool,optional overwrite: Overwrite on conflict, defaults to False + :param str path: Source file or directory path. + :param str destination: Destination directory path. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :raises cterasdk.exceptions.io.edge.CopyError: Raised on error to copy a file or directory. """ if destination is None: raise ValueError('Copy destination was not specified.') - return await Copy(io.copy, self._edge, self.normalize(path), self.normalize(destination), overwrite).a_execute() + return await Copy(io.copy, self._edge, io.listdir, path, destination, overwrite).a_execute() async def move(self, path, destination=None, overwrite=False): """ - Move a file or a folder + Move a file or directory. - :param str path: Source file or folder path - :param str destination: Destination folder path - :param bool,optional overwrite: Overwrite on conflict, defaults to False + :param str path: Source file or directory path. + :param str destination: Destination directory path. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :raises cterasdk.exceptions.io.edge.MoveError: Raised on error to move a file or directory. """ if destination is None: raise ValueError('Move destination was not specified.') - return await Move(io.move, self._edge, self.normalize(path), self.normalize(destination), overwrite).a_execute() + return await Move(io.move, self._edge, io.listdir, path, destination, overwrite).a_execute() - async def delete(self, path): + async def rename(self, path, name, overwrite=False): """ - Delete a file + Rename a file or directory. - :param str path: File path + :param str path: Path of the file or directory to rename. + :param str name: New name for the file or directory. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :returns: Remote object path. + :rtype: str + :raises cterasdk.exceptions.io.edge.RenameError: Raised on error to rename a file or directory. """ - return await Delete(io.delete, self._edge, self.normalize(path)).a_execute() + return await Rename(io.move, self._edge, io.listdir, path, name, overwrite).a_execute() - @staticmethod - def normalize(path): - return EdgePath('/', path) + async def delete(self, path): + """ + Delete a file or directory. + + :param str path: File or directory path. + :raises cterasdk.exceptions.io.edge.DeleteError: Raised on error to delete a file or directory. + """ + return await Delete(io.delete, self._edge, path).a_execute() diff --git a/cterasdk/cio/common.py b/cterasdk/cio/common.py index a4f579a1..bc9af643 100644 --- a/cterasdk/cio/common.py +++ b/cterasdk/cio/common.py @@ -1,5 +1,6 @@ +import urllib.parse from pathlib import PurePosixPath -from ..objects.uri import quote +from ..common import Object from ..common.utils import utf8_decode from ..convert.serializers import toxmlstr @@ -33,13 +34,17 @@ def reference(self): def relative(self): return self._reference.as_posix() + @property + def relative_encode(self): + return urllib.parse.quote(self._reference.as_posix()) + @property def name(self): return self._reference.name @property def parent(self): - return self.__class__(self._scope.as_posix(), self._reference.parent.as_posix()) + return self.__class__(self._reference.parent.as_posix()) # pylint: disable=no-value-for-parameter @property def absolute(self): @@ -47,26 +52,59 @@ def absolute(self): @property def absolute_encode(self): - return f'/{self.scope.joinpath(quote(self.reference.as_posix())).as_posix()}' + return f'/{self.scope.joinpath(urllib.parse.quote(self.reference.as_posix())).as_posix()}' @property def absolute_parent(self): return self.parent.as_posix() + @property + def extension(self): + return self.reference.suffix + def join(self, p): """ Join Path. :param str p: Path. """ - return self.__class__(self._scope.as_posix(), self.reference.joinpath(p).as_posix()) + return self.__class__(self.reference.joinpath(p).as_posix()) # pylint: disable=no-value-for-parameter @property def parts(self): return self.reference.parts + def __eq__(self, p): + return self.absolute == p.absolute + + def __str__(self): + return self.relative + + +class BaseResource(Object): + """ + Class for a Filesystem Resource. + + :ivar str name: Resource name + :ivar cterasdk.cio.common.BasePath path: Path Object + :ivar bool is_dir: ``True`` if directory, ``False`` otherwise + :ivar int size: Size + :ivar datetime.datetime last_modified: Last Modified + :ivar str extension: Extension + """ + def __init__(self, name, path, is_dir, size, last_modified): + super().__init__(name=name, path=path, is_dir=is_dir, size=size, last_modified=last_modified, extension=path.extension) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"'is_dir': {self.is_dir}, " + f"'size': {self.size}, " + f"'path': {self.path}}})" + ) + def __str__(self): - return self.absolute + return str(self.path) def encode_stream(fd, size): diff --git a/cterasdk/cio/core/__init__.py b/cterasdk/cio/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cterasdk/cio/core.py b/cterasdk/cio/core/commands.py similarity index 67% rename from cterasdk/cio/core.py rename to cterasdk/cio/core/commands.py index 4a95bf28..183e1396 100644 --- a/cterasdk/cio/core.py +++ b/cterasdk/cio/core/commands.py @@ -5,226 +5,95 @@ import asyncio from abc import abstractmethod from contextlib import contextmanager -from ..objects.uri import quote, unquote -from ..common import Object, DateTimeUtils -from ..core.enum import ProtectionLevel, CollaboratorType, SearchType, PortalAccountType, FileAccessMode, \ +from ...common import Object, DateTimeUtils +from ...core.enum import ProtectionLevel, CollaboratorType, SearchType, PortalAccountType, FileAccessMode, \ UploadError, ResourceScope, ResourceError -from ..core.types import PortalAccount, UserAccount, GroupAccount, Collaborator -from .. import exceptions -from ..lib.iterator import DefaultResponse -from ..lib.storage import synfs, asynfs, commonfs -from . import common -from .actions import PortalCommand +from ...core.types import PortalAccount, UserAccount, GroupAccount, Collaborator +from ... import exceptions +from ...lib.storage import synfs, asynfs, commonfs +from .types import SrcDstParam, CreateShareParam, ActionResourcesParam, FetchResourcesError, \ + FetchResourcesParamBuilder, FetchResourcesResponse, PortalResource, PreviousVersion, automatic_resolution +from ..common import encode_request_parameter, encode_stream +from ..actions import PortalCommand logger = logging.getLogger('cterasdk.core') -class CorePath(common.BasePath): - """Path for CTERA Portal""" +def split_file_directory(listdir, receiver, destination): + """ + Split a path into its parent directory and final component. - def __init__(self, scope, reference): - """ - Initialize a CTERA Portal Path. + :returns: + tuple[str, str]: A ``(parent_directory, name)`` tuple when: - :param str scope: Scope. - :param str reference: Reference. - """ - if isinstance(reference, Object): - super().__init__(*CorePath._from_server_object(reference)) - elif isinstance(reference, str): - super().__init__(scope, reference) - elif reference is None: - super().__init__(scope, '') - else: - message = 'Path validation failed: ensure the path exists and is correctly formatted.' - logger.error(message) - raise ValueError(message) - - @staticmethod - def _from_server_object(reference): - """ - Parse Path from Server Object. - - :param object reference: Path. - :returns: Base Path and Relative Path - :rtype: tuple(str) - """ - classname = reference.__dict__.get('_classname', None) - - href = None - if classname == 'ResourceInfo': - href = reference.href - elif classname == 'SnapshotResp': - href = f'{reference.url}{reference.path}' - else: - raise ValueError(f'Could not determine server object: {classname}') - - href = unquote(href) - match = re.search('^/?(ServicesPortal|admin)/webdav', href) - start, end = match.span() - return (href[start: end], href[end + 1:]) - - @property - def absolute(self): - reference = self.relative - previous_versions = 'PreviousVersions/' - if previous_versions in reference: - index = reference.index(previous_versions) + len(previous_versions) - return f'{self.scope.as_posix()}/{quote(reference[:index]) + reference[index:]}' - return super().absolute - - @staticmethod - def instance(scope, entries): - if isinstance(entries, list): - return [CorePath(scope, e) for e in entries] - if isinstance(entries, tuple): - source, destination = entries - return (CorePath(scope, source), CorePath(scope, destination)) - return CorePath(scope, entries) - - -class SrcDstParam(Object): - - __instance = None - - @staticmethod - def instance(src, dest=None): - SrcDstParam(src, dest) - return SrcDstParam.__instance - - def __init__(self, src, dest=None): - super().__init__() - self._classname = self.__class__.__name__ - self.src = src - self.dest = dest - SrcDstParam.__instance = self # pylint: disable=unused-private-member - - -class ResourceActionCursor(Object): - - def __init__(self): - super().__init__() - self._classname = self.__class__.__name__ - - -class ActionResourcesParam(Object): - - __instance = None - - @staticmethod - def instance(): - ActionResourcesParam() - return ActionResourcesParam.__instance - - def __init__(self): - super().__init__() - self._classname = self.__class__.__name__ - self.urls = [] - self.startFrom = None - ActionResourcesParam.__instance = self # pylint: disable=unused-private-member - - def add(self, param): - self.urls.append(param) - - def start_from(self, cursor): - self.startFrom = cursor - - -class CreateShareParam(Object): - - __instance = None - - @staticmethod - def instance(path, access, expire_on): - CreateShareParam(path, access, expire_on) - return CreateShareParam.__instance - - def __init__(self, path, access, expire_on): - super().__init__() - self._classname = self.__class__.__name__ - self.url = path - self.share = Object() - self.share._classname = 'ShareConfig' - self.share.accessMode = access - self.share.protectionLevel = 'publicLink' - self.share.expiration = expire_on - self.share.invitee = Object() - self.share.invitee._classname = 'Collaborator' - self.share.invitee.type = 'external' - CreateShareParam.__instance = self # pylint: disable=unused-private-member - - -class FetchResourcesParam(Object): - - def __init__(self): - super().__init__() - self._classname = 'FetchResourcesParam' - self.start = 0 - self.limit = 100 - - def increment(self): - self.start = self.start + self.limit + * The path refers to an existing file + * The path refers to an existing directory + * The parent directory of the path exists + :raises cterasdk.exceptions.io.core.GetMetadataError: If neither the path nor its parent directory exist. + """ + is_dir, resource = EnsureDirectory(listdir, receiver, destination, True).execute() + if not is_dir: + is_dir, resource = EnsureDirectory(listdir, receiver, destination.parent).execute() + return resource, destination.parent, destination.name + return resource, destination, None -class FetchResourcesParamBuilder: - def __init__(self): - self.param = FetchResourcesParam() +async def a_split_file_directory(listdir, receiver, destination): + """ + Split a path into its parent directory and final component. - def root(self, root): - self.param.root = root # pylint: disable=attribute-defined-outside-init - return self + :returns: + tuple[str, str]: A ``(parent_directory, name)`` tuple when: - def depth(self, depth): - self.param.depth = depth # pylint: disable=attribute-defined-outside-init - return self + * The path refers to an existing file + * The path refers to an existing directory + * The parent directory of the path exists - def searchCriteria(self, criteria): - self.param.searchCriteria = criteria # pylint: disable=attribute-defined-outside-init - return self + :raises cterasdk.exceptions.io.core.GetMetadataError: If neither the path nor its parent directory exist. + """ + is_dir, resource = await EnsureDirectory(listdir, receiver, destination, True).a_execute() + if not is_dir: + is_dir, resource = await EnsureDirectory(listdir, receiver, destination.parent).a_execute() + return resource, destination.parent, destination.name + return resource, destination, None - def include_deleted(self): - self.param.includeDeleted = True # pylint: disable=attribute-defined-outside-init - return self - def limit(self, limit): - self.param.limit = limit - return self +class PathResolver: - def build(self): - return self.param - - -class FetchResourcesError(Exception): - - def __init__(self, error): - super().__init__() - self.error = error + def __init__(self, listdir, receiver, destination, default): + self._listdir = listdir + self._receiver = receiver + self._destination = destination + self._default = default + async def a_resolve(self): + resource, parent, name = await a_split_file_directory(self._listdir, self._receiver, self._destination) + return resource, self._resolve(parent, name) -class FetchResourcesResponse(DefaultResponse): + def resolve(self): + resource, parent, name = split_file_directory(self._listdir, self._receiver, self._destination) + return resource, self._resolve(parent, name) - def __init__(self, response): - super().__init__(response) - if response.errorType is not None: - raise FetchResourcesError(response.errorType) + def _resolve(self, parent, name): + if name is not None: + return parent.join(name) + if self._default is not None: + return parent.join(self._default) + return self._destination - @property - def objects(self): - return self._response.items - -def destination_prerequisite_conditions(destination, name): - if any(c in name for c in ['\\', '/', ':', '?', '&', '<', '>', '"', '|']): - raise exceptions.io.core.FilenameError(destination.join(name).relative) +def destination_prerequisite_conditions(destination): + if any(c in destination.name for c in ['\\', '/', ':', '?', '&', '<', '>', '"', '|']): + raise exceptions.io.core.FilenameError(destination.relative) class EnsureDirectory(PortalCommand): def __init__(self, function, receiver, path, suppress_error=False): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.suppress_error = suppress_error def _execute(self): @@ -251,98 +120,58 @@ def ensure_writeable(resource, directory): class Upload(PortalCommand): - def __init__(self, function, receiver, metadata_function, name, destination, size, fd): + def __init__(self, function, receiver, listdir, destination, fd, name, size): super().__init__(function, receiver) - destination_prerequisite_conditions(destination, name) - self.metadata_function = metadata_function - self.name = name - self.destination = destination + self.destination = automatic_resolution(destination, receiver.context) + self._resolver = PathResolver(listdir, receiver, self.destination, name) self.size = size self.fd = fd + self._resource = None def get_parameter(self): - fd, size = common.encode_stream(self.fd, self.size) + fd, size = encode_stream(self.fd, self.size) param = dict( - name=self.name, - Filename=self.name, - fullpath=self._receiver.io.builder(CorePath(self.destination.reference, self.name).absolute_encode), + name=self.destination.name, + Filename=self.destination.name, + fullpath=self._receiver.io.builder(self.destination.relative_encode), fileSize=size, file=fd ) return param - def _validate_destination(self): - is_dir, resource = EnsureDirectory(self.metadata_function, self._receiver, self.destination, True).execute() - if not is_dir: - is_dir, resource = EnsureDirectory(self.metadata_function, self._receiver, self.destination.parent).execute() - self.name, self.destination = self.destination.name, self.destination.parent - ensure_writeable(resource, self.destination) - return resource.cloudFolderInfo.uid - - async def _a_validate_destination(self): - is_dir, resource = await EnsureDirectory(self.metadata_function, self._receiver, self.destination, True).a_execute() - if not is_dir: - is_dir, resource = await EnsureDirectory(self.metadata_function, self._receiver, self.destination.parent).a_execute() - self.name, self.destination = self.destination.name, self.destination.parent - ensure_writeable(resource, self.destination) - return resource.cloudFolderInfo.uid - def _before_command(self): - logger.info('Uploading: %s', self.destination.join(self.name).relative) + destination_prerequisite_conditions(self.destination) + ensure_writeable(self._resource, self.destination.parent) + logger.info('Uploading: %s', self.destination) def _execute(self): - cloudfolder = self._validate_destination() + self._resource, self.destination = self._resolver.resolve() with self.trace_execution(): - return self._function(self._receiver, str(cloudfolder), self.get_parameter()) + return self._function(self._receiver, str(self._resource.cloudFolderInfo.uid), self.get_parameter()) + + async def _a_execute(self): + self._resource, self.destination = await self._resolver.a_resolve() + with self.trace_execution(): + return await self._function(self._receiver, str(self._resource.cloudFolderInfo.uid), self.get_parameter()) def _handle_response(self, r): - path = self.destination.join(self.name).relative + path = self.destination.relative if r.rc: + error = exceptions.io.core.UploadError(r.msg, path) logger.error('Upload failed: %s', path) if r.msg in [UploadError.UserQuotaViolation, UploadError.PortalQuotaViolation, UploadError.FolderQuotaViolation]: - raise exceptions.io.core.QuotaError(path) + raise error from exceptions.io.core.QuotaError(path) if r.msg == UploadError.RejectedByPolicy: - raise exceptions.io.core.FileRejectedError(path) + raise error from exceptions.io.core.FileRejectedError(path) if r.msg == UploadError.WindowsACL: - raise exceptions.io.core.NTACLError(path) + raise error from exceptions.io.core.NTACLError(path) if r.msg.startswith(UploadError.NoStorageBucket): - raise exceptions.io.core.StorageBackendError(path) - raise exceptions.io.core.WriteError(r.msg, path) + raise error from exceptions.io.core.StorageBackendError(path) + raise error if not r.rc and r.msg == 'OK': logger.info('Upload success. Saved to: %s', path) return path - async def _a_execute(self): - cloudfolder = await self._a_validate_destination() - with self.trace_execution(): - return await self._function(self._receiver, str(cloudfolder), self.get_parameter()) - - -class UploadFile(PortalCommand): - - def __init__(self, function, receiver, metadata_function, path, destination): - super().__init__(function, receiver) - self._metadata_function = metadata_function - self.path = path - self.destination = destination - - def _get_properties(self): - return commonfs.properties(self.path) - - def _execute(self): - metadata = self._get_properties() - with open(self.path, 'rb') as handle: - with self.trace_execution(): - return Upload(self._function, self._receiver, self._metadata_function, metadata['name'], - self.destination, metadata['size'], handle).execute() - - async def _a_execute(self): - metadata = self._get_properties() - with open(self.path, 'rb') as handle: - with self.trace_execution(): - return await Upload(self._function, self._receiver, self._metadata_function, metadata['name'], - self.destination, metadata['size'], handle).a_execute() - def _is_sharing_on_user_behalf(path): return re.match(r'^Users/[^/]+(?=/)', path.relative) is not None @@ -482,13 +311,13 @@ class Open(PortalCommand): def __init__(self, function, receiver, path): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) def get_parameter(self): return self.path.reference def _before_command(self): - logger.info('Getting handle: %s', self.path.relative) + logger.info('Getting handle: %s', self.path) def _execute(self): with self.trace_execution(): @@ -499,22 +328,25 @@ async def _a_execute(self): return await self._function(self._receiver, self.get_parameter()) def _handle_exception(self, e): + path = self.path.relative + error = exceptions.io.edge.OpenError(path) if isinstance(e, exceptions.transport.NotFound): - raise exceptions.io.core.FileNotFoundException(self.path.relative) from e + raise error from exceptions.io.core.FileNotFoundException(path) + raise error class Download(PortalCommand): def __init__(self, function, receiver, path, destination): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.destination = destination def get_parameter(self): return commonfs.determine_directory_and_filename(self.path.reference, destination=self.destination) def _before_command(self): - logger.info('Downloading: %s', self.path.relative) + logger.info('Downloading: %s', self.path) def _execute(self): directory, name = self.get_parameter() @@ -531,9 +363,10 @@ async def _a_execute(self): class OpenMany(PortalCommand): - def __init__(self, function, receiver, directory, *objects): + def __init__(self, function, receiver, resource, directory, *objects): super().__init__(function, receiver) - self.directory = directory + self.uid = str(resource.cloudFolderInfo.uid) + self.directory = automatic_resolution(directory, receiver.context) self.objects = objects def _before_command(self): @@ -546,42 +379,43 @@ def get_parameter(self): param.password = None param.portalName = None param.showDeleted = False - return common.encode_request_parameter(param) + return encode_request_parameter(param) def _execute(self): with self.trace_execution(): - return self._function(self._receiver, self.get_parameter(), self.directory) + return self._function(self._receiver, self.uid, self.get_parameter()) async def _a_execute(self): with self.trace_execution(): - return await self._function(self._receiver, self.get_parameter(), self.directory) + return await self._function(self._receiver, self.uid, self.get_parameter()) class DownloadMany(PortalCommand): - def __init__(self, function, receiver, target, objects, destination): + def __init__(self, function, receiver, resource, directory, objects, destination): super().__init__(function, receiver) - self.target = target + self.resource = resource + self.directory = automatic_resolution(directory, receiver.context) self.objects = objects self.destination = destination def get_parameter(self): - return commonfs.determine_directory_and_filename(self.target.reference, self.objects, destination=self.destination, archive=True) + return commonfs.determine_directory_and_filename(self.directory.reference, self.objects, destination=self.destination, archive=True) def _before_command(self): for o in self.objects: - logger.info('Downloading: %s', self.target.join(o).relative) + logger.info('Downloading: %s', self.directory.join(o).relative) def _execute(self): directory, name = self.get_parameter() with self.trace_execution(): - with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: + with OpenMany(self._function, self._receiver, self.resource, self.directory, *self.objects) as handle: return synfs.write(directory, name, handle) async def _a_execute(self): directory, name = self.get_parameter() with self.trace_execution(): - async with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: + async with OpenMany(self._function, self._receiver, self.resource, self.directory, *self.objects) as handle: return await asynfs.write(directory, name, handle) @@ -590,7 +424,7 @@ class ListDirectory(PortalCommand): def __init__(self, function, receiver, path, depth, include_deleted, search_criteria, limit): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.depth = depth self.include_deleted = include_deleted self.search_criteria = search_criteria @@ -608,7 +442,7 @@ def get_parameter(self): return builder.build() def _before_command(self): - logger.info('Listing directory: %s', self.path.relative) + logger.info('Listing directory: %s', self.path) def _execute(self): with self.trace_execution(): @@ -623,14 +457,15 @@ class ResourceIterator(ListDirectory): def execute(self): try: - yield from super().execute() + for o in super().execute(): + yield PortalResource.from_server_object(o) except FetchResourcesError as e: self._fetch_resources_error(e) async def a_execute(self): try: async for o in self._execute(): - yield o + yield PortalResource.from_server_object(o) except FetchResourcesError as e: self._fetch_resources_error(e) @@ -642,9 +477,10 @@ def _execute(self): return self._fetch_resources() def _fetch_resources_error(self, e): + error = exceptions.io.core.ListDirectoryError(self.path.relative) if e.error == ResourceError.DestinationNotExists: - raise exceptions.io.core.FolderNotFoundError(self.path.relative) from e - raise exceptions.io.core.ListDirectoryError(self.path.relative) + raise error from exceptions.io.core.FolderNotFoundError(self.path.relative) + raise error from e class GetMetadata(ListDirectory): @@ -654,7 +490,7 @@ def __init__(self, function, receiver, path, suppress_error=False): self.suppress_error = suppress_error def _before_command(self): - logger.info('Getting metadata: %s', self.path.relative) + logger.info('Getting metadata: %s', self.path) def _execute(self): with self.trace_execution(): @@ -667,11 +503,19 @@ async def _a_execute(self): def _handle_response(self, r): if r.root is None: if not self.suppress_error: - raise exceptions.io.core.ObjectNotFoundError(self.path.relative) + cause = exceptions.io.core.ObjectNotFoundError(self.path.relative) + raise exceptions.io.core.GetMetadataError(self.path.relative) from cause return False, None return True, r.root +class GetProperties(GetMetadata): + + def _handle_response(self, r): + _, metadata = super()._handle_response(r) + return PortalResource.from_server_object(metadata) + + class GetPermalink(GetMetadata): def _handle_response(self, r): @@ -686,41 +530,50 @@ def __init__(self, function, receiver, path, include_deleted): self._receiver = receiver self.path = path self.include_deleted = include_deleted - self.tree = [CorePath.instance(path.scope, path.relative)] + self.tree = [path] def _generator(self): + logger.info('Traversing: %s', self.path or '.') while len(self.tree) > 0: yield self.tree.pop(0) - def _before_generate(self): - EnsureDirectory(self._function, self._receiver, CorePath.instance(self.path.scope, self.path.relative)) - logger.info('Traversing: %s', self.path.relative) - def generate(self): - self._before_generate() for path in self._generator(): - for o in ResourceIterator(self._function, self._receiver, path, None, self.include_deleted, None, None).execute(): - yield self._process_object(o) + try: + print('Enumerating: ', path or '.') + for o in ResourceIterator(self._function, self._receiver, path, None, self.include_deleted, None, None).execute(): + yield self._process_object(o) + except exceptions.io.core.ListDirectoryError as e: + RecursiveIterator._suppress_error(e) async def a_generate(self): for path in self._generator(): - async for o in ResourceIterator(self._function, self._receiver, path, None, self.include_deleted, None, None).a_execute(): - yield self._process_object(o) + try: + async for o in ResourceIterator(self._function, self._receiver, path, None, self.include_deleted, None, None).a_execute(): + yield self._process_object(o) + except exceptions.io.core.ListDirectoryError as e: + RecursiveIterator._suppress_error(e) def _process_object(self, o): - if o.isFolder: - self.tree.append(CorePath.instance(self.path.scope, o)) + if o.is_dir: + self.tree.append(o.path.relative) return o + @staticmethod + def _suppress_error(e): + if not isinstance(e.__cause__, exceptions.io.core.FolderNotFoundError): + raise e + logger.warning("Could not list directory contents: %s. No such directory.", e.path) + class ListVersions(PortalCommand): def __init__(self, function, receiver, path): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) def _before_command(self): - logger.info('Getting versions: %s', self.path.relative) + logger.info('Listing versions: %s', self.path) def get_parameter(self): return self.path.absolute @@ -733,8 +586,11 @@ async def _a_execute(self): with self.trace_execution(): return await self._function(self._receiver, self.get_parameter()) + def _handle_response(self, r): + return [PreviousVersion.from_server_object(v) for v in r] + def _handle_exception(self, e): - raise exceptions.io.core.GetSnapshotsError(self.path.relative) from e + raise exceptions.io.core.GetVersionsError(self.path.relative) from e class CreateDirectory(PortalCommand): @@ -742,7 +598,7 @@ class CreateDirectory(PortalCommand): def __init__(self, function, receiver, path, parents=False): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.parents = parents def get_parameter(self): @@ -752,13 +608,13 @@ def get_parameter(self): return param def _before_command(self): - logger.info('Creating directory: %s', self.path.relative) + logger.info('Creating directory: %s', self.path) def _parents_generator(self): if self.parents: parts = self.path.parts for i in range(1, len(parts)): - yield CorePath.instance(self.path.scope, '/'.join(parts[:i])) + yield automatic_resolution('/'.join(parts[:i]), self._receiver.context) else: yield self.path @@ -767,41 +623,51 @@ def _execute(self): for path in self._parents_generator(): try: CreateDirectory(self._function, self._receiver, path).execute() - except exceptions.io.core.FileConflictError: - pass - return self._function(self._receiver, self.get_parameter()) + except exceptions.io.core.CreateDirectoryError as e: + CreateDirectory._suppress_file_conflict_error(e) + with self.trace_execution(): + return self._function(self._receiver, self.get_parameter()) async def _a_execute(self): if self.parents: for path in self._parents_generator(): try: await CreateDirectory(self._function, self._receiver, path).a_execute() - except exceptions.io.core.FileConflictError: - pass - return await self._function(self._receiver, self.get_parameter()) + except exceptions.io.core.CreateDirectoryError as e: + CreateDirectory._suppress_file_conflict_error(e) + with self.trace_execution(): + return await self._function(self._receiver, self.get_parameter()) + + @staticmethod + def _suppress_file_conflict_error(e): + if not isinstance(e.__cause__, exceptions.io.core.FileConflictError): + raise e def _handle_response(self, r): path = self.path.relative if r is None or r == 'Ok': return path + + error, cause = exceptions.io.core.CreateDirectoryError(path), None if r == ResourceError.FileWithTheSameNameExist: - raise exceptions.io.core.FileConflictError(path) + cause = exceptions.io.core.FileConflictError(path) if r == ResourceError.DestinationNotExists: - raise exceptions.io.core.FolderNotFoundError(self.path.parent.relative) + cause = exceptions.io.core.FolderNotFoundError(self.path.parent.relative) if r == ResourceError.ReservedName: - raise exceptions.io.core.ReservedNameError(path) + cause = exceptions.io.core.ReservedNameError(path) if r == ResourceError.InvalidName: - raise exceptions.io.core.FilenameError(path) + cause = exceptions.io.core.FilenameError(path) if r == ResourceError.PermissionDenied: - raise exceptions.io.core.ReservedNameError(path) - raise exceptions.io.core.CreateDirectoryError(path) + cause = exceptions.io.core.ReservedNameError(path) + + raise error from cause class GetShareMetadata(PortalCommand): def __init__(self, function, receiver, path): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) def _before_command(self): logger.info('Share metadata: %s', self.path.relative) @@ -819,8 +685,9 @@ async def _a_execute(self): def _handle_exception(self, e): path = self.path.relative + error = exceptions.io.core.GetShareMetadataError(path) if e.error.response.error.msg == 'Resource does not exist': - raise exceptions.io.core.ObjectNotFoundError(path) from e + raise error from exceptions.io.core.ObjectNotFoundError(path) raise exceptions.io.core.GetShareMetadataError(path) from e @@ -828,12 +695,12 @@ class Link(PortalCommand): def __init__(self, function, receiver, path, access, expire_in): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.access = access self.expire_in = expire_in def _before_command(self): - logger.info('Creating Public Link: %s', self.path.relative) + logger.info('Creating Public Link: %s', self.path) def get_parameter(self): access = {'RO': 'ReadOnly', 'RW': 'ReadWrite', 'PO': 'PreviewOnly'}.get(self.access) @@ -854,9 +721,10 @@ def _handle_response(self, r): def _handle_exception(self, e): path = self.path.relative + error = exceptions.io.core.CreateLinkError(path) if e.error.response.error.msg == 'Resource does not exist': - raise exceptions.io.core.ObjectNotFoundError(path) from e - raise exceptions.io.core.CreateLinkError(path) from e + raise error from exceptions.io.core.ObjectNotFoundError(path) + raise error from e class FilterShareMembers(PortalCommand): @@ -903,16 +771,16 @@ def _enumerate_members(self, collaborators): def _execute(self): collaborators = [] with self._enumerate_members(collaborators) as members: - for member in enumerate(members): - members.collaborator = SearchMember(self._function, self._receiver, member.account, self.cloud_folder_uid).execute() + for member in members: + member.collaborator = SearchMember(self._function, self._receiver, member.account, self.cloud_folder_uid).execute() return collaborators async def _a_execute(self): collaborators = [] with self._enumerate_members(collaborators) as members: - for member in enumerate(members): - members.collaborator = await SearchMember(self._function, - self._receiver, member.account, self.cloud_folder_uid).a_execute() + for member in members: + member.collaborator = await SearchMember(self._function, + self._receiver, member.account, self.cloud_folder_uid).a_execute() return collaborators @@ -920,7 +788,7 @@ class Share(PortalCommand): def __init__(self, function, receiver, path, members, as_project, allow_reshare, allow_sync): super().__init__(function, receiver) - self.path = path + self.path = automatic_resolution(path, receiver.context) self.members = members self.as_project = as_project self.allow_reshare = allow_reshare @@ -1027,7 +895,10 @@ def __init__(self, function, receiver, path): super().__init__(function, receiver, path, [], True, True, True) def _before_command(self): - logger.info('Revoking Share: %s', self.path.relative) + logger.info('Revoking Share: %s', self.path) + + def _handle_response(self, r): + return None class TaskCommand(PortalCommand): @@ -1074,7 +945,7 @@ def _handle_response(self, r): return self._task_complete(r) if r.failed or r.completed_with_warnings: - return self._task_complete(r) + return self._task_error(r) return r @@ -1085,47 +956,24 @@ def _task_error(self, task): # pylint: disable=no-self-use return task -class Rename(TaskCommand): - - def __init__(self, function, receiver, path, new_name, block): - super().__init__(function, receiver, block) - self.path = path - self.new_path = self.path.parent.join(new_name) - - def _progress_str(self): - return 'Renaming' - - def get_parameter(self): - param = ActionResourcesParam.instance() - param.add(SrcDstParam.instance(src=self.path.absolute_encode, dest=self.new_path.absolute_encode)) - return param - - def _before_command(self): - logger.info('%s: %s, to: %s', self._progress_str(), self.path.relative, self.new_path.relative) - - def _task_complete(self, task): - return self.new_path.relative - - class MultiResourceCommand(TaskCommand): def __init__(self, function, receiver, block, *paths): super().__init__(function, receiver, block) - self.paths = paths + self.paths = list(automatic_resolution(paths, receiver.context)) def get_parameter(self): param = ActionResourcesParam.instance() - paths = [self.paths] if not isinstance(self.paths, tuple) else self.paths - for path in paths: + for path in self.paths: param.add(SrcDstParam.instance(src=path.absolute_encode)) return param def _before_command(self): for path in self.paths: - logger.info('%s: %s', self._progress_str(), path.relative) + logger.info('%s: %s', self._progress_str(), path) def _task_complete(self, task): - return [path.relative for path in self.paths] + return [str(path) for path in self.paths] class Delete(MultiResourceCommand): @@ -1152,13 +1000,12 @@ class ResolverCommand(TaskCommand): def __init__(self, function, receiver, block, *paths, destination=None, resolver=None, cursor=None): super().__init__(function, receiver, block) - self.paths = paths - self.destination = destination + self.paths = list(automatic_resolution(paths, receiver.context)) + self.destination = automatic_resolution(destination, receiver.context) self.resolver = resolver self.cursor = cursor def get_parameter(self): - param = ActionResourcesParam.instance() if self.cursor: param.startFrom = self.cursor @@ -1195,7 +1042,7 @@ def _progress_str(self): def execute(self): try: return super().execute() - except (exceptions.io.core.CopyError, exceptions.io.core.MoveError) as e: + except (exceptions.io.core.CopyError, exceptions.io.core.MoveError, exceptions.io.core.RenameError) as e: if self.resolver: return self._try_with_resolver(e.cursor) return self._handle_exception(e) @@ -1203,11 +1050,36 @@ def execute(self): async def a_execute(self): try: return await super().a_execute() - except (exceptions.io.core.CopyError, exceptions.io.core.MoveError) as e: + except (exceptions.io.core.CopyError, exceptions.io.core.MoveError, exceptions.io.core.RenameError) as e: if self.resolver: return await self._a_try_with_resolver(e.cursor) return self._handle_exception(e) + @property + @abstractmethod + def _error_object(self): + raise NotImplementedError('Subclass must implement the "_error_object" property.') + + def _task_error(self, task): + cursor = task.cursor + error = self._error_object(self.paths, cursor) + + if task.error_type == ResourceError.Conflict: # file conflict + resource = automatic_resolution(cursor.destResource).relative + raise error from exceptions.io.core.FileConflictError(resource) + + if not task.unknown_object(): # file not found + resource = automatic_resolution(cursor).relative + raise error from exceptions.io.core.ObjectNotFoundError(resource) + + if task.progress_str == ResourceError.DestinationNotExists: # destination directory not found + directory = self.destination if self.destination is not None else dict(self.paths).get( + automatic_resolution(cursor.srcResource).relative, None + ) + raise error from exceptions.io.core.FolderNotFoundError(directory.relative) + + raise self._error_object(self.paths, cursor) + class Copy(ResolverCommand): @@ -1222,12 +1094,9 @@ async def _a_try_with_resolver(self, cursor): return await Copy(self._function, self._receiver, self.block, *self.paths, destination=self.destination, resolver=self.resolver, cursor=cursor).a_execute() - def _task_error(self, task): - cursor = task.cursor - if task.error_type == ResourceError.Conflict: - dest = CorePath.instance('', cursor.destResource).relative - raise exceptions.io.core.CopyError(self.paths, cursor) from exceptions.io.core.FileConflictError(dest) - raise exceptions.io.core.CopyError(self.paths, cursor) + @property + def _error_object(self): + return exceptions.io.core.CopyError class Move(ResolverCommand): @@ -1243,9 +1112,36 @@ async def _a_try_with_resolver(self, cursor): return await Move(self._function, self._receiver, self.block, *self.paths, destination=self.destination, resolver=self.resolver, cursor=cursor).a_execute() - def _task_error(self, task): - cursor = task.cursor - if task.error_type == ResourceError.Conflict: - dest = CorePath.instance('', cursor.destResource).relative - raise exceptions.io.core.MoveError(self.paths, cursor) from exceptions.io.core.FileConflictError(dest) - raise exceptions.io.core.MoveError(self.paths, cursor) + @property + def _error_object(self): + return exceptions.io.core.MoveError + + +class Rename(Move): + + def _progress_str(self): + return 'Renaming' + + def __init__(self, function, receiver, block, path, new_name, resolver, cursor=None): + super().__init__(function, receiver, block, + *[(path, automatic_resolution(path, receiver.context).parent.join(new_name))], + resolver=resolver, cursor=cursor + ) + + def _try_with_resolver(self, cursor): + source, destination = self.paths[0] + return Rename(self._function, self._receiver, self.block, source, destination.name, + resolver=self.resolver, cursor=cursor).execute() + + async def _a_try_with_resolver(self, cursor): + source, destination = self.paths[0] + return await Rename(self._function, self._receiver, self.block, source, destination.name, + resolver=self.resolver, cursor=cursor).a_execute() + + @property + def _error_object(self): + return exceptions.io.core.RenameError + + def _handle_response(self, r): + response = super()._handle_response(r) + return self.paths[0][1].relative if self.block else response diff --git a/cterasdk/cio/core/types.py b/cterasdk/cio/core/types.py new file mode 100644 index 00000000..ccba3881 --- /dev/null +++ b/cterasdk/cio/core/types.py @@ -0,0 +1,408 @@ +import re +import urllib.parse +from datetime import datetime +from ...common import Object +from ..common import BasePath, BaseResource +from ...lib.iterator import DefaultResponse + + +class PortalPath(BasePath): + + @staticmethod + def from_server_object(server_object): + """ + Parse Path from Server Object. + + :param object server_object: Server Object + """ + classname = getattr(server_object, '_classname', None) + + if classname == 'ResourceInfo': + return PortalPath.from_resource(server_object) + + if classname == 'SnapshotResp': + return PortalPath.from_snapshot(server_object) + + if classname == 'ResourceActionCursor': + return PortalPath.from_cursor(server_object) + + raise ValueError(f"Unsupported server object type: '{classname}'") + + @staticmethod + def from_resource(resource): + """ + Create Path Object from 'ResourceInfo' Class Object. + + :param object resource: Resource Info Object + """ + return PortalPath.from_str(urllib.parse.unquote(resource.href)) + + @staticmethod + def from_snapshot(snapshot): + """ + Create Path Object from 'SnapshotResp' Class Object. + + :param object snapshot: Snapshot Response Object + """ + return PortalPath.from_str(urllib.parse.unquote(snapshot.url + snapshot.path)) + + @staticmethod + def from_cursor(cursor): + """ + Create Path Object from 'ResourceActionCursor' Class Object. + + :param object cursor: Resource Action Cursor Object + """ + return PortalPath.from_str(urllib.parse.unquote(cursor.upperLevelUrl)) + + @staticmethod + def from_str(path): + """ + Create Path Object from String. + + :param str path: Path + """ + return PortalPath._parse_from_str(path or '') + + @staticmethod + def _parse_from_str(path): + """ + Path Object from String. + + :param str path: Path + """ + groups = [f'(?P<{o.__name__}>{namespace})' for namespace, o in Namespaces.items()] + regex = re.compile(f"^{'|'.join(groups)}") + match = re.match(regex, path) + if match: + return Namespaces[match.group()](path[match.end():]) + raise ValueError(f'Could not determine object path: {path}') + + +class ServicesPortalPath(PortalPath): + """ + ServicesPortal Path Object + """ + Namespace = '/ServicesPortal/webdav' + + def __init__(self, reference): + super().__init__(ServicesPortalPath.Namespace, reference) + + +class GlobalAdminPath(PortalPath): + """ + Global Admin Path Object + """ + Namespace = '/admin/webdav' + + def __init__(self, reference): + super().__init__(GlobalAdminPath.Namespace, reference) + + +Namespaces = { + ServicesPortalPath.Namespace: ServicesPortalPath, + GlobalAdminPath.Namespace: GlobalAdminPath +} + + +def resolve(path, namespace=None): + """ + Resolve Path + + :param object path: Path + :param cterasdk.cio.core.types.PortalPath,optional namespace: Path Object + """ + if isinstance(path, PortalPath): + return path + + if isinstance(path, (PortalResource, PreviousVersion)): + return path.path + + if isinstance(path, Object): + return PortalPath.from_server_object(path) + + if namespace: + if path is None or isinstance(path, str): + return namespace(path or '') + + raise ValueError(f'Error: Could not resolve path: {path}. Type: {type(path)}') + + +def create_generator(paths, namespace=None): + """ + Create Path Object Generator Object. + + :param object paths: List or a tuple + :param cterasdk.cio.core.types.PortalPath,optional namespace: Path Object + """ + def wrapper(): + for path in paths: + if isinstance(path, tuple): + yield resolve(path[0], namespace), resolve(path[1], namespace) + else: + yield resolve(path, namespace) + return wrapper() + + +def automatic_resolution(p, context=None): + """ + Automatic Resolution of Path Object + + :param object p: Path + :param str,optional context: Context (e.g. 'ServicesPortal' or 'admin') + """ + namespace = Namespaces.get(f'/{context}/webdav', None) + + if isinstance(p, (list, tuple)): + return create_generator(p, namespace) + + return resolve(p, namespace) + + +class SrcDstParam(Object): + + __instance = None + + @staticmethod + def instance(src, dest=None): + SrcDstParam(src, dest) + return SrcDstParam.__instance + + def __init__(self, src, dest=None): + super().__init__() + self._classname = self.__class__.__name__ + self.src = src + self.dest = dest + SrcDstParam.__instance = self # pylint: disable=unused-private-member + + +class ResourceActionCursor(Object): + + def __init__(self): + super().__init__() + self._classname = self.__class__.__name__ + + +class ActionResourcesParam(Object): + + __instance = None + + @staticmethod + def instance(): + ActionResourcesParam() + return ActionResourcesParam.__instance + + def __init__(self): + super().__init__() + self._classname = self.__class__.__name__ + self.urls = [] + self.startFrom = None + ActionResourcesParam.__instance = self # pylint: disable=unused-private-member + + def add(self, param): + self.urls.append(param) + + def start_from(self, cursor): + self.startFrom = cursor + + +class CreateShareParam(Object): + + __instance = None + + @staticmethod + def instance(path, access, expire_on): + CreateShareParam(path, access, expire_on) + return CreateShareParam.__instance + + def __init__(self, path, access, expire_on): + super().__init__() + self._classname = self.__class__.__name__ + self.url = path + self.share = Object() + self.share._classname = 'ShareConfig' + self.share.accessMode = access + self.share.protectionLevel = 'publicLink' + self.share.expiration = expire_on + self.share.invitee = Object() + self.share.invitee._classname = 'Collaborator' + self.share.invitee.type = 'external' + CreateShareParam.__instance = self # pylint: disable=unused-private-member + + +class FetchResourcesParam(Object): + + def __init__(self): + super().__init__() + self._classname = 'FetchResourcesParam' + self.start = 0 + self.limit = 100 + + def increment(self): + self.start = self.start + self.limit + + +class FetchResourcesParamBuilder: + + def __init__(self): + self.param = FetchResourcesParam() + + def root(self, root): + self.param.root = root # pylint: disable=attribute-defined-outside-init + return self + + def depth(self, depth): + self.param.depth = depth # pylint: disable=attribute-defined-outside-init + return self + + def searchCriteria(self, criteria): + self.param.searchCriteria = criteria # pylint: disable=attribute-defined-outside-init + return self + + def include_deleted(self): + self.param.includeDeleted = True # pylint: disable=attribute-defined-outside-init + return self + + def limit(self, limit): + self.param.limit = limit + return self + + def build(self): + return self.param + + +class FetchResourcesError(Exception): + + def __init__(self, error): + super().__init__() + self.error = error + + +class FetchResourcesResponse(DefaultResponse): + + def __init__(self, response): + super().__init__(response) + if response.errorType is not None: + raise FetchResourcesError(response.errorType) + + @property + def objects(self): + return self._response.items + + +class VolumeOwner(Object): + """ + Class for a Cloud Volume Owner. + + :ivar int id: Owner ID + :ivar str id: Owner Full Name. + :ivar str namespace: User namespace. + """ + def __init__(self, i, name): + super().__init__(id=i, name=name) + + @property + def user_namespace(self): + return f'/Users/{self.name}' + + +class PortalVolume(Object): + """ + Class for a Portal Cloud Volume. + + :ivar int id: Cloud Drive Folder ID + :ivar str name: Cloud Drive Folder Name + :ivar int group: Folder Group ID + :ivar bool protected: Passphrase-Protected + :ivar cterasdk.core.types.VolumeOwner owner: Volume owner information. + """ + def __init__(self, i, name, group, protected, owner): + super().__init__(id=i, name=name, group=group, protected=protected, owner=owner) + + @staticmethod + def from_server_object(server_object): + return PortalVolume( + server_object.uid, + server_object.name, + server_object.groupUid, + server_object.passphraseProtected, + VolumeOwner(server_object.ownerUid, server_object.ownerFriendlyName) + ) + + +class PreviousVersion(Object): + """ + Class Representing a Previous Version + + :ivar bool current: Current + :ivar cterasdk.cio.types.PortalPath path: Path Object + :ivar datetime.datetime start_time: Snapshot start time + :ivar datetime.datetime end_time: Snapshot end time + """ + def __init__(self, server_object): + super().__init__( + path=PortalPath.from_snapshot(server_object), + current=server_object.current, + start_time=datetime.fromisoformat(server_object.startTimestamp), + end_time=datetime.fromisoformat(server_object.calculatedTimestamp) + ) + + @staticmethod + def from_server_object(server_object): + return PreviousVersion(server_object) + + def __repr__(self): + return str(self) + + def __str__(self): + return ( + f"{self.__class__.__name__}(" + f"{{'start_time': {self.start_time.isoformat()}, " + f"'end_time': {self.end_time.isoformat()}, " + f"'current': {self.current}, " + f"'path': {self.path}}})" + ) + + +class PortalResource(BaseResource): + """ + Class for a Portal Filesystem Resource. + + :ivar int,optional id: Resource ID, defaults to ``None`` if not exists + :ivar str name: Resource name + :ivar cterasdk.cio.types.PortalPath path: Path Object + :ivar bool is_dir: ``True`` if directory, ``False`` otherwise + :ivar bool deleted: ``True`` if deleted, ``False`` otherwise + :ivar int size: Size + :ivar datetime.datetime last_modified: Last Modified + :ivar str extension: Extension + :ivar str permalink: Permalink + :ivar cterasdk.core.types.Volume,optional volume: Volume information. + """ + def __init__(self, i, name, path, is_dir, deleted, size, permalink, last_modified, volume): + super().__init__( + name, path, is_dir, size, + None if last_modified is None else datetime.fromisoformat(last_modified), + ) + self.id = i + self.deleted = deleted + self.permalink = permalink + self.volume = PortalVolume.from_server_object(volume) if volume else None + + @staticmethod + def from_server_object(server_object): + return PortalResource( + getattr(server_object, 'fileId', None), + server_object.name, + PortalPath.from_resource(server_object), + server_object.isFolder, + server_object.isDeleted, + server_object.size, + server_object.permalink, + server_object.lastmodified, + server_object.cloudFolderInfo + ) + + @property + def with_user_namespace(self): + return PortalPath(self.volume.owner.user_namespace if self.volume else '/', self.path.relative).absolute diff --git a/cterasdk/cio/edge.py b/cterasdk/cio/edge.py deleted file mode 100644 index 737f60d2..00000000 --- a/cterasdk/cio/edge.py +++ /dev/null @@ -1,461 +0,0 @@ -import logging -from datetime import datetime -from pathlib import Path -from ..common import Object -from ..edge.enum import ResourceError -from ..objects.uri import unquote -from . import common -from .. import exceptions -from .actions import EdgeCommand -from ..lib.storage import synfs, asynfs, commonfs - - -logger = logging.getLogger('cterasdk.edge') - - -class EdgePath(common.BasePath): - """Path for CTERA Edge Filer""" - - def __init__(self, scope, reference): - """ - Initialize a CTERA Edge Filer Path. - - :param str scope: Scope. - :param str reference: Reference. - """ - if isinstance(reference, Object): - super().__init__(scope, reference.path) - elif isinstance(reference, str): - super().__init__(scope, reference) - elif reference is None: - super().__init__(scope, '') - else: - message = 'Path validation failed: ensure the path exists and is correctly formatted.' - logger.error(message) - raise ValueError(message) - - @staticmethod - def instance(scope, reference): - if isinstance(reference, tuple): - source, destination = reference - return (EdgePath(scope, source), EdgePath(scope, destination)) - return EdgePath(scope, reference) - - -class Open(EdgeCommand): - """Open file""" - - def __init__(self, function, receiver, path): - super().__init__(function, receiver) - self.path = path - - def get_parameter(self): - return self.path.absolute - - def _before_command(self): - logger.info('Getting handle: %s', self.path.relative) - - def _execute(self): - with self.trace_execution(): - return self._function(self._receiver, self.get_parameter()) - - async def _a_execute(self): - with self.trace_execution(): - return await self._function(self._receiver, self.get_parameter()) - - -class Download(EdgeCommand): - - def __init__(self, function, receiver, path, destination): - super().__init__(function, receiver) - self.path = path - self.destination = destination - - def get_parameter(self): - return commonfs.determine_directory_and_filename(self.path.reference, destination=self.destination) - - def _before_command(self): - logger.info('Downloading: %s', self.path.relative) - - def _execute(self): - directory, name = self.get_parameter() - with self.trace_execution(): - with Open(self._function, self._receiver, self.path) as handle: - return synfs.write(directory, name, handle) - - async def _a_execute(self): - directory, name = self.get_parameter() - with self.trace_execution(): - async with Open(self._function, self._receiver, self.path) as handle: - return await asynfs.write(directory, name, handle) - - -class OpenMany(EdgeCommand): - - def __init__(self, function, receiver, directory, *objects): - super().__init__(function, receiver) - self.directory = directory - self.objects = objects - - def _before_command(self): - logger.info('Getting handle: %s', [self.directory.join(o).relative for o in self.objects]) - - def get_parameter(self): - param = Object() - param.paths = ['/'.join([self.directory.absolute, item]) for item in self.objects] - param.snapshot = Object() - param._classname = 'BackupRepository' # pylint: disable=protected-access - param.snapshot.location = 1 - param.snapshot.timestamp = None - param.snapshot.path = None - logger.info('Getting directory handle: %s', self.directory.reference) - return common.encode_request_parameter(param) - - def _execute(self): - with self.trace_execution(): - return self._function(self._receiver, self.get_parameter()) - - async def _a_execute(self): - with self.trace_execution(): - return await self._function(self._receiver, self.get_parameter()) - - -class DownloadMany(EdgeCommand): - - def __init__(self, function, receiver, target, objects, destination): - super().__init__(function, receiver) - self.target = target - self.objects = objects - self.destination = destination - - def get_parameter(self): - return commonfs.determine_directory_and_filename(self.target.reference, self.objects, destination=self.destination, archive=True) - - def _before_command(self): - for o in self.objects: - logger.info('Downloading: %s', self.target.join(o).relative) - - def _execute(self): - directory, name = self.get_parameter() - with self.trace_execution(): - with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: - return synfs.write(directory, name, handle) - - async def _a_execute(self): - directory, name = self.get_parameter() - with self.trace_execution(): - async with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: - return await asynfs.write(directory, name, handle) - - -def decode_reference(href): - namespace = '/localFiles' - return unquote(href[href.index(namespace)+len(namespace) + 1:]) - - -class ListDirectory(EdgeCommand): - """List""" - - def __init__(self, function, receiver, path, depth=None): - super().__init__(function, receiver) - self.path = path - self.depth = depth if depth is not None else 1 - - def _before_command(self): - logger.info('Listing directory: %s', self.path.relative) - - def _execute(self): - with self.trace_execution(): - return self._function(self._receiver, self.path.absolute, self.depth) - - async def _a_execute(self): - with self.trace_execution(): - return await self._function(self._receiver, self.path.absolute, self.depth) - - def _handle_response(self, r): - entries = [] - for e in r: - path = decode_reference(e.href) - if path and self.path != path: - is_dir = e.getcontenttype == 'httpd/unix-directory' - param = Object( - path=path, - name=Path(path).name, - is_dir=is_dir, - is_file=not is_dir, - created_at=e.creationdate, - last_modified=datetime.strptime(e.getlastmodified, "%a, %d %b %Y %H:%M:%S GMT").isoformat(), - size=e.getcontentlength - ) - entries.append(param) - return entries if self.depth > 0 else entries[0] - - -class RecursiveIterator: - - def __init__(self, function, receiver, path): - self._function = function - self._receiver = receiver - self.path = path - self.tree = [EdgePath.instance(path.scope, path.relative)] - - def _generator(self): - while len(self.tree) > 0: - yield self.tree.pop(0) - - def _before_generate(self): - EnsureDirectory(self._function, self._receiver, EdgePath.instance(self.path.scope, self.path.relative)) - logger.info('Traversing: %s', self.path.relative) - - def generate(self): - self._before_generate() - for path in self._generator(): - for o in ListDirectory(self._function, self._receiver, path).execute(): - if path.relative != o.path: - yield self._process_object(o) - - async def a_generate(self): - self._before_generate() - for path in self._generator(): - for o in await ListDirectory(self._function, self._receiver, path).a_execute(): - if path.relative != o.path: - yield self._process_object(o) - - def _process_object(self, o): - if o.is_dir: - self.tree.append(EdgePath.instance(self.path.scope, o)) - return o - - -class GetMetadata(ListDirectory): - - def __init__(self, function, receiver, path, suppress_error=False): - super().__init__(function, receiver, path, 0) - self.suppress_error = suppress_error - - def _before_command(self): - logger.info('Getting metadata: %s', self.path.relative) - - def _handle_response(self, r): - return True, super()._handle_response(r) - - def _handle_exception(self, e): - if not self.suppress_error: - if isinstance(e, exceptions.transport.NotFound): - raise exceptions.io.edge.FileNotFoundException(self.path.relative) from e - return False, None - - -class EnsureDirectory(EdgeCommand): - - def __init__(self, function, receiver, path, suppress_error=False): - super().__init__(function, receiver) - self.path = path - self.suppress_error = suppress_error - - def _execute(self): - return GetMetadata(self._function, self._receiver, self.path, self.suppress_error).execute() - - async def _a_execute(self): - return await GetMetadata(self._function, self._receiver, self.path, self.suppress_error).a_execute() - - def _handle_response(self, r): - exists, resource = r if r is not None else (False, None) - if (not exists or not resource.is_dir) and not self.suppress_error: - raise exceptions.io.edge.NotADirectoryException(self.path.relative) - return resource.is_dir if exists else False, resource - - -class CreateDirectory(EdgeCommand): - """Create Directory""" - - def __init__(self, function, receiver, path, parents=False): - super().__init__(function, receiver) - self.path = path - self.parents = parents - - def get_parameter(self): - param = Object() - param.name = self.path.name - param.parentPath = self.path.parent.absolute_encode - return param - - def _before_command(self): - logger.info('Creating directory: %s', self.path.relative) - - def _parents_generator(self): - if self.parents: - parts = self.path.parts - for i in range(1, len(parts)): - yield EdgePath.instance(self.path.scope, '/'.join(parts[:i])) - else: - yield self.path - - def _execute(self): - with self.trace_execution(): - if self.parents: - for path in self._parents_generator(): - try: - CreateDirectory(self._function, self._receiver, path).execute() - except (exceptions.io.edge.FileConflictError, exceptions.io.edge.ROFSError): - pass - return self._function(self._receiver, self.path.absolute) - - async def _a_execute(self): - with self.trace_execution(): - if self.parents: - for path in self._parents_generator(): - try: - await CreateDirectory(self._function, self._receiver, path).a_execute() - except (exceptions.io.edge.FileConflictError, exceptions.io.edge.ROFSError): - pass - return await self._function(self._receiver, self.path.absolute) - - def _handle_response(self, r): - if r == 'OK': - return self.path.relative - raise exceptions.io.edge.CreateDirectoryError(self.path.relative) - - def _handle_exception(self, e): - if e.error.response.error.msg == ResourceError.FileExists: - raise exceptions.io.edge.FileConflictError(self.path.relative) - if e.error.response.error.msg == ResourceError.Forbidden: - raise exceptions.io.edge.ROFSError(self.path.relative) - - -class Copy(EdgeCommand): - """Copy""" - - def __init__(self, function, receiver, path, destination=None, overwrite=False): - super().__init__(function, receiver) - self.path = path - self.destination = destination - self.overwrite = overwrite - - def get_parameter(self): - if isinstance(self.path, tuple): - self.path, self.destination = self.path[0], self.path[1] - else: - self.path, self.destination = self.path, self.destination.join(self.path.name) - return (self.path.absolute, self.destination.absolute) - - def _before_command(self): - logger.info('%s: %s to: %s', 'Copying', self.path.relative, self.destination.relative) - - def _execute(self): - source, destination = self.get_parameter() - with self.trace_execution(): - return self._function(self._receiver, source, destination, overwrite=self.overwrite) - - async def _a_execute(self): - source, destination = self.get_parameter() - with self.trace_execution(): - return await self._function(self._receiver, source, destination, overwrite=self.overwrite) - - -class Move(Copy): - """Move""" - - def _before_command(self): - logger.info('%s: %s to: %s', 'Moving', self.path.relative, self.destination.relative) - - -class Rename(Move): - """Rename""" - - def __init__(self, function, receiver, path, new_name, overwrite=False): - super().__init__(function, receiver, path, None, overwrite) - self.new_name = new_name - - def get_parameter(self): - return (self.path.absolute, self.path.parent.join(self.new_name).absolute) - - def _before_command(self): - logger.info('Renaming: %s to: %s', self.path.relative, self.path.parent.join(self.new_name).relative) - - -class Delete(EdgeCommand): - - def __init__(self, function, receiver, path): - super().__init__(function, receiver) - self.path = path - - def _before_command(self): - logger.info('Deleting: %s', self.path.relative) - - def _execute(self): - with self.trace_execution(): - self._function(self._receiver, self.path.absolute) - - async def _a_execute(self): - with self.trace_execution(): - await self._function(self._receiver, self.path.absolute) - - -class Upload(EdgeCommand): - - def __init__(self, function, receiver, metadata_function, name, destination, fd): - super().__init__(function, receiver) - self._metadata_function = metadata_function - self.name = name - self.destination = destination - self.fd = fd - - def get_parameter(self): - fd, *_ = common.encode_stream(self.fd, 0) - param = dict( - name=self.name, - fullpath=f'{self.destination.absolute}/{self.name}', - filedata=fd - ) - return param - - def _validate_destination(self): - is_dir, *_ = EnsureDirectory(self._metadata_function, self._receiver, self.destination, True).execute() - if not is_dir: - is_dir, *_ = EnsureDirectory(self._metadata_function, self._receiver, self.destination.parent).execute() - self.name, self.destination = self.destination.name, self.destination.parent - - async def _a_validate_destination(self): - is_dir, *_ = await EnsureDirectory(self._metadata_function, self._receiver, self.destination, True).a_execute() - if not is_dir: - is_dir, *_ = await EnsureDirectory(self._metadata_function, self._receiver, self.destination.parent).a_execute() - self.name, self.destination = self.destination.name, self.destination.parent - - def _before_command(self): - logger.info('Uploading: %s', self.destination.join(self.name).relative) - - def _execute(self): - self._validate_destination() - with self.trace_execution(): - return self._function(self._receiver, self.get_parameter()) - - async def _a_execute(self): - await self._a_validate_destination() - with self.trace_execution(): - return await self._function(self._receiver, self.get_parameter()) - - -class UploadFile(EdgeCommand): - - def __init__(self, function, receiver, metadata_function, path, destination): - super().__init__(function, receiver) - self._metadata_function = metadata_function - self.path = path - self.destination = destination - - def _get_properties(self): - return commonfs.properties(self.path) - - def _execute(self): - metadata = self._get_properties() - with open(self.path, 'rb') as handle: - with self.trace_execution(): - return Upload(self._function, self._receiver, self._metadata_function, metadata['name'], self.destination, handle).execute() - - async def _a_execute(self): - metadata = self._get_properties() - with open(self.path, 'rb') as handle: - with self.trace_execution(): - return await Upload(self._function, self._receiver, self._metadata_function, - metadata['name'], self.destination, handle).a_execute() diff --git a/cterasdk/cio/edge/__init__.py b/cterasdk/cio/edge/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cterasdk/cio/edge/commands.py b/cterasdk/cio/edge/commands.py new file mode 100644 index 00000000..99316f56 --- /dev/null +++ b/cterasdk/cio/edge/commands.py @@ -0,0 +1,511 @@ +import logging +from ..common import Object +from ...edge.enum import ResourceError +from ..common import encode_request_parameter, encode_stream +from ... import exceptions +from ..actions import EdgeCommand +from ...lib.storage import synfs, asynfs, commonfs +from .types import EdgeResource, automatic_resolution + + +logger = logging.getLogger('cterasdk.edge') + + +def split_file_directory(listdir, receiver, destination): + """ + Split a path into its parent directory and final component. + + :returns: + tuple[str, str]: A ``(parent_directory, name)`` tuple when: + + * The path refers to an existing file + * The path refers to an existing directory + * The parent directory of the path exists + + :raises cterasdk.exceptions.io.edge.GetMetadataError: If neither the path nor its parent directory exist. + """ + is_dir, *_ = EnsureDirectory(listdir, receiver, destination, True).execute() + if not is_dir: + is_dir, *_ = EnsureDirectory(listdir, receiver, destination.parent).execute() + return destination.parent, destination.name + return destination, None + + +async def a_split_file_directory(listdir, receiver, destination): + """ + Split a path into its parent directory and final component. + + :returns: + tuple[str, str]: A ``(parent_directory, name)`` tuple when: + + * The path refers to an existing file + * The path refers to an existing directory + * The parent directory of the path exists + + :raises cterasdk.exceptions.io.edge.GetMetadataError: If neither the path nor its parent directory exist. + """ + is_dir, *_ = await EnsureDirectory(listdir, receiver, destination, True).a_execute() + if not is_dir: + is_dir, *_ = await EnsureDirectory(listdir, receiver, destination.parent).a_execute() + return destination.parent, destination.name + return destination, None + + +class Open(EdgeCommand): + """Open file""" + + def __init__(self, function, receiver, path): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + + def get_parameter(self): + return self.path.absolute + + def _before_command(self): + logger.info('Getting handle: %s', self.path) + + def _execute(self): + with self.trace_execution(): + return self._function(self._receiver, self.get_parameter()) + + async def _a_execute(self): + with self.trace_execution(): + return await self._function(self._receiver, self.get_parameter()) + + def _handle_exception(self, e): + path = self.path.relative + error = exceptions.io.edge.OpenError(path) + if isinstance(e, exceptions.transport.NotFound): + raise error from exceptions.io.edge.FileNotFoundException(path) + raise error + + +class Download(EdgeCommand): + + def __init__(self, function, receiver, path, destination): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + self.destination = destination + + def get_parameter(self): + return commonfs.determine_directory_and_filename(self.path.reference, destination=self.destination) + + def _before_command(self): + logger.info('Downloading: %s', self.path.relative) + + def _execute(self): + directory, name = self.get_parameter() + with self.trace_execution(): + with Open(self._function, self._receiver, self.path) as handle: + return synfs.write(directory, name, handle) + + async def _a_execute(self): + directory, name = self.get_parameter() + with self.trace_execution(): + async with Open(self._function, self._receiver, self.path) as handle: + return await asynfs.write(directory, name, handle) + + +class OpenMany(EdgeCommand): + + def __init__(self, function, receiver, directory, *objects): + super().__init__(function, receiver) + self.directory = directory + self.objects = objects + + def _before_command(self): + logger.info('Getting handle: %s', [self.directory.join(o).relative for o in self.objects]) + + def get_parameter(self): + param = Object() + param.paths = ['/'.join([self.directory.absolute, item]) for item in self.objects] + param.snapshot = Object() + param._classname = 'BackupRepository' # pylint: disable=protected-access + param.snapshot.location = 1 + param.snapshot.timestamp = None + param.snapshot.path = None + logger.info('Getting directory handle: %s', self.directory.reference) + return encode_request_parameter(param) + + def _execute(self): + with self.trace_execution(): + return self._function(self._receiver, self.get_parameter()) + + async def _a_execute(self): + with self.trace_execution(): + return await self._function(self._receiver, self.get_parameter()) + + +class DownloadMany(EdgeCommand): + + def __init__(self, function, receiver, target, objects, destination): + super().__init__(function, receiver) + self.target = automatic_resolution(target) + self.objects = objects + self.destination = destination + + def get_parameter(self): + return commonfs.determine_directory_and_filename(self.target.reference, self.objects, destination=self.destination, archive=True) + + def _before_command(self): + for o in self.objects: + logger.info('Downloading: %s', self.target.join(o).relative) + + def _execute(self): + directory, name = self.get_parameter() + with self.trace_execution(): + with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: + return synfs.write(directory, name, handle) + + async def _a_execute(self): + directory, name = self.get_parameter() + with self.trace_execution(): + async with OpenMany(self._function, self._receiver, self.target, *self.objects) as handle: + return await asynfs.write(directory, name, handle) + + +class ListDirectory(EdgeCommand): + """List""" + + def __init__(self, function, receiver, path, depth=None): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + self.depth = depth if depth is not None else 1 + + def _before_command(self): + logger.info('Listing directory: %s', self.path) + + def _execute(self): + with self.trace_execution(): + return self._function(self._receiver, self.path.absolute, self.depth) + + async def _a_execute(self): + with self.trace_execution(): + return await self._function(self._receiver, self.path.absolute, self.depth) + + def _handle_response(self, r): + if self.depth <= 0: + return EdgeResource.from_server_object(r[0]) + return [EdgeResource.from_server_object(e) for e in r if self.path.relative != EdgeResource.decode_reference(e.href)] + + def _handle_exception(self, e): + path = self.path.relative + error = exceptions.io.edge.ListDirectoryError(path) + if isinstance(e, exceptions.transport.NotFound): + raise error from exceptions.io.edge.FolderNotFoundError(path) + raise error from e + + +class RecursiveIterator: + + def __init__(self, function, receiver, path): + self._function = function + self._receiver = receiver + self.path = automatic_resolution(path) + self.tree = [self.path] + + def _generator(self): + logger.info('Traversing: %s', self.path) + while len(self.tree) > 0: + yield self.tree.pop(0) + + def generate(self): + for path in self._generator(): + try: + for o in ListDirectory(self._function, self._receiver, path).execute(): + if path.relative != o.path.relative: + yield self._process_object(o) + except exceptions.io.edge.ListDirectoryError as e: + RecursiveIterator._suppress_error(e) + + async def a_generate(self): + for path in self._generator(): + try: + for o in await ListDirectory(self._function, self._receiver, path).a_execute(): + if path.relative != o.path.relative: + yield self._process_object(o) + except exceptions.io.edge.ListDirectoryError as e: + RecursiveIterator._suppress_error(e) + + def _process_object(self, o): + if o.is_dir: + self.tree.append(o.path) + return o + + @staticmethod + def _suppress_error(e): + if not isinstance(e.__cause__, exceptions.io.edge.FolderNotFoundError): + raise e + logger.warning("Could not list directory contents: %s. No such directory.", e.path) + + +class GetMetadata(ListDirectory): + + def __init__(self, function, receiver, path, suppress_error=False): + super().__init__(function, receiver, path, 0) + self.suppress_error = suppress_error + + def _before_command(self): + logger.info('Getting metadata: %s', self.path.relative) + + def _handle_response(self, r): + return True, super()._handle_response(r) + + def _handle_exception(self, e): + path = self.path.relative + if not self.suppress_error: + if isinstance(e, exceptions.transport.NotFound): + cause = exceptions.io.edge.ObjectNotFoundError(path) + raise exceptions.io.edge.GetMetadataError(path) from cause + return False, None + + +class EnsureDirectory(EdgeCommand): + + def __init__(self, function, receiver, path, suppress_error=False): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + self.suppress_error = suppress_error + + def _execute(self): + return GetMetadata(self._function, self._receiver, self.path, self.suppress_error).execute() + + async def _a_execute(self): + return await GetMetadata(self._function, self._receiver, self.path, self.suppress_error).a_execute() + + def _handle_response(self, r): + exists, resource = r if r is not None else (False, None) + if (not exists or not resource.is_dir) and not self.suppress_error: + raise exceptions.io.edge.NotADirectoryException(self.path.relative) + return resource.is_dir if exists else False, resource + + +class CreateDirectory(EdgeCommand): + """Create Directory""" + + def __init__(self, function, receiver, path, parents=False): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + self.parents = parents + + def get_parameter(self): + param = Object() + param.name = self.path.name + param.parentPath = self.path.parent.absolute_encode + return param + + def _before_command(self): + logger.info('Creating directory: %s', self.path.relative) + + def _parents_generator(self): + if self.parents: + parts = self.path.parts + for i in range(1, len(parts)): + yield automatic_resolution('/'.join(parts[:i])) + else: + yield self.path + + def _execute(self): + if self.parents: + for path in self._parents_generator(): + try: + CreateDirectory(self._function, self._receiver, path).execute() + except exceptions.io.edge.CreateDirectoryError as e: + CreateDirectory._suppress_error(e) + with self.trace_execution(): + return self._function(self._receiver, self.path.absolute) + + async def _a_execute(self): + if self.parents: + for path in self._parents_generator(): + try: + await CreateDirectory(self._function, self._receiver, path).a_execute() + except exceptions.io.edge.CreateDirectoryError as e: + CreateDirectory._suppress_error(e) + with self.trace_execution(): + return await self._function(self._receiver, self.path.absolute) + + def _handle_response(self, r): + if r is None or not r or r == 'OK': + return self.path.relative + raise exceptions.io.edge.CreateDirectoryError(self.path.relative) + + def _handle_exception(self, e): + path = self.path.relative + error = exceptions.io.edge.CreateDirectoryError(path) + if e.error.response.error.msg == ResourceError.FileExists: + raise error from exceptions.io.edge.FileConflictError(path) + if e.error.response.error.msg == ResourceError.Forbidden: + raise error from exceptions.io.edge.ROFSError(path) + raise error from e + + @staticmethod + def _suppress_error(e): + if not isinstance(e.__cause__, (exceptions.io.edge.FileConflictError, exceptions.io.edge.ROFSError)): + raise e + + +class Copy(EdgeCommand): + """Copy""" + + def __init__(self, function, receiver, listdir, path, destination, overwrite): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + self.destination = automatic_resolution(destination) + self._resolver = PathResolver(listdir, receiver, self.destination, self.path.name) + self.overwrite = overwrite + + def get_parameter(self): + return self.path.absolute, self.destination.absolute + + def _before_command(self): + if self.path == self.destination: + logger.info('No-op copy. Source and destination refer to the same file: %s', self.path.relative) + raise self._error_object() from exceptions.io.edge.FileConflictError(self.path.relative) + logger.info('%s: %s to: %s', 'Copying', self.path.relative, self.destination.relative) + + def _execute(self): + self.destination = self._resolver.resolve() + source, destination = self.get_parameter() + with self.trace_execution(): + return self._function(self._receiver, source, destination, overwrite=self.overwrite) + + async def _a_execute(self): + self.destination = await self._resolver.a_resolve() + source, destination = self.get_parameter() + with self.trace_execution(): + return await self._function(self._receiver, source, destination, overwrite=self.overwrite) + + def _error_object(self): + return exceptions.io.edge.CopyError(self.path.relative, self.destination.relative) + + def _file_conflict(self): + return exceptions.io.edge.FileConflictError(self.destination.relative) + + def _handle_response(self, r): + return self.destination.relative + + def _handle_exception(self, e): + error = self._error_object() + if isinstance(e, (exceptions.transport.PreConditionFailed, exceptions.transport.Conflict)): + raise error from self._file_conflict() + raise error + + +class Move(Copy): + """Move""" + + def _before_command(self): + logger.info('%s: %s to: %s', 'Moving', self.path.relative, self.destination.relative) + + def _error_object(self): + return exceptions.io.edge.MoveError(self.path.relative, self.destination.relative) + + +class Rename(Move): + """Rename""" + + def __init__(self, function, receiver, listdir, path, new_name, overwrite): + super().__init__(function, receiver, listdir, path, automatic_resolution(path).parent.join(new_name), overwrite) + self.new_name = new_name + + def _before_command(self): + logger.info('%s: %s to: %s', 'Renaming', self.path.relative, self.destination.relative) + + def _error_object(self): + return exceptions.io.edge.RenameError(self.path.relative, self.new_name) + + def _file_conflict(self): + return exceptions.io.edge.FileConflictError(self.destination.relative) + + +class Delete(EdgeCommand): + + def __init__(self, function, receiver, path): + super().__init__(function, receiver) + self.path = automatic_resolution(path) + + def _before_command(self): + logger.info('Deleting: %s', self.path.relative) + + def _execute(self): + with self.trace_execution(): + self._function(self._receiver, self.path.absolute) + + async def _a_execute(self): + with self.trace_execution(): + await self._function(self._receiver, self.path.absolute) + + def _handle_response(self, r): + return self.path.relative + + def _handle_exception(self, e): + path = self.path.relative + error = exceptions.io.edge.DeleteError(path) + if isinstance(e, exceptions.transport.NotFound): + raise error from exceptions.io.edge.ObjectNotFoundError(path) + raise error + + +class PathResolver: + + def __init__(self, listdir, receiver, destination, default): + self._listdir = listdir + self._receiver = receiver + self._destination = destination + self._default = default + + async def a_resolve(self): + parent, name = await a_split_file_directory(self._listdir, self._receiver, self._destination) + return self._resolve(parent, name) + + def resolve(self): + parent, name = split_file_directory(self._listdir, self._receiver, self._destination) + return self._resolve(parent, name) + + def _resolve(self, parent, name): + if name is not None: + return parent.join(name) + if self._default is not None: + return parent.join(self._default) + return self._destination + + +class Upload(EdgeCommand): + + def __init__(self, function, receiver, listdir, destination, fd, name): + super().__init__(function, receiver) + self.destination = automatic_resolution(destination) + self._resolver = PathResolver(listdir, receiver, self.destination, name) + self.fd = fd + + def get_parameter(self): + fd, *_ = encode_stream(self.fd, 0) + param = dict( + name=self.destination.name, + fullpath=f'{self.destination.absolute}', + filedata=fd + ) + return param + + def _before_command(self): + logger.info('Uploading: %s', self.destination.relative) + + def _execute(self): + self.destination = self._resolver.resolve() + with self.trace_execution(): + return self._function(self._receiver, self.get_parameter()).xml() + + async def _a_execute(self): + self.destination = await self._resolver.a_resolve() + with self.trace_execution(): + response = await self._function(self._receiver, self.get_parameter()) + return await response.xml() + + def _handle_response(self, r): + if r.rc != 0: + raise exceptions.io.edge.UploadError(r.msg, self.destination.relative) + return self.destination.relative + + def _handle_exception(self, e): + raise exceptions.io.edge.UploadError(e.error.msg, self.destination.relative) from e diff --git a/cterasdk/cio/edge/types.py b/cterasdk/cio/edge/types.py new file mode 100644 index 00000000..5251848e --- /dev/null +++ b/cterasdk/cio/edge/types.py @@ -0,0 +1,92 @@ +import urllib.parse +from datetime import datetime +from ..common import BasePath, BaseResource + + +class EdgePath(BasePath): + """ + Edge Filer Path Object + """ + Namespace = '/' + + def __init__(self, reference): + super().__init__(EdgePath.Namespace, reference or '.') + + +class EdgeResource(BaseResource): + """ + Class for a Edge Filer Filesystem Resource. + + :ivar str name: Resource name + :ivar cterasdk.cio.types.EdgePath path: Path Object + :ivar bool is_dir: ``True`` if directory, ``False`` otherwise + :ivar int size: Size + :ivar datetime.datetime created_at: Last Modified + :ivar datetime.datetime last_modified: Last Modified + :ivar str extension: Extension + """ + Scheme = 'ctera-edge' + + def __init__(self, path, is_dir, size, created_at, last_modified): + super().__init__(path.name, path, is_dir, size, last_modified) + self.created_at = created_at + + @staticmethod + def decode_reference(href): + namespace = '/localFiles' + return urllib.parse.unquote(href[href.index(namespace)+len(namespace) + 1:]) + + @staticmethod + def from_server_object(server_object): + return EdgeResource( + EdgePath(EdgeResource.decode_reference(server_object.href)), + server_object.getcontenttype == 'httpd/unix-directory', + server_object.getcontentlength, + datetime.fromisoformat(server_object.creationdate), + datetime.strptime(server_object.getlastmodified, "%a, %d %b %Y %H:%M:%S GMT") + ) + + +def resolve(path): + """ + Resolve Path + + :param object path: Path + """ + if isinstance(path, EdgePath): + return path + + if isinstance(path, EdgeResource): + return path.path + + if path is None or isinstance(path, str): + return EdgePath(path) + + raise ValueError(f'Error: Could not resolve path: {path}. Type: {type(path)}') + + +def create_generator(paths): + """ + Create Path Object Generator Object. + + :param object paths: List or a tuple + """ + def wrapper(): + for path in paths: + if isinstance(path, tuple): + yield resolve(path[0]), resolve(path[1]) + else: + yield resolve(path) + return wrapper() + + +def automatic_resolution(p): + """ + Automatic Resolution of Path Object + + :param object p: Path + """ + if isinstance(p, (list, tuple)): + return create_generator(p) + + return resolve(p) diff --git a/cterasdk/clients/errors.py b/cterasdk/clients/errors.py index 275a5e8d..7425d7ea 100644 --- a/cterasdk/clients/errors.py +++ b/cterasdk/clients/errors.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from http import HTTPStatus from ..exceptions.transport import ( - BadRequest, Unauthorized, Forbidden, NotFound, NotAllowed, PreConditionFailed, Unprocessable, + BadRequest, Unauthorized, Forbidden, NotFound, NotAllowed, Conflict, PreConditionFailed, Unprocessable, InternalServerError, BadGateway, ServiceUnavailable, GatewayTimeout, HTTPError ) from ..common import Object @@ -68,6 +68,7 @@ def raise_error(status, error): HTTPStatus.FORBIDDEN: Forbidden, HTTPStatus.NOT_FOUND: NotFound, HTTPStatus.METHOD_NOT_ALLOWED: NotAllowed, + HTTPStatus.CONFLICT: Conflict, HTTPStatus.PRECONDITION_FAILED: PreConditionFailed, HTTPStatus.UNPROCESSABLE_ENTITY: Unprocessable, HTTPStatus.INTERNAL_SERVER_ERROR: InternalServerError, diff --git a/cterasdk/common/__init__.py b/cterasdk/common/__init__.py index a3de2ff0..e16302ec 100644 --- a/cterasdk/common/__init__.py +++ b/cterasdk/common/__init__.py @@ -6,3 +6,4 @@ from .types import PolicyRule, PolicyRuleConverter, StringCriteriaBuilder, IntegerCriteriaBuilder, DateTimeCriteriaBuilder, \ PredefinedListCriteriaBuilder, CustomListCriteriaBuilder, ThrottlingRuleBuilder, ThrottlingRule, FilterBackupSet, \ FileFilterBuilder, ApplicationBackupSet, TimeRange # noqa: E402, F401 +from .modules import BaseModule # noqa: E402, F401 diff --git a/cterasdk/common/modules.py b/cterasdk/common/modules.py new file mode 100644 index 00000000..5f74ea48 --- /dev/null +++ b/cterasdk/common/modules.py @@ -0,0 +1,21 @@ +import logging +from abc import ABC, abstractmethod +from .utils import Version + + +logger = logging.getLogger('cterasdk.common') + + +class BaseModule(ABC): + + @abstractmethod + def initialize_version(self, software_version): + raise NotImplementedError("Subclass must implement the 'initialize_version' method.") + + +def initialize(module_class, receiver): + session = receiver.session() + software_version = session.software_version if session.software_version else Version('0') + concrete_class = module_class().initialize_version(software_version) + logger.debug('Initializing: %s', concrete_class.__name__) + return concrete_class(receiver) diff --git a/cterasdk/core/files/browser.py b/cterasdk/core/files/browser.py index ed4f2e31..2202ecc5 100644 --- a/cterasdk/core/files/browser.py +++ b/cterasdk/core/files/browser.py @@ -1,298 +1,355 @@ -import logging - from .. import query -from ...cio.core import CorePath, Open, OpenMany, Upload, UploadFile, Download, \ - DownloadMany, UnShare, CreateDirectory, GetMetadata, ListVersions, RecursiveIterator, \ +from ...cio.core.commands import Open, OpenMany, Upload, Download, EnsureDirectory, \ + DownloadMany, UnShare, CreateDirectory, GetMetadata, GetProperties, ListVersions, RecursiveIterator, \ Delete, Recover, Rename, GetShareMetadata, Link, Copy, Move, ResourceIterator, GetPermalink -from ...exceptions.transport import HTTPError from ...lib.storage import commonfs from ..base_command import BaseCommand from . import io -logger = logging.getLogger('cterasdk.core') - - class FileBrowser(BaseCommand): - - def __init__(self, core): - super().__init__(core) - self._scope = f'/{self._core.context}/webdav' + """CTERA Portal File Browser API.""" def handle(self, path): """ - Get File Handle. + Get a file handle. - :param str path: Path to a file + :param str path: Path to a file. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.core.OpenError: Raised on error to obtain a file handle. """ - return Open(io.handle, self._core, self.normalize(path)).execute() + return Open(io.handle, self._core, path).execute() def handle_many(self, directory, *objects): """ - Get a Zip Archive File Handle. + Get a ZIP archive file handle. - :param str directory: Path to a folder - :param args objects: List of files and folders + :param str directory: Path to a folder. + :param args objects: Files and folders to include. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. """ - return OpenMany(io.handle_many, self._core, self.normalize(directory), *objects).execute() + with EnsureDirectory(io.listdir, self._core, directory) as (_, resource): + return OpenMany(io.handle_many, self._core, resource, directory, *objects).execute() def download(self, path, destination=None): """ - Download a file + Download a file. - :param str path: Path - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str path: Path. + :param str, optional destination: File destination. If a directory is provided, the original filename is preserved. + Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str + :raises cterasdk.exceptions.io.core.OpenError: Raised on error to obtain a file handle. """ - return Download(io.handle, self._core, self.normalize(path), destination).execute() + return Download(io.handle, self._core, path, destination).execute() - def download_many(self, target, objects, destination=None): + def download_many(self, directory, objects, destination=None): """ Download selected files and/or directories as a ZIP archive. .. warning:: - The provided list of objects is not validated. Only existing files and directories - will be included in the resulting ZIP file. + Only existing files and directories will be included in the resulting ZIP file. - :param str target: - Path to the cloud folder containing the files and directories to download. - :param list[str] objects: - List of file and/or directory names to include in the download. - :param str destination: - Optional. Path to the destination file or directory. If a directory is provided, - the original filename will be preserved. Defaults to the default download directory. + :param str directory: Path to a folder. + :param list[str] objects: List of files and / or directory names to download. + :param str destination: Optional path to destination file or directory. Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. """ - return DownloadMany(io.handle_many, self._core, self.normalize(target), objects, destination).execute() + with EnsureDirectory(io.listdir, self._core, directory) as (_, resource): + return DownloadMany(io.handle_many, self._core, resource, directory, objects, destination).execute() - def listdir(self, path=None, depth=None, include_deleted=False): + def listdir(self, path=None, include_deleted=False): """ - List Directory + List directory contents. - :param str,optional path: Path, defaults to the Cloud Drive root - :param bool,optional include_deleted: Include deleted files, defaults to False + :param str, optional path: Path. Defaults to the Cloud Drive root. + :param bool, optional include_deleted: Include deleted files. Defaults to False. + :returns: Directory contents. + :rtype: list[cterasdk.cio.core.types.PortalResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. + :raises cterasdk.exceptions.io.core.ListDirectoryError: Raised on error fetching directory contents. """ - return ResourceIterator(query.iterator, self._core, self.normalize(path), depth, include_deleted, None, None).execute() + with EnsureDirectory(io.listdir, self._core, path): + return ResourceIterator(query.iterator, self._core, path, None, include_deleted, None, None).execute() + + def properties(self, path): + """ + Get object properties. + + :param str path: Path. + :returns: Object properties. + :rtype: cterasdk.cio.core.types.PortalResource + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error retrieving object metadata. + """ + _, metadata = GetProperties(io.listdir, self._core, path, False).execute() + return metadata def exists(self, path): """ - Check if item exists + Check whether an item exists. - :param str path: Path + :param str path: Path. + :returns: True if the item exists, False otherwise. + :rtype: bool """ - with GetMetadata(io.listdir, self._core, self.normalize(path), True) as (exists, *_): + with GetMetadata(io.listdir, self._core, path, True) as (exists, *_): return exists def versions(self, path): """ - List snapshots of a file or directory + List snapshots of a file or directory. - :param str path: Path + :param str path: Path. + :returns: List of versions. + :rtype: list[cterasdk.cio.core.types.PreviousVersion] + :raises cterasdk.exceptions.io.core.GetVersionsError: Raised on error retrieving versions. """ - return ListVersions(io.versions, self._core, self.normalize(path)).execute() + return ListVersions(io.versions, self._core, path).execute() def walk(self, path=None, include_deleted=False): """ - Walk Directory Contents + Walk directory contents. - :param str,optional path: Path to walk, defaults to the root directory - :param bool,optional include_deleted: Include deleted files, defaults to False + :param str, optional path: Path to walk. Defaults to the root directory. + :param bool, optional include_deleted: Include deleted files. Defaults to False. + :returns: A generator of file-system objects. + :rtype: Iterator[cterasdk.cio.core.types.PortalResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. """ - return RecursiveIterator(query.iterator, self._core, self.normalize(path), include_deleted).generate() + with EnsureDirectory(io.listdir, self._core, path): + return RecursiveIterator(query.iterator, self._core, path, include_deleted).generate() def public_link(self, path, access='RO', expire_in=30): """ - Create a public link to a file or a folder + Create a public link to a file or folder. - :param str path: The path of the file to create a link to - :param str,optional access: Access policy of the link, defaults to 'RO' - :param int,optional expire_in: Number of days until the link expires, defaults to 30 + :param str path: Path of the file or folder. + :param str, optional access: Access policy of the link. Defaults to 'RO'. + :param int, optional expire_in: Days until link expires. Defaults to 30. + :returns: Public link. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateLinkError: Raised on failure to generate public link. """ - return Link(io.public_link, self._core, self.normalize(path), access, expire_in).execute() + return Link(io.public_link, self._core, path, access, expire_in).execute() def copy(self, *paths, destination=None, resolver=None, cursor=None, wait=True): """ - Copy one or more files or folders + Copy one or more files or folders. - :param list[str] paths: List of paths - :param str destination: Destination - :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` - :param cterasdk.common.object.Object cursor: Resume copy from cursor - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to copy. + :param str destination: Destination path. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults to None. + :param cterasdk.common.object.Object cursor: Resume copy from cursor. + :param bool, optional wait: Wait for task completion. Defaults to True. + :returns: Task status object, or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.CopyError: Raised on failure to copy resources. """ try: - return Copy(io.copy, self._core, wait, *[self.normalize(path) for path in paths], - destination=self.normalize(destination), resolver=resolver, cursor=cursor).execute() + return Copy(io.copy, self._core, wait, *paths, destination=destination, resolver=resolver, cursor=cursor).execute() except ValueError: raise ValueError('Copy destination was not specified.') def permalink(self, path): """ - Get Permalink for Path. + Get permalink for a path. :param str path: Path. + :returns: Permalink. + :rtype: str + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error retrieving object metadata. """ - return GetPermalink(io.listdir, self._core, self.normalize(path)).execute() - - def normalize(self, entries): - return CorePath.instance(self._scope, entries) + return GetPermalink(io.listdir, self._core, path).execute() class CloudDrive(FileBrowser): + """CloudDrive extends FileBrowser with upload and share functionality.""" - def upload(self, name, destination, handle, size=None): + def upload(self, destination, handle, name=None, size=None): """ Upload from file handle. - :param str name: File name. - :param str destination: Path to remote directory. - :param object handle: File handle, String, or Bytes. - :param str,optional size: File size, defaults to content length + :param str destination: Remote path. + :param object handle: File-like handle. + :param str, optional size: File size. Defaults to content length. + :param str, optional name: Filename to use if it cannot be derived from ``destination`` + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.core.UploadError: Raised on upload failure. """ - return Upload(io.upload, self._core, io.listdir, name, self.normalize(destination), size, handle).execute() + return Upload(io.upload, self._core, io.listdir, destination, handle, name, size).execute() def upload_file(self, path, destination): """ Upload a file. - :param str path: Local path - :param str destination: Remote path + :param str path: Local path. + :param str destination: Remote path. + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.core.UploadError: Raised on upload failure. """ - return UploadFile(io.upload, self._core, io.listdir, path, self.normalize(destination)).execute() + _, name = commonfs.split_file_directory(path) + with open(path, 'rb') as handle: + return self.upload(destination, handle, name, commonfs.properties(path)['size']) def mkdir(self, path): """ - Create a new directory + Create a directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateDirectoryError: Raised on error creating directory. """ - return CreateDirectory(io.mkdir, self._core, self.normalize(path)).execute() + return CreateDirectory(io.mkdir, self._core, path).execute() def makedirs(self, path): """ - Create a directory recursively + Recursively create a directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.core.CreateDirectoryError: Raised on error creating directory. """ - return CreateDirectory(io.mkdir, self._core, self.normalize(path), True).execute() + return CreateDirectory(io.mkdir, self._core, path, True).execute() - def rename(self, path, name, *, wait=True): + def rename(self, path, name, *, resolver=None, wait=True): """ - Rename a file + Rename a file or folder. - :param str path: Path of the file or directory to rename - :param str name: The name to rename to - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param str path: Path of the file or directory. + :param str name: New name. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults None. + :param bool, optional wait: Wait for task completion. Defaults to True. + :returns: Task status object, or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.RenameError: Raised on error renaming object. """ - return Rename(io.move, self._core, self.normalize(path), name, wait).execute() + return Rename(io.move, self._core, wait, path, name, resolver).execute() def delete(self, *paths, wait=True): """ - Delete one or more files or folders + Delete one or more files or folders. - :param str path: Path - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to delete. + :param bool, optional wait: Wait for task completion. Defaults to True. + :returns: Task status object, or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.DeleteError: Raised on error deleting resources. """ - return Delete(io.delete, self._core, wait, *[self.normalize(path) for path in paths]).execute() + return Delete(io.delete, self._core, wait, *paths).execute() def undelete(self, *paths, wait=True): """ - Recover one or more files or folders + Recover one or more files or folders. - :param str path: Path - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to recover. + :param bool, optional wait: Wait for task completion. Defaults to True. + :returns: Task status object, or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.RecoverError: Raised on error recovering resources. """ - return Recover(io.undelete, self._core, wait, *[self.normalize(path) for path in paths]).execute() + return Recover(io.undelete, self._core, wait, *paths).execute() def move(self, *paths, destination=None, resolver=None, cursor=None, wait=True): """ - Move one or more files or folders + Move one or more files or folders. - :param list[str] paths: List of paths - :param str destination: Destination - :param cterasdk.core.types.ConflictResolver resolver: Conflict resolver, defaults to ``None`` - :param cterasdk.common.object.Object cursor: Resume copy from cursor - :param bool,optional wait: ``True`` Wait for task to complete, or ``False`` to return an awaitable task object. - :returns: Task status object, or an awaitable task object + :param list[str] paths: Paths to move. + :param str destination: Destination path. + :param cterasdk.core.types.ConflictResolver, optional resolver: Conflict resolver. Defaults to None. + :param cterasdk.common.object.Object cursor: Resume move from cursor. + :param bool, optional wait: Wait for task completion. Defaults to True. + :returns: Task status object, or awaitable task. :rtype: cterasdk.common.object.Object or :class:`cterasdk.lib.tasks.AwaitablePortalTask` + :raises cterasdk.exceptions.io.core.MoveError: Raised on error moving resources. """ try: - return Move(io.move, self._core, wait, *[self.normalize(path) for path in paths], - destination=self.normalize(destination), resolver=resolver, cursor=cursor).execute() + return Move(io.move, self._core, wait, *paths, destination=destination, resolver=resolver, cursor=cursor).execute() except ValueError: raise ValueError('Move destination was not specified.') def get_share_info(self, path): """ - Get share settings and recipients + Get share settings and recipients. - :param str path: Path + :param str path: Path. + :returns: Share metadata. + :rtype: object + :raises cterasdk.exceptions.io.core.GetShareMetadataError: Raised on error obtaining share metadata. """ - return GetShareMetadata(io.list_shares, self._core, self.normalize(path)).execute() + return GetShareMetadata(io.list_shares, self._core, path).execute() def share(self, path, recipients, as_project=True, allow_reshare=True, allow_sync=True): """ - Share a file or a folder + Share a file or folder. - :param str path: The path of the file or folder to share - :param list[cterasdk.core.types.Collaborator] recipients: A list of share recipients - :param bool,optional as_project: Share as a team project, defaults to True when the item is a cloud folder else False - :param bool,optional allow_reshare: Allow recipients to re-share this item, defaults to True - :param bool,optional allow_sync: Allow recipients to sync this item, defaults to True when the item is a cloud folder else False - :return: A list of all recipients added to the collaboration share + :param str path: Path of the file or folder to share. + :param list[cterasdk.core.types.Collaborator] recipients: Share recipients. + :param bool, optional as_project: Share as a team project. Defaults True if cloud folder. + :param bool, optional allow_reshare: Allow re-share. Defaults True. + :param bool, optional allow_sync: Allow sync. Defaults True if cloud folder. + :returns: Current list of share members. :rtype: list[cterasdk.core.types.Collaborator] """ - return io.share(self._core, self.normalize(path), recipients, as_project, allow_reshare, allow_sync) + return io.share(self._core, path, recipients, as_project, allow_reshare, allow_sync) def add_share_recipients(self, path, recipients): """ - Add share recipients + Add share recipients. - :param str path: The path of the file or folder - :param list[cterasdk.core.types.Collaborator] recipients: A list of share recipients - :return: A list of all recipients added + :param str path: Path of file/folder. + :param list[cterasdk.core.types.Collaborator] recipients: Recipients to add. + :returns: Current list of share members. :rtype: list[cterasdk.core.types.Collaborator] """ - return io.add_share_recipients(self._core, self.normalize(path), recipients) + return io.add_share_recipients(self._core, path, recipients) def remove_share_recipients(self, path, accounts): """ - Remove share recipients + Remove share recipients. - :param str path: The path of the file or folder - :param list[cterasdk.core.types.PortalAccount] accounts: A list of portal user or group accounts - :return: A list of all share recipients removed - :rtype: list[cterasdk.core.types.PortalAccount] + :param str path: Path of file/folder. + :param list[cterasdk.core.types.PortalAccount] accounts: Accounts to remove. + :returns: Current list of share members. + :rtype: list[cterasdk.core.types.Collaborator] """ - return io.remove_share_recipients(self._core, self.normalize(path), accounts) + return io.remove_share_recipients(self._core, path, accounts) def unshare(self, path): """ - Unshare a file or a folder + Unshare a file or folder. + + :param str path: Path of file/folder. """ - return UnShare(io.update_share, self._core, self.normalize(path)).execute() + return UnShare(io.update_share, self._core, path).execute() class Backups(FileBrowser): + """Backup management API.""" def device_config(self, device, destination=None): """ - Download a device configuration file + Download a device configuration file. - :param str device: The device name - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str device: Device name. + :param str, optional destination: File destination. If a directory is provided, original filename is preserved. + Defaults to default downloads directory. + :returns: Path to local file. + :rtype: str + :raises cterasdk.exceptions.io.core.OpenError: Raised on error obtaining file handle. """ - try: - destination = destination if destination is not None else f'{commonfs.downloads()}/{device}.xml' - return self.download(f'backups/{device}/Device Configuration/db.xml', destination) - except HTTPError as error: - logger.error('Error downloading device configuration: %s. Reason: %s', device, error.response.reason) - raise error + destination = destination if destination is not None else f'{commonfs.downloads()}/{device}.xml' + return Download(io.handle, self._core, f'backups/{device}/Device Configuration/db.xml', destination).execute() diff --git a/cterasdk/core/files/io.py b/cterasdk/core/files/io.py index 9d105fb3..dd222faf 100644 --- a/cterasdk/core/files/io.py +++ b/cterasdk/core/files/io.py @@ -1,4 +1,4 @@ -from ...cio import core as fs +from ...cio.core import commands as fs def listdir(core, param): @@ -33,9 +33,8 @@ def handle(core, param): return core.io.download(param) -def handle_many(core, param, directory): - with fs.EnsureDirectory(listdir, core, directory) as (_, resource): - return core.io.download_zip(str(resource.cloudFolderInfo.uid), param) +def handle_many(core, cloudfolder, param): + return core.io.download_zip(cloudfolder, param) def upload(core, cloudfolder, param): diff --git a/cterasdk/edge/directoryservice.py b/cterasdk/edge/directoryservice.py index 82c33f45..112fd6d9 100644 --- a/cterasdk/edge/directoryservice.py +++ b/cterasdk/edge/directoryservice.py @@ -58,7 +58,7 @@ def _check_domain_connectivity(self, domain): port = 389 domain_controllers = self.get_static_domain_controller() domain_controllers = re.findall(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b', domain_controllers) if domain_controllers else [domain] - connection_results = self._edge.network.diagnose([TCPService(host, port) for host in domain_controllers]) + connection_results = self._edge.network.diag.diagnose([TCPService(host, port) for host in domain_controllers]) for connection_result in connection_results: if not connection_result.is_open: logger.error("Connection failed. No traffic allowed over port %(port)s", dict(port=connection_result.port)) diff --git a/cterasdk/edge/files/browser.py b/cterasdk/edge/files/browser.py index 34c3b8b6..6742d93c 100644 --- a/cterasdk/edge/files/browser.py +++ b/cterasdk/edge/files/browser.py @@ -1,63 +1,98 @@ from ..base_command import BaseCommand -from ...cio.edge import EdgePath, ListDirectory, RecursiveIterator, GetMetadata, Open, OpenMany, Upload, \ - UploadFile, CreateDirectory, Copy, Move, Delete, Download, DownloadMany, Rename +from ...cio.edge.commands import ListDirectory, RecursiveIterator, GetMetadata, Open, OpenMany, Upload, \ + CreateDirectory, Copy, Move, Delete, Download, DownloadMany, Rename, EnsureDirectory +from ...lib.storage import commonfs from . import io class FileBrowser(BaseCommand): - """ Edge Filer File Browser APIs """ + """Edge Filer File Browser API.""" - def listdir(self, path): + def listdir(self, path=None): """ - List Directory + List directory contents. - :param str path: Path + :param str path: Path. Defaults to the root directory. + :returns: Directory contents. + :rtype: list[cterasdk.cio.edge.types.EdgeResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. + :raises cterasdk.exceptions.io.edge.ListDirectoryError: Raised on error to fetch directory contents. """ - return ListDirectory(io.listdir, self._edge, self.normalize(path)).execute() + with EnsureDirectory(io.listdir, self._edge, path): + return ListDirectory(io.listdir, self._edge, path).execute() def walk(self, path=None): """ - Walk Directory Contents + Walk directory contents. - :param str,optional path: Path to walk, defaults to the root directory + :param str, optional path: Path to walk. Defaults to the root directory. + :returns: A generator of file-system objects. + :rtype: Iterator[cterasdk.cio.edge.types.EdgeResource] + :raises cterasdk.exceptions.io.core.GetMetadataError: If the directory was not found. + :raises cterasdk.exceptions.io.core.NotADirectoryException: If the target path is not a directory. """ - return RecursiveIterator(io.listdir, self._edge, self.normalize(path)).generate() + with EnsureDirectory(io.listdir, self._edge, path): + return RecursiveIterator(io.listdir, self._edge, path).generate() + + def properties(self, path): + """ + Get object properties. + + :param str path: Path. + :returns: Object properties. + :rtype: cterasdk.cio.edge.types.EdgeResource + :raises cterasdk.exceptions.io.core.GetMetadataError: Raised on error to obtain object metadata. + """ + with GetMetadata(io.listdir, self._edge, path, False) as (_, metadata): + return metadata def exists(self, path): """ - Check if item exists + Check whether an item exists. - :param str path: Path + :param str path: Path. + :returns: ``True`` if the item exists, ``False`` otherwise. + :rtype: bool """ - with GetMetadata(io.listdir, self._edge, self.normalize(path), True) as (exists, *_): + with GetMetadata(io.listdir, self._edge, path, True) as (exists, *_): return exists def handle(self, path): """ - Get File Handle. + Get a file handle. - :param str path: Path to a file + :param str path: Path to a file. + :returns: File handle. + :rtype: object + :raises cterasdk.exceptions.io.edge.OpenError: Raised on error to obtain a file handle. """ - return Open(io.handle, self._edge, self.normalize(path)).execute() + return Open(io.handle, self._edge, path).execute() def handle_many(self, directory, *objects): """ - Get a Zip Archive File Handle. + Get a ZIP archive file handle. - :param str directory: Path to a folder - :param args objects: List of files and folders + :param str directory: Path to a folder. + :param args objects: Files and folders to include. + :returns: File handle. + :rtype: object """ return OpenMany(io.handle_many, self._edge, directory, *objects).execute() def download(self, path, destination=None): """ - Download a file + Download a file. - :param str path: The file path on the Edge Filer - :param str,optional destination: - File destination, if it is a directory, the original filename will be kept, defaults to the default directory + :param str path: The file path on the Edge Filer. + :param str, optional destination: + File destination. If a directory is provided, the original filename is preserved. + Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str + :raises cterasdk.exceptions.io.edge.OpenError: Raised on error to obtain a file handle. """ - return Download(io.handle, self._edge, self.normalize(path), destination).execute() + return Download(io.handle, self._edge, path, destination).execute() def download_many(self, target, objects, destination=None): """ @@ -73,82 +108,103 @@ def download_many(self, target, objects, destination=None): List of file and/or directory names to include in the download. :param str destination: Optional. Path to the destination file or directory. If a directory is provided, - the original filename will be preserved. Defaults to the default download directory. + the original filename is preserved. Defaults to the default download directory. + :returns: Path to the local file. + :rtype: str """ - return DownloadMany(io.handle_many, self._edge, self.normalize(target), objects, destination).execute() + return DownloadMany(io.handle_many, self._edge, target, objects, destination).execute() - def upload(self, name, destination, handle): + def upload(self, destination, handle, name=None): """ - Upload from file handle. + Upload from a file handle. - :param str name: File name. - :param str destination: Path to remote directory. - :param object handle: Handle. + :param str destination: Remote path. + :param object handle: File-like handle. + :param str, optional name: Filename to use if it cannot be derived from ``destination`` + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.edge.UploadError: Raised on upload failure. """ - return Upload(io.upload, self._edge, io.listdir, name, self.normalize(destination), handle).execute() + return Upload(io.upload, self._edge, io.listdir, destination, handle, name).execute() def upload_file(self, path, destination): """ Upload a file. - :param str path: Local path - :param str destination: Remote path + :param str path: Local path. + :param str destination: Remote path. + :returns: Remote file path. + :rtype: str + :raises cterasdk.exceptions.io.edge.UploadError: Raised on upload failure. """ - return UploadFile(io.upload, self._edge, io.listdir, path, self.normalize(destination)).execute() + _, name = commonfs.split_file_directory(path) + with open(path, 'rb') as handle: + return self.upload(destination, handle, name) def mkdir(self, path): """ - Create a new directory + Create a new directory. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.edge.CreateDirectoryError: Raised on error to create a directory. """ - return CreateDirectory(io.mkdir, self._edge, self.normalize(path)).execute() + return CreateDirectory(io.mkdir, self._edge, path).execute() def makedirs(self, path): """ - Create a directory recursively + Create a directory recursively. - :param str path: Directory path + :param str path: Directory path. + :returns: Remote directory path. + :rtype: str + :raises cterasdk.exceptions.io.edge.CreateDirectoryError: Raised on error to create a directory. """ - return CreateDirectory(io.mkdir, self._edge, self.normalize(path), True).execute() + return CreateDirectory(io.mkdir, self._edge, path, True).execute() def copy(self, path, destination=None, overwrite=False): """ - Copy a file or a folder + Copy a file or directory. - :param str path: Source file or folder path - :param str destination: Destination folder path - :param bool,optional overwrite: Overwrite on conflict, defaults to False + :param str path: Source file or directory path. + :param str destination: Destination directory path. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :raises cterasdk.exceptions.io.edge.CopyError: Raised on error to copy a file or directory. """ - return Copy(io.copy, self._edge, self.normalize(path), self.normalize(destination), overwrite).execute() + return Copy(io.copy, self._edge, io.listdir, path, destination, overwrite).execute() def move(self, path, destination=None, overwrite=False): """ - Move a file or a folder + Move a file or directory. - :param str path: Source file or folder path - :param str destination: Destination folder path - :param bool,optional overwrite: Overwrite on conflict, defaults to False + :param str path: Source file or directory path. + :param str destination: Destination directory path. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :raises cterasdk.exceptions.io.edge.MoveError: Raised on error to move a file or directory. """ - return Move(io.move, self._edge, self.normalize(path), self.normalize(destination), overwrite).execute() + return Move(io.move, self._edge, io.listdir, path, destination, overwrite).execute() - def rename(self, path, name): + def rename(self, path, new_name, overwrite=False): """ - Rename a file + Rename a file or directory. - :param str path: Path of the file or directory to rename - :param str name: The name to rename to + :param str path: Path of the file or directory to rename. + :param str new_name: New name for the file or directory. + :param bool, optional overwrite: Overwrite on conflict. Defaults to ``False``. + :returns: Remote object path. + :rtype: str + :raises cterasdk.exceptions.io.edge.RenameError: Raised on error to rename a file or directory. """ - return Rename(io.move, self._edge, self.normalize(path), name).execute() + return Rename(io.move, self._edge, io.listdir, path, new_name, overwrite).execute() def delete(self, path): """ - Delete a file + Delete a file or directory. - :param str path: File path + :param str path: File or directory path. + :returns: Deleted object path. + :rtype: str + :raises cterasdk.exceptions.io.edge.DeleteError: Raised on error to delete a file or directory. """ - return Delete(io.delete, self._edge, self.normalize(path)).execute() - - @staticmethod - def normalize(path): - return EdgePath.instance('/', path) + return Delete(io.delete, self._edge, path).execute() diff --git a/cterasdk/edge/network.py b/cterasdk/edge/network.py index e85943c2..90514256 100644 --- a/cterasdk/edge/network.py +++ b/cterasdk/edge/network.py @@ -1,56 +1,94 @@ import logging +from abc import abstractmethod +from contextlib import contextmanager import ipaddress -from ..exceptions import CTERAException from ..exceptions.common import TaskException from .enum import Mode, IPProtocol, Traffic -from .types import TCPConnectResult, StaticRoute -from ..common import Object +from .types import TCPConnectResult, StaticRoute, NetworkInterface +from ..common import Object, BaseModule from .base_command import BaseCommand logger = logging.getLogger('cterasdk.edge') +class NetworkModule(BaseModule): + + def initialize_version(self, software_version): + return Network711 if software_version > '7.9' else LegacyNetwork + + class Network(BaseCommand): - """ Edge Filer Network configuration APIs """ - def __init__(self, portal): - super().__init__(portal) - self.proxy = Proxy(self._edge) - self.mtu = MTU(self._edge) - self.routes = StaticRoutes(self._edge) + def __init__(self, edge): + super().__init__(edge) self.hosts = Hosts(self._edge) + self.proxy = Proxy(self._edge) + self.diag = Diagnostics(self._edge) - def get_status(self): + def interface(self, name): """ - Retrieve the network interface status + Get Network Interface + + :param str name: Interface name """ - return self._edge.api.get('/status/network/ports/0') + for interface in self.interfaces: + if interface.name == name: + return interface + raise ValueError(f'Could not find interface: {name}.') - def ifconfig(self): + def port(self, name): """ - Retrieve the ip configuration + Get Port + + :param str name: Interface name """ - return self.ipconfig() + return self.interface(name).port - def ipconfig(self): + def deduce_port(self, interface): + return interface if interface in [0, 1] else self.port(interface) + + @property + def interfaces(self): """ - Retrieve the ip configuration + Get Network Interfaces. + + :returns: A list of network interfaces + :rtype: list[cterasdk.edge.types.NetworkInterface] """ - return self._edge.api.get('/config/network/ports/0') + return [NetworkInterface(i, interface.name, interface.ethernet.mac) + for i, interface in enumerate(self._edge.api.get('/status/network/ports'))] - def set_static_ipaddr(self, address, subnet, gateway, primary_dns_server, secondary_dns_server=None): + def status(self, interface): + """ + Get Interface Status. + + :param object interface: Interface name or port number + """ + return self._edge.api.get(f'/status/network/ports/{self.deduce_port(interface)}') + + def ipconfig(self, interface): + """ + Get Interface Configuration + + :param object interface: Interface name or port number + """ + return self._edge.api.get(f'/config/network/ports/{self.deduce_port(interface)}') + + def set_static_ipaddr(self, interface, address, subnet, gateway, primary_dns_server, secondary_dns_server=None): """ Set a Static IP Address + :param object interface: Interface name or port number :param str address: The static address :param str subnet: The subnet for the static address :param str gateway: The default gateway :param str primary_dns_server: The primary DNS server :param str,optinal secondary_dns_server: The secondary DNS server, defaults to None """ - ip = self._edge.api.get('/config/network/ports/0/ip') + port = self.deduce_port(interface) + ip = self._edge.api.get(f'/config/network/ports/{port}/ip') ip.DHCPMode = Mode.Disabled ip.address = address ip.netmask = subnet @@ -61,112 +99,92 @@ def set_static_ipaddr(self, address, subnet, gateway, primary_dns_server, second if secondary_dns_server is not None: ip.DNSServer2 = secondary_dns_server - logger.info('Configuring a static ip address.') - - self._edge.api.put('/config/network/ports/0/ip', ip) - + logger.info('Updating network configuration: IP Configuration') + self._edge.api.put(f'/config/network/ports/{port}/ip', ip) logger.info( - 'Network settings updated. %s', + 'Network configuration updated. %s', {'address': address, 'subnet': subnet, 'gateway': gateway, 'DNS1': primary_dns_server, 'DNS2': secondary_dns_server} ) - def set_static_nameserver(self, primary_dns_server, secondary_dns_server=None): + def set_static_nameserver(self, interface, primary_dns_server, secondary_dns_server=None): """ - Set the DNS Server addresses statically + Set Static DNS Servers - :param str primary_dns_server: The primary DNS server - :param str,optinal secondary_dns_server: The secondary DNS server, defaults to None + :param object interface: Interface name or port number + :param str primary_dns_server: Primary DNS server + :param str,optional secondary_dns_server: Secondary DNS server, defaults to None """ - ip = self._edge.api.get('/config/network/ports/0/ip') + port = self.deduce_port(interface) + ip = self._edge.api.get(f'/config/network/ports/{port}/ip') ip.autoObtainDNS = False ip.DNSServer1 = primary_dns_server if secondary_dns_server is not None: ip.DNSServer2 = secondary_dns_server - logger.info('Configuring nameserver settings.') - - self._edge.api.put('/config/network/ports/0/ip', ip) + logger.info('Updating network configuration: DNS.') + self._edge.api.put(f'/config/network/ports/{port}/ip', ip) + logger.info('Network configuration updated. %s', {'DNS1': primary_dns_server, 'DNS2': secondary_dns_server}) - logger.info('Nameserver settings updated. %s', {'DNS1': primary_dns_server, 'DNS2': secondary_dns_server}) - - def enable_dhcp(self): + def enable_dhcp(self, interface): """ Enable DHCP + + :param object interface: Interface name or port number """ - ip = self._edge.api.get('/config/network/ports/0/ip') + port = self.deduce_port(interface) + ip = self._edge.api.get(f'/config/network/ports/{port}/ip') ip.DHCPMode = Mode.Enabled ip.autoObtainDNS = True + logger.info('Updating network configuration: Enabling DHCP.') + self._edge.api.put(f'/config/network/ports/{port}/ip', ip) + logger.info('Network configuration updated. Enabled DHCP.') - logger.info('Enabling DHCP.') - self._edge.api.put('/config/network/ports/0/ip', ip) +class Network711(Network): + """ Edge Filer v7.11 Network API """ - logger.info('Network settings updated. Enabled DHCP.') + def __init__(self, edge): + super().__init__(edge) + self.mtu = MTU711(self._edge) + self.routes = StaticRoutes711(self._edge) - def diagnose(self, services): - """ - Test a TCP connection to a host over a designated port - :param list[cterasdk.edge.types.TCPService] services: List of services, identified by a host and a port - :returns: A list of named-tuples including the host, port and a boolean value indicating whether TCP connection can be established - :rtype: list[cterasdk.edge.types.TCPConnectResult] - """ - return [self.tcp_connect(service) for service in services] +class LegacyNetwork(Network): + """ Edge Filer Legacy Network API """ - def tcp_connect(self, service): - """ - Test a TCP connection between the Edge Filer and the provided host address + def __init__(self, edge): + super().__init__(edge) + self.mtu = LegacyMTU(self._edge) + self.routes = LegacyStaticRoutes(self._edge) - :param cterasdk.edge.types.TCPService service: A service, identified by a host and a port - :returns: A named-tuple including the host, port and a boolean value indicating whether TCP connection can be established - :rtype: cterasdk.edge.types.TCPConnectResult + def get_status(self): """ - param = Object() - param.address = service.host - param.port = service.port - - logger.info("Testing connection. %s", {'host': service.host, 'port': service.port}) + Retrieve the network interface status + """ + return super().status(0) - ref = self._edge.api.execute("/status/network", "tcpconnect", param) - try: - task = self._edge.tasks.wait(ref) - logger.debug("Connection status: %s", task.result.rc) - if task.result.rc == "Open": - return TCPConnectResult(service.host, service.port, True) - except TaskException: - pass + def ifconfig(self): + """ + Retrieve the IP address settings + """ + return super().ipconfig(0) - logger.warning("Couldn't establish TCP connection. %s", {'address': service.host, 'port': service.port}) + def ipconfig(self): # pylint: disable=arguments-differ + """ + Retrieve the IP address settings + """ + return super().ipconfig(0) - return TCPConnectResult(service.host, service.port, False) + def set_static_ipaddr(self, address, subnet, gateway, # pylint: disable=arguments-differ + primary_dns_server, secondary_dns_server=None): + return super().set_static_ipaddr(0, address, subnet, gateway, primary_dns_server, secondary_dns_server) - def iperf(self, address, port=5201, threads=1, protocol=IPProtocol.TCP, direction=Traffic.Upload, timeout=None): - """ - Invoke a network throughput test + def set_static_nameserver(self, primary_dns_server, secondary_dns_server=None): # pylint: disable=arguments-differ + return super().set_static_nameserver(0, primary_dns_server, secondary_dns_server) - :param str address: The host running the iperf server - :param int,optional port: The iperf server port, defaults to 5201 - :param int,optional threads: The number of threads, defaults to 1 - :param cterasdk.edge.enum.IPProtocol,optional protocol: IP protocol, defaults to `'TCP'` - :param cterasdk.edge.enum.Traffic,optional direction: Traffic direction, defaults to `'Upload'` - :param float,optional timeout: Timeout (in seconds). - :returns: A string containing the iperf output - :rtype: str - """ - param = Object() - param._classname = 'IperfParam' # pylint: disable=protected-access - param.address = address - param.port = port - param.threads = threads - param.reverse = direction == Traffic.Download - param.protocol = None if protocol == IPProtocol.TCP else IPProtocol.UDP - ref = self._edge.api.execute("/status/network", "iperf", param) - try: - task = self._edge.tasks.wait(ref, timeout) - return task.result.res - except TaskException as error: - return error.task.result.res + def enable_dhcp(self): # pylint: disable=arguments-differ + return super().enable_dhcp(0) class Proxy(BaseCommand): @@ -228,75 +246,81 @@ def disable(self): return self._configure(False) -class MTU(BaseCommand): - """Edge Filer MTU Configuration APIs""" +class BaseMTU(BaseCommand): + + def reset(self, interface): # pylint: disable=arguments-differ + return self._update_max_transmission_unit(interface, False, 1500) + + def modify(self, interface, size): # pylint: disable=arguments-differ + return self._update_max_transmission_unit(interface, True, size) + + def _update_max_transmission_unit(self, interface, jumbo, size): + port = self._edge.network.deduce_port(interface) + settings = self._edge.api.get(f'/config/network/ports/{port}/ethernet') + settings.jumbo = jumbo + settings.mtu = size + logger.info('Configuring %s MTU for interface: %s', size, port) + return self._edge.api.put(f'/config/network/ports/{port}/ethernet', settings) + + +class MTU711(BaseMTU): + """Multi Network Interface MTU Configuration""" + + +class LegacyMTU(BaseMTU): + """Single Network Interface MTU Configuration""" - def reset(self): + def reset(self): # pylint: disable=arguments-differ """ - Set the default maximum transmission unit (MTU) settings + Reset to defaults. """ - return self._configure(False, 1500) + super().reset(0) - def modify(self, mtu): + def modify(self, size): # pylint: disable=arguments-differ """ - Set a custom network maximum transmission unit (MTU) + Set Custom Network Maximum Transmission Unit (MTU) - :param int mtu: Maximum transmission unit + :param int size: Maximum Transmission Unit """ - return self._configure(True, mtu) - - def _configure(self, jumbo, mtu): - settings = self._edge.api.get('/config/network/ports/0/ethernet') - settings.jumbo = jumbo - settings.mtu = mtu - logger.info('Configuring MTU. %s', {'MTU': mtu}) - return self._edge.api.put('/config/network/ports/0/ethernet', settings) + super().modify(0, size) -class StaticRoutes(BaseCommand): - """Edge Filer Static Route Configuration APIs""" +class BaseStaticRoutes(BaseCommand): + """Edge Filer Static Routes Configuration""" - def get(self): - """ - Get routes. - """ - return [StaticRoute(r.DestIpMask, r.GwIP) for r in self._edge.api.get('/config/network/static_routes')] + @staticmethod + @contextmanager + def _validate_route(gateway, network): + yield str(ipaddress.ip_address(gateway)), ipaddress.ip_network(network) - def add(self, gateway, network): + def add(self, interface, route_gateway, route_network): # pylint: disable=arguments-differ """ Add a route. - :param str gateway: Gateway IP address - :param str network: Network (CIDR) + :param object interface: Interface name or port number + :param str route_gateway: Gateway IP address + :param str route_network: Network (CIDR) """ - ipaddress.ip_address(gateway) - ipaddress.ip_network(network) - param = Object() - param.GwIP = gateway - param.DestIpMask = network.replace('/', '_') - try: - logger.info('Adding route for network: %s, to: %s', network, param.GwIP) - self._edge.api.add('/config/network/static_routes', param) - logger.info('Route added for network: %s, to: %s', network, param.GwIP) - return StaticRoute(network, gateway) - except CTERAException as error: - logger.error("Static route creation failed.") - raise CTERAException('Static route creation failed') from error + with BaseStaticRoutes._validate_route(route_gateway, route_network) as (gateway, network): + ip_network = str(network) + logger.info('Adding route for network: %s, to: %s', ip_network, gateway) + response = self._add_route(interface, gateway, network) + logger.info('Route added for network: %s, to: %s', ip_network, gateway) + return response + + @abstractmethod + def _add_route(self, interface, gateway, network): + raise NotImplementedError('Subclass must implement the "_add_route" method.') - def delete(self, network): + def delete(self, route): """ Delete a route. - :param str network: Subnet mask (CIDR) + :param cterasdk.edge.types.StaticRoute route: Static Route """ - ipaddress.ip_network(network) - try: - logger.info('Deleting route for: %s', network) - self._edge.api.delete(f'/config/network/static_routes/{network.replace("/", "_")}') - logger.info('Route deleted. Subnet: %s', network) - except CTERAException as error: - logger.error("Static route deletion failed.") - raise CTERAException('Static route deletion failed') from error + logger.info('Deleting route. Network: %s, Gateway: %s', route.network, route.gateway) + self._edge.api.delete(f'/config/network/static_routes/{route.id}') + logger.info('Route deleted. Network: %s, Gateway: %s', route.network, route.gateway) def clear(self): logger.info('Clearing route table.') @@ -304,6 +328,59 @@ def clear(self): logger.info('Route table cleared.') +class StaticRoutes711(BaseStaticRoutes): + """Edge Filer 7.11 Static Routes Configuration""" + + def get(self): + """ + Get Routes. + """ + routes = [] + for port, interface in enumerate(self._edge.api.get('/config/network/ports')): + for route in interface.ipv4StaticRoutes: + ip_network = str(ipaddress.IPv4Network(f'{route.destination}/{route.netmask}', False)) + routes.append(StaticRoute(interface._uuid, port, # pylint: disable=protected-access + interface.name, ip_network, route.gateway)) + return routes + + def _add_route(self, interface, gateway, network): + port = self._edge.network.deduce_port(interface) + param = Object() + param.destination = str(network.network_address) + param.netmask = str(network.netmask) + param.gateway = gateway + return self._edge.api.add(f'/config/network/ports/{port}/ipv4StaticRoutes', param) + + +class LegacyStaticRoutes(BaseStaticRoutes): + """Legacy Static Routes Configuration""" + + def get(self): + """ + Get Routes. + """ + network = self._edge.api.get('/config/network') + return [StaticRoute( + r._uuid, 0, network.ports[0].name, # pylint: disable=protected-access + str(ipaddress.IPv4Network(r.DestIpMask.replace('_', '/'), False)), r.GwIP + ) for r in network.static_routes] + + def add(self, gateway, network): # pylint: disable=arguments-differ + """ + Add a route. + + :param str gateway: Gateway IP address + :param str network: Network (CIDR) + """ + return super().add(0, gateway, network) + + def _add_route(self, interface, gateway, network): # pylint: disable=unused-argument + param = Object() + param.GwIP = gateway + param.DestIpMask = str(network).replace('/', '_') + return self._edge.api.add('/config/network/static_routes', param) + + class Hosts(BaseCommand): """Edge Filer Static Route Configuration APIs""" @@ -325,3 +402,70 @@ def add(self, ipaddr, hostname): def delete(self, hostname): return self._edge.api.delete(f'/config/network/hostsFileEntries/{hostname}') + + +class Diagnostics(BaseCommand): + + def diagnose(self, services): + """ + Test a TCP connection to a host over a designated port + + :param list[cterasdk.edge.types.TCPService] services: List of services, identified by a host and a port + :returns: A list of named-tuples including the host, port and a boolean value indicating whether TCP connection can be established + :rtype: list[cterasdk.edge.types.TCPConnectResult] + """ + return [self.tcp_connect(service) for service in services] + + def tcp_connect(self, service): + """ + Test a TCP connection between the Edge Filer and the provided host address + + :param cterasdk.edge.types.TCPService service: A service, identified by a host and a port + :returns: A named-tuple including the host, port and a boolean value indicating whether TCP connection can be established + :rtype: cterasdk.edge.types.TCPConnectResult + """ + param = Object() + param.address = service.host + param.port = service.port + + logger.info("Testing connection. %s", {'host': service.host, 'port': service.port}) + + ref = self._edge.api.execute("/status/network", "tcpconnect", param) + try: + task = self._edge.tasks.wait(ref) + logger.debug("Connection status: %s", task.result.rc) + if task.result.rc == "Open": + return TCPConnectResult(service.host, service.port, True) + except TaskException: + pass + + logger.warning("Couldn't establish TCP connection. %s", {'address': service.host, 'port': service.port}) + + return TCPConnectResult(service.host, service.port, False) + + def iperf(self, address, port=5201, threads=1, protocol=IPProtocol.TCP, direction=Traffic.Upload, timeout=None): + """ + Invoke a network throughput test + + :param str address: The host running the iperf server + :param int,optional port: The iperf server port, defaults to 5201 + :param int,optional threads: The number of threads, defaults to 1 + :param cterasdk.edge.enum.IPProtocol,optional protocol: IP protocol, defaults to `'TCP'` + :param cterasdk.edge.enum.Traffic,optional direction: Traffic direction, defaults to `'Upload'` + :param float,optional timeout: Timeout (in seconds). + :returns: A string containing the iperf output + :rtype: str + """ + param = Object() + param._classname = 'IperfParam' # pylint: disable=protected-access + param.address = address + param.port = port + param.threads = threads + param.reverse = direction == Traffic.Download + param.protocol = None if protocol == IPProtocol.TCP else IPProtocol.UDP + ref = self._edge.api.execute("/status/network", "iperf", param) + try: + task = self._edge.tasks.wait(ref, timeout) + return task.result.res + except TaskException as error: + return error.task.result.res diff --git a/cterasdk/edge/services.py b/cterasdk/edge/services.py index 53d23fbf..eb809444 100644 --- a/cterasdk/edge/services.py +++ b/cterasdk/edge/services.py @@ -150,7 +150,7 @@ def _validate_license(ctera_license): raise error def _check_cttp_traffic(self, address, port=995): - tcp_connect_result = self._edge.network.tcp_connect(TCPService(address, port)) + tcp_connect_result = self._edge.network.diag.tcp_connect(TCPService(address, port)) if not tcp_connect_result.is_open: logger.error("Unable to establish connection over port %s", str(tcp_connect_result.port)) raise ConnectionError(f'Unable to establish CTTP connection {tcp_connect_result.host}:{tcp_connect_result.port}') diff --git a/cterasdk/edge/shares.py b/cterasdk/edge/shares.py index 1cf6a5a0..453488a7 100644 --- a/cterasdk/edge/shares.py +++ b/cterasdk/edge/shares.py @@ -1,7 +1,7 @@ import logging from . import enum -from ..cio.edge import EdgePath +from ..cio.edge.types import automatic_resolution from ..common import Object from ..exceptions import CTERAException, InputError from .base_command import BaseCommand @@ -60,7 +60,7 @@ def add(self, param = Object() param.name = name - parts = EdgePath('/', directory).parts + parts = automatic_resolution(directory).parts volume = parts[0] self._validate_root_directory(volume) param.volume = volume @@ -239,7 +239,7 @@ def modify( """ share = self.get(name=name) if directory is not None: - parts = EdgePath('/', directory).parts + parts = automatic_resolution(directory).parts volume = parts[0] self._validate_root_directory(volume) share.volume = volume diff --git a/cterasdk/edge/ssl.py b/cterasdk/edge/ssl.py index 33fe2662..60de2576 100644 --- a/cterasdk/edge/ssl.py +++ b/cterasdk/edge/ssl.py @@ -2,19 +2,16 @@ from .base_command import BaseCommand from ..lib import X509Certificate, PrivateKey, create_certificate_chain -from ..common import Object +from ..common import Object, BaseModule logger = logging.getLogger('cterasdk.edge') -def initialize(edge): - """ - Conditional intialization of the Edge Filer SSL Module. - """ - if edge.session().software_version > '7.8': - return SSLv78(edge) - return SSLv1(edge) +class SSLModule(BaseModule): + + def initialize_version(self, software_version): + return SSLv78 if software_version > '7.8' else SSLv1 class SSL(BaseCommand): diff --git a/cterasdk/edge/types.py b/cterasdk/edge/types.py index 6abae58f..c3812cba 100644 --- a/cterasdk/edge/types.py +++ b/cterasdk/edge/types.py @@ -18,8 +18,11 @@ 'established to the target host over the specified port' -StaticRoute = namedtuple('StaticRoute', ('network', 'gateway')) -StaticRoute.__doc__ = 'Tuple holding the network and gateway of a static route' +StaticRoute = namedtuple('StaticRoute', ('id', 'port', 'interface', 'network', 'gateway')) +StaticRoute.__doc__ = 'Object holding a network interface static route' +StaticRoute.id.__doc__ = 'Route ID' +StaticRoute.port.__doc__ = 'Port number' +StaticRoute.interface.__doc__ = 'Interface name' StaticRoute.network.__doc__ = 'Network (CIDR)' StaticRoute.gateway.__doc__ = 'Gateway IP address' @@ -428,3 +431,15 @@ def from_server_object(server_object): 'disconnected_hours': server_object.CloudConnectFailHours, } return AlertSettings(**params) + + +class NetworkInterface(Object): + """ + Interface Status + + :ivar int port: Port number + :ivar str name: Interface name + :ivar str mac: MAC address + """ + def __init__(self, port, name, mac): + super().__init__(port=port, name=name, mac=mac) diff --git a/cterasdk/exceptions/io/base.py b/cterasdk/exceptions/io/base.py index 04c42b5c..9a270ff1 100644 --- a/cterasdk/exceptions/io/base.py +++ b/cterasdk/exceptions/io/base.py @@ -3,3 +3,14 @@ class BaseIOError(IOError): """Base CTERA IO Error""" + + +class PathError(BaseIOError): + """ + Object I/O Error + + :ivar str path: Path + """ + def __init__(self, errno, strerror, filename, filename2=None): + super().__init__(errno, strerror, filename, None, filename2) + self.path = filename diff --git a/cterasdk/exceptions/io/core.py b/cterasdk/exceptions/io/core.py index d21d75ef..f900dfe9 100644 --- a/cterasdk/exceptions/io/core.py +++ b/cterasdk/exceptions/io/core.py @@ -1,123 +1,143 @@ import errno -from .base import BaseIOError, EREMOTEIO +from .base import BaseIOError, PathError, EREMOTEIO -class FileConflictError(BaseIOError): +class FileConflictError(PathError): def __init__(self, filename): super().__init__(errno.EEXIST, 'File exists', filename) -class ObjectNotFoundError(BaseIOError): +class ObjectNotFoundError(PathError): def __init__(self, filename): super().__init__(errno.ENOENT, 'Object not found', filename) -class FileNotFoundException(BaseIOError): +class FileNotFoundException(PathError): def __init__(self, filename): super().__init__(errno.ENOENT, 'File not found', filename) -class FolderNotFoundError(BaseIOError): +class FolderNotFoundError(PathError): def __init__(self, filename): super().__init__(errno.ENOENT, 'Folder not found', filename) -class NotADirectoryException(BaseIOError): +class NotADirectoryException(PathError): def __init__(self, filename): super().__init__(errno.ENOTDIR, 'Not a directory', filename) -class WriteError(BaseIOError): +class OpenError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to open file', filename) + + +class UploadError(PathError): def __init__(self, strerror, filename): - super().__init__(EREMOTEIO, f'Write failed. Reason: {strerror}', filename) + super().__init__(EREMOTEIO, f'Upload failed. Reason: {strerror}', filename) -class ROFSError(BaseIOError): +class ROFSError(PathError): def __init__(self, filename): super().__init__(errno.EROFS, 'Write access denied. Target path is read-only', filename) -class PrivilegeError(BaseIOError): +class PrivilegeError(PathError): def __init__(self, filename): super().__init__(errno.EACCES, 'Access denied. No permission to access resource', filename) -class NTACLError(BaseIOError): +class NTACLError(PathError): def __init__(self, filename): super().__init__(errno.EACCES, 'Access denied. Unable to access Windows ACL-enabled volume', filename) -class QuotaError(BaseIOError): +class QuotaError(PathError): def __init__(self, filename): super().__init__(errno.EDQUOT, 'Write failed. Out of quota', filename) -class StorageBackendError(BaseIOError): +class StorageBackendError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Storage backend unavailable', filename) -class FileRejectedError(BaseIOError): +class FileRejectedError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Rejected by policy', filename) -class FilenameError(BaseIOError): +class FilenameError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'File name contains characters that are not allowed "\\ / : ? & < > \" |".', filename) -class ReservedNameError(BaseIOError): +class ReservedNameError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Specified name is reserved by the system', filename) -class ListDirectoryError(BaseIOError): +class ListDirectoryError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to list directory', filename) -class GetSnapshotsError(BaseIOError): +class GetVersionsError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to enumerate versions', filename) -class CreateDirectoryError(BaseIOError): +class CreateDirectoryError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to create folder', filename) -class GetShareMetadataError(BaseIOError): +class GetMetadataError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to retrieve object metadata', filename) + + +class GetShareMetadataError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to retrieve collaboration-share metadata', filename) -class CreateLinkError(BaseIOError): +class CreateLinkError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to create public link', filename) +class RenameError(PathError): + + def __init__(self, paths, cursor): + source, destination = paths[0] + super().__init__(EREMOTEIO, 'Failed to rename object', str(source), str(destination.name)) + self.cursor = cursor + + class BatchError(BaseIOError): - """Base job error""" + """Task Error""" def __init__(self, strerror, paths, cursor): super().__init__(EREMOTEIO, strerror) diff --git a/cterasdk/exceptions/io/edge.py b/cterasdk/exceptions/io/edge.py index 99099a81..37a27bcd 100644 --- a/cterasdk/exceptions/io/edge.py +++ b/cterasdk/exceptions/io/edge.py @@ -1,32 +1,104 @@ import errno -from .base import BaseIOError, EREMOTEIO +from .base import PathError, EREMOTEIO -class FileConflictError(BaseIOError): +class FileConflictError(PathError): def __init__(self, filename): super().__init__(errno.EEXIST, 'File exists', filename) -class FileNotFoundException(BaseIOError): +class ObjectNotFoundError(PathError): + + def __init__(self, filename): + super().__init__(errno.ENOENT, 'Object not found', filename) + + +class FileNotFoundException(PathError): def __init__(self, filename): super().__init__(errno.ENOENT, 'File not found', filename) -class NotADirectoryException(BaseIOError): +class FolderNotFoundError(PathError): + + def __init__(self, filename): + super().__init__(errno.ENOENT, 'Folder not found', filename) + + +class GetMetadataError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to retrieve object metadata', filename) + + +class NotADirectoryException(PathError): def __init__(self, filename): super().__init__(errno.ENOTDIR, 'Not a directory', filename) -class ROFSError(BaseIOError): +class ROFSError(PathError): def __init__(self, filename): super().__init__(errno.EROFS, 'Write access denied. Target path is read-only', filename) -class CreateDirectoryError(BaseIOError): +class ListDirectoryError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to list directory', filename) + + +class CreateDirectoryError(PathError): def __init__(self, filename): super().__init__(EREMOTEIO, 'Failed to create directory', filename) + + +class OpenError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to open file', filename) + + +class DeleteError(PathError): + + def __init__(self, filename): + super().__init__(EREMOTEIO, 'Failed to delete object', filename) + + +class RenameError(PathError): + + def __init__(self, filename, filename2): + super().__init__(EREMOTEIO, 'Failed to rename object', filename, filename2) + + +class CopyError(PathError): + """ + Copy Error. + + :ivar str path: Path + :ivar str destination: Destination + """ + def __init__(self, path, destination): + super().__init__(EREMOTEIO, 'Failed to copy object', path, destination) + self.destination = destination + + +class MoveError(PathError): + """ + Move Error. + + :ivar str path: Path + :ivar str destination: Destination + """ + def __init__(self, path, destination): + super().__init__(EREMOTEIO, 'Failed to move object', path, destination) + self.destination = destination + + +class UploadError(PathError): + + def __init__(self, strerror, filename): + super().__init__(EREMOTEIO, f'Upload failed. Reason: {strerror}', filename) diff --git a/cterasdk/exceptions/transport.py b/cterasdk/exceptions/transport.py index 5e0ea324..ada6ffbb 100644 --- a/cterasdk/exceptions/transport.py +++ b/cterasdk/exceptions/transport.py @@ -48,6 +48,12 @@ def __init__(self, error): super().__init__(HTTPStatus.METHOD_NOT_ALLOWED, error) +class Conflict(HTTPError): + + def __init__(self, error): + super().__init__(HTTPStatus.CONFLICT, error) + + class PreConditionFailed(HTTPError): def __init__(self, error): diff --git a/cterasdk/lib/storage/commonfs.py b/cterasdk/lib/storage/commonfs.py index 689fa920..bf5d3dd1 100644 --- a/cterasdk/lib/storage/commonfs.py +++ b/cterasdk/lib/storage/commonfs.py @@ -194,15 +194,19 @@ def write_new_version(directory, name, *, ctx=None): def split_file_directory(location): """ - Split file and directory. + Split a path into its parent directory and final component. - :param str path: Path + :param str path: The path to a file or directory. + + :returns: + tuple[str, str]: A ``(parent_directory, name)`` tuple when: + + * The path refers to an existing file + * The path refers to an existing directory + * The parent directory of the path exists - Returns: - 1. (parent directory, file name), if a file exists - 2. (parent directory, file name), if a directory exists - 3. (parent directory, file name), if the parent directory exists - 4. Raises ``FileNotFoundError`` if neither the object nor the parent directory exist + :raises FileNotFoundError: + If neither the path nor its parent directory exist. """ p = expanduser(location) if p.exists(): diff --git a/cterasdk/lib/tasks.py b/cterasdk/lib/tasks.py index 3658118b..f19b5c87 100644 --- a/cterasdk/lib/tasks.py +++ b/cterasdk/lib/tasks.py @@ -74,6 +74,12 @@ def __init__(self, task): self.progress_str = task.progstring self.tenant = task.portalUid + def unknown_object(self): + match = re.search(r'(?<=^Resource,\ )(.+(?=\ ,\ ))', self.progress_str) + if match and self.progress_str[match.end() + 3:] == "doesn't exist.": + return False + return True + class FilesystemTask(PortalTask): """ diff --git a/cterasdk/objects/synchronous/core.py b/cterasdk/objects/synchronous/core.py index 7e5cfa7f..0b1769de 100644 --- a/cterasdk/objects/synchronous/core.py +++ b/cterasdk/objects/synchronous/core.py @@ -108,9 +108,9 @@ def public_info(self): @property def _omit_fields(self): - return super()._omit_fields + ['activation', 'admins', 'cloudfs', 'credentials', 'devices', 'directoryservice', 'domains', 'files', - 'firmwares', 'groups', 'logs', 'plans', 'reports', 'roles', 'settings', 'tasks', 'templates', - 'users'] + return super()._omit_fields + ['activation', 'admins', 'backups', 'cloudfs', 'credentials', 'devices', 'directoryservice', + 'domains', 'files', 'firmwares', 'groups', 'logs', 'plans', 'reports', 'roles', 'settings', + 'storage_classes', 'tasks', 'templates', 'users'] class GlobalAdmin(Portal): # pylint: disable=too-many-instance-attributes diff --git a/cterasdk/objects/synchronous/edge.py b/cterasdk/objects/synchronous/edge.py index 788d8db7..dbe2c9ba 100644 --- a/cterasdk/objects/synchronous/edge.py +++ b/cterasdk/objects/synchronous/edge.py @@ -3,8 +3,10 @@ from ..services import Management from ..endpoints import EndpointBuilder from .. import authenticators +from ...common import modules from ...lib.session.edge import Session + from ...edge import ( afp, aio, antivirus, array, audit, backup, cache, cli, config, connection, ctera_migrate, dedup, directoryservice, drive, files, firmware, ftp, groups, licenses, login, @@ -93,7 +95,7 @@ def __init__(self, host=None, port=None, https=True, Portal=None, *, base=None): self.licenses = licenses.Licenses(self) self.logs = logs.Logs(self) self.mail = mail.Mail(self) - self.network = network.Network(self) + self.network = modules.initialize(network.NetworkModule, self) self.nfs = nfs.NFS(self) self.ntp = ntp.NTP(self) self.power = power.Power(self) @@ -105,7 +107,7 @@ def __init__(self, host=None, port=None, https=True, Portal=None, *, base=None): self.smb = smb.SMB(self) self.snmp = snmp.SNMP(self) self.ssh = ssh.SSH(self) - self.ssl = ssl.SSL(self) + self.ssl = modules.initialize(ssl.SSLModule, self) self.support = support.Support(self) self.sync = sync.Sync(self) self.syslog = syslog.Syslog(self) @@ -116,7 +118,8 @@ def __init__(self, host=None, port=None, https=True, Portal=None, *, base=None): self.volumes = volumes.Volumes(self) def _after_login(self): - self.ssl = ssl.initialize(self) + self.ssl = modules.initialize(ssl.SSLModule, self) + self.network = modules.initialize(network.NetworkModule, self) @property def migrate(self): @@ -158,8 +161,8 @@ def remote_access(self): @property def _omit_fields(self): - return super()._omit_fields + ['afp', 'aio', 'array', 'audit', 'backup', 'cache', 'cli', 'config', 'ctera_migrate', 'dedup', - 'directoryservice', 'drive', 'files', 'firmware', 'ftp', 'groups', 'licenses', 'logs', 'mail', - 'network', 'nfs', 'ntp', 'power', 'ransom_protect', 'rsync', 'services', 'shares', 'shell', + return super()._omit_fields + ['afp', 'aio', 'array', 'audit', 'antivirus', 'backup', 'cache', 'cli', 'config', 'ctera_migrate', + 'dedup', 'directoryservice', 'drive', 'files', 'firmware', 'ftp', 'groups', 'licenses', 'logs', + 'mail', 'network', 'nfs', 'ntp', 'power', 'ransom_protect', 'rsync', 'services', 'shares', 'shell', 'smb', 'snmp', 'ssh', 'ssl', 'support', 'sync', 'syslog', 'tasks', 'telnet', 'timezone', 'users', 'volumes'] diff --git a/docs/source/UserGuides/Edge/Configuration.rst b/docs/source/UserGuides/Edge/Configuration.rst index a41b2c55..95ef3092 100644 --- a/docs/source/UserGuides/Edge/Configuration.rst +++ b/docs/source/UserGuides/Edge/Configuration.rst @@ -843,68 +843,81 @@ Windows File Sharing (CIFS/SMB) Network ======= -.. automethod:: cterasdk.edge.network.Network.set_static_ipaddr - :noindex: +Version 7.11+ +------------- -.. code-block:: python +.. automethod:: cterasdk.edge.network.Network.interface + :noindex: - edge.network.set_static_ipaddr('10.100.102.4', '255.255.255.0', '10.100.102.1', '10.100.102.1') +.. automethod:: cterasdk.edge.network.Network.port + :noindex: - edge.show('/status/network/ports/0/ip') # will print the IP configuration +.. automethod:: cterasdk.edge.network.Network.deduce_port + :noindex: -.. automethod:: cterasdk.edge.network.Network.set_static_nameserver +.. automethod:: cterasdk.edge.network.Network.status :noindex: -.. code-block:: python +.. automethod:: cterasdk.edge.network.Network.ipconfig + :noindex: - edge.network.set_static_nameserver('10.100.102.1') # to set the primary name server +.. automethod:: cterasdk.edge.network.Network.set_static_ipaddr + :noindex: - edge.network.set_static_nameserver('10.100.102.1', '10.100.102.254') # to set both primary and secondary +.. automethod:: cterasdk.edge.network.Network.set_static_nameserver + :noindex: .. automethod:: cterasdk.edge.network.Network.enable_dhcp :noindex: -.. code-block:: python +.. automethod:: cterasdk.edge.network.MTU711.reset + :noindex: - edge.network.enable_dhcp() +.. automethod:: cterasdk.edge.network.MTU711.modify + :noindex: -Proxy Settings --------------- +.. automethod:: cterasdk.edge.network.StaticRoutes711.get + :noindex: -.. automethod:: cterasdk.edge.network.Proxy.get_configuration +.. automethod:: cterasdk.edge.network.StaticRoutes711.add :noindex: -.. code-block:: python +.. automethod:: cterasdk.edge.network.StaticRoutes711.delete + :noindex: - configuration = edge.network.proxy.get_configuration() - print(configuration) +.. automethod:: cterasdk.edge.network.StaticRoutes711.clear + :noindex: -.. automethod:: cterasdk.edge.network.Proxy.is_enabled + +Pre-Version 7.11 +---------------- + +.. automethod:: cterasdk.edge.network.LegacyNetwork.set_static_ipaddr :noindex: .. code-block:: python - if edge.network.proxy.is_enabled(): - print('Proxy Server is Enabled') + edge.network.set_static_ipaddr('10.100.102.4', '255.255.255.0', '10.100.102.1', '10.100.102.1') -.. automethod:: cterasdk.edge.network.Proxy.modify + edge.show('/status/network/ports/0/ip') # will print the IP configuration + +.. automethod:: cterasdk.edge.network.LegacyNetwork.set_static_nameserver :noindex: .. code-block:: python - edge.network.proxy.modify('192.168.11.11', 8081, 'proxy-user', 'proxy-user-password') + edge.network.set_static_nameserver('10.100.102.1') # to set the primary name server -.. automethod:: cterasdk.edge.network.Proxy.disable + edge.network.set_static_nameserver('10.100.102.1', '10.100.102.254') # to set both primary and secondary + +.. automethod:: cterasdk.edge.network.LegacyNetwork.enable_dhcp :noindex: .. code-block:: python - edge.network.proxy.disable() - -MTU ---- + edge.network.enable_dhcp() -.. automethod:: cterasdk.edge.network.MTU.modify +.. automethod:: cterasdk.edge.network.LegacyMTU.modify :noindex: .. code-block:: python @@ -913,24 +926,21 @@ MTU edge.network.mtu.modify(9000) # configure 'jumbo' frames (MTU: 9000) -.. automethod:: cterasdk.edge.network.MTU.reset +.. automethod:: cterasdk.edge.network.LegacyMTU.reset :noindex: .. code-block:: python edge.network.mtu.reset() # disable custom mtu configuration and restore default setting (1500) -Static Routes -------------- - -.. automethod:: cterasdk.edge.network.StaticRoutes.get +.. automethod:: cterasdk.edge.network.LegacyStaticRoutes.get :noindex: .. code-block:: python edge.network.routes.get() -.. automethod:: cterasdk.edge.network.StaticRoutes.add +.. automethod:: cterasdk.edge.network.LegacyStaticRoutes.add :noindex: .. code-block:: python @@ -939,14 +949,14 @@ Static Routes edge.network.routes.add('10.100.102.4', '172.18.100.0/24') -.. automethod:: cterasdk.edge.network.StaticRoutes.delete +.. automethod:: cterasdk.edge.network.LegacyStaticRoutes.delete :noindex: .. code-block:: python edge.network.routes.delete('192.168.55.7/32') -.. automethod:: cterasdk.edge.network.StaticRoutes.clear +.. automethod:: cterasdk.edge.network.LegacyStaticRoutes.clear :noindex: .. code-block:: python @@ -954,6 +964,38 @@ Static Routes # remove all static routes - (clean) edge.network.routes.clear() +Proxy Settings +-------------- + +.. automethod:: cterasdk.edge.network.Proxy.get_configuration + :noindex: + +.. code-block:: python + + configuration = edge.network.proxy.get_configuration() + print(configuration) + +.. automethod:: cterasdk.edge.network.Proxy.is_enabled + :noindex: + +.. code-block:: python + + if edge.network.proxy.is_enabled(): + print('Proxy Server is Enabled') + +.. automethod:: cterasdk.edge.network.Proxy.modify + :noindex: + +.. code-block:: python + + edge.network.proxy.modify('192.168.11.11', 8081, 'proxy-user', 'proxy-user-password') + +.. automethod:: cterasdk.edge.network.Proxy.disable + :noindex: + +.. code-block:: python + + edge.network.proxy.disable() Hosts ----- @@ -983,13 +1025,13 @@ Hosts Diagnostics ----------- -.. automethod:: cterasdk.edge.network.Network.tcp_connect +.. automethod:: cterasdk.edge.network.Diagnostics.tcp_connect :noindex: .. code-block:: python cttp_service = edge_types.TCPService('tenant.ctera.com', 995) - result = edge.network.tcp_connect(cttp_service) + result = edge.network.diag.tcp_connect(cttp_service) if result.is_open: print('Success') # do something... @@ -997,9 +1039,9 @@ Diagnostics print('Failure') ldap_service = edge_types.TCPService('dc.ctera.com', 389) - edge.network.tcp_connect(ldap_service) + edge.network.diag.tcp_connect(ldap_service) -.. automethod:: cterasdk.edge.network.Network.diagnose +.. automethod:: cterasdk.edge.network.Diagnostics.diagnose :noindex: .. code-block:: python @@ -1008,23 +1050,23 @@ Diagnostics services.append(edge_types.TCPService('192.168.90.1', 389)) # LDAP services.append(edge_types.TCPService('ctera.portal.com', 995)) # CTTP services.append(edge_types.TCPService('ctera.portal.com', 443)) # HTTPS - result = edge.network.diagnose(services) + result = edge.network.diag.diagnose(services) for result in results: print(result.host, result.port, result.is_open) -.. automethod:: cterasdk.edge.network.Network.iperf +.. automethod:: cterasdk.edge.network.Diagnostics.iperf :noindex: .. code-block:: python - edge.network.iperf('192.168.1.145') # iperf server: 192.168.1.145, threads: 1, measure upload over TCP port 5201 + edge.network.diag.iperf('192.168.1.145') # iperf server: 192.168.1.145, threads: 1, measure upload over TCP port 5201 - edge.network.iperf('192.168.1.145', port=85201, threads=5) # Customized port and number of threads + edge.network.diag.iperf('192.168.1.145', port=85201, threads=5) # Customized port and number of threads - edge.network.iperf('192.168.1.145', direction=edge_enum.Traffic.Download) # Measure download speed + edge.network.diag.iperf('192.168.1.145', direction=edge_enum.Traffic.Download) # Measure download speed - edge.network.iperf('192.168.1.145', protocol=edge_enum.IPProtocol.UDP) # Use UDP + edge.network.diag.iperf('192.168.1.145', protocol=edge_enum.IPProtocol.UDP) # Use UDP Antivirus diff --git a/docs/source/UserGuides/Edge/Files.rst b/docs/source/UserGuides/Edge/Files.rst index 03a8a1d3..a82de3df 100644 --- a/docs/source/UserGuides/Edge/Files.rst +++ b/docs/source/UserGuides/Edge/Files.rst @@ -2,108 +2,231 @@ File Browser ============ +Preface: Authentication +======================= + +Before using the Edge Filer API, you need to authenticate with your Edge device. Below are examples for synchronous and asynchronous usage. + +Synchronous login +----------------- + +Set ``cterasdk.settings.edge.syn.settings.connector.ssl = False`` to disable SSL verification. + +.. code:: python + + from cterasdk.edge import Edge + + username = "admin" + password = "secret" + + with Edge('172.54.3.149') as edge: + edge.login(username, password) + # Now you can access files via edge.files + +Asynchronous login +------------------ + +Set ``cterasdk.settings.edge.asyn.settings.connector.ssl = False`` to disable SSL verification. + +.. code:: python + + import asyncio + from cterasdk.asynchronous.edge import AsyncEdge + + async def main(): + username = "admin" + password = "secret" + + async with AsyncEdge('172.54.3.149') as edge: + await edge.login(username, password) + # Now you can access files via edge.files + + asyncio.run(main()) + Synchronous API =============== -List ----- +Listing Files and Directories +----------------------------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.listdir :noindex: .. code:: python + # List contents of the root directory for item in edge.files.listdir('/'): - print(item.name, item.fullpath) + print(item.name, str(item), item.is_dir, item.size) + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.walk + :noindex: + +.. code:: python + + # Walk through all files and directories recursively + for item in edge.files.walk('/'): + print(item.name, str(item), item.is_dir, item.size) + +Inspecting Files +---------------- + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.properties + :noindex: + +.. code:: python + + # Get file or directory properties + resource = edge.files.properties('cloud/users/Service Account/My Files/Keystone Project.docx') + print(resource.name, str(resource), resource.is_dir, resource.size, resource.last_modified) + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.exists + :noindex: + +.. code:: python + + # Check if a file exists + if edge.files.exists('cloud/users/Service Account/My Files/Keystone Project.docx'): + print("File exists") + +File Handles +------------ + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.handle + :noindex: + +.. code:: python + + # Get a handle to a single file + handle = edge.files.handle('cloud/users/Service Account/My Files/Keystone Project.docx') + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.handle_many + :noindex: + +.. code:: python + + # Get a handle to multiple files/folders as a ZIP + handle = edge.files.handle_many('cloud/users/Service Account/My Files', 'Keystone Project.docx', 'Keystone Model.docx') -Download --------- +Downloading Files +----------------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.download :noindex: .. code:: python - edge.files.download('cloud/users/Service Account/My Files/Documents/Sample.docx') + # Download a single file to default location + local_path = edge.files.download('cloud/users/Service Account/My Files/Keystone Project.docx') .. automethod:: cterasdk.edge.files.browser.FileBrowser.download_many :noindex: .. code:: python - edge.files.download_many('network-share/docs', ['Sample.docx', 'Summary.xlsx']) + # Download multiple files as a ZIP archive + zip_path = edge.files.download_many('network-share/docs', ['Keystone Project.docx', 'Keystone Model.docx']) -Create Directory ----------------- +Creating Directories +-------------------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.mkdir :noindex: .. code:: python + # Create a single directory edge.files.mkdir('cloud/users/Service Account/My Files/Documents') - .. automethod:: cterasdk.edge.files.browser.FileBrowser.makedirs :noindex: .. code:: python + # Create directories recursively edge.files.makedirs('cloud/users/Service Account/My Files/The/quick/brown/fox') - -Copy ----- +File Operations: Copy, Move, Rename, Delete +------------------------------------------- .. automethod:: cterasdk.edge.files.browser.FileBrowser.copy :noindex: .. code:: python - """ - Copy the 'Documents' folder from Bruce Wayne to Alice Wonderland - The full path of the documents folder after the copy: 'cloud/users/Alice Wonderland/My Files/Documents' - """ - edge.files.copy('cloud/users/Bruce Wayne/My Files/Documents', destination='cloud/users/Alice Wonderland/My Files') + # Copy a file to a directory + edge.files.copy('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents') - """Copy the file Summary.xlsx to another directory, and overwrite on conflict""" - edge.files.copy('cloud/users/Bruce Wayne/My Files/Summary.xlsx', destination='cloud/users/Bruce Wayne/Spreadsheets', overwrite=True) + # Copy a file and rename the file at the destination + edge.files.copy('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents/Keystone 2026.docx') - -Move ----- + # Copy a file to a directory and overwrite if it exists + edge.files.copy('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents', overwrite=True) .. automethod:: cterasdk.edge.files.browser.FileBrowser.move :noindex: .. code:: python - """ - Move the 'Documents' folder from Bruce Wayne to Alice Wonderland - The full path of the documents folder after the move: 'cloud/users/Alice Wonderland/My Files/Documents' - """ - edge.files.move('cloud/users/Bruce Wayne/My Files/Documents', destination='cloud/users/Alice Wonderland/My Files') + # Move a file to a directory + edge.files.move('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents') + + # Move a file and rename the file at the destination + edge.files.move('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents/Keystone 2026.docx') - """Move the file Summary.xlsx to another directory, and overwrite on conflict""" - edge.files.move('cloud/users/Bruce Wayne/My Files/Summary.xlsx', destination='cloud/users/Bruce Wayne/Spreadsheets', overwrite=True) + # Move a file to a directory and overwrite if it exists + edge.files.move('Keystone-Corporation/Project Keystone.docx', destination='Keystone-Coporation/Documents', overwrite=True) -Delete ------- +.. automethod:: cterasdk.edge.files.browser.FileBrowser.rename + :noindex: + +.. code:: python + + # Rename a file or directory + edge.files.rename('Keystone-Corporation/Project Keystone.docx', 'Keystone 2026.docx') .. automethod:: cterasdk.edge.files.browser.FileBrowser.delete :noindex: .. code:: python - edge.files.delete('cloud/users/Service Account/My Files/Documents') + # Delete a file or directory + edge.files.delete('Keystone-Corporation/Project Keystone.docx') +Uploading Files +--------------- -Asynchronous API -================ +.. automethod:: cterasdk.edge.files.browser.FileBrowser.upload + :noindex: + +.. code:: python + + # Upload from file handle + with open('Keystone Project.docx', 'rb') as f: + remote_path = edge.files.upload('cloud/users/Service Account/My Files/Keystone Project.docx', f) + + # Upload from string or bytes + remote_path = edge.files.upload('cloud/users/Service Account/My Files/Keystone Notes.txt', 'File contents here') + +.. automethod:: cterasdk.edge.files.browser.FileBrowser.upload_file + :noindex: + +.. code:: python + + # Upload from a local path to a directory + remote_path = edge.files.upload_file('./Keystone Project.docx', 'cloud/users/Service Account/My Files') + + # Upload from a local path and rename the file at the destination + remote_path = edge.files.upload_file('./Keystone Project.docx', 'cloud/users/Service Account/My Files/Keystone 2026.docx') Asynchronous API ================ +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.listdir + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.walk + :noindex: + .. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.handle :noindex: @@ -116,26 +239,32 @@ Asynchronous API .. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.download_many :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.listdir +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.copy +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload_file :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.move +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.mkdir :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.makedirs :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.upload_file +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.copy :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.mkdir +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.move :noindex: -.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.makedirs +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.rename :noindex: .. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.delete :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.exists + :noindex: + +.. automethod:: cterasdk.asynchronous.edge.files.browser.FileBrowser.properties + :noindex: diff --git a/docs/source/UserGuides/Miscellaneous/Changelog.rst b/docs/source/UserGuides/Miscellaneous/Changelog.rst index 58f89840..65619737 100644 --- a/docs/source/UserGuides/Miscellaneous/Changelog.rst +++ b/docs/source/UserGuides/Miscellaneous/Changelog.rst @@ -1,6 +1,34 @@ Changelog ========= +2.20.26 +------- + +Improvements +^^^^^^^^^^^^ +* Added support for Edge Filer v7.11 dual-interface network configuration, including MTU and static routes +* Refactored file browser extensions for Edge Filer and Portal +* Improved exception handling for file access errors +* Updated documentation for file browser extensions +* Listing previous versions, files, and directories now returns Python objects +* Added automatic path inference from multiple input types +* Introduced a version-based module initialization template + +Bug Fixes +^^^^^^^^^ +* Added ``backups`` to the list of omitted fields to allow printing the Portal object +* Fixed handling of HTTP ``Conflict`` errors + +Related issues and pull requests on GitHub: `#329 `_ + +2.20.25 +------- + +Security +^^^^^^^^ +* Upgraded library dependencies to address security vulnerabilities + + 2.20.24 ------- @@ -290,7 +318,7 @@ Improvements ^^^^^^^^^^^^ * Support for overriding timeout settings on a per-request basis. -* Increased the ``sock_read`` timeout to 2 minutes when invoking :py:func:`cterasdk.edge.network.Network.tcp_connect`. +* Increased the ``sock_read`` timeout to 2 minutes when invoking :py:func:`cterasdk.edge.network.Diagnostics.tcp_connect`. Related issues and pull requests on GitHub: `#302 `_ diff --git a/docs/source/UserGuides/Miscellaneous/Exceptions.rst b/docs/source/UserGuides/Miscellaneous/Exceptions.rst index 79883d76..9cd4fd39 100644 --- a/docs/source/UserGuides/Miscellaneous/Exceptions.rst +++ b/docs/source/UserGuides/Miscellaneous/Exceptions.rst @@ -37,46 +37,223 @@ Session I/O --- -.. autoclass:: cterasdk.exceptions.io.RemoteStorageError +Edge Filer +^^^^^^^^^^ + +.. autoclass:: cterasdk.exceptions.io.edge.FileConflictError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.ObjectNotFoundError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.FileNotFoundException + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.FolderNotFoundError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.GetMetadataError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.NotADirectoryException + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.ROFSError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.ListDirectoryError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.CreateDirectoryError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.OpenError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.DeleteError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.RenameError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.CopyError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.MoveError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.edge.UploadError + :noindex: + :members: + :show-inheritance: + +Portal +^^^^^^ + +.. autoclass:: cterasdk.exceptions.io.core.FileConflictError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.ObjectNotFoundError :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.ResourceNotFoundError +.. autoclass:: cterasdk.exceptions.io.core.FileNotFoundException :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.NotADirectory +.. autoclass:: cterasdk.exceptions.io.core.FolderNotFoundError :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.ResourceExistsError +.. autoclass:: cterasdk.exceptions.io.core.NotADirectoryException :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.PathValidationError +.. autoclass:: cterasdk.exceptions.io.core.OpenError :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.NameSyntaxError +.. autoclass:: cterasdk.exceptions.io.core.UploadError :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.ReservedNameError +.. autoclass:: cterasdk.exceptions.io.core.ROFSError :noindex: :members: :show-inheritance: -.. autoclass:: cterasdk.exceptions.io.RestrictedPathError +.. autoclass:: cterasdk.exceptions.io.core.PrivilegeError :noindex: :members: :show-inheritance: +.. autoclass:: cterasdk.exceptions.io.core.NTACLError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.QuotaError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.StorageBackendError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.FileRejectedError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.FilenameError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.ReservedNameError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.ListDirectoryError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.GetVersionsError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.CreateDirectoryError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.GetMetadataError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.GetShareMetadataError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.CreateLinkError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.RenameError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.BatchError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.DeleteError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.RecoverError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.CopyError + :noindex: + :members: + :show-inheritance: + +.. autoclass:: cterasdk.exceptions.io.core.MoveError + :noindex: + :members: + :show-inheritance: + + Notification Service -------------------- @@ -119,6 +296,11 @@ HTTP Transport :members: :show-inheritance: +.. autoclass:: cterasdk.exceptions.transport.Conflict + :noindex: + :members: + :show-inheritance: + .. autoclass:: cterasdk.exceptions.transport.PreConditionFailed :noindex: :members: diff --git a/docs/source/UserGuides/Portal/Files.rst b/docs/source/UserGuides/Portal/Files.rst index f0b7f35c..e8dd8abd 100644 --- a/docs/source/UserGuides/Portal/Files.rst +++ b/docs/source/UserGuides/Portal/Files.rst @@ -1,174 +1,224 @@ -============ +=============== File Browser -============ +=============== -This article outlines the file-browser APIs available in the CTERA Portal, enabling programmatic access to files and directories. +This article describes the file-browser APIs available in the CTERA Portal, which provide programmatic access to files and directories. -The API supports both **synchronous** and **asynchronous** implementations, allowing developers to choose the most suitable model -for their integration use case—whether real-time interactions or background processing. +The APIs support both **synchronous** and **asynchronous** execution models, enabling developers to choose the approach +best suited to their integration needs, from real-time operations to background processing. +Preface: Authentication +======================= -Synchronous API -=============== +All file-browser operations require an authenticated session with the CTERA Portal. +Authentication is performed by creating a portal context and calling ``login`` with valid credentials. + +Once authenticated, the ``files`` attribute provides access to the file-browser APIs. + +Synchronous Authentication +-------------------------- + +Set ``cterasdk.settings.core.syn.settings.connector.ssl = False`` to disable SSL verification. + +Authenticate as a tenant user using ``ServicesPortal``: + +.. code-block:: python + + with ServicesPortal('tenant.ctera.com') as user: + user.login(username, password) + files = user.files + +Authenticate as a global administrator using ``GlobalAdmin``: + +.. code-block:: python + + with GlobalAdmin('global.ctera.com') as admin: + admin.login(username, password) + files = admin.files + +Asynchronous Authentication +--------------------------- +Set ``cterasdk.settings.core.asyn.settings.connector.ssl = False`` to disable SSL verification. -User Roles ----------- +Authenticate as a tenant user using ``AsyncServicesPortal``: + +.. code-block:: python + + async with AsyncServicesPortal('tenant.ctera.com') as user: + await user.login(username, password) + files = user.files + +Authenticate as a global administrator using ``AsyncGlobalAdmin``: + +.. code-block:: python + + async with AsyncGlobalAdmin('global.ctera.com') as admin: + await admin.login(username, password) + files = admin.files + +User Roles and Permissions +========================== The file access APIs are available to the following user roles: -- **Global Administrators** with the `Access End User Folders` permission enabled. -- **Team Portal Administrators** with the `Access End User Folders` permission enabled. +- **Global Administrators** with the ``Access End User Folders`` permission enabled. +- **Team Portal Administrators** with the ``Access End User Folders`` permission enabled. - **End Users**, accessing their personal cloud drive folders. -For more information, See: `Customizing Administrator Roles `_ +For more information about configuring administrator permissions, see +`Customizing Administrator Roles `_. -List ----- +Key Objects +=========== -.. automethod:: cterasdk.core.files.browser.FileBrowser.listdir +This section describes the core objects returned by the file-browser APIs. + +.. autoclass:: cterasdk.cio.core.types.PortalResource + :members: + :undoc-members: :noindex: -.. code:: python +.. autoclass:: cterasdk.cio.core.types.PortalVolume + :members: + :undoc-members: + :noindex: - """List directories as a Global Administrator""" - with GlobalAdmin('tenant.ctera.com') as admin: - admin.login('admin-user', 'admin-pass') +.. autoclass:: cterasdk.cio.core.types.VolumeOwner + :members: + :undoc-members: + :noindex: - """List all sub-directories""" - for f in admin.files.listdir('Users/John Smith/My Files'): - if f.isFolder: - print({ - f.name, - f.href, - f.permalink # a URL that links directly to a specific file - }) +.. autoclass:: cterasdk.cio.core.types.PreviousVersion + :members: + :undoc-members: + :noindex: -.. code:: python +Synchronous API +=============== - """List directories as a Team Portal Administrator or End User""" - with ServicesPortal('tenant.ctera.com') as user: - user.login('username', 'user-password') - for f in user.files.listdir('My Files/Documents'): - if not f.isFolder: - print({ - f.name, - f.href, - f.size, - f.lastmodified, - f.permalink # a URL that links directly to a specific file - }) - - """List all deleted files""" - deleted_files = [f.href for f in user.files.listdir('My Files/Documents', include_deleted=True) if f.isDeleted] - print(deleted_files) +Listing Files and Directories +----------------------------- -.. automethod:: cterasdk.core.files.browser.FileBrowser.walk +.. automethod:: cterasdk.core.files.browser.FileBrowser.listdir :noindex: -.. code:: python +.. code-block:: python - with GlobalAdmin('tenant.ctera.com') as admin: - admin.login('admin-user', 'admin-pass') - for element in admin.files.walk('Users/John Smith/My Files'): - print(element.name) # traverse John Smith's 'My Files' directory and print the name of all files and folders + resources = files.listdir('My Files') + for r in resources: + print(r.name, r.is_dir) - with ServicesPortal('tenant.ctera.com') as user: - user.login('username', 'user-password') - for element in user.files.walk('My Files/Documents'): - print(element.name) # as a user, traverse all and print the name of all files and folders in 'My Files/Documents' +.. automethod:: cterasdk.core.files.browser.FileBrowser.walk + :noindex: + +.. code-block:: python -Versions --------- + for resource in files.walk('My Files'): + if not resource.is_dir and resource.extension == 'pdf': + files.download(resource.path) + +Listing Files from Previous Versions +------------------------------------ .. automethod:: cterasdk.core.files.browser.FileBrowser.versions :noindex: -.. code:: python +.. code-block:: python - versions = admin.files.versions('Users/John Smith/My Files/Documents') - for version in versions: - if not version.current: - for item in admin.files.listdir(version): # list items from previous versions - print(version.calculatedTimestamp, item.name) + # List all versions of a file + versions = files.versions('My Files/Keystone Project.docx') + for v in versions: + print(v.start_time, v.end_time, v.current) + # List files in a previous version + prev_version = next(v for v in versions if not v.current) + for f in files.listdir(prev_version.path): + print(f'File in previous version: {f.path}, Size: {f.size}, Last modified: {f.last_modified}') - versions = user.files.versions('My Files/Documents') - for version in versions: - if not version.current: - for item in user.files.listdir(version): # list items from previous versions - print(version.calculatedTimestamp, item.name) + # Download files from a previous version + for f in files.listdir(prev_version.path): + local_path = files.download(f.path) + print(local_path) -Download --------- +Inspecting Files +---------------- -.. automethod:: cterasdk.core.files.browser.FileBrowser.download +.. automethod:: cterasdk.core.files.browser.FileBrowser.properties :noindex: -.. code:: python +.. code-block:: python - admin.files.download('Users/John Smith/My Files/Documents/Sample.docx') + metadata = files.properties('My Files/Keystone Project.docx') + print(metadata.size, metadata.last_modified) - user.files.download('My Files/Documents/Sample.docx') - -.. automethod:: cterasdk.core.files.browser.FileBrowser.download_many +.. automethod:: cterasdk.core.files.browser.FileBrowser.exists :noindex: -.. code:: python +.. code-block:: python - admin.files.download_many('Users/John Smith/My Files/Documents', ['Sample.docx', 'Wizard Of Oz.docx']) + exists = files.exists('My Files/Keystone Project.docx') - user.files.download_many('My Files/Documents', ['Sample.docx', 'Wizard Of Oz.docx']) - -Copy ----- +Retrieve a Permalink +-------------------- -.. automethod:: cterasdk.core.files.browser.FileBrowser.copy +.. automethod:: cterasdk.core.files.browser.FileBrowser.permalink :noindex: - To resolve file conflicts, use :py:class:`cterasdk.core.types.ConflictResolver` +.. code-block:: python + + permalink = files.permalink('My Files/Keystone Project.docx') + print(f'Permalink: {permalink}') -.. code:: python +Create a Public Link +-------------------- - admin.files.copy(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx'], destination='Users/John Smith/The/quick/brown/fox') +.. automethod:: cterasdk.core.files.browser.FileBrowser.public_link + :noindex: - user.files.copy(*['My Files/Documents/Sample.docx', 'My Files/Documents/Burndown.xlsx'], destination='The/quick/brown/fox') +.. code-block:: python + public_url = files.public_link('My Files/Keystone Project.docx', access='RO', expire_in=7) + print(f'Public link: {public_url}') -Create Public Link ------------------- + preview_link = files.public_link('My Files/Keystone Market Overview.pdf', access='PO', expire_in=7) + print(f'Preview-only link: {preview_link}') -.. automethod:: cterasdk.core.files.browser.FileBrowser.public_link + public_url_rw = files.public_link('My Files', access='RW', expire_in=30) + print(f'Public read-write link: {public_url_rw}') + +File Handles +------------ + +.. automethod:: cterasdk.core.files.browser.FileBrowser.handle :noindex: -.. code:: python +.. code-block:: python - """ - Access: - - RW: Read Write - - RO: Read Only - - NA: No Access - """ + handle = files.handle('My Files/Keystone Project.docx') - """Create a Read Only public link to a file that expires in 30 days""" - user.files.public_link('My Files/Documents/Sample.docx') +.. automethod:: cterasdk.core.files.browser.FileBrowser.handle_many - """Create a Read Write public link to a folder that expires in 45 days""" - user.files.public_link('My Files/Documents/Sample.docx', 'RW', 45) +.. code-block:: python + handle = files.handle_many('My Files', 'Keystone Project.docx', 'Images', 'Notes.txt') -Get Permalink -------------- +Downloading Files +----------------- -.. automethod:: cterasdk.core.files.browser.FileBrowser.permalink +.. automethod:: cterasdk.core.files.browser.FileBrowser.download :noindex: -.. code:: python +.. code-block:: python + + local_path = files.download('My Files/Keystone Project.docx') - user.files.permalink('My Files/Documents/Sample.docx') # file +.. automethod:: cterasdk.core.files.browser.FileBrowser.download_many + :noindex: - user.files.permalink('My Files/Documents') # folder +.. code-block:: python + zip_path = files.download_many('My Files', ['Keystone Project.docx', 'Images'], destination='/tmp/MyFiles.zip') Create Directories ------------------ @@ -176,97 +226,149 @@ Create Directories .. automethod:: cterasdk.core.files.browser.CloudDrive.mkdir :noindex: -.. code:: python +.. code-block:: python - admin.files.mkdir('Users/John Smith/My Files/Documents') - - user.files.mkdir('My Files/Documents') + new_dir = files.mkdir('My Files/NewProject') + print(f'Created directory: {new_dir}') .. automethod:: cterasdk.core.files.browser.CloudDrive.makedirs :noindex: -.. code:: python - - admin.files.makedirs('Users/John Smith/My Files/The/quick/brown/fox') +.. code-block:: python - user.files.makedirs('The/quick/brown/fox') + nested_dir = files.makedirs('My Files/Projects/2026/Q1') + print(f'Created nested directories: {nested_dir}') -Rename ------- +Uploading Files +--------------- -.. automethod:: cterasdk.core.files.browser.CloudDrive.rename +.. automethod:: cterasdk.core.files.browser.CloudDrive.upload_file :noindex: -.. code:: python +.. code-block:: python - admin.files.rename('Users/John Smith/My Files/Documents/Sample.docx', 'Wizard Of Oz.docx') + # Upload from a local path to a directory + remote_path = files.upload_file('/tmp/Keystone Project.docx', 'My Files') + print(f'File uploaded to: {remote_path}') - user.files.makedirs('My Files/Documents/Sample.docx', 'Wizard Of Oz.docx') + # Upload from a local path and rename the file at the destination + remote_path = files.upload_file('/tmp/Keystone Project.docx', 'My Files/Keystone 2026.docx') + print(f'File uploaded to: {remote_path}') -Delete ------- - -.. automethod:: cterasdk.core.files.browser.CloudDrive.delete +.. automethod:: cterasdk.core.files.browser.CloudDrive.upload :noindex: -.. code:: python +.. code-block:: python - admin.files.delete(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx']) + name = 'Keystone Project.docx' + destination = 'My Files' - user.files.delete(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx']) + # Upload from file handle + with open('/tmp/Keystone Project.docx', 'rb') as f: + remote_path = files.upload(name, destination, f) + print(f'File uploaded from handle to: {remote_path}') -Undelete --------- + # Upload from string or bytes + remote_path = files.upload(name, destination, handle=b'Sample content for ProjectPlan.') + print(f'File uploaded from bytes to: {remote_path}') -.. automethod:: cterasdk.core.files.browser.CloudDrive.undelete +Renaming Files and Folders +-------------------------- + +.. automethod:: cterasdk.core.files.browser.CloudDrive.rename :noindex: -.. code:: python +.. code-block:: python - admin.files.undelete(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx']) + remote_path = files.rename('My Files/Keystone Project.docx', 'Keystone Project 2026.docx') + print(f'Renamed file: {remote_path}') - user.files.undelete(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx']) +Copying and Moving Files and Folders +------------------------------------ -Move ----- +.. automethod:: cterasdk.core.files.browser.FileBrowser.copy + :noindex: + +.. code-block:: python + + # Copy files into a destination directory + result = files.copy('My Files/Keystone Project.docx', 'My Files/Keystone Notes.txt', destination='Archive') + print(f'Files copied: {result}') + + # Copy multiple files at once while renaming them. Requires explicitly defining the target path + result = files.copy( + ('My Files/Keystone Project.docx', 'Archive/Keystone Project 2026.docx'), + ('My Files/Keystone Notes.txt', 'Archive/Keystone Notes 2026.txt') + ) + print(f'Files copied with explicit paths: {result}') .. automethod:: cterasdk.core.files.browser.CloudDrive.move :noindex: - To resolve file conflicts, use :py:class:`cterasdk.core.types.ConflictResolver` +.. code-block:: python + + # Move files into a destination directory + result = files.move('My Files/Keystone Project.docx', 'My Files/Keystone Notes.txt', destination='Archive') + print(f'Files moved: {result}') + + # Move multiple files at once while renaming them. Requires explicitly defining the target path + result = files.move( + ('My Files/Keystone Project.docx', 'Archive/Keystone 2026.docx'), + ('My Files/Keystone Notes.txt', 'Archive/Keystone Notes 2026.txt') + ) + print(f'Files moved with explicit paths: {result}') -.. code:: python +Delete or Recovering Files and Folders +-------------------------------------- - admin.files.move(*['Users/John Smith/My Files/Documents/Sample.docx', 'Users/John Smith/My Files/Documents/Wizard Of Oz.docx'], destination='Users/John Smith/The/quick/brown/fox') +.. automethod:: cterasdk.core.files.browser.CloudDrive.delete + :noindex: - user.files.move(*['My Files/Documents/Sample.docx', 'My Files/Documents/Wizard Of Oz.docx'], destination='The/quick/brown/fox') +.. code-block:: python -Upload ------- + result = files.delete('My Files/Project Keystone.docx') + print(f'Deleted file: {result}') -.. automethod:: cterasdk.core.files.browser.CloudDrive.upload + result = files.delete('My Files/Project Keystone.docx', 'My Files/Keystone Notes.txt', 'Archive/Keystone') + print(f'Deleted multiple files/folders: {result}') -.. code:: python +.. automethod:: cterasdk.core.files.browser.CloudDrive.undelete + :noindex: - admin.files.upload(r'C:\Users\admin\Downloads\Tree.jpg', 'Users/John Smith/My Files/Images') +.. code-block:: python - user.files.upload(r'C:\Users\admin\Downloads\Tree.jpg', 'My Files/Images') + result = files.undelete('My Files/Project Keystone.docx') + print(f'Recovered file: {result}') + result = files.undelete('My Files/Project Keystone.docx', 'My Files/Keystone Notes.txt', 'Archive/Keystone') + print(f'Recovered multiple files/folders: {result}') Collaboration Shares -------------------- -.. automethod:: cterasdk.core.files.browser.CloudDrive.share +This section describes the main objects used for managing collaboration shares. + +.. autoclass:: cterasdk.core.types.UserAccount + :members: + :undoc-members: :noindex: -.. code:: python +.. autoclass:: cterasdk.core.types.GroupAccount + :members: + :undoc-members: + :noindex: - """ - Share with a local user and a local group. - - Grant the local user with read only access for 30 days - - Grant the local group with read write access with no expiration - """ + +.. autoclass:: cterasdk.core.types.Collaborator + :members: + :undoc-members: + :noindex: + +.. automethod:: cterasdk.core.files.browser.CloudDrive.share + :noindex: + +.. code-block:: python alice = core_types.UserAccount('alice') engineers = core_types.GroupAccount('Engineers') @@ -276,35 +378,16 @@ Collaboration Shares user.files.share('Codebase', [alice_rcpt, engineers_rcpt]) -.. - -.. code:: python +.. code-block:: python - - """ - Share with an external recipient - - Grant the external user with preview only access for 10 days - """ jsmith = core_types.Collaborator.external('jsmith@hotmail.com').expire_in(10).preview_only() user.files.share('My Files/Projects/2020/ProjectX', [jsmith]) - """ - Share with an external recipient, and require 2 factor authentication - - Grant the external user with read only access for 5 days, and require 2 factor authentication over e-mail - """ jsmith = core_types.Collaborator.external('jsmith@hotmail.com', True).expire_in(5).read_only() user.files.share('My Files/Projects/2020/ProjectX', [jsmith]) -.. - -.. code:: python +.. code-block:: python - - """ - Share with a domain groups - - Grant the Albany domain group with read write access with no expiration - - Grant the Cleveland domain group with read only access with no expiration - """ albany_group = core_types.GroupAccount('Albany', 'ctera.com') cleveland_group = core_types.GroupAccount('Cleveland', 'ctera.com') @@ -316,27 +399,17 @@ Collaboration Shares .. automethod:: cterasdk.core.files.browser.CloudDrive.add_share_recipients :noindex: -.. code:: python - +.. code-block:: python - """ - Add collaboration shares members. - - - Grant the 'Engineering' local group with read-write permission - """ engineering = core_types.GroupAccount('Engineering') engineering_rcpt = core_types.Collaborator.local_group(engineering).read_write() user.files.add_share_recipients('My Files/Projects/2020/ProjectX', [engineering_rcpt]) -.. note:: if the share recipients provided as an argument already exist, they will be skipped and not updated - .. automethod:: cterasdk.core.files.browser.CloudDrive.remove_share_recipients :noindex: -.. code:: python +.. code-block:: python - - """Remove 'Alice' and 'Engineering' from the List of Recipients""" alice = core_types.UserAccount('alice') engineering = core_types.GroupAccount('Engineering') user.files.remove_share_recipients('My Files/Projects/2020/ProjectX', [alice, engineering]) @@ -344,62 +417,61 @@ Collaboration Shares .. automethod:: cterasdk.core.files.browser.CloudDrive.unshare :noindex: -.. code:: python +.. code-block:: python - """ - Unshare a file or a folder - """ user.files.unshare('Codebase') user.files.unshare('My Files/Projects/2020/ProjectX') user.files.unshare('Cloud/Albany') - Managing S3 Credentials ----------------------- -Starting CTERA 8.0, CTERA Portal features programmatic access via the S3 protocol, also known as *CTERA Fusion* -For more information on how to enable CTERA Fusion and the supported extensions of the S3 protocol, please refer to the following `article `_ +CTERA Portal supports programmatic access to cloud storage via the S3 protocol, also known as *CTERA Fusion*. +This allows users and administrators to manage files and folders using standard S3 tools and SDKs, such as the Amazon SDK for Python (`boto3 `_). -The following section includes examples on how to instantiate an S3 client using the Amazon SDK for Python `boto3 `_ +For details on enabling CTERA Fusion and supported S3 features, +see the `CTERA KB article `_. -.. code:: python +The following example demonstrates how to create S3 credentials and interact with the portal using `boto3`. - credentials = user.credentials.s3.create() # if logged in as a user - # credentials = admin.credentials.s3.create(core_types.UserAccount('username', 'domain')) # if logged in as a Global Admin +.. code-block:: python - """Instantiate the boto3 client""" - client = boto3.client( - 's3', - endpoint_url=https://domain.ctera.com:8443, # your CTERA Portal tenant domain - aws_access_key_id=credentials.accessKey, - aws_secret_access_key=credentials.secretKey, - verify=False # disable certificate verification (Optional) - ) + import boto3 + + bucket = 'my-bucket-name' + local_file = './ProjectOverview.docx' + remote_key = 'documents/ProjectOverview.docx' + download_file = './ProjectOverview_Copy.docx' + + # CTERA Fusion: Create S3 credentials (user or admin) + creds = user.credentials.s3.create() # or admin.credentials.s3.create(core_types.UserAccount('username', 'domain')) + + # Instantiate boto3 client + client = boto3.client('s3', endpoint_url='https://tenant.ctera.com:8443', aws_access_key_id=creds.accessKey, + aws_secret_access_key=creds.secretKey, verify=False) - """List Buckets""" - response = client.list_buckets() - for bucket in response['Buckets']: - print(bucket['Name']) + # List buckets + for b in client.list_buckets()['Buckets']: + print(b['Name']) - """Upload a file""" - client.upload_file(r'./document.docx', 'my-bucket-name', 'data-management-document.docx') + # Upload a file + client.upload_file(local_file, bucket, remote_key) - """List files""" - response = client.list_objects_v2(Bucket='my-bucket-name') - for item in response['Contents']: - print(item['Key'], item['LastModified']) + # List files in a bucket + for item in client.list_objects_v2(Bucket=bucket).get('Contents', []): + print(item['Key'], item['LastModified']) - """List files, using Pagination""" - paginator = client.get_paginator('list_objects_v2') - for page in paginator.paginate(Bucket='my-bucket-name'): - for item in page['Contents']: - print(item['Key'], item['LastModified']) + # List files with pagination + for page in client.get_paginator('list_objects_v2').paginate(Bucket=bucket): + for item in page.get('Contents', []): + print(item['Key'], item['LastModified']) - """Download a file""" - client.download_file(r'./data-management-document.docx', 'my-bucket-name', 'data-management-document-copy.docx') + # Download a file + client.download_file(bucket, remote_key, download_file) - # for more information, please refer to the Amazon SDK for Python (boto3) documentation. +.. note:: + For more details on using the Amazon SDK for Python (`boto3`), refer to the official `boto3 documentation `_. Asynchronous API ================ @@ -458,52 +530,14 @@ Asynchronous API .. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.undelete :noindex: +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.share + :noindex: -.. code:: python - - """Access a Global Administrator""" - async with AsyncGlobalAdmin('global.ctera.com') as admin: - await admin.login('username', 'password') - await admin.portals.browse('corp') # access files in the 'corp' Team Portal tenant - - """Create directories recursively""" - await admin.files.makedirs('Users/John Smith/My Files/the/quick/brown/fox') - - """Create a 'Documents' directory""" - await admin.files.mkdir('Users/John Smith/Documents') - - """Walk 'John Smith's My Files directory""" - async for i in admin.files.walk('Users/John Smith/My Files'): - print(i.name, i.size, i.lastmodified, i.permalink) - - """List all files in a directory""" - documents = [i.name async for i in admin.files.listdir('Users/John Smith/Documents') if i.isfile] - - """Rename a directory""" - await admin.files.rename('Users/John Smith/Documents', 'Documents360') - - """Download""" - await admin.files.download('Users/John Smith/My Files/Sunrise.png') - await admin.files.download('Users/John Smith/My Files/Sunrise.png', 'c:/users/jsmith/downloads/Patagonia.png') - - await admin.files.download_many('Users/John Smith/Pictures', ['Sunrise.png', 'Gelato.pptx']) - await admin.files.download_many('Users/John Smith/Pictures', ['Sunrise.png', 'Gelato.pptx'], 'c:/users/jsmith/downloads/Images.zip') - - """Upload""" - await admin.files.upload_file('c:/users/jsmith/downloads/Sunset.png', '/Users/John Smith/Pictures') - - """Public Link""" - url = await admin.files.public_link('Users/John Smith/Pictures/Sunrise.png') - print(url) - -.. code:: python - - """Access a Team Portal Administrator or End User""" - async with AsyncservicesPortal('tenant.ctera.com') as user: - await user.login('username', 'password') +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.add_share_recipients + :noindex: - """Create directories as an End User""" - await user.files.makedirs('My Files/the/quick/brown/fox') # Create a directory in your own account +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.remove_share_recipients + :noindex: - """Create directories as Team Portal Administrator""" - await user.files.makedirs('Users/John Smith/My Files/the/quick/brown/fox') # Create a directory in a user's account +.. automethod:: cterasdk.asynchronous.core.files.browser.CloudDrive.unshare + :noindex: diff --git a/docs/source/api/cterasdk.cio.core.commands.rst b/docs/source/api/cterasdk.cio.core.commands.rst new file mode 100644 index 00000000..ffc78f3a --- /dev/null +++ b/docs/source/api/cterasdk.cio.core.commands.rst @@ -0,0 +1,7 @@ +cterasdk.cio.core.commands module +================================= + +.. automodule:: cterasdk.cio.core.commands + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.cio.core.rst b/docs/source/api/cterasdk.cio.core.rst new file mode 100644 index 00000000..efaec7e0 --- /dev/null +++ b/docs/source/api/cterasdk.cio.core.rst @@ -0,0 +1,15 @@ +cterasdk.cio.core package +========================= + +.. automodule:: cterasdk.cio.core + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + cterasdk.cio.core.commands + cterasdk.cio.core.types diff --git a/docs/source/api/cterasdk.cio.core.types.rst b/docs/source/api/cterasdk.cio.core.types.rst new file mode 100644 index 00000000..81edc9c2 --- /dev/null +++ b/docs/source/api/cterasdk.cio.core.types.rst @@ -0,0 +1,7 @@ +cterasdk.cio.core.types module +============================== + +.. automodule:: cterasdk.cio.core.types + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.cio.edge.commands.rst b/docs/source/api/cterasdk.cio.edge.commands.rst new file mode 100644 index 00000000..ab11ea4e --- /dev/null +++ b/docs/source/api/cterasdk.cio.edge.commands.rst @@ -0,0 +1,7 @@ +cterasdk.cio.edge.commands module +================================= + +.. automodule:: cterasdk.cio.edge.commands + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.cio.edge.rst b/docs/source/api/cterasdk.cio.edge.rst new file mode 100644 index 00000000..15a7e446 --- /dev/null +++ b/docs/source/api/cterasdk.cio.edge.rst @@ -0,0 +1,15 @@ +cterasdk.cio.edge package +========================= + +.. automodule:: cterasdk.cio.edge + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + cterasdk.cio.edge.commands + cterasdk.cio.edge.types diff --git a/docs/source/api/cterasdk.cio.edge.types.rst b/docs/source/api/cterasdk.cio.edge.types.rst new file mode 100644 index 00000000..fb89d6e8 --- /dev/null +++ b/docs/source/api/cterasdk.cio.edge.types.rst @@ -0,0 +1,7 @@ +cterasdk.cio.edge.types module +============================== + +.. automodule:: cterasdk.cio.edge.types + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.cio.rst b/docs/source/api/cterasdk.cio.rst new file mode 100644 index 00000000..b5c09c95 --- /dev/null +++ b/docs/source/api/cterasdk.cio.rst @@ -0,0 +1,16 @@ +cterasdk.cio package +==================== + +.. automodule:: cterasdk.cio + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + cterasdk.cio.core + cterasdk.cio.edge + diff --git a/docs/source/api/cterasdk.exceptions.io.base.rst b/docs/source/api/cterasdk.exceptions.io.base.rst new file mode 100644 index 00000000..8210101d --- /dev/null +++ b/docs/source/api/cterasdk.exceptions.io.base.rst @@ -0,0 +1,7 @@ +cterasdk.exceptions.io.base module +================================== + +.. automodule:: cterasdk.exceptions.io.base + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.exceptions.io.core.rst b/docs/source/api/cterasdk.exceptions.io.core.rst new file mode 100644 index 00000000..f2a1fc03 --- /dev/null +++ b/docs/source/api/cterasdk.exceptions.io.core.rst @@ -0,0 +1,7 @@ +cterasdk.exceptions.io.core module +================================== + +.. automodule:: cterasdk.exceptions.io.core + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.exceptions.io.edge.rst b/docs/source/api/cterasdk.exceptions.io.edge.rst new file mode 100644 index 00000000..97617f17 --- /dev/null +++ b/docs/source/api/cterasdk.exceptions.io.edge.rst @@ -0,0 +1,7 @@ +cterasdk.exceptions.io.edge module +================================== + +.. automodule:: cterasdk.exceptions.io.edge + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/cterasdk.exceptions.io.rst b/docs/source/api/cterasdk.exceptions.io.rst index c3ff092a..1ea8bb04 100644 --- a/docs/source/api/cterasdk.exceptions.io.rst +++ b/docs/source/api/cterasdk.exceptions.io.rst @@ -1,7 +1,16 @@ -cterasdk.exceptions.io module -============================= +cterasdk.exceptions.io package +============================== .. automodule:: cterasdk.exceptions.io :members: :undoc-members: :show-inheritance: + +Submodules +---------- + +.. toctree:: + + cterasdk.exceptions.io.base + cterasdk.exceptions.io.core + cterasdk.exceptions.io.edge diff --git a/docs/source/api/cterasdk.lib.platform.rst b/docs/source/api/cterasdk.lib.platform.rst deleted file mode 100644 index 9166bd2c..00000000 --- a/docs/source/api/cterasdk.lib.platform.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.lib.platform module -============================ - -.. automodule:: cterasdk.lib.platform - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.lib.rst b/docs/source/api/cterasdk.lib.rst index 6bf07915..ee2036d5 100644 --- a/docs/source/api/cterasdk.lib.rst +++ b/docs/source/api/cterasdk.lib.rst @@ -14,10 +14,8 @@ Submodules cterasdk.lib.cmd cterasdk.lib.consent cterasdk.lib.iterator - cterasdk.lib.platform cterasdk.lib.registry cterasdk.lib.storage cterasdk.lib.tasks cterasdk.lib.tempfile cterasdk.lib.tracker - cterasdk.lib.version diff --git a/docs/source/api/cterasdk.lib.version.rst b/docs/source/api/cterasdk.lib.version.rst deleted file mode 100644 index a337b750..00000000 --- a/docs/source/api/cterasdk.lib.version.rst +++ /dev/null @@ -1,7 +0,0 @@ -cterasdk.lib.version module -=========================== - -.. automodule:: cterasdk.lib.version - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/cterasdk.rst b/docs/source/api/cterasdk.rst index 0f460f5c..384937d3 100644 --- a/docs/source/api/cterasdk.rst +++ b/docs/source/api/cterasdk.rst @@ -14,6 +14,7 @@ Subpackages cterasdk.asynchronous cterasdk.clients cterasdk.common + cterasdk.cio cterasdk.convert cterasdk.core cterasdk.direct diff --git a/tests/ut/core/user/test_browser.py b/tests/ut/core/user/test_browser.py index bb58530b..1e1ebc04 100644 --- a/tests/ut/core/user/test_browser.py +++ b/tests/ut/core/user/test_browser.py @@ -1,6 +1,7 @@ from unittest import mock from urllib.parse import quote from datetime import datetime, timedelta +import munch from cterasdk.common.object import Object from cterasdk.core.tasks import AwaitablePortalTask @@ -19,24 +20,53 @@ def setUp(self): self.new_filename = 'Summary.txt' def test_versions(self): - response = 'snapshots-response-object' - self._init_global_admin(execute_response=response) + directory = 'Users/John Smith/My Files' + self._init_global_admin(execute_response=[munch.Munch({ + 'url': TestSynchronousFileBrowser.scope, + 'path': directory, + 'current': True, + 'startTimestamp': datetime.now().isoformat(), + 'calculatedTimestamp': datetime.now().isoformat() + })]) ret = self._global_admin.files.versions(self.directory) self._global_admin.api.execute.assert_called_once_with('', 'listSnapshots', f'{TestSynchronousFileBrowser.scope}/{self.directory}') - self.assertEqual(ret, response) + self.assertEqual(str(ret[0].path), directory) def test_listdir(self): + self.patch_call("cterasdk.cio.core.commands.EnsureDirectory.execute") for include_deleted in [True, False]: self._init_global_admin(execute_response=Object(**{ 'errorType': None, 'hasMore': False, - 'items': [self.filename] + 'items': [TestSynchronousFileBrowser._generate_resource(self.filename)] })) - filename = next(self._global_admin.files.listdir(self.directory, include_deleted=include_deleted)) + item = next(self._global_admin.files.listdir(self.directory, include_deleted=include_deleted)) self._global_admin.api.execute.assert_called_once_with('', 'fetchResources', mock.ANY) param = self._global_admin.api.execute.call_args[0][2] self.assertEqual(param.root, f'{TestSynchronousFileBrowser.scope}/{self.directory}') - self.assertEqual(filename, self.filename) + self.assertEqual(item.name, self.filename) + + @staticmethod + def _generate_resource(name): + resource_info = Object() + resource_info._classname = 'ResourceInfo' # pylint: disable=protected-access + resource_info.href = TestSynchronousFileBrowser.scope + '/' + name + resource_info.name = name + resource_info.fileId = 1 + resource_info.isFolder = False + resource_info.isDeleted = False + resource_info.size = 1 + resource_info.permalink = 'xyz' + resource_info.lastmodified = datetime.now().isoformat() + resource_info.cloudFolderInfo = Object( + uid=1, + name='Volume name', + groupUid=1, + passphraseProtected=False, + ownerUid=1, + ownerFriendlyName='First Last' + ) + return resource_info def test_mkdir(self): self._init_global_admin() diff --git a/tests/ut/core/user/test_listdir.py b/tests/ut/core/user/test_listdir.py index 0ced034a..9a274047 100644 --- a/tests/ut/core/user/test_listdir.py +++ b/tests/ut/core/user/test_listdir.py @@ -1,4 +1,5 @@ from unittest import mock +from datetime import datetime from cterasdk.common import Object from tests.ut.core.user import base_user @@ -14,12 +15,13 @@ def setUp(self): self._path = 'Documents' def test_list_directory_str_arg(self): + self.patch_call("cterasdk.cio.core.commands.EnsureDirectory.execute") self._init_services() self._services.api.execute = mock.MagicMock(side_effect=BaseCoreServicesFilesList._fetch_resources_side_effect) iterator = self._services.files.listdir(self._path) files = BaseCoreServicesFilesList.files[0] + BaseCoreServicesFilesList.files[1] for item in iterator: - self.assertEqual(item.href, f'{self._base}/{files.pop(0)}') + self.assertEqual(item.path.absolute, f'{BaseCoreServicesFilesList.basepath}/{files.pop(0)}') self._services.api.execute.assert_has_calls( [ mock.call('', 'fetchResources', mock.ANY), @@ -54,14 +56,28 @@ def _fetch_resources_side_effect(path, name, param): @staticmethod def _fetch_resources_response(response, files): for file in files: - resource_info = BaseCoreServicesFilesList._create_resource_info(file) response.errorType = None - response.items.append(resource_info) + response.items.append(BaseCoreServicesFilesList._create_resource_info(file)) return response @staticmethod - def _create_resource_info(path): + def _create_resource_info(name): resource_info = Object() resource_info._classname = 'ResourceInfo' # pylint: disable=protected-access - resource_info.href = BaseCoreServicesFilesList.basepath + '/' + path + resource_info.href = BaseCoreServicesFilesList.basepath + '/' + name + resource_info.name = name + resource_info.fileId = 1 + resource_info.isFolder = False + resource_info.isDeleted = False + resource_info.size = 1 + resource_info.permalink = 'xyz' + resource_info.lastmodified = datetime.now().isoformat() + resource_info.cloudFolderInfo = Object( + uid=1, + name='Volume name', + groupUid=1, + passphraseProtected=False, + ownerUid=1, + ownerFriendlyName='First Last' + ) return resource_info diff --git a/tests/ut/core/user/test_versions.py b/tests/ut/core/user/test_versions.py index a1c1deb0..0ac407f8 100644 --- a/tests/ut/core/user/test_versions.py +++ b/tests/ut/core/user/test_versions.py @@ -1,3 +1,5 @@ +from datetime import datetime +import munch from tests.ut.core.user import base_user @@ -5,8 +7,16 @@ class BaseCoreServicesFilesVersions(base_user.BaseCoreServicesTest): def test_list_versions(self): directory = 'My Files' - execute_response = 'Success' - self._init_services(execute_response=execute_response) + self._init_services(execute_response=[self._create_snapshot_response(directory, True)]) ret = self._services.files.versions(directory) self._services.api.execute.assert_called_once_with('', 'listSnapshots', f'{self._base}/{directory}') - self.assertEqual(ret, execute_response) + self.assertEqual(str(ret[0].path), directory) + + def _create_snapshot_response(self, path, current): + return munch.Munch({ + 'url': self._base, + 'path': path, + 'current': current, + 'startTimestamp': datetime.now().isoformat(), + 'calculatedTimestamp': datetime.now().isoformat() + }) diff --git a/tests/ut/edge/test_browser.py b/tests/ut/edge/test_browser.py index 19043cda..58510e68 100644 --- a/tests/ut/edge/test_browser.py +++ b/tests/ut/edge/test_browser.py @@ -17,6 +17,8 @@ def setUp(self): self._target = 'target/folder' self._target_fullpath = f'/{self._target}/{self._filename}' self._default_download_dir = cterasdk.settings.io.downloads + ensure_directory_mock = self.patch_call("cterasdk.cio.edge.commands.EnsureDirectory.execute") + ensure_directory_mock.return_value = (True,) def test_download_as_zip_success(self): pass # self._files.download_as_zip() diff --git a/tests/ut/edge/test_directory_service.py b/tests/ut/edge/test_directory_service.py index b5d87a4e..2db84490 100644 --- a/tests/ut/edge/test_directory_service.py +++ b/tests/ut/edge/test_directory_service.py @@ -40,11 +40,11 @@ def test_connect(self): self._init_filer() get_response_side_effect = TestEdgeDirectoryService._get_response_side_effect(self._get_workgroup_param(), None) self._filer.api.get = mock.MagicMock(side_effect=get_response_side_effect) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) directoryservice.DirectoryService(self._filer).connect(self._domain, self._username, self._password, check_connection=True) - self._filer.network.tcp_connect.assert_called_once_with(self._ldap_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._ldap_service) self._filer.api.get.assert_has_calls([ mock.call('/config/fileservices/cifs/passwordServer'), mock.call('/config/fileservices/cifs') @@ -65,11 +65,11 @@ def test_connect_with_ou_path(self): ou_path = "ou=North America,DC=ctera,DC=local" get_response_side_effect = TestEdgeDirectoryService._get_response_side_effect(self._get_workgroup_param(), None) self._filer.api.get = mock.MagicMock(side_effect=get_response_side_effect) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) directoryservice.DirectoryService(self._filer).connect(self._domain, self._username, self._password, ou_path, check_connection=True) - self._filer.network.tcp_connect.assert_called_once_with(self._ldap_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._ldap_service) self._filer.api.get.assert_has_calls([ mock.call('/config/fileservices/cifs/passwordServer'), mock.call('/config/fileservices/cifs') @@ -98,13 +98,13 @@ def test_connect_join_failure(self): self._init_filer() get_response_side_effect = TestEdgeDirectoryService._get_response_side_effect(self._get_workgroup_param(), None) self._filer.api.get = mock.MagicMock(side_effect=get_response_side_effect) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, True)) self._filer.api.execute = mock.MagicMock(side_effect=TaskException('Task failed', self._task_id)) with self.assertRaises(exceptions.CTERAException): directoryservice.DirectoryService(self._filer).connect(self._domain, self._username, self._password, check_connection=True) - self._filer.network.tcp_connect.assert_called_once_with(self._ldap_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._ldap_service) self._filer.api.get.assert_has_calls([ mock.call('/config/fileservices/cifs/passwordServer'), @@ -123,13 +123,13 @@ def test_connect_join_failure(self): def test_connect_connection_error(self): self._init_filer(get_response=None) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, False)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._domain, self._ldap_port, False)) with self.assertRaises(ConnectionError) as error: directoryservice.DirectoryService(self._filer).connect(self._domain, self._username, self._password, check_connection=True) self._filer.api.get.assert_called_once_with('/config/fileservices/cifs/passwordServer') - self._filer.network.tcp_connect.assert_called_once_with(self._ldap_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._ldap_service) self.assertEqual(f'Unable to establish LDAP connection {self._domain}:{self._ldap_port}', str(error.exception)) def test_get_advanced_mapping(self): diff --git a/tests/ut/edge/test_network.py b/tests/ut/edge/test_network.py index f4da20a7..faac3bf6 100644 --- a/tests/ut/edge/test_network.py +++ b/tests/ut/edge/test_network.py @@ -2,11 +2,10 @@ import munch from cterasdk.edge import network -from cterasdk.edge.types import TCPService, TCPConnectResult +from cterasdk.edge.types import TCPService, TCPConnectResult, StaticRoute from cterasdk.edge.enum import Mode, IPProtocol, Traffic from cterasdk.common import Object from cterasdk.exceptions.common import TaskException -from cterasdk import exceptions from tests.ut.edge import base_edge @@ -39,7 +38,9 @@ def setUp(self): self._mtu = 1320 self._static_route_gateway = '192.168.0.150' - self._static_route_network = '172.64.28.15/32' + self._static_route_netmask = '255.255.255.255' + self._static_route_destination = '172.64.28.15' + self._static_route_network = f'{self._static_route_destination}/32' self._proxy_address = '192.168.27.131' self._proxy_port = 3192 @@ -52,28 +53,28 @@ def setUp(self): def test_network_status(self): get_response = 'Success' self._init_filer(get_response=get_response) - ret = network.Network(self._filer).get_status() + ret = network.LegacyNetwork(self._filer).get_status() self._filer.api.get('/status/network/ports/0') self.assertEqual(ret, get_response) def test_ifconfig(self): get_response = 'Success' self._init_filer(get_response=get_response) - ret = network.Network(self._filer).ifconfig() + ret = network.LegacyNetwork(self._filer).ifconfig() self._filer.api.get('/config/network/ports/0') self.assertEqual(ret, get_response) def test_ipconfig(self): get_response = 'Success' self._init_filer(get_response=get_response) - ret = network.Network(self._filer).ipconfig() + ret = network.LegacyNetwork(self._filer).ipconfig() self._filer.api.get('/config/network/ports/0') self.assertEqual(ret, get_response) def test_set_static_ip_addr(self): get_response = self._dhcp_ip self._init_filer(get_response=get_response) - network.Network(self._filer).set_static_ipaddr( + network.LegacyNetwork(self._filer).set_static_ipaddr( self._static_ip.address, self._static_ip.netmask, self._static_ip.gateway, @@ -90,7 +91,7 @@ def test_set_static_ip_addr(self): def test_enable_dhcp(self): get_response = self._static_ip self._init_filer(get_response=get_response) - network.Network(self._filer).enable_dhcp() + network.LegacyNetwork(self._filer).enable_dhcp() self._filer.api.get.assert_called_once_with('/config/network/ports/0/ip') self._filer.api.put.assert_called_once_with('/config/network/ports/0/ip', mock.ANY) @@ -104,7 +105,7 @@ def test_set_static_primary_dns_server(self): self._dhcp_ip.secondary_dns_server = None get_response = self._dhcp_ip self._init_filer(get_response=get_response) - network.Network(self._filer).set_static_nameserver(self._static_ip.DNSServer1) + network.LegacyNetwork(self._filer).set_static_nameserver(self._static_ip.DNSServer1) self._filer.api.get.assert_called_once_with('/config/network/ports/0/ip') self._filer.api.put.assert_called_once_with('/config/network/ports/0/ip', mock.ANY) @@ -116,7 +117,7 @@ def test_set_static_primary_dns_server(self): def test_set_static_primary_and_secondary_dns_servers(self): get_response = self._dhcp_ip self._init_filer(get_response=get_response) - network.Network(self._filer).set_static_nameserver(self._static_ip.DNSServer1, self._static_ip.DNSServer2) + network.LegacyNetwork(self._filer).set_static_nameserver(self._static_ip.DNSServer1, self._static_ip.DNSServer2) self._filer.api.get.assert_called_once_with('/config/network/ports/0/ip') self._filer.api.put.assert_called_once_with('/config/network/ports/0/ip', mock.ANY) @@ -135,7 +136,7 @@ def test_tcp_connect_success(self): task.result.rc = 'Open' self._filer.tasks.wait = mock.MagicMock(return_value=task) - ret = network.Network(self._filer).tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) + ret = network.LegacyNetwork(self._filer).diag.tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) self._filer.api.execute.assert_called_once_with('/status/network', 'tcpconnect', mock.ANY) self._filer.tasks.wait.assert_called_once_with(self._task_id) @@ -155,7 +156,7 @@ def test_tcp_connect_failure(self): task.result.rc = 'BadAddress' self._filer.tasks.wait = mock.MagicMock(return_value=task) - ret = network.Network(self._filer).tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) + ret = network.LegacyNetwork(self._filer).diag.tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) self._filer.api.execute.assert_called_once_with('/status/network', 'tcpconnect', mock.ANY) self._filer.tasks.wait.assert_called_once_with(self._task_id) @@ -175,7 +176,7 @@ def test_iperf_success(self): task.result.res = 'Success' self._filer.tasks.wait = mock.MagicMock(return_value=task) - ret = network.Network(self._filer).iperf(self._static_ip.address) + ret = network.LegacyNetwork(self._filer).diag.iperf(self._static_ip.address) expected_param = self._get_iperf_param() actual_param = self._filer.api.execute.call_args[0][2] @@ -198,7 +199,7 @@ def test_tcp_connect_task_error(self): self._init_filer(execute_response=execute_response) self._filer.tasks.wait = mock.MagicMock(side_effect=TaskException('Task failed', self._task_id)) - ret = network.Network(self._filer).tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) + ret = network.LegacyNetwork(self._filer).diag.tcp_connect(TCPService(self._tcp_connect_address, self._tcp_connect_port)) self._filer.api.execute.assert_called_once_with('/status/network', 'tcpconnect', mock.ANY) self._filer.tasks.wait.assert_called_once_with(self._task_id) @@ -212,7 +213,7 @@ def test_tcp_connect_task_error(self): def test_edge_set_mtu(self): get_response = TestEdgeNetwork._get_ethernet_object() self._init_filer(get_response=get_response) - network.Network(self._filer).mtu.modify(self._mtu) + network.LegacyNetwork(self._filer).mtu.modify(self._mtu) self._filer.api.put.assert_called_once_with('/config/network/ports/0/ethernet', mock.ANY) expected_param = TestEdgeNetwork._get_ethernet_object(jumbo=True, mtu=self._mtu) actual_param = self._filer.api.put.call_args[0][1] @@ -221,7 +222,7 @@ def test_edge_set_mtu(self): def test_edge_reset_mtu(self): get_response = TestEdgeNetwork._get_ethernet_object(jumbo=True, mtu=1320) self._init_filer(get_response=get_response) - network.Network(self._filer).mtu.reset() + network.LegacyNetwork(self._filer).mtu.reset() self._filer.api.put.assert_called_once_with('/config/network/ports/0/ethernet', mock.ANY) expected_param = TestEdgeNetwork._get_ethernet_object() actual_param = self._filer.api.put.call_args[0][1] @@ -240,10 +241,10 @@ def _get_tcp_connect_object(self): tcp_connect_param.port = self._tcp_connect_port return tcp_connect_param - def test_add_static_routes(self): + def test_add_legacy_static_route(self): add_response = 'success' self._init_filer(add_response=add_response) - ret = network.Network(self._filer).routes.add(self._static_route_gateway, self._static_route_network) + ret = network.LegacyNetwork(self._filer).routes.add(self._static_route_gateway, self._static_route_network) self._filer.api.add.assert_called_once_with('/config/network/static_routes', mock.ANY) expected_param = Object(**{ 'GwIP': self._static_route_gateway, @@ -251,61 +252,70 @@ def test_add_static_routes(self): }) actual_param = self._filer.api.add.call_args[0][1] self._assert_equal_objects(actual_param, expected_param) - self.assertEqual(ret.network, self._static_route_network) - self.assertEqual(ret.gateway, self._static_route_gateway) + self.assertEqual(ret, add_response) - def test_add_static_routes_raise(self): - self._filer.api.add = mock.MagicMock(side_effect=exceptions.CTERAException()) - with self.assertRaises(exceptions.CTERAException) as error: - network.Network(self._filer).routes.add(self._static_route_gateway, self._static_route_network) - self.assertEqual('Static route creation failed', str(error.exception)) + def test_add_v711_static_route(self): + add_response = 'success' + self._init_filer(add_response=add_response, get_response=[ + Object(name='LAN0', ethernet=Object(mac='xyz')), + Object(name='LAN1', ethernet=Object(mac='abc')) + ]) + ret = network.Network711(self._filer).routes.add('LAN1', self._static_route_gateway, self._static_route_network) + self._filer.api.add.assert_called_once_with('/config/network/ports/1/ipv4StaticRoutes', mock.ANY) + expected_param = Object(**{ + 'destination': self._static_route_destination, + 'netmask': self._static_route_netmask, + 'gateway': self._static_route_gateway + }) + actual_param = self._filer.api.add.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + self.assertEqual(ret, add_response) def test_get_all_static_routes(self): - get_response = [Object(**{ + get_response = Object() + get_response.ports = [Object(name='LAN')] + get_response.static_routes = [Object(**{ + '_uuid': 'abc', 'GwIP': self._static_route_gateway, 'DestIpMask': self._static_route_network })] self._init_filer(get_response=get_response) - ret = network.Network(self._filer).routes.get() - self._filer.api.get.assert_called_once_with('/config/network/static_routes') - self.assertEqual(ret[0].gateway, get_response[0].GwIP) - self.assertEqual(ret[0].network, get_response[0].DestIpMask) + ret = network.LegacyNetwork(self._filer).routes.get() + self._filer.api.get.assert_called_once_with('/config/network') + self.assertEqual(ret[0].gateway, get_response.static_routes[0].GwIP) + self.assertEqual(ret[0].network, get_response.static_routes[0].DestIpMask) def test_remove_static_route(self): self._init_filer() - network.Network(self._filer).routes.delete(self._static_route_network) - self._filer.api.delete.assert_called_once_with(f'/config/network/static_routes/{self._static_route_network.replace("/", "_")}') - - def test_remove_static_route_raise(self): - self._filer.api.delete = mock.MagicMock(side_effect=exceptions.CTERAException()) - with self.assertRaises(exceptions.CTERAException) as error: - network.Network(self._filer).routes.delete(self._static_route_network) - self.assertEqual('Static route deletion failed', str(error.exception)) + route_uuid = '123' + route = StaticRoute(route_uuid, 0, 'LAN', self._static_route_destination, self._static_route_gateway) + network.LegacyNetwork(self._filer).routes.delete(route) + self._filer.api.delete.assert_called_once_with(f'/config/network/static_routes/{route_uuid}') def test_clean_all_static_routes_success(self): execute_response = 'Success' self._init_filer(execute_response=execute_response) - network.Network(self._filer).routes.clear() + network.LegacyNetwork(self._filer).routes.clear() self._filer.api.execute.assert_called_once_with('/config/network', 'cleanStaticRoutes') def test_get_proxy_config(self): get_response = 'Success' self._init_filer(get_response=get_response) - ret = network.Network(self._filer).proxy.get_configuration() + ret = network.LegacyNetwork(self._filer).proxy.get_configuration() self._filer.api.get.assert_called_once_with('/config/network/proxy') self.assertEqual(ret, get_response) def test_is_proxy_enabled(self): for expected_response, configuration in [(False, 'NoProxy'), (True, 'Manual')]: self._init_filer(get_response=configuration) - ret = network.Network(self._filer).proxy.is_enabled() + ret = network.LegacyNetwork(self._filer).proxy.is_enabled() self._filer.api.get.assert_called_once_with('/config/network/proxy/configurationMode') self.assertEqual(ret, expected_response) def test_disable_proxy(self): put_response = 'Success' self._init_filer(put_response=put_response) - ret = network.Network(self._filer).proxy.disable() + ret = network.LegacyNetwork(self._filer).proxy.disable() actual_param = self._filer.api.put.call_args[0][1] expected_param = TestEdgeNetwork._create_proxy_param(False) self._assert_equal_objects(actual_param, expected_param) @@ -314,7 +324,7 @@ def test_disable_proxy(self): def test_modify_proxy(self): put_response = 'Success' self._init_filer(put_response=put_response) - ret = network.Network(self._filer).proxy.modify(self._proxy_address, self._proxy_port, self._proxy_user, self._proxy_pass) + ret = network.LegacyNetwork(self._filer).proxy.modify(self._proxy_address, self._proxy_port, self._proxy_user, self._proxy_pass) actual_param = self._filer.api.put.call_args[0][1] expected_param = TestEdgeNetwork._create_proxy_param(True, self._proxy_address, self._proxy_port, self._proxy_user, self._proxy_pass) @@ -338,14 +348,14 @@ def _create_proxy_param(enabled=None, address=None, port=None, username=None, pa def test_get_hosts_file(self): get_response = 'Success' self._init_filer(get_response=get_response) - ret = network.Network(self._filer).hosts.get() + ret = network.LegacyNetwork(self._filer).hosts.get() self._filer.api.get.assert_called_once_with('/config/network/hostsFileEntries') self.assertEqual(ret, get_response) def test_add_hosts_file_entry(self): add_response = 'Success' self._init_filer(add_response=add_response) - ret = network.Network(self._filer).hosts.add(self._hosts_ipaddr, self._hosts_hostname) + ret = network.LegacyNetwork(self._filer).hosts.add(self._hosts_ipaddr, self._hosts_hostname) self._filer.api.add.assert_called_once_with('/config/network/hostsFileEntries', mock.ANY) actual_param = self._filer.api.add.call_args[0][1] expected_param = munch.Munch(dict(ip=self._hosts_ipaddr, hostName=self._hosts_hostname)) @@ -355,6 +365,6 @@ def test_add_hosts_file_entry(self): def test_delete_hosts_file_entry(self): delete_response = 'Success' self._init_filer(delete_response=delete_response) - ret = network.Network(self._filer).hosts.delete(self._hosts_hostname) + ret = network.LegacyNetwork(self._filer).hosts.delete(self._hosts_hostname) self._filer.api.delete.assert_called_once_with(f'/config/network/hostsFileEntries/{self._hosts_hostname}') self.assertEqual(ret, delete_response) diff --git a/tests/ut/edge/test_services.py b/tests/ut/edge/test_services.py index 2dfd3212..4b7fb86c 100644 --- a/tests/ut/edge/test_services.py +++ b/tests/ut/edge/test_services.py @@ -42,10 +42,10 @@ def test_activate_default_args_success(self): self._init_filer() self._filer.api.execute = mock.MagicMock(side_effect=TestEdgeServices._mock_execute_connect_ok) self._filer.tasks.wait = mock.MagicMock() - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) services.Services(self._filer).activate(self._server, self._user, self._code) - self._filer.network.tcp_connect.assert_called_once_with(self._cttp_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._cttp_service) self._filer.api.execute.assert_called_once_with('/status/services', 'attachAndSave', mock.ANY) self._filer.tasks.wait.assert_called_once_with(TestEdgeServices._background_task_id) expected_param = self._get_attach_and_save_param(False, use_activation_code=True) @@ -66,11 +66,11 @@ def test_connect_default_args_success(self): self._init_filer() self._filer.api.execute = mock.MagicMock(side_effect=TestEdgeServices._mock_execute_connect_ok) self._filer.tasks.wait = mock.MagicMock() - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) services.Services(self._filer).connect(self._server, self._user, self._password) - self._filer.network.tcp_connect.assert_called_once_with(self._cttp_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._cttp_service) self._filer.api.execute.assert_has_calls( [ mock.call('/status/services', 'isWebSsoEnabled', mock.ANY), @@ -97,20 +97,20 @@ def test_connect_default_args_success(self): 1) def test_connect_tcp_connect_error(self): - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, False)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, False)) with self.assertRaises(ConnectionError) as error: services.Services(self._filer).connect(self._server, self._user, self._password) - self._filer.network.tcp_connect.assert_called_once_with(self._cttp_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._cttp_service) self.assertEqual(f'Unable to establish CTTP connection {self._server}:{self._cttp_port}', str(error.exception)) def test_connect_require_sso_failure(self): self._init_filer(execute_response=TestEdgeServices._check_web_sso_require_sso()) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) with self.assertRaises(exceptions.CTERAException) as error: services.Services(self._filer).connect(self._server, self._user, self._password) - self._filer.network.tcp_connect.assert_called_once_with(self._cttp_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._cttp_service) self._filer.api.execute.assert_called_once_with('/status/services', 'isWebSsoEnabled', mock.ANY) expected_param = self._get_is_web_sso_param(False) actual_param = self._filer.api.execute.call_args[0][2] @@ -122,12 +122,12 @@ def test_connect_default_args_task_failure(self): self._filer.api.execute = mock.MagicMock(side_effect=TestEdgeServices._mock_execute_connect_ok) task_error_side_effect = TestEdgeServices._get_task_error() self._filer.tasks.wait = mock.MagicMock(side_effect=task_error_side_effect) - self._filer.network.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) + self._filer.network.diag.tcp_connect = mock.MagicMock(return_value=TCPConnectResult(self._server, self._cttp_port, True)) with self.assertRaises(exceptions.CTERAException) as error: services.Services(self._filer).connect(self._server, self._user, self._password) - self._filer.network.tcp_connect.assert_called_once_with(self._cttp_service) + self._filer.network.diag.tcp_connect.assert_called_once_with(self._cttp_service) self._filer.api.execute.assert_has_calls( [ mock.call('/status/services', 'isWebSsoEnabled', mock.ANY),