Saimon/upgrade depdencies to resolve CVE (#327)

Author: saimonation

cterasdk/common/__init__.py

@@ -1,7 +1,7 @@
 from .item import Item  # noqa: E402, F401
 from .object import Object, Device, delete_attrs  # noqa: E402, F401
-from .datetime_utils import DateTimeUtils, from_iso_format  # noqa: E402, F401
-from .utils import merge, union, parse_base_object_ref, convert_size, df_military_time, DataUnit, parse_to_ipaddress, \
+from .datetime_utils import DateTimeUtils  # noqa: E402, F401
+from .utils import merge, union, parse_base_object_ref, convert_size, df_military_time, DataUnit, \
                    utf8_decode, utf8_encode, Version  # noqa: E402, F401
 from .types import PolicyRule, PolicyRuleConverter, StringCriteriaBuilder, IntegerCriteriaBuilder, DateTimeCriteriaBuilder, \
                    PredefinedListCriteriaBuilder, CustomListCriteriaBuilder, ThrottlingRuleBuilder, ThrottlingRule, FilterBackupSet, \

cterasdk/common/datetime_utils.py

@@ -19,14 +19,3 @@ def get_expiration_date(expiration):
         elif isinstance(expiration, datetime.date):
             expiration_date = expiration
         return expiration_date  # pylint: disable=possibly-used-before-assignment
-
-
-def from_iso_format(time):
-    """
-    Parse datetime object from ISO 8601 format
-
-    :param str time: Timestamp
-    :returns: Datetime object
-    :rtype: datetime.datetime
-    """
-    return datetime.datetime.fromisoformat(time)

cterasdk/common/utils.py

@@ -1,7 +1,6 @@
 import re
 import socket
 import logging
-import ipaddress
 
 from datetime import datetime
 from packaging.version import parse as parse_version
@@ -154,28 +153,6 @@ def parse_base_object_ref(base_object_ref):
     return BaseObjectRef(**arguments)
 
 
-def parse_to_ipaddress(address):
-    """
-    Parse an ip or network address into ipaddress object
-
-    :param str address: ip (10.0.0.5) or network address (192.168.44.0/28)
-    :return: ipaddress.IPV4Address/IPV6Address or ipaddress.IPV4Network/IPV6Network
-    """
-    try:
-        try:
-            ip_addrr = ipaddress.ip_address(address)
-            logger.debug('ip address validated. %s', {'ip': str(ip_addrr)})
-            return ip_addrr
-        except (ValueError, TypeError):
-            ip_network = ipaddress.ip_network(address)
-            logger.debug('ip network validated. %s', {'network': str(ip_network)})
-            return ip_network
-    except (ValueError, TypeError):
-        err = ValueError(f'{address} does not appear to be an IPv4 or IPv6 network or ip address')
-        logger.error('Incorrect entry, please use IPv4 or IPv6 CIDR Formats. %s', {'Error': err})
-        raise err
-
-
 class Version:
     """Software Version"""
 
@@ -222,10 +199,11 @@ def utf8_encode(message):
     return message.encode('utf-8')
 
 
-def tcp_connect(host, port):
+def tcp_connect(host, port, *, timeout=None):
     logger.debug('Testing connection. %s', {'host': host, 'port': port})
     message = f"Connection error to remote host {host} on port {port}."
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
     rc = None
     try:
         rc = sock.connect_ex((host, port))

cterasdk/direct/cli.py

@@ -0,0 +1,89 @@
+import sys
+import asyncio
+import logging
+import argparse
+import aiofiles
+import yarl
+
+
+from .. import settings
+from ..common import utils
+from ..lib.storage import commonfs
+from .client import DirectIO
+from ..exceptions.direct import StreamError, DirectIOError
+from ..exceptions.transport import TLSError
+
+
+logging.basicConfig(format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", level=logging.ERROR)
+
+
+logger = logging.getLogger('cterasdk.direct')
+
+
+def validate_endpoint(endpoint):
+    baseurl, port = yarl.URL(endpoint), 443
+    logger.debug('Validating connection to host: %s on port: %s', baseurl.host, port)
+    utils.tcp_connect(baseurl.host, port, timeout=5)
+    return f'{baseurl}'
+
+
+def validate_directory_and_filename(path):
+    directory, filename = commonfs.generate_file_destination(path)
+    assert all([directory, filename]), f'Error: Could not resolve file path: {path}'
+
+
+async def download_from_object_storage(options, file_id, path):
+    async with DirectIO(**options) as client:
+        streamer = await client.streamer(file_id)
+        try:
+            async with aiofiles.open(path, 'wb') as fd:
+                async for block in streamer.start():
+                    await fd.seek(block.offset)
+                    await fd.write(block.data)
+        except StreamError as e:
+            print(f'Download failed. Cause: {e.__cause__}', file=sys.stderr)
+
+
+def download_object():
+    parser = argparse.ArgumentParser(
+        description="Download a file from CTERA Portal via CTERA Direct I/O."
+    )
+
+    arguments = [
+        ("--endpoint", "-e", {"type": str, "required": True, "help": "CTERA Portal (e.g. corp.acme.ctera.com)"}),
+        ("--path", "-p", {"type": str, "required": True, "help": "File path (e.g. ./download.zip)"}),
+        ("--access", "-a", {"type": str, "help": "Access Key (optional)"}),
+        ("--secret", "-s", {"type": str, "help": "Secret Key (optional)"}),
+        ("--bearer", "-b", {"type": str, "help": "Bearer token (optional)"}),
+        ("--file-id", "-f", {"required": True, "type": int, "help": "File ID (numeric)"}),
+        ("--no-verify-ssl", "-k", {"action": "store_true", "help": "Disable SSL verification"}),
+        ("--debug", "-d", {"action": "store_true", "help": "Enable debug logging"}),
+    ]
+
+    for lopt, sopt, options in arguments:
+        parser.add_argument(lopt, sopt, **options)
+
+    args = parser.parse_args()
+
+    try:
+
+        if args.debug:
+            logger.setLevel(logging.DEBUG)
+
+        options = {
+            'baseurl': validate_endpoint(args.endpoint),
+            'access_key_id': args.access,
+            'secret_access_key': args.secret,
+            'bearer': args.bearer
+        }
+
+        validate_directory_and_filename(args.path)
+
+        settings.io.direct.api.settings.connector.ssl = not args.no_verify_ssl
+        settings.io.direct.storage.settings.connector.ssl = not args.no_verify_ssl
+
+        asyncio.run(download_from_object_storage(options, args.file_id, args.path))
+    except ConnectionError:
+        print(f'Error: Could not establish connection to host: {args.endpoint}:443', file=sys.stderr)
+    except (AssertionError, TLSError, DirectIOError) as e:
+        print(e, file=sys.stderr)

cterasdk/direct/client.py

@@ -2,7 +2,7 @@
 import cterasdk.settings
 
 from . import filters
-from .credentials import KeyPair, Bearer, create_bearer_token
+from .credentials import KeyPair, Bearer
 from .lib import get_chunks, decrypt_encryption_key, process_chunks
 from .types import ByteRange
 from .stream import Streamer
@@ -29,10 +29,9 @@ def __init__(self, baseurl=None, access_key_id=None, secret_access_key=None, bea
                               authenticator=lambda *_: True)
         self._client = AsyncClient(DefaultBuilder(), settings=cterasdk.settings.io.direct.storage.settings, authenticator=lambda *_: True)
         self._credentials = Bearer(bearer) if bearer else KeyPair(access_key_id, secret_access_key)
-        self._bearer = create_bearer_token(self._credentials)
 
     async def _chunks(self, file_id):
-        metadata = await get_chunks(self._api, self._bearer, file_id)
+        metadata = await get_chunks(self._api, self._credentials.bearer, file_id)
         if metadata.encrypted:
             metadata.encryption_key = decrypt_encryption_key(
                 metadata.file_id,

cterasdk/direct/credentials.py

@@ -1,4 +1,6 @@
+import os
 import logging
+from abc import abstractmethod
 
 
 logger = logging.getLogger('cterasdk.direct')
@@ -7,38 +9,33 @@
 class BaseCredentials:
     """Base Credentials for CTERA Direct IO"""
 
+    @property
+    def bearer(self):
+        return f'Bearer {self._bearer()}'
+
+    @abstractmethod
+    def _bearer(self):
+        raise NotImplementedError("Subclass must implemenet the '_bearer' method.")
+
 
 class KeyPair(BaseCredentials):
     """Access and Secret Key Pair"""
 
     def __init__(self, access_key_id, secret_access_key):
-        self.access_key_id = access_key_id
-        self.secret_access_key = secret_access_key
+        logger.debug('Initializing client using Key Pair.')
+        self.access_key_id = access_key_id if access_key_id else os.getenv('cterasdk.io.direct.access_key_id')
+        self.secret_access_key = secret_access_key if secret_access_key else os.getenv('cterasdk.io.direct.secret_access_key')
+
+    def _bearer(self):
+        return self.access_key_id
 
 
 class Bearer(BaseCredentials):
     """Bearer Token"""
 
     def __init__(self, bearer):
-        self.bearer = bearer
-
-
-def create_bearer_token(credentials):
-    """
-    Create Authorization Header.
-
-    :param cterasdk.direct.credentials.BaseCredentials credentials: Credentials
-    :returns: Authorization header as a dictionary.
-    :rtype: dict
-    """
-    token = None
-
-    if isinstance(credentials, Bearer):
-        logger.debug('Initializing client using Bearer token')
-        token = f'Bearer {credentials.bearer}'
-
-    elif isinstance(credentials, KeyPair):
-        logger.debug('Initializing client using Key Pair.')
-        token = f'Bearer {credentials.access_key_id}'
+        logger.debug('Initializing client using Bearer token.')
+        self.bearer = bearer if bearer else os.getenv('cterasdk.io.direct.bearer')
 
-    return token
+    def _bearer(self):
+        return self.bearer

cterasdk/direct/stream.py

@@ -42,8 +42,8 @@ async def start(self):
                 yield fragment
                 self._offset = fragment.offset + fragment.length
         except DirectIOAPIError as error:
-            raise StreamError(error.filename, self._offset)
+            raise StreamError(error.filename, self._offset) from error
         except BlockError as error:
-            raise StreamError(error.block.file_id, self._offset)
+            raise StreamError(error.block.file_id, self._offset) from error
         finally:
             self.stop()

cterasdk/edge/network.py

@@ -1,10 +1,11 @@
 import logging
+import ipaddress
 
 from ..exceptions import CTERAException
 from ..exceptions.common import TaskException
 from .enum import Mode, IPProtocol, Traffic
-from .types import TCPConnectResult
-from ..common import Object, parse_to_ipaddress
+from .types import TCPConnectResult, StaticRoute
+from ..common import Object
 from .base_command import BaseCommand
 
 
@@ -257,55 +258,50 @@ class StaticRoutes(BaseCommand):
 
     def get(self):
         """
-        Get All Static Routes
+        Get routes.
         """
-        return self._edge.api.get('/config/network/static_routes')
+        return [StaticRoute(r.DestIpMask, r.GwIP) for r in self._edge.api.get('/config/network/static_routes')]
 
-    def add(self, source_ip, destination_ip_mask):
+    def add(self, gateway, network):
         """
-        Add a Static Route
+        Add a route.
 
-        :param str source_ip: The source IP (192.168.15.55)
-        :param str destination_ip_mask: The destination IP and CIDR block (10.5.0.1/32)
+        :param str gateway: Gateway IP address
+        :param str network: Network (CIDR)
         """
+        ipaddress.ip_address(gateway)
+        ipaddress.ip_network(network)
+        param = Object()
+        param.GwIP = gateway
+        param.DestIpMask = network.replace('/', '_')
         try:
-            param = Object()
-            param.GwIP = str(parse_to_ipaddress(source_ip))
-            param.DestIpMask = str(parse_to_ipaddress(destination_ip_mask)).replace("/", "_")
-            res = self._edge.api.add('/config/network/static_routes', param)
-            logger.info(
-                "Static route updated. %s", {'Source': param.GwIP, 'Destination': destination_ip_mask})
-            return res
+            logger.info('Adding route for network: %s, to: %s', network, param.GwIP)
+            self._edge.api.add('/config/network/static_routes', param)
+            logger.info('Route added for network: %s, to: %s', network, param.GwIP)
+            return StaticRoute(network, gateway)
         except CTERAException as error:
             logger.error("Static route creation failed.")
             raise CTERAException('Static route creation failed') from error
 
-    def remove(self, destination_ip_mask):
+    def delete(self, network):
         """
-        Remove a Static Route
+        Delete a route.
 
-        :param str destination_ip_mask: The destination IP and CIDR block (10.5.0.1/32)
+        :param str network: Subnet mask (CIDR)
         """
+        ipaddress.ip_network(network)
         try:
-            dest_ip_mask = str(parse_to_ipaddress(destination_ip_mask)).replace("/", "_")
-            response = self._edge.api.delete(f'/config/network/static_routes/{dest_ip_mask}')
-            logger.info(
-                "Static route deleted. %s", {'Destination': dest_ip_mask})
-            return response
+            logger.info('Deleting route for: %s', network)
+            self._edge.api.delete(f'/config/network/static_routes/{network.replace("/", "_")}')
+            logger.info('Route deleted. Subnet: %s', network)
         except CTERAException as error:
             logger.error("Static route deletion failed.")
             raise CTERAException('Static route deletion failed') from error
 
     def clear(self):
-        """
-        Clear All Static routes
-        """
-        try:
-            self._edge.api.execute('/config/network', 'cleanStaticRoutes')
-            logger.info('Static routes were deleted successfully')
-        except CTERAException as error:
-            logger.error("Failed to clear static routes")
-            raise CTERAException('Failed to clear static routes') from error
+        logger.info('Clearing route table.')
+        self._edge.api.execute('/config/network', 'cleanStaticRoutes')
+        logger.info('Route table cleared.')
 
 
 class Hosts(BaseCommand):

cterasdk/edge/types.py

@@ -18,6 +18,12 @@
                                    'established to the target host over the specified port'
 
 
+StaticRoute = namedtuple('StaticRoute', ('network', 'gateway'))
+StaticRoute.__doc__ = 'Tuple holding the network and gateway of a static route'
+StaticRoute.network.__doc__ = 'Network (CIDR)'
+StaticRoute.gateway.__doc__ = 'Gateway IP address'
+
+
 class UserGroupEntry():
     """
     User or Group Entry

docs/source/UserGuides/Edge/Configuration.rst

@@ -928,27 +928,23 @@ Static Routes
 
 .. code-block:: python
 
-   # get static routes
    edge.network.routes.get()
 
 .. automethod:: cterasdk.edge.network.StaticRoutes.add
    :noindex:
 
 .. code-block:: python
 
-   # add static route from 10.10.12.1 to 192.168.55.7/32
    edge.network.routes.add('10.10.12.1', '192.168.55.7/32')
 
-   # add static route from 10.100.102.4 to 172.18.100.0/24
    edge.network.routes.add('10.100.102.4', '172.18.100.0/24')
 
-.. automethod:: cterasdk.edge.network.StaticRoutes.remove
+.. automethod:: cterasdk.edge.network.StaticRoutes.delete
    :noindex:
 
 .. code-block:: python
 
-   # remove static route 192.168.55.7/32
-   edge.network.routes.remove('192.168.55.7/32')
+   edge.network.routes.delete('192.168.55.7/32')
 
 .. automethod:: cterasdk.edge.network.StaticRoutes.clear
    :noindex: