Merge pull request #103 from UpCloudLtd/storage-uploads-and-defaults

Storage uploads and defaults

Author: ajmyyra

.gitignore

@@ -11,6 +11,7 @@ docs/html/
 .tox
 .cache
 .vscode/
+.idea/
 
 # coverage
 .coverage

docs/Storage.md

@@ -50,12 +50,14 @@ Storage can be created with the CloudManager's `.create_storage(size=10, tier="m
 
 ```python
 
-storage1 = manager.create_storage(	size=10,
-									tier="maxiops",
-									title="my storage disk",
-									zone='fi-hel1' )
+storage1 = manager.create_storage(
+    zone='fi-hel1',
+    size=10,
+    tier="maxiops",
+    title="my storage disk"
+)
 
-storage2 = manager.create_storage(100, zone='fi-hel1')
+storage2 = manager.create_storage(zone='de-fra1', size=100)
 
 ```
 
@@ -81,6 +83,58 @@ storage.destroy()
 
 ```
 
+## Import
+
+Storages can be imported either by passing a URL or by uploading the file. Currently .iso, .raw and .img formats
+are supported. Other formats like QCOW2 or VMDK should be converted before uploading
+(with e.g. [`qemu-img convert`](https://linux.die.net/man/1/qemu-img)).
+
+Uploaded storage is expected to be uncompressed. It is possible to upload gzip (`application/gzip`)
+or LZMA2 (`application/x-xz`) compressed files, but you need to specify a separate `content_type`
+when calling the `upload_file_for_storage_import` function.
+
+Warning: the size of the import cannot exceed the size of the storage.
+The data will be written starting from the beginning of the storage,
+and the storage will not be truncated before starting to write.
+
+Storages can be uploaded by providing a URL. Note that the upload is not
+done by the time `create_storage_import` returns, and you need to poll its
+status with `get_storage_import_details`.
+```python
+
+new_storage = manager.create_storage(size=20, zone='nl-ams1')
+storage_import = manager.create_storage_import(
+    storage=new_storage.uuid,
+    source='http_import',
+    source_location='https://username:[email protected]/path/to/data.raw',
+)
+
+import_details = manager.get_storage_import_details(new_storage.uuid)
+
+```
+
+Other way is to upload a storage directly. After finishing, you should confirm
+that the storage has been processed with `get_storage_import_details`.
+```python
+
+new_storage = manager.create_storage(size=20, zone='de-fra1', title='New imported storage')
+storage_import = manager.create_storage_import(storage=new_storage.uuid, source='direct_upload')
+
+manager.upload_file_for_storage_import(
+    storage_import=storage_import,
+    file='/path/to/your/storage.img',
+)
+
+import_details = manager.get_storage_import_details(new_storage.uuid)
+
+```
+
+Ongoing imports can also be cancelled:
+```python
+
+manager.cancel_storage_import(new_storage.uuid)
+
+```
 
 ## Clone
 

test/test_storage.py

@@ -30,7 +30,9 @@ def test_get_templates(self, manager):
     @responses.activate
     def test_storage_create(self, manager):
         Mock.mock_post("storage")
-        storage = manager.create_storage(666, "maxiops", "My data collection", "fi-hel1")
+        storage = manager.create_storage(
+            zone="fi-hel1", size=666, tier="maxiops", title="My data collection"
+        )
         assert type(storage).__name__ == "Storage"
         assert storage.size == 666
         assert storage.tier == "maxiops"

upcloud_api/cloud_manager/storage_mixin.py

@@ -1,9 +1,9 @@
-from typing import Optional, Union
+from os import PathLike
+from typing import BinaryIO, Optional, Union
 
 from upcloud_api.api import API
 from upcloud_api.storage import Storage
 from upcloud_api.storage_import import StorageImport
-from upcloud_api.utils import get_raw_data_from_file
 
 
 class StorageManager:
@@ -41,10 +41,11 @@ def get_storage(self, storage: str) -> Storage:
 
     def create_storage(
         self,
+        zone: str,
         size: int = 10,
         tier: str = 'maxiops',
         title: str = 'Storage disk',
-        zone: str = 'fi-hel1',
+        *,
         backup_rule: Optional[dict] = None,
     ) -> Storage:
         """
@@ -184,33 +185,52 @@ def create_storage_import(
         Creates an import task to import data into an existing storage.
         Source types: http_import or direct_upload.
         """
+        if source not in ("http_import", "direct_upload"):
+            raise Exception(f"invalid storage import source: {source}")
+
         url = f'/storage/{storage}/import'
         body = {'storage_import': {'source': source}}
         if source_location:
             body['storage_import']['source_location'] = source_location
         res = self.api.post_request(url, body)
         return StorageImport(**res['storage_import'])
 
-    def upload_file_for_storage_import(self, storage_import, file):
+    def upload_file_for_storage_import(
+        self,
+        storage_import: StorageImport,
+        file: Union[str, PathLike, BinaryIO],
+        timeout: int = 30,
+        content_type: str = 'application/octet-stream',
+    ):
         """
         Uploads a file directly to UpCloud's uploader session.
         """
-        # TODO: this should not buffer the entire `file` into memory
-
-        # This is importing and using `requests` directly since there doesn't
-        # seem to be a point in adding a `.api.raw_request()` call to the `API` class.
-        # That could be changed.
 
         import requests
 
-        resp = requests.put(
-            url=storage_import.direct_upload_url,
-            data=get_raw_data_from_file(file),
-            headers={'Content-type': 'application/octet-stream'},
-            timeout=600,
-        )
-        resp.raise_for_status()
-        return resp.json()
+        # This is importing and using `requests` directly since there doesn't
+        # seem to be a point in adding a `.api.raw_request()` call to the `API` class.
+        # That could be changed if there starts to be more of these cases.
+
+        f = file
+        needs_closing = False
+        if not hasattr(file, 'read'):
+            f = open(file, 'rb')
+            needs_closing = True
+
+        try:
+            resp = requests.put(
+                url=storage_import.direct_upload_url,
+                data=f,
+                headers={'Content-type': content_type},
+                timeout=timeout,
+            )
+
+            resp.raise_for_status()
+            return resp.json()
+        finally:
+            if needs_closing:
+                f.close()
 
     def get_storage_import_details(self, storage: str) -> StorageImport:
         """

upcloud_api/utils.py

@@ -22,13 +22,3 @@ def try_it_n_times(operation, expected_error_codes, custom_error='operation fail
             sleep(3)
         if i >= n - 1:
             raise UpCloudClientError(custom_error)
-
-
-def get_raw_data_from_file(file: str) -> bytes:
-    """
-    Helper function to get raw file data for uploading.
-    """
-    with open(file, 'rb') as file:
-        data = file.read()
-    file.close()
-    return data