Merge pull request #26 from cloudify-incubator/fortigate

Add example for directly download volume content from http

Author: 0lvin

.circleci/config.yml

@@ -59,7 +59,7 @@ jobs:
           command: pip install --upgrade pip==9.0.1
       - run:
           name: Install virtualenv
-          command: pip install virtualenv
+          command: pip install virtualenv==16.7.9
       - run:
           name: Init virtualenv
           command: virtualenv env

CHANGELOG.txt

@@ -1,11 +1,14 @@
 releases:
 
-  v0.1: Implement minimal functionality for create network and virtual machine
-        based on embeded template.
+  v0.1:
+   * Implement minimal functionality for create network and virtual machine
+     based on embeded template.
 
-  v0.2: Add arm support and autogenerate mac addresses in examples
+  v0.2:
+   * Add arm support and autogenerate mac addresses in examples
 
-  v0.3: Fix network validation and use virtio devices in embeded templates
+  v0.3:
+   * Fix network validation and use virtio devices in embeded templates
 
   v0.4:
    * Add cluster example with external ip support
@@ -15,25 +18,28 @@ releases:
   v0.4.1: Add support for disk/network backup/snapshots
 
   v0.5.0:
-    * Support for reuse external network
-    * Support for kvm virtualization in embeded examples
+   * Support for reuse external network
+   * Support for kvm virtualization in embeded examples
 
   v0.6.0:
-    * Support for dump full vm snapshot to fs on backup
-    * Update start action for fill in `network` in vm runtime properties
-    * Cluster Example: Automatically add libvirt host to trusted
-    * Support for reuse external vm
-    * move use_external_resource to top of instance properties
-    * add `update` action for sync vm size to values from runtime properties.
-    * rename `memory_size` to `memory_maxsize`
+   * Support for dump full vm snapshot to fs on backup
+   * Update start action for fill in `network` in vm runtime properties
+   * Cluster Example: Automatically add libvirt host to trusted
+   * Support for reuse external vm
+   * move use_external_resource to top of instance properties
+   * add `update` action for sync vm size to values from runtime properties.
+   * rename `memory_size` to `memory_maxsize`
 
   v0.7.0:
-    * Support storage pool creation.
-    * Rename `params.resource_id` to `params.name`.
-    * Support creation snapshot on external resources.
+   * Support storage pool creation.
+   * Rename `params.resource_id` to `params.name`.
+   * Support creation snapshot on external resources.
 
   v0.8.0:
-    * Support storage volume creation.
+   * Support storage volume creation.
 
   v0.8.1:
-    * Fix volume wipe code.
+   * Fix volume wipe code.
+
+  v0.9.0:
+   * Support download image contwent from external resource

cloudify_libvirt/common.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 import os
 import uuid
-from jinja2 import Template
 from pkg_resources import resource_filename
 
 from cloudify import ctx
 from cloudify import exceptions as cfy_exc
 
+from cloudify_common_sdk import filters
+
 
 def get_libvirt_params(**kwargs):
     libvirt_auth = kwargs.get('libvirt_auth')
@@ -66,11 +67,10 @@ def gen_xml_template(kwargs, template_params, default_template):
         with open(template_resource) as object_desc:
             template_content = object_desc.read()
 
-    template_engine = Template(template_content)
     params = {"ctx": ctx}
     if template_params:
         params.update(template_params)
-    xmlconfig = template_engine.render(params)
+    xmlconfig = filters.render_template(template_content, params)
     ctx.logger.debug(repr(xmlconfig))
     return xmlconfig
 

cloudify_libvirt/iso9660_tasks.py

@@ -13,38 +13,14 @@
 # limitations under the License.
 
 import libvirt
-import pycdlib
 import os
-import re
-from io import BytesIO
 
 from cloudify import ctx
 from cloudify.decorators import operation
 from cloudify import exceptions as cfy_exc
-import cloudify_libvirt.common as common
-
-
-def _joliet_name(name):
-    if name[0] == "/":
-        name = name[1:]
-    return "/{}".format(name[:64])
-
-
-def _name_cleanup(name):
-    return re.sub('[^A-Z0-9_]{1}', r'_', name.upper())
+import cloudify_common_sdk.iso9660 as iso9660
 
-
-def _iso_name(name):
-    if name[0] == "/":
-        name = name[1:]
-
-    name_splited = name.split('.')
-    if len(name_splited[-1]) <= 3 and len(name_splited) > 1:
-        return "/{}.{};1".format(
-            _name_cleanup("_".join(name_splited[:-1])[:8]),
-            _name_cleanup(name_splited[-1]))
-    else:
-        return "/{}.;1".format(_name_cleanup(name[:8]))
+import cloudify_libvirt.common as common
 
 
 @operation
@@ -68,22 +44,16 @@ def create(**kwargs):
                 'Failed to find the volume: {}'.format(repr(e))
             )
 
-        iso = pycdlib.PyCdlib()
-        iso.new(vol_ident='cidata', joliet=3, rock_ridge='1.09')
+        outiso = iso9660.create_iso(
+            vol_ident=template_params.get('vol_ident', 'cidata'),
+            sys_ident=template_params.get('sys_ident', ""),
+            get_resource=ctx.get_resource,
+            files=template_params.get('files', {}),
+            files_raw=template_params.get('files_raw', {}))
 
-        fstree = template_params.get('files', {})
-        for name in fstree:
-            file_bufer = BytesIO()
-            file_bufer.write(fstree[name].encode())
-            iso.add_fp(file_bufer, len(fstree[name]),
-                       _iso_name(name), rr_name=name,
-                       joliet_path=_joliet_name(name))
-
-        outiso = BytesIO()
-        iso.write_fp(outiso)
         outiso.seek(0, os.SEEK_END)
         iso_size = outiso.tell()
-        iso.close()
+        outiso.seek(0, os.SEEK_SET)
 
         ctx.logger.info("ISO size: {}".format(repr(iso_size)))
 

cloudify_libvirt/pool_tasks.py

@@ -252,6 +252,11 @@ def delete(**kwargs):
                 'Failed to find the pool: {}'.format(repr(e))
             )
 
+        if pool.delete() < 0:
+            raise cfy_exc.RecoverableError(
+                'Can not delete pool.'
+            )
+
         if pool.undefine() < 0:
             raise cfy_exc.NonRecoverableError(
                 'Can not undefine pool.'

cloudify_libvirt/tests/test_domain.py

@@ -134,6 +134,19 @@ def test_configure(self):
         self.assertEqual(
             _ctx.instance.runtime_properties['resource_id'], "domain_name"
         )
+        # check rerun after create
+        with mock.patch(
+            "cloudify_libvirt.domain_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            connect.lookupByName = mock.Mock(
+                side_effect=domain_tasks.libvirt.libvirtError("e"))
+            with self.assertRaisesRegexp(
+                NonRecoverableError,
+                'Failed to find the domain:'
+            ):
+                domain_tasks.configure(ctx=_ctx,
+                                       template_resource="template_resource")
         # with params from inputs
         _ctx.instance.runtime_properties['resource_id'] = None
         _ctx.instance.runtime_properties['params'] = {}

cloudify_libvirt/tests/test_iso9660_tasks.py

@@ -20,23 +20,6 @@
 
 class TestIso9660Task(LibVirtCommonTest):
 
-    def test_joliet_name(self):
-        self.assertEqual("/abc", iso9660_tasks._joliet_name("abc"))
-        self.assertEqual("/" + "*" * 64, iso9660_tasks._joliet_name("*" * 128))
-        self.assertEqual("/" + "*" * 64,
-                         iso9660_tasks._joliet_name("/" + "*" * 128))
-
-    def test_iso_name(self):
-        self.assertEqual("/ABC.;1", iso9660_tasks._iso_name("abc"))
-        self.assertEqual("/12345678.;1",
-                         iso9660_tasks._iso_name("1234567890.abcdef"))
-        self.assertEqual("/" + "_" * 8 + ".;1",
-                         iso9660_tasks._iso_name("*" * 128))
-        self.assertEqual("/" + "_" * 8 + ".;1",
-                         iso9660_tasks._iso_name("/" + "*" * 128))
-        self.assertEqual("/12345678.123;1",
-                         iso9660_tasks._iso_name("12345678.123"))
-
     def test_create(self):
         # check correct handle exception with empty connection
         self._check_correct_connect(

cloudify_libvirt/tests/test_network.py

@@ -553,6 +553,20 @@ def test_create(self):
             _ctx.instance.runtime_properties['resource_id'], "network_name"
         )
 
+        # rerun on created
+        connect.networkLookupByName = mock.Mock(
+            side_effect=network_tasks.libvirt.libvirtError("e"))
+        with mock.patch(
+            "cloudify_libvirt.network_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            with self.assertRaisesRegexp(
+                NonRecoverableError,
+                'Failed to find the network.'
+            ):
+                network_tasks.create(ctx=_ctx,
+                                     template_resource="template_resource")
+
 
 if __name__ == '__main__':
     unittest.main()

cloudify_libvirt/tests/test_pool.py

@@ -579,7 +579,22 @@ def test_delete(self):
                 "Can not undefine pool."
             ):
                 pool_tasks.delete(ctx=_ctx)
+
+        # remove with delete issues
+        pool.delete = mock.Mock(return_value=-1)
+        pool.undefine = mock.Mock(return_value=0)
+        with mock.patch(
+            "cloudify_libvirt.pool_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            with self.assertRaisesRegexp(
+                RecoverableError,
+                "Can not delete pool."
+            ):
+                pool_tasks.delete(ctx=_ctx)
+
         # remove witout issues
+        pool.delete = mock.Mock(return_value=0)
         pool.undefine = mock.Mock(return_value=0)
         with mock.patch(
             "cloudify_libvirt.pool_tasks.libvirt.open",

cloudify_libvirt/tests/test_volume.py

@@ -351,7 +351,58 @@ def test_create(self):
             _ctx.instance.runtime_properties['resource_id'], "volume_name"
         )
 
+        # failed check size of download
+        _ctx.instance.runtime_properties['resource_id'] = None
+        _ctx.instance.runtime_properties['params'] = {}
+        _ctx.node.properties['params'] = {}
+        with mock.patch(
+            "cloudify_libvirt.volume_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            # empty
+            head_response = mock.Mock()
+            head_response.headers = {'Content-Length': 0}
+            with mock.patch(
+                "cloudify_libvirt.volume_tasks.requests.head",
+                mock.Mock(return_value=head_response)
+            ):
+                with self.assertRaisesRegexp(
+                    NonRecoverableError,
+                    "Failed to download volume."
+                ):
+                    volume_tasks.create(
+                        ctx=_ctx,
+                        template_resource="template_resource",
+                        params={
+                            'pool': 'empty',
+                            'url': "https://fake.org/centos.iso"})
+
+        # sucessful check size of download
+        _ctx.instance.runtime_properties['resource_id'] = None
+        _ctx.instance.runtime_properties['params'] = {}
+        _ctx.node.properties['params'] = {}
+        with mock.patch(
+            "cloudify_libvirt.volume_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            head_response = mock.Mock()
+            head_response.headers = {'Content-Length': 512,
+                                     'Accept-Ranges': 'bytes'}
+            with mock.patch(
+                "cloudify_libvirt.volume_tasks.requests.head",
+                mock.Mock(return_value=head_response)
+            ):
+                volume_tasks.create(
+                    ctx=_ctx,
+                    template_resource="template_resource",
+                    params={
+                        'pool': 'empty',
+                        'url': "https://fake.org/centos.iso"})
+
         # failed on create
+        _ctx.instance.runtime_properties['resource_id'] = None
+        _ctx.instance.runtime_properties['params'] = {}
+        _ctx.node.properties['params'] = {}
         pool.createXML = mock.Mock(return_value=None)
         with mock.patch(
             "cloudify_libvirt.volume_tasks.libvirt.open",
@@ -444,6 +495,60 @@ def test_start_wipe(self):
                                     'allocation': 1
                                 })
 
+    def test_start_download(self):
+        # download
+        _ctx = self._create_ctx()
+        _ctx.instance.runtime_properties['resource_id'] = 'volume'
+        _ctx.instance.runtime_properties['params'] = {'pool': 'pool_name'}
+
+        volume = mock.Mock()
+        volume.name = mock.Mock(return_value="volume")
+        volume.upload = mock.Mock()
+        pool = mock.Mock()
+        pool.name = mock.Mock(return_value="pool")
+        pool.storageVolLookupByName = mock.Mock(return_value=volume)
+
+        connect = self._create_fake_connection()
+
+        connect.storagePoolLookupByName = mock.Mock(return_value=pool)
+        with mock.patch(
+            "cloudify_libvirt.volume_tasks.libvirt.open",
+            mock.Mock(return_value=connect)
+        ):
+            # empty
+            head_response = mock.Mock()
+            head_response.headers = {'Content-Length': 0}
+            with mock.patch(
+                "cloudify_libvirt.volume_tasks.requests.head",
+                mock.Mock(return_value=head_response)
+            ):
+                with self.assertRaisesRegexp(
+                    NonRecoverableError,
+                    "Failed to download volume."
+                ):
+                    volume_tasks.start(
+                        ctx=_ctx,
+                        params={
+                            'url': "https://fake.org/centos.iso"})
+
+            # 512 for download
+            head_response = mock.Mock()
+            head_response.headers = {'Content-Length': 512,
+                                     'Accept-Ranges': 'bytes'}
+            head_response.iter_content = mock.Mock(return_value=["\0" * 256])
+            with mock.patch(
+                "cloudify_libvirt.volume_tasks.requests.head",
+                mock.Mock(return_value=head_response)
+            ):
+                with mock.patch(
+                    "cloudify_libvirt.volume_tasks.requests.get",
+                    mock.Mock(return_value=head_response)
+                ):
+                    volume_tasks.start(
+                        ctx=_ctx,
+                        params={
+                            'url': "https://fake.org/centos.iso"})
+
     def test_stop(self):
         # check correct handle exception with empty connection
         self._test_check_correct_connect_action(