Merge pull request #290 from xcp-ng/dtt-coalesce

feat(coalesce): Add tests for coalesce

Author: ydirson

conftest.py

@@ -94,6 +94,13 @@ def pytest_addoption(parser):
              "4KiB blocksize to be formatted and used in storage tests. "
              "Set it to 'auto' to let the fixtures auto-detect available disks."
     )
+    parser.addoption(
+        "--image-format",
+        action="append",
+        default=[],
+        help="Format of VDI to execute tests on."
+        "Example: vhd,qcow2"
+    )
 
 def pytest_configure(config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -106,6 +113,12 @@ def pytest_generate_tests(metafunc):
             vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default
         metafunc.parametrize("vm_ref", vms, indirect=True, scope="module")
 
+    if "image_format" in metafunc.fixturenames:
+        image_format = metafunc.config.getoption("image_format")
+        if len(image_format) == 0:
+            image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
+        metafunc.parametrize("image_format", image_format, scope="session")
+
 def pytest_collection_modifyitems(items, config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.
@@ -304,14 +317,21 @@ def host_no_ipv6(host):
     if is_ipv6(host.hostname_or_ip):
         pytest.skip(f"This test requires an IPv4 XCP-ng")
 
+@pytest.fixture(scope="session")
+def shared_sr(host):
+    sr = host.pool.first_shared_sr()
+    assert sr, "No shared SR available on hosts"
+    logging.info(">> Shared SR on host present: {} of type {}".format(sr.uuid, sr.get_type()))
+    yield sr
+
 @pytest.fixture(scope='session')
 def local_sr_on_hostA1(hostA1):
     """ A local SR on the pool's master. """
     srs = hostA1.local_vm_srs()
     assert len(srs) > 0, "a local SR is required on the pool's master"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostA1 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostA1 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')
@@ -321,7 +341,7 @@ def local_sr_on_hostA2(hostA2):
     assert len(srs) > 0, "a local SR is required on the pool's second host"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostA2 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostA2 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')
@@ -331,7 +351,7 @@ def local_sr_on_hostB1(hostB1):
     assert len(srs) > 0, "a local SR is required on the second pool's master"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostB1 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostB1 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')

lib/basevm.py

@@ -1,6 +1,6 @@
 import logging
 
-from typing import TYPE_CHECKING, Any, Literal, Optional, overload
+from typing import TYPE_CHECKING, Any, List, Literal, Optional, overload
 
 if TYPE_CHECKING:
     import lib.host
@@ -59,7 +59,7 @@ def name(self) -> str:
     def _disk_list(self):
         raise NotImplementedError()
 
-    def vdi_uuids(self, sr_uuid=None):
+    def vdi_uuids(self, sr_uuid: Optional[str] = None) -> List[str]:
         output = self._disk_list()
         if output == '':
             return []

lib/host.py

@@ -67,6 +67,7 @@ def __init__(self, pool: 'lib.pool.Pool', hostname_or_ip):
         self.uuid = self.inventory['INSTALLATION_UUID']
         self.xcp_version = version.parse(self.inventory['PRODUCT_VERSION'])
         self.xcp_version_short = f"{self.xcp_version.major}.{self.xcp_version.minor}"
+        self._dom0: Optional[VM] = None
 
     def __str__(self):
         return self.hostname_or_ip
@@ -711,3 +712,20 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def get_dom0_uuid(self):
+        return self.inventory["CONTROL_DOMAIN_UUID"]
+
+    def get_dom0_vm(self) -> VM:
+        if not self._dom0:
+            self._dom0 = VM(self.get_dom0_uuid(), self)
+        return self._dom0
+
+    def get_sr_from_vdi_uuid(self, vdi_uuid: str) -> Optional[SR]:
+        sr_uuid = self.xe("vdi-param-get", {
+            "param-name": "sr-uuid",
+            "uuid": vdi_uuid,
+        })
+        if not sr_uuid:
+            return None
+        return SR(sr_uuid, self.pool)

lib/snapshot.py

@@ -2,7 +2,18 @@
 
 from lib.basevm import BaseVM
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from lib.host import Host
+    from lib.vm import VM
+
+
 class Snapshot(BaseVM):
+    def __init__(self, uuid: str, host: "Host", vm: "VM"):
+        self.basevm = vm
+        super(Snapshot, self).__init__(uuid, host)
+
     def _disk_list(self):
         return self.host.xe('snapshot-disk-list', {'uuid': self.uuid, 'vbd-params': ''},
                             minimal=True)
@@ -21,3 +32,4 @@ def exists(self):
     def revert(self):
         logging.info("Revert to snapshot %s", self.uuid)
         self.host.xe('snapshot-revert', {'uuid': self.uuid})
+        self.basevm.create_vdis_list() # We reset the base VM object VDIs list because it changed following the revert

lib/sr.py

@@ -2,15 +2,24 @@
 import time
 
 import lib.commands as commands
-from lib.common import prefix_object_name, safe_split, strtobool, wait_for, wait_for_not
+from lib.common import (
+    prefix_object_name,
+    safe_split,
+    strtobool,
+    wait_for,
+    wait_for_not,
+)
 from lib.vdi import VDI
 
+from typing import Optional
+
 class SR:
     def __init__(self, uuid, pool):
         self.uuid = uuid
         self.pool = pool
         self._is_shared = None # cached value for is_shared()
         self._main_host = None # cached value for main_host()
+        self._type = None # cache value for get_type()
 
     def pbd_uuids(self):
         return safe_split(self.pool.master.xe('pbd-list', {'sr-uuid': self.uuid}, minimal=True))
@@ -152,13 +161,21 @@ def is_shared(self):
                                                             {'uuid': self.uuid, 'param-name': 'shared'}))
         return self._is_shared
 
-    def create_vdi(self, name_label, virtual_size=64):
+    def get_type(self) -> str:
+        if self._type is None:
+            self._type = self.pool.master.xe("sr-param-get", {"uuid": self.uuid, "param-name": "type"})
+        return self._type
+
+    def create_vdi(self, name_label: str, virtual_size: int = 64, image_format: Optional[str] = None) -> VDI:
         logging.info("Create VDI %r on SR %s", name_label, self.uuid)
-        vdi_uuid = self.pool.master.xe('vdi-create', {
+        args = {
             'name-label': prefix_object_name(name_label),
             'virtual-size': str(virtual_size),
-            'sr-uuid': self.uuid
-        })
+            'sr-uuid': self.uuid,
+        }
+        if image_format:
+            args["sm-config:image-format"] = image_format
+        vdi_uuid = self.pool.master.xe('vdi-create', args)
         return VDI(vdi_uuid, sr=self)
 
     def run_quicktest(self):

lib/vdi.py

@@ -25,11 +25,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         # TODO: use a different approach when migration is possible
         if sr is None:
             assert host
-            sr_uuid = host.pool.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_sr_from_vdi_uuid(self.uuid)
         else:
             self.sr = sr
 
@@ -50,6 +46,12 @@ def readonly(self) -> bool:
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 
+    def get_parent(self) -> Optional[str]:
+        return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True)
+
+    def get_image_format(self) -> Optional[str]:
+        return self.param_get("sm-config", key="image-format", accept_unknown_key=True)
+
     @overload
     def param_get(self, param_name: str, key: Optional[str] = ...,
                   accept_unknown_key: Literal[False] = ...) -> str:

lib/vm.py

@@ -19,6 +19,7 @@
 )
 from lib.snapshot import Snapshot
 from lib.vbd import VBD
+from lib.vdi import VDI
 from lib.vif import VIF
 
 from typing import TYPE_CHECKING, List, Literal, Optional, Union, overload
@@ -33,6 +34,7 @@ def __init__(self, uuid: str, host: 'Host'):
         self.previous_host = None # previous host when migrated or being migrated
         self.is_windows = self.param_get('platform', 'device_id', accept_unknown_key=True) == '0002'
         self.is_uefi = self.param_get('HVM-boot-params', 'firmware', accept_unknown_key=True) == 'uefi'
+        self.create_vdis_list()
 
     def power_state(self) -> str:
         return self.param_get('power-state')
@@ -273,19 +275,74 @@ def migrate(self, target_host, sr=None, network=None):
 
         self.previous_host = self.host
         self.host = target_host
+        self.create_vdis_list()
 
     def snapshot(self, ignore_vdis=None):
         logging.info("Snapshot VM")
         args = {'uuid': self.uuid, 'new-name-label': 'Snapshot of %s' % self.uuid}
         if ignore_vdis:
             args['ignore-vdi-uuids'] = ','.join(ignore_vdis)
-        return Snapshot(self.host.xe('vm-snapshot', args), self.host)
+        snap_uuid = self.host.xe('vm-snapshot', args)
+        return Snapshot(snap_uuid, self.host, self)
 
     def checkpoint(self) -> Snapshot:
         logging.info("Checkpoint VM")
         return Snapshot(self.host.xe('vm-checkpoint', {'uuid': self.uuid,
                                                        'new-name-label': 'Checkpoint of %s' % self.uuid}),
-                        self.host)
+                        self.host, self)
+
+    def connect_vdi(self, vdi: VDI, device: str = "autodetect") -> str:
+        logging.info(f">> Plugging VDI {vdi.uuid} on VM {self.uuid}")
+        vbd_uuid = self.host.xe("vbd-create", {
+            "vdi-uuid": vdi.uuid,
+            "vm-uuid": self.uuid,
+            "device": device,
+        })
+        try:
+            self.host.xe("vbd-plug", {"uuid": vbd_uuid})
+        except commands.SSHCommandFailed:
+            self.host.xe("vbd-destroy", {"uuid": vbd_uuid})
+            raise
+
+        self.vdis.append(vdi)
+
+        return vbd_uuid
+
+    def disconnect_vdi(self, vdi: VDI):
+        logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {self.uuid}")
+        assert vdi in self.vdis, "VDI {vdi.uuid} not in VM {self.uuid} VDI list"
+        vbd_uuid = self.host.xe("vbd-list", {
+            "vdi-uuid": vdi.uuid,
+            "vm-uuid": self.uuid
+        }, minimal=True)
+        try:
+            self.host.xe("vbd-unplug", {"uuid": vbd_uuid})
+        except commands.SSHCommandFailed as e:
+            if e.stdout == f"The device is not currently attached\ndevice: {vbd_uuid}":
+                logging.info(f"VBD {vbd_uuid} already unplugged")
+            else:
+                raise
+        self.host.xe("vbd-destroy", {"uuid": vbd_uuid})
+        self.vdis.remove(vdi)
+
+    def destroy_vdi(self, vdi_uuid: str) -> None:
+        for vdi in self.vdis:
+            if vdi.uuid == vdi_uuid:
+                self.vdis.remove(vdi)
+                super().destroy_vdi(vdi_uuid)
+                break
+
+    def create_vdis_list(self) -> None:
+        """ Used to redo the VDIs list of the VM when reverting a snapshot. """
+        try:
+            self.vdis = [VDI(vdi_uuid, host=self.host) for vdi_uuid in self.vdi_uuids()]
+        except commands.SSHCommandFailed as e:
+            # Doesn't work with Dom0 since `vm-disk-list` doesn't work on it so we create empty list
+            if e.stdout == "Error: No matching VMs found":
+                logging.info("Couldn't get disks list. We are Dom0. Continuing...")
+                self.vdis = []
+            else:
+                raise
 
     def vifs(self):
         _vifs = []

tests/storage/coalesce/__init__.py

@@ -0,0 +1,53 @@
+import pytest
+
+import logging
+
+from lib.vdi import VDI
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr = local_sr_on_hostA1
+    vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format)
+    logging.info(">> Created VDI {} of type {}".format(vdi.uuid, image_format))
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi.uuid))
+    vdi.destroy()
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    dom0 = host.get_dom0_vm()
+    dom0.connect_vdi(vdi_on_local_sr)
+
+    yield vdi_on_local_sr
+
+    dom0.disconnect_vdi(vdi_on_local_sr)
+
+@pytest.fixture(scope="function")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    """
+    A tapdev is a blockdevice allowing access to a VDI from the Dom0.
+
+    It is usually used to give access to the VDI to Qemu for emulating devices
+    before PV driver are loaded in the guest.
+    """
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"