feat(coalesce): Add tests for coalesce

Add a vdi-type parameter to parametrize the vdi_type fixture,
it default to vhd at the moment, will be used to run test on other vdi
type, e.g. qcow2.

The tests create a VDI, connect it to Dom0 then use the tapdev to write
random data somewhere in it.
It then compare the data of the VDI to the original data we have to see
if it has changed.
The test create snapshot/clone before deleting them and waiting for the
coalesce to have happened by observing sm-config:vhd-parent before
checking the integrity of the data.

Signed-off-by: Damien Thenot <[email protected]>

Author: Nambrok

conftest.py

@@ -75,6 +75,13 @@ def pytest_addoption(parser):
              "4KiB blocksize to be formatted and used in storage tests. "
              "Set it to 'auto' to let the fixtures auto-detect available disks."
     )
+    parser.addoption(
+        "--image-format",
+        action="append",
+        default=[],
+        help="Format of VDI to execute tests on."
+        "Example: vhd,qcow2"
+    )
 
 def pytest_configure(config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -87,6 +94,12 @@ def pytest_generate_tests(metafunc):
             vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default
         metafunc.parametrize("vm_ref", vms, indirect=True, scope="module")
 
+    if "image_format" in metafunc.fixturenames:
+        image_format = metafunc.config.getoption("image_format")
+        if len(image_format) == 0:
+            image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
+        metafunc.parametrize("image_format", image_format, scope="session")
+
 def pytest_collection_modifyitems(items, config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.

lib/host.py

@@ -6,6 +6,7 @@
 import subprocess
 import tempfile
 import uuid
+from typing import Optional
 
 from packaging import version
 from typing import Dict, List, Literal, Optional, overload, TYPE_CHECKING, Union
@@ -658,3 +659,16 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def get_dom0_uuid(self):
+        output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"])
+        return output.split("=")[1].replace("'", "")
+
+    def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]:
+        sr_uuid = self.xe("vdi-param-get",
+                          {"param-name": "sr-uuid",
+                           "uuid": vdi_uuid,
+                           })
+        if sr_uuid is None:
+            return None
+        return SR(sr_uuid, self.pool)

lib/vdi.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set, strtobool
 from typing import Literal, Optional, overload, TYPE_CHECKING
@@ -23,11 +24,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         # TODO: use a different approach when migration is possible
         if sr is None:
             assert host
-            sr_uuid = host.pool.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_sr_from_vdi_uuid(self.uuid)
         else:
             self.sr = sr
 
@@ -48,6 +45,9 @@ def readonly(self) -> bool:
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 
+    def get_parent(self) -> Optional[str]:
+        return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True)
+
     @overload
     def param_get(self, param_name: str, key: Optional[str] = ...,
                   accept_unknown_key: Literal[False] = ...) -> str:

tests/storage/coalesce/conftest.py

@@ -0,0 +1,62 @@
+import pytest
+import logging
+
+from lib.vdi import VDI
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = host.xe("vdi-create",
+                       {"sr-uuid": sr_uuid,
+                        "name-label": "testVDI",
+                        "virtual-size": str(MAX_LENGTH),
+                        "sm-config:type": image_format,
+                        })
+    logging.info(">> Created VDI {} of type {}".format(vdi_uuid, image_format))
+
+    vdi = VDI(vdi_uuid, host=host)
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi_uuid))
+    host.xe("vdi-destroy", {"uuid": vdi_uuid})
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    vdi_uuid = vdi_on_local_sr.uuid
+    dom0_uuid = host.get_dom0_uuid()
+    logging.info(f">> Plugging VDI {vdi_uuid} on Dom0")
+    vbd_uuid = host.xe("vbd-create",
+                       {"vdi-uuid": vdi_uuid,
+                        "vm-uuid": dom0_uuid,
+                        "device": "autodetect",
+                        })
+    host.xe("vbd-plug", {"uuid": vbd_uuid})
+
+    yield vdi_on_local_sr
+
+    logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0")
+    host.xe("vbd-unplug", {"uuid": vbd_uuid})
+    host.xe("vbd-destroy", {"uuid": vbd_uuid})
+
+@pytest.fixture(scope="class")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"

tests/storage/coalesce/test_coalesce.py

@@ -0,0 +1,93 @@
+import logging
+import time
+
+from lib.host import Host
+
+def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int):
+    """
+    if offset == 0:
+        off = "0"
+    else:
+        off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3
+    """
+    bs = 1
+    off = int(offset / bs)
+    count = length / bs
+    count += length % bs
+    count = int(count)
+    cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"]
+    host.ssh(cmd)
+
+def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str:
+    cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file]
+    if checksum:
+        cmd = cmd + ["|", "sha256sum"]
+    return host.ssh(cmd)
+
+def get_hashed_data(host: Host, file: str, offset: int, length: int):
+    return get_data(host, file, offset, length, True).split()[0]
+
+def snapshot_vdi(host: Host, vdi_uuid: str):
+    vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid})
+    logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}")
+    return vdi_snap
+
+def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool:
+    logging.info("Getting data from VDI and file")
+    vdi_checksum = get_hashed_data(host, tapdev, offset, length)
+    file_checksum = get_hashed_data(host, data_file, 0, length)
+    logging.info(f"VDI: {vdi_checksum}")
+    logging.info(f"FILE: {file_checksum}")
+
+    return vdi_checksum == file_checksum
+
+def test_write_data(host, tapdev, data_file_on_host):
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    vdi_snap = snapshot_vdi(host, vdi_uuid)
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    logging.info("Removing VDI snapshot")
+    host.xe("vdi-destroy", {"uuid": vdi_snap})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid})
+    logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}")
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    host.xe("vdi-destroy", {"uuid": clone_uuid})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)