feat(coalesce): Add tests for coalesce

Add a vdi-type parameter to parametrize the vdi_type fixture,
it default to vhd at the moment, will be used to run test on other vdi
type, e.g. qcow2.

The tests create a VDI, connect it to Dom0 then use the tapdev to write
random data somewhere in it.
It then compare the data of the VDI to the original data we have to see
if it has changed.
The test create snapshot/clone before deleting them and waiting for the
coalesce to have happened by observing sm-config:vhd-parent before
checking the integrity of the data.

Signed-off-by: Damien Thenot <[email protected]>

Author: Nambrok

conftest.py

@@ -94,6 +94,13 @@ def pytest_addoption(parser):
              "4KiB blocksize to be formatted and used in storage tests. "
              "Set it to 'auto' to let the fixtures auto-detect available disks."
     )
+    parser.addoption(
+        "--image-format",
+        action="append",
+        default=[],
+        help="Format of VDI to execute tests on."
+        "Example: vhd,qcow2"
+    )
 
 def pytest_configure(config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -106,6 +113,12 @@ def pytest_generate_tests(metafunc):
             vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default
         metafunc.parametrize("vm_ref", vms, indirect=True, scope="module")
 
+    if "image_format" in metafunc.fixturenames:
+        image_format = metafunc.config.getoption("image_format")
+        if len(image_format) == 0:
+            image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
+        metafunc.parametrize("image_format", image_format, scope="session")
+
 def pytest_collection_modifyitems(items, config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.

lib/vdi.py

@@ -25,11 +25,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         # TODO: use a different approach when migration is possible
         if sr is None:
             assert host
-            sr_uuid = host.pool.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_sr_from_vdi_uuid(self.uuid)
         else:
             self.sr = sr
 

tests/storage/coalesce/__init__.py

@@ -0,0 +1,53 @@
+import pytest
+
+import logging
+
+from lib.vdi import VDI
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr = local_sr_on_hostA1
+    vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format)
+    logging.info(">> Created VDI {} of type {}".format(vdi.uuid, image_format))
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi.uuid))
+    vdi.destroy()
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    dom0 = host.get_dom0_VM()
+    dom0.connect_vdi(vdi_on_local_sr)
+
+    yield vdi_on_local_sr
+
+    dom0.disconnect_vdi(vdi_on_local_sr)
+
+@pytest.fixture(scope="function")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    """
+    A tapdev is a blockdevice allowing access to a VDI from the Dom0.
+
+    It is usually used to give access to the VDI to Qemu for emulating devices
+    before PV driver are loaded in the guest.
+    """
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"

tests/storage/coalesce/conftest.py

@@ -0,0 +1,42 @@
+import pytest
+
+import logging
+
+from .utils import compare_data, copy_data_to_tapdev, operation_on_vdi, wait_for_vdi_coalesce
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from lib.host import Host
+    from lib.vdi import VDI
+
+class Test:
+    def test_write_data(self, host: "Host", tapdev: str, data_file_on_host: str):
+        length = 1 * 1024 * 1024
+        offset = 0
+
+        logging.info("Copying data to tapdev")
+        copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+        logging.info("Comparing data to tapdev")
+        assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+    @pytest.mark.parametrize("vdi_op", ["snapshot", "clone"])
+    def test_coalesce(self, host: "Host", tapdev: str, vdi_with_vbd_on_dom0: "VDI", data_file_on_host: str, vdi_op):
+        vdi = vdi_with_vbd_on_dom0
+        vdi_uuid = vdi.uuid
+        length = 1 * 1024 * 1024
+        offset = 0
+
+        new_vdi = operation_on_vdi(host, vdi_uuid, vdi_op)
+
+        logging.info("Copying data to tapdev")
+        copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+        logging.info(f"Removing VDI {vdi_op}")
+        host.xe("vdi-destroy", {"uuid": new_vdi})
+
+        wait_for_vdi_coalesce(vdi)
+
+        logging.info("Comparing data to tapdev")
+        assert compare_data(host, tapdev, data_file_on_host, offset, length)

tests/storage/coalesce/test_coalesce.py

@@ -0,0 +1,48 @@
+import logging
+
+from lib.common import wait_for_not
+from lib.host import Host
+from lib.vdi import VDI
+
+from typing import Literal
+
+def wait_for_vdi_coalesce(vdi: VDI):
+    wait_for_not(lambda: vdi.get_parent(), msg="Waiting for coalesce")
+    logging.info("Coalesce done")
+
+def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int):
+    # if offset == 0:
+    #     off = "0"
+    # else:
+    #     off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3
+
+    bs = 1
+    off = int(offset / bs)
+    count = length / bs
+    count += length % bs
+    count = int(count)
+    cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"]
+    host.ssh(cmd)
+
+def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str:
+    cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file]
+    if checksum:
+        cmd = cmd + ["|", "sha256sum"]
+    return host.ssh(cmd)
+
+def get_hashed_data(host: Host, file: str, offset: int, length: int):
+    return get_data(host, file, offset, length, True).split()[0]
+
+def operation_on_vdi(host: Host, vdi_uuid: str, vdi_op: Literal["snapshot", "clone"]) -> str:
+    new_vdi = host.xe(f"vdi-{vdi_op}", {"uuid": vdi_uuid})
+    logging.info(f"{vdi_op.capitalize()} VDI {vdi_uuid}: {new_vdi}")
+    return new_vdi
+
+def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool:
+    logging.info("Getting data from VDI and file")
+    vdi_checksum = get_hashed_data(host, tapdev, offset, length)
+    file_checksum = get_hashed_data(host, data_file, 0, length)
+    logging.info(f"VDI: {vdi_checksum}")
+    logging.info(f"FILE: {file_checksum}")
+
+    return vdi_checksum == file_checksum