From f00ad131869dcd80f0742ae8aa9d1d45a1a0a1ec Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Mon, 31 Mar 2025 11:33:02 +0200 Subject: [PATCH 01/19] feat(coalesce): Add tests for coalesce Add a vdi-type parameter to parametrize the vdi_type fixture, it default to vhd at the moment, will be used to run test on other vdi type, e.g. qcow2. The tests create a VDI, connect it to Dom0 then use the tapdev to write random data somewhere in it. It then compare the data of the VDI to the original data we have to see if it has changed. The test create snapshot/clone before deleting them and waiting for the coalesce to have happened by observing sm-config:vhd-parent before checking the integrity of the data. Signed-off-by: Damien Thenot --- conftest.py | 13 ++++ lib/host.py | 14 ++++ lib/vdi.py | 10 +-- tests/storage/coalesce/conftest.py | 62 +++++++++++++++++ tests/storage/coalesce/test_coalesce.py | 93 +++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 tests/storage/coalesce/conftest.py create mode 100644 tests/storage/coalesce/test_coalesce.py diff --git a/conftest.py b/conftest.py index fc1031e48..059068bf6 100644 --- a/conftest.py +++ b/conftest.py @@ -86,6 +86,13 @@ def pytest_addoption(parser): "4KiB blocksize to be formatted and used in storage tests. " "Set it to 'auto' to let the fixtures auto-detect available disks." ) + parser.addoption( + "--image-format", + action="append", + default=[], + help="Format of VDI to execute tests on." + "Example: vhd,qcow2" + ) def pytest_configure(config): global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner') @@ -98,6 +105,12 @@ def pytest_generate_tests(metafunc): vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default metafunc.parametrize("vm_ref", vms, indirect=True, scope="module") + if "image_format" in metafunc.fixturenames: + image_format = metafunc.config.getoption("image_format") + if len(image_format) == 0: + image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd + metafunc.parametrize("image_format", image_format, scope="session") + def pytest_collection_modifyitems(items, config): # Automatically mark tests based on fixtures they require. # Check pytest.ini or pytest --markers for marker descriptions. diff --git a/lib/host.py b/lib/host.py index d7031a9bc..44a63cfd4 100644 --- a/lib/host.py +++ b/lib/host.py @@ -6,6 +6,7 @@ import subprocess import tempfile import uuid +from typing import Optional from packaging import version from typing import Dict, List, Literal, Optional, overload, TYPE_CHECKING, Union @@ -699,3 +700,16 @@ def enable_hsts_header(self): def disable_hsts_header(self): self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf']) self.restart_toolstack(verify=True) + + def get_dom0_uuid(self): + output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"]) + return output.split("=")[1].replace("'", "") + + def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]: + sr_uuid = self.xe("vdi-param-get", + {"param-name": "sr-uuid", + "uuid": vdi_uuid, + }) + if sr_uuid is None: + return None + return SR(sr_uuid, self.pool) diff --git a/lib/vdi.py b/lib/vdi.py index d260ebe03..47c123e67 100644 --- a/lib/vdi.py +++ b/lib/vdi.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set, strtobool from typing import Literal, Optional, overload, TYPE_CHECKING @@ -23,11 +24,7 @@ def __init__(self, uuid, *, host=None, sr=None): # TODO: use a different approach when migration is possible if sr is None: assert host - sr_uuid = host.pool.get_vdi_sr_uuid(uuid) - # avoid circular import - # FIXME should get it from Host instead - from lib.sr import SR - self.sr = SR(sr_uuid, host.pool) + self.sr = host.get_sr_from_vdi_uuid(self.uuid) else: self.sr = sr @@ -48,6 +45,9 @@ def readonly(self) -> bool: def __str__(self): return f"VDI {self.uuid} on SR {self.sr.uuid}" + def get_parent(self) -> Optional[str]: + return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True) + @overload def param_get(self, param_name: str, key: Optional[str] = ..., accept_unknown_key: Literal[False] = ...) -> str: diff --git a/tests/storage/coalesce/conftest.py b/tests/storage/coalesce/conftest.py new file mode 100644 index 000000000..a0e13577c --- /dev/null +++ b/tests/storage/coalesce/conftest.py @@ -0,0 +1,62 @@ +import pytest +import logging + +from lib.vdi import VDI + +MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB + +@pytest.fixture(scope="module") +def vdi_on_local_sr(host, local_sr_on_hostA1, image_format): + sr_uuid = local_sr_on_hostA1.uuid + vdi_uuid = host.xe("vdi-create", + {"sr-uuid": sr_uuid, + "name-label": "testVDI", + "virtual-size": str(MAX_LENGTH), + "sm-config:type": image_format, + }) + logging.info(">> Created VDI {} of type {}".format(vdi_uuid, image_format)) + + vdi = VDI(vdi_uuid, host=host) + + yield vdi + + logging.info("<< Destroying VDI {}".format(vdi_uuid)) + host.xe("vdi-destroy", {"uuid": vdi_uuid}) + +@pytest.fixture(scope="module") +def vdi_with_vbd_on_dom0(host, vdi_on_local_sr): + vdi_uuid = vdi_on_local_sr.uuid + dom0_uuid = host.get_dom0_uuid() + logging.info(f">> Plugging VDI {vdi_uuid} on Dom0") + vbd_uuid = host.xe("vbd-create", + {"vdi-uuid": vdi_uuid, + "vm-uuid": dom0_uuid, + "device": "autodetect", + }) + host.xe("vbd-plug", {"uuid": vbd_uuid}) + + yield vdi_on_local_sr + + logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0") + host.xe("vbd-unplug", {"uuid": vbd_uuid}) + host.xe("vbd-destroy", {"uuid": vbd_uuid}) + +@pytest.fixture(scope="class") +def data_file_on_host(host): + filename = "/root/data.bin" + logging.info(f">> Creating data file {filename} on host") + size = 1 * 1024 * 1024 # 1MiB + assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size" + + host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"]) + + yield filename + + logging.info("<< Deleting data file") + host.ssh(["rm", filename]) + +@pytest.fixture(scope="module") +def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0): + sr_uuid = local_sr_on_hostA1.uuid + vdi_uuid = vdi_with_vbd_on_dom0.uuid + yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}" diff --git a/tests/storage/coalesce/test_coalesce.py b/tests/storage/coalesce/test_coalesce.py new file mode 100644 index 000000000..a4a4844e0 --- /dev/null +++ b/tests/storage/coalesce/test_coalesce.py @@ -0,0 +1,93 @@ +import logging +import time + +from lib.host import Host + +def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int): + """ + if offset == 0: + off = "0" + else: + off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3 + """ + bs = 1 + off = int(offset / bs) + count = length / bs + count += length % bs + count = int(count) + cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"] + host.ssh(cmd) + +def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str: + cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file] + if checksum: + cmd = cmd + ["|", "sha256sum"] + return host.ssh(cmd) + +def get_hashed_data(host: Host, file: str, offset: int, length: int): + return get_data(host, file, offset, length, True).split()[0] + +def snapshot_vdi(host: Host, vdi_uuid: str): + vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid}) + logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}") + return vdi_snap + +def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool: + logging.info("Getting data from VDI and file") + vdi_checksum = get_hashed_data(host, tapdev, offset, length) + file_checksum = get_hashed_data(host, data_file, 0, length) + logging.info(f"VDI: {vdi_checksum}") + logging.info(f"FILE: {file_checksum}") + + return vdi_checksum == file_checksum + +def test_write_data(host, tapdev, data_file_on_host): + length = 1 * 1024 * 1024 + offset = 0 + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + assert compare_data(host, tapdev, data_file_on_host, offset, length) + +def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 + + vdi_snap = snapshot_vdi(host, vdi_uuid) + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + logging.info("Removing VDI snapshot") + host.xe("vdi-destroy", {"uuid": vdi_snap}) + + logging.info("Waiting for coalesce") + while vdi.get_parent() is not None: + time.sleep(1) + logging.info("Coalesce done") + + assert compare_data(host, tapdev, data_file_on_host, offset, length) + +def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 + + clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid}) + logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}") + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + host.xe("vdi-destroy", {"uuid": clone_uuid}) + + logging.info("Waiting for coalesce") + while vdi.get_parent() is not None: + time.sleep(1) + logging.info("Coalesce done") + + assert compare_data(host, tapdev, data_file_on_host, offset, length) From b27d85b98894922c487ed04df2e27e8c93eaf5f4 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Tue, 20 May 2025 11:39:52 +0200 Subject: [PATCH 02/19] feat(coalesce): add Timeout for coalesce Adapt waiting for coalesce to fail with a timeout if coalesce doesn't happen using wait_for. Signed-off-by: Damien Thenot --- tests/storage/coalesce/test_coalesce.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/storage/coalesce/test_coalesce.py b/tests/storage/coalesce/test_coalesce.py index a4a4844e0..3cfd11fbb 100644 --- a/tests/storage/coalesce/test_coalesce.py +++ b/tests/storage/coalesce/test_coalesce.py @@ -1,7 +1,12 @@ import logging -import time +from lib.common import wait_for_not from lib.host import Host +from lib.vdi import VDI + +def wait_for_vdi_coalesce(vdi: VDI): + wait_for_not(lambda: vdi.get_parent(), msg="Waiting for coalesce") + logging.info("Coalesce done") def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int): """ @@ -64,10 +69,7 @@ def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): logging.info("Removing VDI snapshot") host.xe("vdi-destroy", {"uuid": vdi_snap}) - logging.info("Waiting for coalesce") - while vdi.get_parent() is not None: - time.sleep(1) - logging.info("Coalesce done") + wait_for_vdi_coalesce(vdi) assert compare_data(host, tapdev, data_file_on_host, offset, length) @@ -83,11 +85,9 @@ def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): logging.info("Copying data to tapdev") copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + logging.info("Removing VDI clone") host.xe("vdi-destroy", {"uuid": clone_uuid}) - logging.info("Waiting for coalesce") - while vdi.get_parent() is not None: - time.sleep(1) - logging.info("Coalesce done") + wait_for_vdi_coalesce(vdi) assert compare_data(host, tapdev, data_file_on_host, offset, length) From 9cbacb4a8d210aef648a6c29953df4d19b411a39 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Tue, 20 May 2025 16:54:24 +0200 Subject: [PATCH 03/19] fix(Host): use pre-parsed inventory for dom0 Signed-off-by: Damien Thenot --- lib/host.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/host.py b/lib/host.py index 44a63cfd4..2c6dd8e64 100644 --- a/lib/host.py +++ b/lib/host.py @@ -702,8 +702,7 @@ def disable_hsts_header(self): self.restart_toolstack(verify=True) def get_dom0_uuid(self): - output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"]) - return output.split("=")[1].replace("'", "") + return self.inventory["CONTROL_DOMAIN_UUID"] def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]: sr_uuid = self.xe("vdi-param-get", From 508a9453189fb16eaddda7fa398e094e9b8f23c3 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 4 Jun 2025 18:06:51 +0200 Subject: [PATCH 04/19] feat(image_format): support image_format vdi.py: add getting the image_format if available sr.py: add choosing a image_format on vdi creation sr.py add get_type to get the type of the SR Signed-off-by: Damien Thenot --- lib/sr.py | 18 ++++++++++++++---- lib/vdi.py | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/sr.py b/lib/sr.py index c616a890c..2c6486884 100644 --- a/lib/sr.py +++ b/lib/sr.py @@ -1,5 +1,6 @@ import logging import time +from typing import Optional import lib.commands as commands @@ -12,6 +13,7 @@ def __init__(self, uuid, pool): self.pool = pool self._is_shared = None # cached value for is_shared() self._main_host = None # cached value for main_host() + self._type = None # cache value for get_type() def pbd_uuids(self): return safe_split(self.pool.master.xe('pbd-list', {'sr-uuid': self.uuid}, minimal=True)) @@ -153,13 +155,21 @@ def is_shared(self): {'uuid': self.uuid, 'param-name': 'shared'})) return self._is_shared - def create_vdi(self, name_label, virtual_size=64): + def get_type(self) -> str: + if self._type is None: + self._type = self.pool.master.xe("sr-param-get", {"uuid": self.uuid, "param-name": "type"}) + return self._type + + def create_vdi(self, name_label: str, virtual_size: int = 64, image_format: Optional[str] = None) -> VDI: logging.info("Create VDI %r on SR %s", name_label, self.uuid) - vdi_uuid = self.pool.master.xe('vdi-create', { + args = { 'name-label': prefix_object_name(name_label), 'virtual-size': str(virtual_size), - 'sr-uuid': self.uuid - }) + 'sr-uuid': self.uuid, + } + if image_format: + args["sm-config:image-format"] = image_format + vdi_uuid = self.pool.master.xe('vdi-create', args) return VDI(vdi_uuid, sr=self) def run_quicktest(self): diff --git a/lib/vdi.py b/lib/vdi.py index 47c123e67..1b7431429 100644 --- a/lib/vdi.py +++ b/lib/vdi.py @@ -48,6 +48,9 @@ def __str__(self): def get_parent(self) -> Optional[str]: return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True) + def get_image_format(self) -> Optional[str]: + return self.param_get("sm-config", key="image-format", accept_unknown_key=True) + @overload def param_get(self, param_name: str, key: Optional[str] = ..., accept_unknown_key: Literal[False] = ...) -> str: From 44a31a82c28bd0f31dfd3e53e83d9436612027c3 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 4 Jun 2025 18:31:16 +0200 Subject: [PATCH 05/19] Add shared_sr + log type of SR Add a fixture `shared_sr` to get a shared SR Also modify local_sr_* function to log the SR type Signed-off-by: Damien Thenot --- conftest.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 059068bf6..9663ab1ca 100644 --- a/conftest.py +++ b/conftest.py @@ -309,6 +309,13 @@ def host_no_ipv6(host): if is_ipv6(host.hostname_or_ip): pytest.skip(f"This test requires an IPv4 XCP-ng") +@pytest.fixture(scope="session") +def shared_sr(host): + sr = host.pool.first_shared_sr() + assert sr, "No shared SR available on hosts" + logging.info(">> Shared SR on host present: {} of type {}".format(sr.uuid, sr.get_type())) + yield sr + @pytest.fixture(scope='session') def local_sr_on_hostA1(hostA1): """ A local SR on the pool's master. """ @@ -316,7 +323,7 @@ def local_sr_on_hostA1(hostA1): assert len(srs) > 0, "a local SR is required on the pool's master" # use the first local SR found sr = srs[0] - logging.info(">> local SR on hostA1 present : %s" % sr.uuid) + logging.info(">> local SR on hostA1 present: {} of type {}".format(sr.uuid, sr.get_type())) yield sr @pytest.fixture(scope='session') @@ -326,7 +333,7 @@ def local_sr_on_hostA2(hostA2): assert len(srs) > 0, "a local SR is required on the pool's second host" # use the first local SR found sr = srs[0] - logging.info(">> local SR on hostA2 present : %s" % sr.uuid) + logging.info(">> local SR on hostA2 present: {} of type {}".format(sr.uuid, sr.get_type())) yield sr @pytest.fixture(scope='session') @@ -336,7 +343,7 @@ def local_sr_on_hostB1(hostB1): assert len(srs) > 0, "a local SR is required on the second pool's master" # use the first local SR found sr = srs[0] - logging.info(">> local SR on hostB1 present : %s" % sr.uuid) + logging.info(">> local SR on hostB1 present: {} of type {}".format(sr.uuid, sr.get_type())) yield sr @pytest.fixture(scope='session') From b91bb643b83e31561e26a54fe0bc33c5125cce2e Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 4 Jun 2025 18:37:21 +0200 Subject: [PATCH 06/19] coalesce: Fix vdi_on_local_sr fixture Now use `SR.create_vdi` to create a VDI with the chosen image_format Signed-off-by: Damien Thenot --- tests/storage/coalesce/conftest.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/storage/coalesce/conftest.py b/tests/storage/coalesce/conftest.py index a0e13577c..365ce86ef 100644 --- a/tests/storage/coalesce/conftest.py +++ b/tests/storage/coalesce/conftest.py @@ -1,27 +1,18 @@ import pytest import logging -from lib.vdi import VDI - MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB @pytest.fixture(scope="module") def vdi_on_local_sr(host, local_sr_on_hostA1, image_format): - sr_uuid = local_sr_on_hostA1.uuid - vdi_uuid = host.xe("vdi-create", - {"sr-uuid": sr_uuid, - "name-label": "testVDI", - "virtual-size": str(MAX_LENGTH), - "sm-config:type": image_format, - }) - logging.info(">> Created VDI {} of type {}".format(vdi_uuid, image_format)) - - vdi = VDI(vdi_uuid, host=host) + sr = local_sr_on_hostA1 + vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format) + logging.info(">> Created VDI {} of type {}".format(vdi.uuid, image_format)) yield vdi - logging.info("<< Destroying VDI {}".format(vdi_uuid)) - host.xe("vdi-destroy", {"uuid": vdi_uuid}) + logging.info("<< Destroying VDI {}".format(vdi.uuid)) + vdi.destroy() @pytest.fixture(scope="module") def vdi_with_vbd_on_dom0(host, vdi_on_local_sr): From 48f7508365f61ac0a9a0d7c06ce139096cd443ad Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Fri, 6 Jun 2025 10:51:42 +0200 Subject: [PATCH 07/19] Re-order test in class Signed-off-by: Damien Thenot --- tests/storage/coalesce/test_coalesce.py | 63 +++++++++++++------------ 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/tests/storage/coalesce/test_coalesce.py b/tests/storage/coalesce/test_coalesce.py index 3cfd11fbb..79de72d20 100644 --- a/tests/storage/coalesce/test_coalesce.py +++ b/tests/storage/coalesce/test_coalesce.py @@ -46,48 +46,49 @@ def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: i return vdi_checksum == file_checksum -def test_write_data(host, tapdev, data_file_on_host): - length = 1 * 1024 * 1024 - offset = 0 +class Test: + def test_write_data(self, host, tapdev, data_file_on_host): + length = 1 * 1024 * 1024 + offset = 0 - logging.info("Copying data to tapdev") - copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) - assert compare_data(host, tapdev, data_file_on_host, offset, length) + assert compare_data(host, tapdev, data_file_on_host, offset, length) -def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): - vdi = vdi_with_vbd_on_dom0 - vdi_uuid = vdi.uuid - length = 1 * 1024 * 1024 - offset = 0 + def test_coalesce(self, host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 - vdi_snap = snapshot_vdi(host, vdi_uuid) + vdi_snap = snapshot_vdi(host, vdi_uuid) - logging.info("Copying data to tapdev") - copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) - logging.info("Removing VDI snapshot") - host.xe("vdi-destroy", {"uuid": vdi_snap}) + logging.info("Removing VDI snapshot") + host.xe("vdi-destroy", {"uuid": vdi_snap}) - wait_for_vdi_coalesce(vdi) + wait_for_vdi_coalesce(vdi) - assert compare_data(host, tapdev, data_file_on_host, offset, length) + assert compare_data(host, tapdev, data_file_on_host, offset, length) -def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): - vdi = vdi_with_vbd_on_dom0 - vdi_uuid = vdi.uuid - length = 1 * 1024 * 1024 - offset = 0 + def test_clone_coalesce(self, host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 - clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid}) - logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}") + clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid}) + logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}") - logging.info("Copying data to tapdev") - copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) - logging.info("Removing VDI clone") - host.xe("vdi-destroy", {"uuid": clone_uuid}) + logging.info("Removing VDI clone") + host.xe("vdi-destroy", {"uuid": clone_uuid}) - wait_for_vdi_coalesce(vdi) + wait_for_vdi_coalesce(vdi) - assert compare_data(host, tapdev, data_file_on_host, offset, length) + assert compare_data(host, tapdev, data_file_on_host, offset, length) From e498f4e4494b040b4be81c72df4387c9fd2c6867 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 2 Jul 2025 17:14:32 +0200 Subject: [PATCH 08/19] lib/basevm.py: add vdis related function Add a list of vdis object to a VM Add get_dom0_vm in host.py Add function to connect/disconnect a VDI from a VM Signed-off-by: Damien Thenot --- lib/basevm.py | 37 +++++++++++++++++++++++++++++++++++-- lib/host.py | 6 ++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/basevm.py b/lib/basevm.py index 31c257035..86bf5ff97 100644 --- a/lib/basevm.py +++ b/lib/basevm.py @@ -1,6 +1,6 @@ import logging -from typing import Any, Literal, Optional, overload, TYPE_CHECKING +from typing import Any, Literal, Optional, overload, TYPE_CHECKING, List import lib.commands as commands if TYPE_CHECKING: @@ -8,6 +8,7 @@ from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set from lib.sr import SR +from lib.vdi import VDI class BaseVM: """ Base class for VM and Snapshot. """ @@ -19,6 +20,15 @@ def __init__(self, uuid: str, host: 'lib.host.Host'): logging.info("New %s: %s", type(self).__name__, uuid) self.uuid = uuid self.host = host + try: + self.vdis = [VDI(vdi_uuid, host=host) for vdi_uuid in self.vdi_uuids()] + except commands.SSHCommandFailed as e: + # Doesn't work with Dom0 since `vm-disk-list` doesn't work on it so we create empty list + if e.stdout == "Error: No matching VMs found": + logging.info("Couldn't get disks list. We are Dom0. Continuing...") + self.vdis = [] + else: + raise @overload def param_get(self, param_name: str, key: Optional[str] = ..., @@ -56,11 +66,31 @@ def name(self) -> str: assert isinstance(n, str) return n + def connect_vdi(self, vdi: VDI, device: str = "autodetect") -> str: + logging.info(f">> Plugging VDI {vdi.uuid} on VM {self.uuid}") + vbd_uuid = self.host.xe("vbd-create", + { + "vdi-uuid": vdi.uuid, + "vm-uuid": self.uuid, + "device": device, + }) + self.host.xe("vbd-plug", {"uuid": vbd_uuid}) + + self.vdis.append(vdi) + + return vbd_uuid + + def disconnect_vdi(self, vdi: VDI): + logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {self.uuid}") + vbd_uuid = self.host.xe("vbd-list", {"vdi-uuid": vdi.uuid, "vm-uuid": self.uuid}, minimal=True) + self.host.xe("vbd-unplug", {"uuid": vbd_uuid}) + self.host.xe("vbd-destroy", {"uuid": vbd_uuid}) + # @abstractmethod def _disk_list(self): raise NotImplementedError() - def vdi_uuids(self, sr_uuid=None): + def vdi_uuids(self, sr_uuid=None) -> List[str]: output = self._disk_list() if output == '': return [] @@ -78,6 +108,9 @@ def vdi_uuids(self, sr_uuid=None): def destroy_vdi(self, vdi_uuid: str) -> None: self.host.xe('vdi-destroy', {'uuid': vdi_uuid}) + for vdi in self.vdis: + if vdi.uuid == vdi_uuid: + self.vdis.remove(vdi) def all_vdis_on_host(self, host): for vdi_uuid in self.vdi_uuids(): diff --git a/lib/host.py b/lib/host.py index 2c6dd8e64..1c5f0b473 100644 --- a/lib/host.py +++ b/lib/host.py @@ -56,6 +56,7 @@ def __init__(self, pool: 'lib.pool.Pool', hostname_or_ip): self.uuid = self.inventory['INSTALLATION_UUID'] self.xcp_version = version.parse(self.inventory['PRODUCT_VERSION']) self.xcp_version_short = f"{self.xcp_version.major}.{self.xcp_version.minor}" + self._dom0: Optional[VM] = None def __str__(self): return self.hostname_or_ip @@ -704,6 +705,11 @@ def disable_hsts_header(self): def get_dom0_uuid(self): return self.inventory["CONTROL_DOMAIN_UUID"] + def get_dom0_VM(self) -> VM: + if not self._dom0: + self._dom0 = VM(self.get_dom0_uuid(), self) + return self._dom0 + def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]: sr_uuid = self.xe("vdi-param-get", {"param-name": "sr-uuid", From 453f3398850364df2744394228d5ae3c82e2ec61 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 2 Jul 2025 17:18:04 +0200 Subject: [PATCH 09/19] feat(coalesce): modify coalesce fixture to use VM function Signed-off-by: Damien Thenot --- tests/storage/coalesce/conftest.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/storage/coalesce/conftest.py b/tests/storage/coalesce/conftest.py index 365ce86ef..a7a32466e 100644 --- a/tests/storage/coalesce/conftest.py +++ b/tests/storage/coalesce/conftest.py @@ -16,21 +16,12 @@ def vdi_on_local_sr(host, local_sr_on_hostA1, image_format): @pytest.fixture(scope="module") def vdi_with_vbd_on_dom0(host, vdi_on_local_sr): - vdi_uuid = vdi_on_local_sr.uuid - dom0_uuid = host.get_dom0_uuid() - logging.info(f">> Plugging VDI {vdi_uuid} on Dom0") - vbd_uuid = host.xe("vbd-create", - {"vdi-uuid": vdi_uuid, - "vm-uuid": dom0_uuid, - "device": "autodetect", - }) - host.xe("vbd-plug", {"uuid": vbd_uuid}) + dom0 = host.get_dom0_VM() + vbd_uuid = dom0.connect_vdi(vdi_on_local_sr) yield vdi_on_local_sr - logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0") - host.xe("vbd-unplug", {"uuid": vbd_uuid}) - host.xe("vbd-destroy", {"uuid": vbd_uuid}) + dom0.disconnect_vdi(vdi_on_local_sr) @pytest.fixture(scope="class") def data_file_on_host(host): From bd8e7c8e50925b2527a849335d9583bd46503136 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 2 Jul 2025 17:21:19 +0200 Subject: [PATCH 10/19] coalesce: move function in a utils.py file It will allow to re-use in other tests Signed-off-by: Damien Thenot --- tests/storage/coalesce/__init__.py | 0 tests/storage/coalesce/test_coalesce.py | 46 +------------------------ tests/storage/coalesce/utils.py | 46 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 45 deletions(-) create mode 100644 tests/storage/coalesce/__init__.py create mode 100644 tests/storage/coalesce/utils.py diff --git a/tests/storage/coalesce/__init__.py b/tests/storage/coalesce/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storage/coalesce/test_coalesce.py b/tests/storage/coalesce/test_coalesce.py index 79de72d20..bd87ace41 100644 --- a/tests/storage/coalesce/test_coalesce.py +++ b/tests/storage/coalesce/test_coalesce.py @@ -1,50 +1,6 @@ import logging -from lib.common import wait_for_not -from lib.host import Host -from lib.vdi import VDI - -def wait_for_vdi_coalesce(vdi: VDI): - wait_for_not(lambda: vdi.get_parent(), msg="Waiting for coalesce") - logging.info("Coalesce done") - -def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int): - """ - if offset == 0: - off = "0" - else: - off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3 - """ - bs = 1 - off = int(offset / bs) - count = length / bs - count += length % bs - count = int(count) - cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"] - host.ssh(cmd) - -def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str: - cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file] - if checksum: - cmd = cmd + ["|", "sha256sum"] - return host.ssh(cmd) - -def get_hashed_data(host: Host, file: str, offset: int, length: int): - return get_data(host, file, offset, length, True).split()[0] - -def snapshot_vdi(host: Host, vdi_uuid: str): - vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid}) - logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}") - return vdi_snap - -def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool: - logging.info("Getting data from VDI and file") - vdi_checksum = get_hashed_data(host, tapdev, offset, length) - file_checksum = get_hashed_data(host, data_file, 0, length) - logging.info(f"VDI: {vdi_checksum}") - logging.info(f"FILE: {file_checksum}") - - return vdi_checksum == file_checksum +from .utils import wait_for_vdi_coalesce, copy_data_to_tapdev, snapshot_vdi, compare_data class Test: def test_write_data(self, host, tapdev, data_file_on_host): diff --git a/tests/storage/coalesce/utils.py b/tests/storage/coalesce/utils.py new file mode 100644 index 000000000..8a461fff6 --- /dev/null +++ b/tests/storage/coalesce/utils.py @@ -0,0 +1,46 @@ +import logging + +from lib.common import wait_for_not +from lib.host import Host +from lib.vdi import VDI + +def wait_for_vdi_coalesce(vdi: VDI): + wait_for_not(lambda: vdi.get_parent(), msg="Waiting for coalesce") + logging.info("Coalesce done") + +def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int): + # if offset == 0: + # off = "0" + # else: + # off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3 + + bs = 1 + off = int(offset / bs) + count = length / bs + count += length % bs + count = int(count) + cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"] + host.ssh(cmd) + +def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str: + cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file] + if checksum: + cmd = cmd + ["|", "sha256sum"] + return host.ssh(cmd) + +def get_hashed_data(host: Host, file: str, offset: int, length: int): + return get_data(host, file, offset, length, True).split()[0] + +def snapshot_vdi(host: Host, vdi_uuid: str): + vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid}) + logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}") + return vdi_snap + +def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool: + logging.info("Getting data from VDI and file") + vdi_checksum = get_hashed_data(host, tapdev, offset, length) + file_checksum = get_hashed_data(host, data_file, 0, length) + logging.info(f"VDI: {vdi_checksum}") + logging.info(f"FILE: {file_checksum}") + + return vdi_checksum == file_checksum From b96297eed335166f0c0b3ae601a9271c8fe6edee Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 2 Jul 2025 17:41:19 +0200 Subject: [PATCH 11/19] pool.py: remove unused import Signed-off-by: Damien Thenot --- lib/pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pool.py b/lib/pool.py index c223397a1..8d2cecd70 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -1,7 +1,7 @@ import logging import os import traceback -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, Optional from packaging import version From c661605827372019ac72a602bc3dcb7716a1cc01 Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Mon, 7 Jul 2025 16:08:38 +0530 Subject: [PATCH 12/19] tests: Updated conftest to handle `image-format` as list of str (e.g. vhd,qcow2) Signed-off-by: Rushikesh Jadhav --- conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 9663ab1ca..ac2e5517d 100644 --- a/conftest.py +++ b/conftest.py @@ -107,8 +107,10 @@ def pytest_generate_tests(metafunc): if "image_format" in metafunc.fixturenames: image_format = metafunc.config.getoption("image_format") - if len(image_format) == 0: - image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd + if len(image_format) == 1: + image_format = image_format[0].split(",") + if not image_format: + image_format = ["vhd"] metafunc.parametrize("image_format", image_format, scope="session") def pytest_collection_modifyitems(items, config): From 5510d9aca6c1e1f684fb58f2b98b20fa37d7755e Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Mon, 7 Jul 2025 16:09:23 +0530 Subject: [PATCH 13/19] tests/storage/ext: Updated ext to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/ext/conftest.py | 7 +++++-- tests/storage/ext/test_ext_sr.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/storage/ext/conftest.py b/tests/storage/ext/conftest.py index 3b09f0819..d7b3c8a54 100644 --- a/tests/storage/ext/conftest.py +++ b/tests/storage/ext/conftest.py @@ -2,9 +2,12 @@ import pytest @pytest.fixture(scope='package') -def ext_sr(host, sr_disk): +def ext_sr(host, sr_disk, image_format): """ An EXT SR on first host. """ - sr = host.sr_create('ext', "EXT-local-SR-test", {'device': '/dev/' + sr_disk}) + sr = host.sr_create('ext', "EXT-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }) yield sr # teardown sr.destroy() diff --git a/tests/storage/ext/test_ext_sr.py b/tests/storage/ext/test_ext_sr.py index 8697219c7..b0fc5cca6 100644 --- a/tests/storage/ext/test_ext_sr.py +++ b/tests/storage/ext/test_ext_sr.py @@ -15,9 +15,12 @@ class TestEXTSRCreateDestroy: def test_create_sr_with_missing_device(self, host): try_to_create_sr_with_missing_device('ext', 'EXT-local-SR-test', host) - def test_create_and_destroy_sr(self, host, sr_disk): + def test_create_and_destroy_sr(self, host, image_format, sr_disk): # Create and destroy tested in the same test to leave the host as unchanged as possible - sr = host.sr_create('ext', "EXT-local-SR-test", {'device': '/dev/' + sr_disk}, verify=True) + sr = host.sr_create('ext', "EXT-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs_fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid) From 77444ee79db4e5582a5d2c307deababf8fb5c520 Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Mon, 16 Jun 2025 19:03:01 +0530 Subject: [PATCH 14/19] lib/host: Avoid using identical source and destination paths in execute_script scp Using the same path for both source and destination may cause scp to fail, especially when the source path (e.g. a temp path on macOS) does not exist on the remote system. Thus, resorting to use `/tmp/` and random name from `sctipt.name` on destination. Signed-off-by: Rushikesh Jadhav --- lib/host.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/host.py b/lib/host.py index 1c5f0b473..58dcab5eb 100644 --- a/lib/host.py +++ b/lib/host.py @@ -203,13 +203,13 @@ def execute_script(self, script_contents, shebang='sh', simple_output=True): script.write('#!/usr/bin/env ' + shebang + '\n') script.write(script_contents) script.flush() - self.scp(script.name, script.name) + self.scp(script.name, "/tmp/" + os.path.basename(script.name)) try: logging.debug(f"[{self}] # Will execute this temporary script:\n{script_contents.strip()}") - return self.ssh([script.name], simple_output=simple_output) + return self.ssh(["/tmp/" + os.path.basename(script.name)], simple_output=simple_output) finally: - self.ssh(['rm', '-f', script.name]) + self.ssh(['rm', '-f', "/tmp/" + os.path.basename(script.name)]) def _get_xensource_inventory(self) -> Dict[str, str]: output = self.ssh(['cat', '/etc/xensource-inventory']) From 5ec3d5ead419801931de7154e5ba878092818b97 Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Mon, 7 Jul 2025 18:23:00 +0530 Subject: [PATCH 15/19] tests/storage/lvm: Updated lvm to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/lvm/conftest.py | 7 +++++-- tests/storage/lvm/test_lvm_sr.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/storage/lvm/conftest.py b/tests/storage/lvm/conftest.py index 3f132c970..58f62ba41 100644 --- a/tests/storage/lvm/conftest.py +++ b/tests/storage/lvm/conftest.py @@ -2,9 +2,12 @@ import pytest @pytest.fixture(scope='package') -def lvm_sr(host, sr_disk): +def lvm_sr(host, sr_disk, image_format): """ An LVM SR on first host. """ - sr = host.sr_create('lvm', "LVM-local-SR-test", {'device': '/dev/' + sr_disk}) + sr = host.sr_create('lvm', "LVM-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }) yield sr # teardown sr.destroy() diff --git a/tests/storage/lvm/test_lvm_sr.py b/tests/storage/lvm/test_lvm_sr.py index 2c791ad42..065543dce 100644 --- a/tests/storage/lvm/test_lvm_sr.py +++ b/tests/storage/lvm/test_lvm_sr.py @@ -15,9 +15,12 @@ class TestLVMSRCreateDestroy: def test_create_sr_with_missing_device(self, host): try_to_create_sr_with_missing_device('lvm', 'LVM-local-SR-test', host) - def test_create_and_destroy_sr(self, host, sr_disk): + def test_create_and_destroy_sr(self, host, image_format, sr_disk): # Create and destroy tested in the same test to leave the host as unchanged as possible - sr = host.sr_create('lvm', "LVM-local-SR-test", {'device': '/dev/' + sr_disk}, verify=True) + sr = host.sr_create('lvm', "LVM-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs_fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid) From 28438a80cb06cb52192be12259a910540cef6ebc Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Mon, 7 Jul 2025 21:11:07 +0530 Subject: [PATCH 16/19] tests/storage/xfs: Updated xfs to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/xfs/conftest.py | 9 ++++++--- tests/storage/xfs/test_xfs_sr.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/storage/xfs/conftest.py b/tests/storage/xfs/conftest.py index ab61079d8..5f750924a 100644 --- a/tests/storage/xfs/conftest.py +++ b/tests/storage/xfs/conftest.py @@ -2,7 +2,7 @@ import pytest @pytest.fixture(scope='package') -def host_with_xfsprogs(host): +def host_with_xfsprogs(host, image_format): assert not host.file_exists('/usr/sbin/mkfs.xfs'), \ "xfsprogs must not be installed on the host at the beginning of the tests" host.yum_save_state() @@ -12,9 +12,12 @@ def host_with_xfsprogs(host): host.yum_restore_saved_state() @pytest.fixture(scope='package') -def xfs_sr(sr_disk, host_with_xfsprogs): +def xfs_sr(sr_disk, host_with_xfsprogs, image_format): """ A XFS SR on first host. """ - sr = host_with_xfsprogs.sr_create('xfs', "XFS-local-SR-test", {'device': '/dev/' + sr_disk}) + sr = host_with_xfsprogs.sr_create('xfs', "XFS-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }) yield sr # teardown sr.destroy() diff --git a/tests/storage/xfs/test_xfs_sr.py b/tests/storage/xfs/test_xfs_sr.py index 6c5c8f100..b7013153b 100644 --- a/tests/storage/xfs/test_xfs_sr.py +++ b/tests/storage/xfs/test_xfs_sr.py @@ -17,23 +17,29 @@ class TestXFSSRCreateDestroy: and VM import. """ - def test_create_xfs_sr_without_xfsprogs(self, host, sr_disk): + def test_create_xfs_sr_without_xfsprogs(self, host, image_format, sr_disk): # This test must be the first in the series in this module assert not host.file_exists('/usr/sbin/mkfs.xfs'), \ "xfsprogs must not be installed on the host at the beginning of the tests" sr = None try: - sr = host.sr_create('xfs', "XFS-local-SR-test", {'device': '/dev/' + sr_disk}) + sr = host.sr_create('xfs', "XFS-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }) except Exception: logging.info("SR creation failed, as expected.") if sr is not None: sr.destroy() assert False, "SR creation should not have succeeded!" - def test_create_and_destroy_sr(self, sr_disk, host_with_xfsprogs): + def test_create_and_destroy_sr(self, sr_disk, host_with_xfsprogs, image_format): # Create and destroy tested in the same test to leave the host as unchanged as possible host = host_with_xfsprogs - sr = host.sr_create('xfs', "XFS-local-SR-test", {'device': '/dev/' + sr_disk}, verify=True) + sr = host.sr_create('xfs', "XFS-local-SR-test", { + 'device': '/dev/' + sr_disk, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid) From 8e9589840f0d80bf725e705832733952b080fa4b Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Tue, 8 Jul 2025 01:35:15 +0530 Subject: [PATCH 17/19] tests/storage/zfs: Updated zfs to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/zfs/conftest.py | 12 +++++++++--- tests/storage/zfs/test_zfs_sr.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/storage/zfs/conftest.py b/tests/storage/zfs/conftest.py index 159b27790..d26b740c7 100644 --- a/tests/storage/zfs/conftest.py +++ b/tests/storage/zfs/conftest.py @@ -13,11 +13,14 @@ def host_without_zfs(host): "zfs must not be installed on the host at the beginning of the tests" @pytest.fixture(scope='package') -def host_with_zfs(host_without_zfs, host_with_saved_yum_state): +def host_with_zfs(host_without_zfs, host_with_saved_yum_state, image_format): host = host_with_saved_yum_state host.yum_install(['zfs']) host.ssh(['modprobe', 'zfs']) yield host + host.ssh(["systemctl", "stop", "zfs-zed"]) + host.ssh(['rmmod', 'zfs']) + host.yum_remove(['zfs']) @pytest.fixture(scope='package') def zpool_vol0(sr_disk_wiped, host_with_zfs): @@ -27,9 +30,12 @@ def zpool_vol0(sr_disk_wiped, host_with_zfs): host_with_zfs.ssh(['zpool', 'destroy', POOL_NAME]) @pytest.fixture(scope='package') -def zfs_sr(host, zpool_vol0): +def zfs_sr(host, image_format, zpool_vol0): """ A ZFS SR on first host. """ - sr = host.sr_create('zfs', "ZFS-local-SR-test", {'location': POOL_PATH}) + sr = host.sr_create('zfs', "ZFS-local-SR-test", { + 'location': POOL_PATH, + 'preferred-image-formats': image_format + }, verify=True) yield sr # teardown sr.destroy() diff --git a/tests/storage/zfs/test_zfs_sr.py b/tests/storage/zfs/test_zfs_sr.py index 123acec8d..0ffd59255 100755 --- a/tests/storage/zfs/test_zfs_sr.py +++ b/tests/storage/zfs/test_zfs_sr.py @@ -19,13 +19,16 @@ class TestZFSSRCreateDestroy: and VM import. """ - def test_create_zfs_sr_without_zfs(self, host): + def test_create_zfs_sr_without_zfs(self, host, image_format): # This test must be the first in the series in this module assert not host.file_exists('/usr/sbin/zpool'), \ "zfs must not be installed on the host at the beginning of the tests" sr = None try: - sr = host.sr_create('zfs', "ZFS-local-SR-test", {'location': POOL_PATH}) + sr = host.sr_create('zfs', "ZFS-local-SR-test", { + 'location': POOL_PATH, + 'preferred-image-formats': image_format + }, verify=True) except Exception: logging.info("SR creation failed, as expected.") if sr is not None: @@ -33,9 +36,12 @@ def test_create_zfs_sr_without_zfs(self, host): assert False, "SR creation should not have succeeded!" @pytest.mark.usefixtures("zpool_vol0") - def test_create_and_destroy_sr(self, host): + def test_create_and_destroy_sr(self, host, image_format): # Create and destroy tested in the same test to leave the host as unchanged as possible - sr = host.sr_create('zfs', "ZFS-local-SR-test", {'location': POOL_PATH}, verify=True) + sr = host.sr_create('zfs', "ZFS-local-SR-test", { + 'location': POOL_PATH, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs_fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid) From cfe0199012853493c1730c62a88b5052efc50678 Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Tue, 8 Jul 2025 19:21:51 +0530 Subject: [PATCH 18/19] tests/storage/zfsvol: Updated zfsvol to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/zfsvol/conftest.py | 11 ++++++++--- tests/storage/zfsvol/test_zfsvol_sr.py | 7 +++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/storage/zfsvol/conftest.py b/tests/storage/zfsvol/conftest.py index 910240cd3..a7581b5ed 100644 --- a/tests/storage/zfsvol/conftest.py +++ b/tests/storage/zfsvol/conftest.py @@ -5,17 +5,22 @@ from pkgfixtures import host_with_saved_yum_state, sr_disk_wiped @pytest.fixture(scope='package') -def host_with_zfsvol(host_with_saved_yum_state): +def host_with_zfsvol(host_with_saved_yum_state, image_format): host = host_with_saved_yum_state host.yum_install(['xcp-ng-xapi-storage-volume-zfsvol']) host.restart_toolstack(verify=True) yield host + host.yum_remove(['xcp-ng-xapi-storage-volume-zfsvol']) + host.restart_toolstack(verify=True) @pytest.fixture(scope='package') -def zfsvol_sr(host, sr_disk_wiped, host_with_zfsvol): +def zfsvol_sr(host, image_format, sr_disk_wiped, host_with_zfsvol): """ A ZFS Volume SR on first host. """ device = '/dev/' + sr_disk_wiped - sr = host.sr_create('zfs-vol', "ZFS-local-SR-test", {'device': device}) + sr = host.sr_create('zfs-vol', "ZFS-local-SR-test", { + 'device': device, + 'preferred-image-formats': image_format + }, verify=True) yield sr # teardown violently - we don't want to require manual recovery when a test fails sr.forget() diff --git a/tests/storage/zfsvol/test_zfsvol_sr.py b/tests/storage/zfsvol/test_zfsvol_sr.py index 92b06880b..981f3dcd8 100755 --- a/tests/storage/zfsvol/test_zfsvol_sr.py +++ b/tests/storage/zfsvol/test_zfsvol_sr.py @@ -19,10 +19,13 @@ class TestZfsvolSRCreateDestroy: and VM import. """ - def test_create_and_destroy_sr(self, sr_disk_wiped, host_with_zfsvol): + def test_create_and_destroy_sr(self, image_format, sr_disk_wiped, host_with_zfsvol): host = host_with_zfsvol # Create and destroy tested in the same test to leave the host as unchanged as possible - sr = host.sr_create('zfs-vol', "ZFS-local-SR-test", {'device': '/dev/' + sr_disk_wiped}, verify=True) + sr = host.sr_create('zfs-vol', "ZFS-local-SR-test", { + 'device': '/dev/' + sr_disk_wiped, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs_fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid) From c421cc48afb3c3dbd24161b9c15aa3861832f933 Mon Sep 17 00:00:00 2001 From: Rushikesh Jadhav Date: Wed, 9 Jul 2025 16:36:52 +0530 Subject: [PATCH 19/19] tests/storage/largeblock: Updated largeblock to handle both vhd and qcow2 vdi image format Signed-off-by: Rushikesh Jadhav --- tests/storage/largeblock/conftest.py | 7 +++++-- tests/storage/largeblock/test_largeblock_sr.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/storage/largeblock/conftest.py b/tests/storage/largeblock/conftest.py index b219d8248..56d91dafc 100644 --- a/tests/storage/largeblock/conftest.py +++ b/tests/storage/largeblock/conftest.py @@ -2,9 +2,12 @@ import pytest @pytest.fixture(scope='package') -def largeblock_sr(host, sr_disk_4k): +def largeblock_sr(host, image_format, sr_disk_4k): """ A LARGEBLOCK SR on first host. """ - sr = host.sr_create('largeblock', "LARGEBLOCK-local-SR-test", {'device': '/dev/' + sr_disk_4k}) + sr = host.sr_create('largeblock', "LARGEBLOCK-local-SR-test", { + 'device': '/dev/' + sr_disk_4k, + 'preferred-image-formats': image_format + }) yield sr # teardown sr.destroy() diff --git a/tests/storage/largeblock/test_largeblock_sr.py b/tests/storage/largeblock/test_largeblock_sr.py index c82853d01..1946e98c8 100644 --- a/tests/storage/largeblock/test_largeblock_sr.py +++ b/tests/storage/largeblock/test_largeblock_sr.py @@ -15,9 +15,12 @@ class TestLARGEBLOCKSRCreateDestroy: def test_create_sr_with_missing_device(self, host): try_to_create_sr_with_missing_device('largeblock', 'LARGEBLOCK-local-SR-test', host) - def test_create_and_destroy_sr(self, host, sr_disk_4k): + def test_create_and_destroy_sr(self, host, image_format, sr_disk_4k): # Create and destroy tested in the same test to leave the host as unchanged as possible - sr = host.sr_create('largeblock', "LARGEBLOCK-local-SR-test", {'device': '/dev/' + sr_disk_4k}, verify=True) + sr = host.sr_create('largeblock', "LARGEBLOCK-local-SR-test", { + 'device': '/dev/' + sr_disk_4k, + 'preferred-image-formats': image_format + }, verify=True) # import a VM in order to detect vm import issues here rather than in the vm_on_xfs_fixture used in # the next tests, because errors in fixtures break teardown vm = host.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid)