Implement fistpoint for smapi in tests

conftest.py

@@ -750,6 +750,15 @@ def nfs_iso_sr(host, nfs_iso_device_config):
     # teardown
     sr.forget()
 
+@pytest.fixture(scope='function')
+def exit_on_fistpoint(host):
+    from lib.fistpoint import FistPoint
+    logging.info(">> Enabling exit on fistpoint")
+    FistPoint.enable_exit_on_fistpoint(host)
+    yield
+    logging.info("<< Disabling exit on fistpoint")
+    FistPoint.disable_exit_on_fistpoint(host)
+
 @pytest.fixture(scope='module')
 def cifs_iso_sr(host, cifs_iso_device_config):
     """ A Samba/CIFS SR. """

lib/fistpoint.py

@@ -0,0 +1,68 @@
+import logging
+
+from lib.host import Host
+
+from typing import Final
+
+FISTPOINT_DIR: Final = "/tmp"
+LVHDRT_EXIT_FIST: Final = "fist_LVHDRT_exit"
+
+
+class FistPoint:
+    """
+    A fistpoint is an action that you can enable in the smapi for tests.
+
+    It allows for example, add a sleep at some point or raise an exception.
+    For example:
+    ```
+    with FistPoint(vm.host, "blktap_activate_inject_failure"):
+        with pytest.raises(SSHCommandFailed):
+            vm.start()
+            vm.shutdown(force=True)
+    ```
+    Activating the fistpoint `blktap_activate_inject_failure` mean that the VDI
+    activation will fail. This fistpoint always raise an exception but most
+    fistpoint just add a sleep at a point in the code.
+    Using the fixture `exit_on_fistpoint` make all fistpoints raise an
+    exception instead by enabling a special fistpoint called `fist_LVHDRT_exit`
+    """
+
+    fistpointName: str
+
+    def __init__(self, host: Host, name: str):
+        self.fistpointName = self._get_name(name)
+        self.host = host
+
+    @staticmethod
+    def enable_exit_on_fistpoint(host: Host):
+        host.create_file(FistPoint._get_path(LVHDRT_EXIT_FIST), "")
+
+    @staticmethod
+    def disable_exit_on_fistpoint(host: Host):
+        host.ssh(["rm", FistPoint._get_path(LVHDRT_EXIT_FIST)])
+
+    @staticmethod
+    def _get_name(name: str) -> str:
+        if name.startswith("fist_"):
+            return name
+        else:
+            return f"fist_{name}"
+
+    @staticmethod
+    def _get_path(name) -> str:
+        return f"{FISTPOINT_DIR}/{name}"
+
+    def enable(self):
+        logging.info(f"Enable fistpoint {self.fistpointName}")
+        self.host.create_file(self._get_path(self.fistpointName), "")
+
+    def disable(self):
+        logging.info(f"Disabling fistpoint {self.fistpointName}")
+        self.host.ssh(["rm", self._get_path(self.fistpointName)])
+
+    def __enter__(self):
+        self.enable()
+        return self
+
+    def __exit__(self, *_):
+        self.disable()

lib/host.py

@@ -711,3 +711,15 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def lvs(self, vgName: Optional[str] = None, ignore_MGT: bool = True) -> List[str]:
+        ret: List[str] = []
+        cmd = ["lvs", "--noheadings", "-o", "LV_NAME"]
+        if vgName:
+            cmd.append(vgName)
+        output = self.ssh(cmd)
+        for line in output.splitlines():
+            if ignore_MGT and "MGT" in line:
+                continue
+            ret.append(line.strip())
+        return ret

lib/vdi.py

@@ -47,6 +47,13 @@ def clone(self):
     def readonly(self) -> bool:
         return strtobool(self.param_get("read-only"))
 
+    def size(self) -> int:
+        return int(self.param_get("virtual-size"))
+
+    def resize(self, new_size: int) -> None:
+        logging.info(f"Resizing VDI {self.uuid} to {new_size}")
+        self.sr.pool.master.xe("vdi-resize", {"uuid": self.uuid, "disk-size": str(new_size)})
+
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 

tests/storage/ext/test_ext_sr.py

@@ -1,6 +1,11 @@
 import pytest
 
+import logging
+
+from lib.commands import SSHCommandFailed
 from lib.common import vm_image, wait_for
+from lib.fistpoint import FistPoint
+from lib.vdi import VDI
 from tests.storage import try_to_create_sr_with_missing_device, vdi_is_open
 
 # Requirements:
@@ -55,6 +60,44 @@ def test_snapshot(self, vm_on_ext_sr):
 
     # *** tests with reboots (longer tests).
 
+    @pytest.mark.small_vm
+    @pytest.mark.big_vm
+    def test_blktap_activate_failure(self, vm_on_ext_sr):
+        from lib.fistpoint import FistPoint
+        vm = vm_on_ext_sr
+        with FistPoint(vm.host, "blktap_activate_inject_failure"), pytest.raises(SSHCommandFailed):
+            vm.start()
+            vm.shutdown(force=True)
+
+    @pytest.mark.small_vm
+    @pytest.mark.big_vm
+    def test_resize(self, vm_on_ext_sr):
+        vm = vm_on_ext_sr
+        vdi = VDI(vm.vdi_uuids()[0], host=vm.host)
+        old_size = vdi.size()
+        new_size = old_size + (1 * 1024 * 1024 * 1024) # Adding a 1GiB to size
+
+        vdi.resize(new_size)
+
+        assert vdi.size() == new_size
+
+    @pytest.mark.small_vm
+    @pytest.mark.big_vm
+    def test_failing_resize(self, host, ext_sr, vm_on_ext_sr, exit_on_fistpoint):
+        vm = vm_on_ext_sr
+        vdi = VDI(vm.vdi_uuids()[0], host=vm.host)
+        old_size = vdi.size()
+        new_size = old_size + (1 * 1024 * 1024 * 1024) # Adding a 1GiB to size
+
+        with FistPoint(vm.host, "LVHDRT_inflate_after_setSize"):
+            try:
+                vdi.resize(new_size)
+            except Exception:
+                logging.info(f"Launching SR scan for {ext_sr} after failure")
+                host.xe("sr-scan", {"uuid": ext_sr})
+
+        assert vdi.size() == new_size
+
     @pytest.mark.reboot
     @pytest.mark.small_vm
     def test_reboot(self, host, ext_sr, vm_on_ext_sr):

tests/storage/lvm/test_lvm_sr.py

@@ -1,6 +1,11 @@
 import pytest
 
+import logging
+
+from lib.commands import SSHCommandFailed
 from lib.common import vm_image, wait_for
+from lib.fistpoint import FistPoint
+from lib.vdi import VDI
 from tests.storage import try_to_create_sr_with_missing_device, vdi_is_open
 
 # Requirements:
@@ -53,6 +58,68 @@ def test_snapshot(self, vm_on_lvm_sr):
         finally:
             vm.shutdown(verify=True)
 
+    @pytest.mark.small_vm
+    @pytest.mark.big_vm
+    def test_failing_resize_on_inflate_after_setSize(self, host, lvm_sr, vm_on_lvm_sr, exit_on_fistpoint):
+        vm = vm_on_lvm_sr
+        lvinflate = ""
+        need_rescan = False
+        vdi = VDI(vm.vdi_uuids()[0], host=vm.host)
+        new_size = vdi.size() + (1 * 1024 * 1024 * 1024) # Adding a 1GiB to size
+
+        with FistPoint(vm.host, "LVHDRT_inflate_after_setSize"):
+            try:
+                vdi.resize(new_size)
+            except SSHCommandFailed as e:
+                logging.info(e)
+                need_rescan = True
+
+        assert need_rescan, "Resize did not make an error"
+        lvlist = host.lvs(f"VG_XenStorage-{lvm_sr.uuid}")
+        for lv in lvlist:
+            if lv.startswith("inflate_"):
+                logging.info(f"Found inflate journal following error: {lv}")
+                lvinflate = lv
+
+        logging.info(f"Launching SR scan for {lvm_sr.uuid} to resolve failure")
+        try:
+            host.xe("sr-scan", {"uuid": lvm_sr.uuid})
+        except SSHCommandFailed as e:
+            assert False, f"Failing to scan following a inflate error {e}"
+        assert lvinflate not in host.lvs(f"VG_XenStorage-{lvm_sr.uuid}"), \
+            "Inflate journal still exist following the scan"
+
+    @pytest.mark.small_vm
+    @pytest.mark.big_vm
+    def test_failing_resize_on_inflate_after_setSizePhys(self, host, lvm_sr, vm_on_lvm_sr, exit_on_fistpoint):
+        vm = vm_on_lvm_sr
+        lvinflate = ""
+        need_rescan = False
+        vdi = VDI(vm.vdi_uuids()[0], host=vm.host)
+        new_size = vdi.size() + (1 * 1024 * 1024 * 1024) # Adding a 1GiB to size
+
+        with FistPoint(vm.host, "LVHDRT_inflate_after_setSizePhys"):
+            try:
+                vdi.resize(new_size)
+            except SSHCommandFailed as e:
+                logging.info(e)
+                need_rescan = True
+
+        assert need_rescan, "Resize did not make an error"
+        lvlist = host.lvs(f"VG_XenStorage-{lvm_sr.uuid}")
+        for lv in lvlist:
+            if lv.startswith("inflate_"):
+                logging.info(f"Found inflate journal following error: {lv}")
+                lvinflate = lv
+
+        logging.info(f"Launching SR scan for {lvm_sr.uuid} to resolve failure")
+        try:
+            host.xe("sr-scan", {"uuid": lvm_sr.uuid})
+        except SSHCommandFailed as e:
+            assert False, f"Failing to scan following a inflate error {e}"
+        assert lvinflate not in host.lvs(f"VG_XenStorage-{lvm_sr.uuid}"), \
+            "Inflate journal still exist following the scan"
+
     # *** tests with reboots (longer tests).
 
     @pytest.mark.reboot