Add support for netinstall

Add a new parameter to allow booting using netinstall. If an ISO only
supports netinstall, it is specified it in the data.py by setting the
`net-only`option to True. This option is set to False by default.

Signed-off-by: Guillaume <[email protected]>
Signed-off-by: Yann Dirson <[email protected]>

Author: gthvn1

data.py-dist

@@ -121,15 +121,33 @@ ISO_IMAGES_CACHE = "/home/user/iso"
 # - absolute URL
 # - path relative to ISO_IMAGES_BASE URL
 # Note the dirname part is ignored when looking in ISO_IMAGES_CACHE, abuse this
-# for local-only ISO with things like "locally-built/my.iso" or "xs/8.3.iso"
+# for local-only ISO with things like "locally-built/my.iso" or "xs/8.3.iso".
+# If 'net-only' is set to 'True' only source of type URL will be possible.
+# By default the parameter is set to False.
 ISO_IMAGES: Dict[str, "IsoImageDef"] = {
     '83nightly': {'path': os.environ.get("XCPNG83_NIGHTLY",
                                          "http://unconfigured.iso"),
                   'unsigned': True},
-    '83rc1': {'path': "8.3/xcp-ng-8.3.0-rc1.iso"},
-    '83b2': {'path': "8.3/xcp-ng-8.3.0-beta2.iso"},
-    '83b1': {'path': "8.3/xcp-ng-8.3.0-beta1.iso"},
-    '821.1': {'path': "8.2/xcp-ng-8.2.1-20231130.iso"},
+    # FIXME: no such symlimk + useless without 'net-url'
+    #'83nightlynet': {'path': "http://pxe/isos/xcp-ng-8.3-ci-netinstall-latest"},
+    #                 'net-url': 'fake",
+    #                 'net-only': True},
+    '83rc1': {'path': "8.3/xcp-ng-8.3.0-rc1.iso",
+              #'net-url': "http://server/installers/xcp-ng/8.3-rc1",
+              },
+    ## FIXME: only a compensation for the lack of 83nightlynet
+    #'83rcnet': {'path': "8.3/xcp-ng-8.3.0-rc1-netinstall.iso",
+    #            'net-url': "http://server/installers/xcp-ng/8.3-rc1",
+    #            'net-only': True},
+    '83b2': {'path': "8.3/xcp-ng-8.3.0-beta2.iso",
+             #'net-url': "http://server/installers/xcp-ng/8.3-beta2",
+             },
+    '83b1': {'path': "8.3/xcp-ng-8.3.0-beta1.iso",
+             #'net-url': "http://server/installers/xcp-ng/8.3-beta1",
+             },
+    '821.1': {'path': "8.2/xcp-ng-8.2.1-20231130.iso",
+              #'net-url': f"http://{PXE_CONFIG_SERVER}/installers/xcp-ng/8.2.1-refreshed/",
+              },
     '821': {'path': "8.2/xcp-ng-8.2.1.iso"},
     '820': {'path': "8.2/xcp-ng-8.2.0.iso"},
     '81': {'path': "8.1/xcp-ng-8.1.0-2.iso"},

lib/typing.py

@@ -3,5 +3,7 @@
 
 IsoImageDef = TypedDict('IsoImageDef',
                         {'path': str,
+                         'net-url': NotRequired[str],
+                         'net-only': NotRequired[bool],
                          'unsigned': NotRequired[bool],
                          })

pytest.ini

@@ -24,6 +24,7 @@ markers =
 
     # * installation-related markers to customize installer run
     answerfile: dict defining an answerfile
+    package_source: source of packages during installation.
 
     # * Test targets related to VMs
     small_vm: tests that it is enough to run just once, using the smallest possible VM.

tests/install/conftest.py

@@ -13,6 +13,30 @@
 from data import (ISO_IMAGES, ISO_IMAGES_BASE, ISO_IMAGES_CACHE,
                   PXE_CONFIG_SERVER, TEST_SSH_PUBKEY, TOOLS)
 
+# Return true if the version of the ISO doesn't support the source type.
+# Note: this is a quick-win hack, to avoid explicit enumeration of supported
+# package_source values for each ISO.
+def skip_package_source(version, package_source):
+    if version not in ISO_IMAGES:
+        return True, "version of ISO {} is unknown".format(version)
+
+    if package_source == "iso":
+        if ISO_IMAGES[version].get('net-only', False):
+            return True, "ISO image is net-only while package_source is local"
+
+        return False, "do not skip"
+
+    if package_source == "net":
+        # Net install is not valid if there is no netinstall URL
+        # FIXME: ISO includes a default URL so we should be able to omit net-url
+        if 'net-url' not in ISO_IMAGES[version].keys():
+            return True, "net-url required for netinstall was not found for {}".format(version)
+
+        return False, "do not skip"
+
+    # If we don't know the source type then it is invalid
+    return True, "unknown source type {}".format(package_source)
+
 @pytest.fixture(scope='function')
 def answerfile(request):
     """
@@ -57,6 +81,10 @@ def answerfile(request):
 @pytest.fixture(scope='function')
 def installer_iso(request):
     iso_key = request.getfixturevalue("iso_version")
+    package_source = request.getfixturevalue("package_source")
+    skip, reason = skip_package_source(iso_key, package_source)
+    if skip:
+        pytest.skip(reason)
     assert iso_key in ISO_IMAGES, f"ISO_IMAGES does not have a value for {iso_key}"
     iso = ISO_IMAGES[iso_key]['path']
     if iso.startswith("/"):

tests/install/test-sequences/inst+upg+rst.lst

@@ -1,2 +1,2 @@
-tests/install/test.py::TestNested::test_restore[uefi-83nightly-83nightly-83nightly-ext]
-tests/install/test.py::TestNested::test_boot_rst[uefi-83nightly-83nightly-83nightly-ext]
+tests/install/test.py::TestNested::test_restore[uefi-83nightly-83nightly-83nightly-iso-ext]
+tests/install/test.py::TestNested::test_boot_rst[uefi-83nightly-83nightly-83nightly-iso-ext]

tests/install/test-sequences/inst+upg.lst

@@ -1,2 +1,2 @@
-tests/install/test.py::TestNested::test_upgrade[uefi-83nightly-83nightly-host1-ext]
-tests/install/test.py::TestNested::test_boot_upg[uefi-83nightly-83nightly-host1-ext]
+tests/install/test.py::TestNested::test_upgrade[uefi-83nightly-83nightly-host1-iso-ext]
+tests/install/test.py::TestNested::test_boot_upg[uefi-83nightly-83nightly-host1-iso-ext]

tests/install/test-sequences/inst.lst

@@ -1,3 +1,3 @@
-tests/install/test.py::TestNested::test_install[uefi-83nightly-ext]
-tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-83nightly-host1-ext]
-tests/install/test.py::TestNested::test_boot_inst[uefi-83nightly-host1-ext]
+tests/install/test.py::TestNested::test_install[uefi-83nightly-iso-ext]
+tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-83nightly-host1-iso-ext]
+tests/install/test.py::TestNested::test_boot_inst[uefi-83nightly-host1-iso-ext]

tests/install/test.py

@@ -9,7 +9,7 @@
 from lib.pool import Pool
 from lib.vdi import VDI
 
-from data import NETWORKS
+from data import ISO_IMAGES, NETWORKS
 assert "MGMT" in NETWORKS
 
 # Requirements:
@@ -41,8 +41,9 @@ def helper_vm_with_plugged_disk(running_vm, create_vms):
 @pytest.mark.dependency()
 class TestNested:
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("iso_version", (
-        "83nightly",
+        "83nightly", "83rcnet",
         "83rc1", "83b2", "83b1",
         "821.1",
         "81", "80", "76", "75",
@@ -75,24 +76,28 @@ class TestNested:
             vifs=[dict(index=0, network_name=NETWORKS["MGMT"])],
         ))
     @pytest.mark.answerfile(
-        lambda install_disk, local_sr: AnswerFile("INSTALL")
+        lambda install_disk, local_sr, package_source, iso_version: AnswerFile("INSTALL")
         .top_setattr({} if local_sr == "nosr" else {"sr-type": local_sr})
         .top_append(
-            {"TAG": "source", "type": "local"},
+            {"TAG": "source", "type": "local"} if package_source == "iso"
+            else {"TAG": "source", "type": "url",
+                  "CONTENTS": ISO_IMAGES[iso_version]['net-url']} if package_source == "net"
+            else {},
             {"TAG": "primary-disk",
              "guest-storage": "no" if local_sr == "nosr" else "yes",
              "CONTENTS": install_disk},
         ))
     def test_install(self, vm_booted_with_installer, install_disk,
-                     firmware, iso_version, local_sr):
+                     firmware, iso_version, package_source, local_sr):
         host_vm = vm_booted_with_installer
         installer.monitor_install(ip=host_vm.ip)
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("machine", ("host1", "host2"))
     @pytest.mark.parametrize("version", (
-        "83nightly",
+        "83nightly", "83rcnet",
         "83rc1", "83b2", "83b1",
         "821.1",
         "81", "80",
@@ -102,12 +107,12 @@ def test_install(self, vm_booted_with_installer, install_disk,
     ))
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda version, firmware, local_sr: [dict(
+        lambda version, firmware, local_sr, package_source: [dict(
             vm="vm1",
-            image_test=f"TestNested::test_install[{firmware}-{version}-{local_sr}]")])
+            image_test=f"TestNested::test_install[{firmware}-{version}-{package_source}-{local_sr}]")])
     @pytest.mark.small_vm
     def test_tune_firstboot(self, create_vms, helper_vm_with_plugged_disk,
-                            firmware, version, machine, local_sr):
+                            firmware, version, machine, local_sr, package_source):
         helper_vm = helper_vm_with_plugged_disk
 
         helper_vm.ssh(["mount /dev/xvdb1 /mnt"])
@@ -151,6 +156,7 @@ def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT', is_restore=Fal
             "83b1": "8.3.0",
             "83b2": "8.3.0",
             "83rc1": "8.3.0",
+            "83rcnet": "8.3.0",
             "83nightly": "8.3.0",
         }[expected_rel_id]
 
@@ -278,9 +284,10 @@ def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT', is_restore=Fal
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("machine", ("host1", "host2"))
     @pytest.mark.parametrize("version", (
-        "83nightly",
+        "83nightly", "83rcnet",
         "83rc1", "83b2", "83b1",
         "821.1",
         "81", "80",
@@ -290,15 +297,17 @@ def _test_firstboot(self, create_vms, mode, *, machine='DEFAULT', is_restore=Fal
     ))
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda firmware, version, machine, local_sr: [
+        lambda firmware, version, machine, local_sr, package_source: [
             dict(vm="vm1",
-                 image_test=f"TestNested::test_tune_firstboot[None-{firmware}-{version}-{machine}-{local_sr}]")])
+                 image_test=("TestNested::test_tune_firstboot"
+                             f"[None-{firmware}-{version}-{machine}-{package_source}-{local_sr}]"))])
     def test_boot_inst(self, create_vms,
-                       firmware, version, machine, local_sr):
+                       firmware, version, machine, package_source, local_sr):
         self._test_firstboot(create_vms, version, machine=machine)
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("machine", ("host1", "host2"))
     @pytest.mark.parametrize("mode", (
         "83nightly-83nightly",
@@ -310,19 +319,21 @@ def test_boot_inst(self, create_vms,
         "80-83nightly",
         "xs8-83nightly",
         "ch821.1-83nightly",
+        "83rcnet-83rcnet",
         "821.1-821.1",
     ))
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda firmware, mode, machine, local_sr: [dict(
+        lambda firmware, mode, machine, package_source, local_sr: [dict(
             vm="vm1",
-            image_test=(f"TestNested::test_upgrade[{firmware}-{mode}-{machine}-{local_sr}]"))])
+            image_test=(f"TestNested::test_upgrade[{firmware}-{mode}-{machine}-{package_source}-{local_sr}]"))])
     def test_boot_upg(self, create_vms,
-                      firmware, mode, machine, local_sr):
+                      firmware, mode, machine, package_source, local_sr):
         self._test_firstboot(create_vms, mode, machine=machine)
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("mode", (
         "83nightly-83nightly-83nightly",
         "83rc1-83nightly-83nightly",
@@ -333,19 +344,21 @@ def test_boot_upg(self, create_vms,
         "80-83nightly-83nightly",
         "xs8-83nightly-83nightly",
         "ch821.1-83nightly-83nightly",
+        "83rcnet-83rcnet-83rcnet", # FIXME
         "821.1-821.1-821.1",
     ))
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda firmware, mode, local_sr: [dict(
+        lambda firmware, mode, package_source, local_sr: [dict(
             vm="vm1",
-            image_test=(f"TestNested::test_restore[{firmware}-{mode}-{local_sr}]"))])
+            image_test=(f"TestNested::test_restore[{firmware}-{mode}-{package_source}-{local_sr}]"))])
     def test_boot_rst(self, create_vms,
-                      firmware, mode, local_sr):
+                      firmware, mode, package_source, local_sr):
         self._test_firstboot(create_vms, mode, is_restore=True)
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize("machine", ("host1", "host2"))
     @pytest.mark.parametrize(("orig_version", "iso_version"), [
         ("83nightly", "83nightly"),
@@ -357,26 +370,31 @@ def test_boot_rst(self, create_vms,
         ("80", "83nightly"),
         ("xs8", "83nightly"),
         ("ch821.1", "83nightly"),
+        ("83rcnet", "83rcnet"), # FIXME
         ("821.1", "821.1"),
     ])
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda firmware, orig_version, machine, local_sr: [dict(
+        lambda firmware, orig_version, machine, package_source, local_sr: [dict(
             vm="vm1",
-            image_test=f"TestNested::test_boot_inst[{firmware}-{orig_version}-{machine}-{local_sr}]")])
+            image_test=f"TestNested::test_boot_inst[{firmware}-{orig_version}-{machine}-{package_source}-{local_sr}]")])
     @pytest.mark.answerfile(
-        lambda install_disk: AnswerFile("UPGRADE").top_append(
-            {"TAG": "source", "type": "local"},
+        lambda install_disk, package_source, iso_version: AnswerFile("UPGRADE").top_append(
+            {"TAG": "source", "type": "local"} if package_source == "iso"
+            else {"TAG": "source", "type": "url",
+                  "CONTENTS": ISO_IMAGES[iso_version]['net-url']} if package_source == "net"
+            else {},
             {"TAG": "existing-installation",
              "CONTENTS": install_disk},
         ))
     def test_upgrade(self, vm_booted_with_installer, install_disk,
-                     firmware, orig_version, iso_version, machine, local_sr):
+                     firmware, orig_version, iso_version, machine, package_source, local_sr):
         host_vm = vm_booted_with_installer
         installer.monitor_upgrade(ip=host_vm.ip)
 
     @pytest.mark.usefixtures("xcpng_chained")
     @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm"))
+    @pytest.mark.parametrize("package_source", ("iso", "net"))
     @pytest.mark.parametrize(("orig_version", "iso_version"), [
         ("83nightly-83nightly", "83nightly"),
         ("83rc1-83nightly", "83nightly"),
@@ -387,19 +405,20 @@ def test_upgrade(self, vm_booted_with_installer, install_disk,
         ("80-83nightly", "83nightly"),
         ("xs8-83nightly", "83nightly"),
         ("ch821.1-83nightly", "83nightly"),
+        ("83rcnet-83rcnet", "83rcnet"), # FIXME
         ("821.1-821.1", "821.1"),
     ])
     @pytest.mark.parametrize("firmware", ("uefi", "bios"))
     @pytest.mark.continuation_of(
-        lambda firmware, orig_version, local_sr: [dict(
+        lambda firmware, orig_version, local_sr, package_source: [dict(
             vm="vm1",
-            image_test=f"TestNested::test_boot_upg[{firmware}-{orig_version}-host1-{local_sr}]")])
+            image_test=f"TestNested::test_boot_upg[{firmware}-{orig_version}-host1-{package_source}-{local_sr}]")])
     @pytest.mark.answerfile(
         lambda install_disk: AnswerFile("RESTORE").top_append(
             {"TAG": "backup-disk",
              "CONTENTS": install_disk},
         ))
     def test_restore(self, vm_booted_with_installer, install_disk,
-                     firmware, orig_version, iso_version, local_sr):
+                     firmware, orig_version, iso_version, package_source, local_sr):
         host_vm = vm_booted_with_installer
         installer.monitor_restore(ip=host_vm.ip)