Fix T2 upgrade path logic (#21222)

What is the motivation for this PR?
Logic error in upgrade_path test for T2 where DUTs were upgrading/downgrading seperately (see details in issue)

How did you do it?
Make whole chassis boot into base image at once at start of test, then upgrade in order.

How did you verify/test it?
Ran regression test on T1 and T2

Author: Javier-Tan

tests/upgrade_path/test_upgrade_path.py

@@ -3,7 +3,7 @@
 from tests.common.helpers.upgrade_helpers import install_sonic, upgrade_test_helper
 from tests.common.helpers.upgrade_helpers import restore_image            # noqa: F401
 from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
-from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image
+from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image, boot_into_base_image_t2
 from tests.common.fixtures.advanced_reboot import get_advanced_reboot   # noqa: F401
 from tests.common.fixtures.consistency_checker.consistency_checker import consistency_checker_provider  # noqa: F401
 from tests.common.platform.device_utils import verify_dut_health, verify_testbed_health    # noqa: F401
@@ -49,9 +49,10 @@ def upgrade_path_lists(request):
 def setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo,
                        upgrade_type, modify_reboot_script=None, allow_fail=False):
     logger.info("Test upgrade path from {} to {}".format(from_image, to_image))
-    cleanup_prev_images(duthost)
-    # Install base image
-    boot_into_base_image(duthost, localhost, from_image, tbinfo)
+    if tbinfo['topo']['type'] != 't2':  # We do this all at once seperately for T2
+        cleanup_prev_images(duthost)
+        # Install base image
+        boot_into_base_image(duthost, localhost, from_image, tbinfo)
 
     # Install target image
     logger.info("Upgrading to {}".format(to_image))
@@ -113,6 +114,10 @@ def test_upgrade_path_t2(localhost, duthosts, ptfhost, upgrade_path_lists,
     upgrade_type = REBOOT_TYPE_COLD
     logger.info("Test upgrade path from {} to {}".format(from_image, to_image))
 
+    for duthost in duthosts:
+        cleanup_prev_images(duthost)
+    boot_into_base_image_t2(duthosts, localhost, from_image, tbinfo)
+
     def upgrade_path_preboot_setup(dut):
         setup_upgrade_test(dut, localhost, from_image, to_image, tbinfo, upgrade_type)
 

tests/upgrade_path/utilities.py

@@ -1,12 +1,38 @@
 import logging
 import re
 from tests.common.errors import RunAnsibleModuleFail
+from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
 from tests.common.helpers.upgrade_helpers import install_sonic, reboot, check_sonic_version
 
 logger = logging.getLogger(__name__)
 
 
 def boot_into_base_image(duthost, localhost, base_image, tbinfo):
+    target_version = _install_base_image(duthost, base_image, tbinfo)
+    # Perform a cold reboot
+    logger.info("Cold reboot the DUT to make the base image as current")
+    # for 6100 devices, sometimes cold downgrade will not work, use soft-reboot here
+    reboot_type = 'soft' if "s6100" in duthost.facts["platform"] else 'cold'
+    reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
+    check_sonic_version(duthost, target_version)
+
+
+def boot_into_base_image_t2(duthosts, localhost, base_image, tbinfo):
+    target_vers = {}
+    with SafeThreadPoolExecutor(max_workers=8) as executor:
+        for duthost in duthosts:
+            future = executor.submit(_install_base_image, duthost, base_image, tbinfo)
+            target_vers[duthost] = future.get()  # Should all be the same, but following best practice
+
+    # Rebooting the supervisor host will reboot all T2 DUTs
+    suphost = duthosts.supervisor_nodes[0]
+    reboot(suphost, localhost, reboot_type='cold', safe_reboot=True)
+
+    for duthost in duthosts:
+        check_sonic_version(duthost, target_vers[duthost])
+
+
+def _install_base_image(duthost, base_image, tbinfo):
     logger.info("Installing {}".format(base_image))
     try:
         target_version = install_sonic(duthost, base_image, tbinfo)
@@ -26,12 +52,8 @@ def boot_into_base_image(duthost, localhost, base_image, tbinfo):
     if duthost.shell("ls /host/old_config/minigraph.xml", module_ignore_errors=True)['rc'] == 0:
         duthost.shell("rm -f /host/old_config/config_db.json")
         duthost.shell("rm -f /host/old_config/golden_config_db.json")
-    # Perform a cold reboot
-    logger.info("Cold reboot the DUT to make the base image as current")
-    # for 6100 devices, sometimes cold downgrade will not work, use soft-reboot here
-    reboot_type = 'soft' if "s6100" in duthost.facts["platform"] else 'cold'
-    reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
-    check_sonic_version(duthost, target_version)
+
+    return target_version
 
 
 def cleanup_prev_images(duthost):