Merge branch 'sonic-net:master' into pfc_testcase_fixes

Author: vikumarks

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -2506,7 +2506,7 @@ generic_config_updater/test_ecn_config_update.py::test_ecn_config_updates:
     reason: "This test is not run on this asic type, topology, or version currently"
     conditions_logical_operator: "OR"
     conditions:
-      - "asic_type in ['cisco-8000']"
+      - "hwsku in ['Cisco-8800-LC-48H-C48']"
       - "topo_type in ['m0', 'mx', 'm1']"
       - "release in ['202211']"
 

tests/generic_config_updater/test_ecn_config_update.py

@@ -60,15 +60,16 @@ def get_asic_db_values(duthost, fields, cli_namespace_prefix):
     asic_db_values = {}
     for wred_object in wred_objects:
         oid = wred_object[wred_object.rfind(':') + 1:]
-        asic_db_values[oid] = {}
         wred_data = duthost.shell('sonic-db-cli {} ASIC_DB hgetall {}'
                                   .format(cli_namespace_prefix, wred_object))["stdout"]
         if "NULL" in wred_data:
             continue
         wred_data = ast.literal_eval(wred_data)
+        values = {}
         for field in fields:
-            value = int(wred_data[WRED_MAPPING[field]])
-            asic_db_values[oid][field] = value
+            values[field] = int(wred_data[WRED_MAPPING[field]])
+        if values:
+            asic_db_values[oid] = values
     return asic_db_values
 
 
@@ -137,8 +138,7 @@ def validate_wred_objects_in_asic_db():
 def get_wred_profiles(duthost, cli_namespace_prefix):
     wred_profiles = duthost.shell(f"sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys \
                                   'WRED_PROFILE|*' | cut -d '|' -f 2")["stdout"]
-    wred_profiles = wred_profiles.split('\n')
-    return wred_profiles
+    return [w for w in wred_profiles.split('\n') if w]
 
 
 @pytest.mark.parametrize("configdb_field", ["green_min_threshold", "green_max_threshold", "green_drop_probability",
@@ -168,12 +168,14 @@ def test_ecn_config_updates(duthost, ensure_dut_readiness, configdb_field, opera
         for field in fields:
             value = int(ecn_data[field])
             if "probability" in field:
-                if 0 <= value <= 99:
+                if 0 <= value < 99:
                     value += 1
-                elif value == 100:
+                elif value >= 99:
                     value -= 1
                 else:
                     raise ValueError("Invalid probability value: {}".format(value))
+            elif "min" in field:
+                value -= 1
             else:
                 value += 1
             new_values[wred_profile][field] = value

tests/generic_config_updater/test_pfcwd_status.py

@@ -153,7 +153,11 @@ def extract_pfcwd_config(duthost, start_pfcwd):
         pfcwd_config: dict of dicts with interface as the 1st level key and 'action', 'detect_time',
                       'restore_time' as the 2nd level keys
     """
-    output = duthost.command('show pfcwd config')
+    # Add flag to show all internal interfaces (for multi-asic systems)
+    cmd = 'show pfcwd config'
+    if duthost.is_multi_asic:
+        cmd += ' -d all'
+    output = duthost.command(cmd)
     pytest_assert('Ethernet' in output['stdout'], 'No ports found in the pfcwd config')
 
     pfcwd_config = defaultdict()

tests/gnmi/grpc_utils.py

@@ -1,5 +1,9 @@
 import sys
 import os
+import grpc
+import logging
+
+from tests.common.helpers.gnmi_utils import GNMIEnvironment
 
 
 def get_gnoi_system_stubs():
@@ -16,3 +20,52 @@ def get_gnoi_system_stubs():
     sys.path.append(os.path.abspath(PROTO_ROOT))
     from gnoi.system import system_pb2_grpc, system_pb2
     return system_pb2_grpc, system_pb2
+
+
+def create_grpc_channel(duthost):
+    """
+    Create a gRPC channel with secure credentials.
+
+    This function replaces the grpc_channel fixture to avoid SSL state sharing
+    issues between forked processes during shell operations.
+
+    Args:
+        duthost: DUT host object
+
+    Returns:
+        grpc.Channel: Configured gRPC channel
+    """
+    # Get DUT gRPC server address and port
+    ip = duthost.mgmt_ip
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    port = env.gnmi_port
+    target = f"{ip}:{port}"
+
+    # Load the TLS certificates
+    with open("gnmiCA.pem", "rb") as f:
+        root_certificates = f.read()
+    with open("gnmiclient.crt", "rb") as f:
+        client_certificate = f.read()
+    with open("gnmiclient.key", "rb") as f:
+        client_key = f.read()
+
+    # Create SSL credentials
+    credentials = grpc.ssl_channel_credentials(
+        root_certificates=root_certificates,
+        private_key=client_key,
+        certificate_chain=client_certificate,
+    )
+
+    # Create gRPC channel
+    logging.info("Creating gRPC secure channel to %s", target)
+    channel = grpc.secure_channel(target, credentials)
+
+    try:
+        grpc.channel_ready_future(channel).result(timeout=10)
+        logging.info("gRPC channel is ready")
+    except grpc.FutureTimeoutError as e:
+        logging.error("Error: gRPC channel not ready: %s", e)
+        channel.close()
+        raise Exception("Failed to connect to gRPC server") from e
+
+    return channel

tests/gnmi/test_gnoi_system_grpc.py

@@ -1,8 +1,7 @@
 import pytest
 import logging
-import time
 
-from tests.gnmi.grpc_utils import get_gnoi_system_stubs
+from tests.gnmi.grpc_utils import get_gnoi_system_stubs, create_grpc_channel
 
 pytestmark = [
     pytest.mark.topology("any"),
@@ -18,24 +17,40 @@
 system_pb2_grpc, system_pb2 = get_gnoi_system_stubs()
 
 
-def test_gnoi_system_time(grpc_channel):
+def test_gnoi_system_time(duthosts, rand_one_dut_hostname):
     """
-    Verify the gNOI System Time API returns the current system time in valid JSON format.
+    Verify the gNOI System Time API returns the current system time.
     """
-    # Use the shared gRPC channel
-    stub = system_pb2_grpc.SystemStub(grpc_channel)
-
-    # Create and send request
-    request = system_pb2.TimeRequest()
-    response = stub.Time(request)
-
-    # Log the response
-    logging.info("Received response: %s", response)
-
-    # Assert the time falls into a reasonable interval
-    current_time_ns = int(time.time() * 1e9)
-    reasonable_interval_ns = 60 * 1e9  # 60 seconds in nanoseconds
-
-    assert (
-        abs(response.time - current_time_ns) < reasonable_interval_ns
-    ), f"System time {response.time} is not within the reasonable interval of current time {current_time_ns}"
+    duthost = duthosts[rand_one_dut_hostname]
+
+    # Get device time in seconds first (before gRPC operations)
+    device_time_result = duthost.shell("date +%s", module_ignore_errors=True)
+    device_time_s = int(device_time_result["stdout"].strip())
+    device_time_ns = device_time_s * int(1e9)
+
+    # Create gRPC channel (no longer a fixture to avoid SSL state sharing)
+    channel = create_grpc_channel(duthost)
+
+    try:
+        # Create gRPC stub
+        stub = system_pb2_grpc.SystemStub(channel)
+
+        # Create and send request
+        request = system_pb2.TimeRequest()
+        response = stub.Time(request)
+
+        # Log the response
+        logging.info("Received response: %s", response)
+        logging.info("Device time from shell: %d", device_time_ns)
+
+        # Assert the gNOI time is close to device shell time
+        reasonable_interval_ns = 60 * 1e9  # 60 seconds in nanoseconds
+
+        time_diff = abs(response.time - device_time_ns)
+        assert time_diff < reasonable_interval_ns, (
+            f"gNOI time {response.time} differs from device time "
+            f"{device_time_ns} by {time_diff}ns (max: {reasonable_interval_ns}ns)"
+        )
+    finally:
+        # Always close the channel to avoid resource leaks
+        channel.close()

tests/upgrade_path/test_upgrade_path.py

@@ -3,7 +3,7 @@
 from tests.common.helpers.upgrade_helpers import install_sonic, upgrade_test_helper
 from tests.common.helpers.upgrade_helpers import restore_image            # noqa: F401
 from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
-from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image
+from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image, boot_into_base_image_t2
 from tests.common.fixtures.advanced_reboot import get_advanced_reboot   # noqa: F401
 from tests.common.fixtures.consistency_checker.consistency_checker import consistency_checker_provider  # noqa: F401
 from tests.common.platform.device_utils import verify_dut_health, verify_testbed_health    # noqa: F401
@@ -49,9 +49,10 @@ def upgrade_path_lists(request):
 def setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo,
                        upgrade_type, modify_reboot_script=None, allow_fail=False):
     logger.info("Test upgrade path from {} to {}".format(from_image, to_image))
-    cleanup_prev_images(duthost)
-    # Install base image
-    boot_into_base_image(duthost, localhost, from_image, tbinfo)
+    if tbinfo['topo']['type'] != 't2':  # We do this all at once seperately for T2
+        cleanup_prev_images(duthost)
+        # Install base image
+        boot_into_base_image(duthost, localhost, from_image, tbinfo)
 
     # Install target image
     logger.info("Upgrading to {}".format(to_image))
@@ -113,6 +114,10 @@ def test_upgrade_path_t2(localhost, duthosts, ptfhost, upgrade_path_lists,
     upgrade_type = REBOOT_TYPE_COLD
     logger.info("Test upgrade path from {} to {}".format(from_image, to_image))
 
+    for duthost in duthosts:
+        cleanup_prev_images(duthost)
+    boot_into_base_image_t2(duthosts, localhost, from_image, tbinfo)
+
     def upgrade_path_preboot_setup(dut):
         setup_upgrade_test(dut, localhost, from_image, to_image, tbinfo, upgrade_type)
 

tests/upgrade_path/utilities.py

@@ -1,12 +1,38 @@
 import logging
 import re
 from tests.common.errors import RunAnsibleModuleFail
+from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
 from tests.common.helpers.upgrade_helpers import install_sonic, reboot, check_sonic_version
 
 logger = logging.getLogger(__name__)
 
 
 def boot_into_base_image(duthost, localhost, base_image, tbinfo):
+    target_version = _install_base_image(duthost, base_image, tbinfo)
+    # Perform a cold reboot
+    logger.info("Cold reboot the DUT to make the base image as current")
+    # for 6100 devices, sometimes cold downgrade will not work, use soft-reboot here
+    reboot_type = 'soft' if "s6100" in duthost.facts["platform"] else 'cold'
+    reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
+    check_sonic_version(duthost, target_version)
+
+
+def boot_into_base_image_t2(duthosts, localhost, base_image, tbinfo):
+    target_vers = {}
+    with SafeThreadPoolExecutor(max_workers=8) as executor:
+        for duthost in duthosts:
+            future = executor.submit(_install_base_image, duthost, base_image, tbinfo)
+            target_vers[duthost] = future.get()  # Should all be the same, but following best practice
+
+    # Rebooting the supervisor host will reboot all T2 DUTs
+    suphost = duthosts.supervisor_nodes[0]
+    reboot(suphost, localhost, reboot_type='cold', safe_reboot=True)
+
+    for duthost in duthosts:
+        check_sonic_version(duthost, target_vers[duthost])
+
+
+def _install_base_image(duthost, base_image, tbinfo):
     logger.info("Installing {}".format(base_image))
     try:
         target_version = install_sonic(duthost, base_image, tbinfo)
@@ -26,12 +52,8 @@ def boot_into_base_image(duthost, localhost, base_image, tbinfo):
     if duthost.shell("ls /host/old_config/minigraph.xml", module_ignore_errors=True)['rc'] == 0:
         duthost.shell("rm -f /host/old_config/config_db.json")
         duthost.shell("rm -f /host/old_config/golden_config_db.json")
-    # Perform a cold reboot
-    logger.info("Cold reboot the DUT to make the base image as current")
-    # for 6100 devices, sometimes cold downgrade will not work, use soft-reboot here
-    reboot_type = 'soft' if "s6100" in duthost.facts["platform"] else 'cold'
-    reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
-    check_sonic_version(duthost, target_version)
+
+    return target_version
 
 
 def cleanup_prev_images(duthost):