Fix gnoi system time test to compare against device time (#21303)

* Fix gnoi system time test to compare against device time

The test was comparing gNOI System Time response with local machine time,
which can fail due to NTP errors or clock drift. Now compares against
the device's own time using shell command.

Use device time from shell command instead of local machine time
for more accurate gNOI System Time API validation.

Replace grpc_channel fixture with create_grpc_channel() function to eliminate
SSL state sharing between forked processes. The fixture maintained persistent
SSL connections that caused segmentation faults when duthost.shell() operations
forked processes that inherited SSL file descriptors.

Changes:
- Add create_grpc_channel() function in grpc_utils.py
- Remove grpc_channel fixture dependency from test
- Add explicit channel cleanup in try/finally block
- Maintain original test functionality while fixing stability

This resolves segfaults in test_gnoi_system_time when combining gRPC operations
with Ansible shell commands due to SSL state corruption in multiprocessing.

Author: hdwhdw

tests/gnmi/grpc_utils.py

@@ -1,5 +1,9 @@
 import sys
 import os
+import grpc
+import logging
+
+from tests.common.helpers.gnmi_utils import GNMIEnvironment
 
 
 def get_gnoi_system_stubs():
@@ -16,3 +20,52 @@ def get_gnoi_system_stubs():
     sys.path.append(os.path.abspath(PROTO_ROOT))
     from gnoi.system import system_pb2_grpc, system_pb2
     return system_pb2_grpc, system_pb2
+
+
+def create_grpc_channel(duthost):
+    """
+    Create a gRPC channel with secure credentials.
+
+    This function replaces the grpc_channel fixture to avoid SSL state sharing
+    issues between forked processes during shell operations.
+
+    Args:
+        duthost: DUT host object
+
+    Returns:
+        grpc.Channel: Configured gRPC channel
+    """
+    # Get DUT gRPC server address and port
+    ip = duthost.mgmt_ip
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    port = env.gnmi_port
+    target = f"{ip}:{port}"
+
+    # Load the TLS certificates
+    with open("gnmiCA.pem", "rb") as f:
+        root_certificates = f.read()
+    with open("gnmiclient.crt", "rb") as f:
+        client_certificate = f.read()
+    with open("gnmiclient.key", "rb") as f:
+        client_key = f.read()
+
+    # Create SSL credentials
+    credentials = grpc.ssl_channel_credentials(
+        root_certificates=root_certificates,
+        private_key=client_key,
+        certificate_chain=client_certificate,
+    )
+
+    # Create gRPC channel
+    logging.info("Creating gRPC secure channel to %s", target)
+    channel = grpc.secure_channel(target, credentials)
+
+    try:
+        grpc.channel_ready_future(channel).result(timeout=10)
+        logging.info("gRPC channel is ready")
+    except grpc.FutureTimeoutError as e:
+        logging.error("Error: gRPC channel not ready: %s", e)
+        channel.close()
+        raise Exception("Failed to connect to gRPC server") from e
+
+    return channel

tests/gnmi/test_gnoi_system_grpc.py

@@ -1,8 +1,7 @@
 import pytest
 import logging
-import time
 
-from tests.gnmi.grpc_utils import get_gnoi_system_stubs
+from tests.gnmi.grpc_utils import get_gnoi_system_stubs, create_grpc_channel
 
 pytestmark = [
     pytest.mark.topology("any"),
@@ -18,24 +17,40 @@
 system_pb2_grpc, system_pb2 = get_gnoi_system_stubs()
 
 
-def test_gnoi_system_time(grpc_channel):
+def test_gnoi_system_time(duthosts, rand_one_dut_hostname):
     """
-    Verify the gNOI System Time API returns the current system time in valid JSON format.
+    Verify the gNOI System Time API returns the current system time.
     """
-    # Use the shared gRPC channel
-    stub = system_pb2_grpc.SystemStub(grpc_channel)
-
-    # Create and send request
-    request = system_pb2.TimeRequest()
-    response = stub.Time(request)
-
-    # Log the response
-    logging.info("Received response: %s", response)
-
-    # Assert the time falls into a reasonable interval
-    current_time_ns = int(time.time() * 1e9)
-    reasonable_interval_ns = 60 * 1e9  # 60 seconds in nanoseconds
-
-    assert (
-        abs(response.time - current_time_ns) < reasonable_interval_ns
-    ), f"System time {response.time} is not within the reasonable interval of current time {current_time_ns}"
+    duthost = duthosts[rand_one_dut_hostname]
+
+    # Get device time in seconds first (before gRPC operations)
+    device_time_result = duthost.shell("date +%s", module_ignore_errors=True)
+    device_time_s = int(device_time_result["stdout"].strip())
+    device_time_ns = device_time_s * int(1e9)
+
+    # Create gRPC channel (no longer a fixture to avoid SSL state sharing)
+    channel = create_grpc_channel(duthost)
+
+    try:
+        # Create gRPC stub
+        stub = system_pb2_grpc.SystemStub(channel)
+
+        # Create and send request
+        request = system_pb2.TimeRequest()
+        response = stub.Time(request)
+
+        # Log the response
+        logging.info("Received response: %s", response)
+        logging.info("Device time from shell: %d", device_time_ns)
+
+        # Assert the gNOI time is close to device shell time
+        reasonable_interval_ns = 60 * 1e9  # 60 seconds in nanoseconds
+
+        time_diff = abs(response.time - device_time_ns)
+        assert time_diff < reasonable_interval_ns, (
+            f"gNOI time {response.time} differs from device time "
+            f"{device_time_ns} by {time_diff}ns (max: {reasonable_interval_ns}ns)"
+        )
+    finally:
+        # Always close the channel to avoid resource leaks
+        channel.close()