Add support for liquid cooling of mellnox api and system health monitor

Signed-off-by: Yuanzhe, Liu <[email protected]>

Author: yuazhe

dockers/docker-platform-monitor/docker-pmon.supervisord.conf.j2

@@ -199,7 +199,20 @@ dependent_startup_wait_for=rsyslogd:running {% if delay_non_critical_daemon %}de
 
 {% if not skip_thermalctld %}
 [program:thermalctld]
-command={% if API_VERSION == 3 and 'thermalctld' not in python2_daemons %}python3 {% else %} python2 {% endif %}/usr/local/bin/thermalctld
+{% set base_command = "python3 /usr/local/bin/thermalctld" %}
+{% set options = "" -%}
+
+{% if enable_liquid_cooling %}
+    {%- set options = options + " --enable_liquid_cooling" %}
+{% endif -%}
+
+{% if liquid_cooling_update_interval %}
+    {%- set options = options + " --liquid_cooling_update_interval {{ liquid_cooling_update_interval }}" %}
+{% endif -%}
+
+{%- set command = base_command ~ options %}
+
+command={{ command }}
 priority=10
 autostart=false
 autorestart=unexpected

platform/mellanox/mlnx-platform-api/sonic_platform/chassis.py

@@ -120,6 +120,8 @@ def __init__(self):
         self._RJ45_port_inited = False
         self._RJ45_port_list = None
 
+        self.liquid_cooling = None
+
         Chassis.chassis_instance = self
 
         self.module_host_mgmt_initializer = module_host_mgmt_initializer.ModuleHostMgmtInitializer()
@@ -1085,6 +1087,20 @@ def is_replaceable(self):
         """
         return False
 
+    
+    ##############################################
+    # LiquidCooling methods
+    ##############################################
+
+    def initialize_liquid_cooling(self):
+        if not self.liquid_cooling:
+            from .liquid_cooling import LiquidCooling
+            self.liquid_cooling = LiquidCooling()
+
+    def get_liquid_cooling(self):
+        self.initialize_liquid_cooling()
+        return self.liquid_cooling
+
 
 class ModularChassis(Chassis):
     def __init__(self):

platform/mellanox/mlnx-platform-api/sonic_platform/device_data.py

@@ -247,6 +247,11 @@ def get_fan_count(cls):
     def is_fan_hotswapable(cls):
         return utils.read_int_from_file('/run/hw-management/config/hotplug_fans') > 0
 
+    @classmethod
+    @utils.read_only_cache()
+    def get_leakage_sensor_count(cls):
+        return len(glob.glob('/var/run/hw-management/system/leakage*'))
+
     @classmethod
     @utils.read_only_cache()
     def get_psu_count(cls):

platform/mellanox/mlnx-platform-api/sonic_platform/liquid_cooling.py

@@ -0,0 +1,77 @@
+#
+# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
+# Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#############################################################################
+# Mellanox
+#
+# Module contains an implementation of SONiC Platform Base API and
+# provides the liquid cooling status which are available in the platform
+#
+#############################################################################
+
+try:
+    from sonic_platform_base.liquid_cooling_base import LeakageSensorBase,LiquidCoolingBase
+    from sonic_py_common.logger import Logger
+    from . import utils
+    import os
+    import glob
+    import logging
+except ImportError as e:
+    raise ImportError(str(e) + "- required module not found")
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+LIQUID_COOLING_SENSOR_PATH = "/var/run/hw-management/system/"
+
+class LeakageSensor(LeakageSensorBase):
+    def __init__(self, name,path):
+        super(LeakageSensor, self).__init__(name)
+        self.path = path
+
+    def is_leak(self):
+        content = utils.read_int_from_file(self.path)
+
+        if content == 1:
+            self.leaking = False
+            return False
+        elif content == 0:
+            self.leaking = True
+            return True
+        else:
+            logger.error(f"Failed to read leakage sensor {self.name} value: {content}")
+            return True
+
+class LiquidCooling(LiquidCoolingBase):
+    """Platform-specific Liquid Cooling class"""
+
+    def __init__(self):
+        
+        sensor_files = glob.glob(os.path.join(LIQUID_COOLING_SENSOR_PATH, "leakage*"))
+
+        sensor_files.sort(key=lambda x: int(x.split("leakage")[-1]))
+
+        self.leakage_sensors_num = len(sensor_files)
+        self.leakage_sensors = []
+
+        for sensor_path in sensor_files:
+            sensor_name = os.path.basename(sensor_path)
+            index = int(sensor_name.replace("leakage", ""))
+            self.leakage_sensors.append(LeakageSensor(sensor_name, sensor_path))
+
+        super(LiquidCooling, self).__init__(self.leakage_sensors_num, self.leakage_sensors)
+

platform/mellanox/mlnx-platform-api/tests/test_liquid_cooling.py

@@ -0,0 +1,52 @@
+import os
+import sys
+from unittest import mock
+from sonic_platform.liquid_cooling import LiquidCooling, LeakageSensor
+
+def test_leakage_sensor_init():
+    sensor = LeakageSensor("leakage1", "/test/path")
+    assert sensor.name == "leakage1"
+    assert sensor.path == "/test/path"
+
+def test_leakage_sensor_is_leak():
+    sensor = LeakageSensor("leakage1", "/test/path")
+    
+    # Test when file exists and content is "1" (no leak)
+    with mock.patch('os.path.exists') as mock_exists:
+        with mock.patch('builtins.open', mock.mock_open(read_data="1")):
+            mock_exists.return_value = True
+            assert sensor.is_leak() is False
+
+    # Test when file exists and content is "0" (leak detected)
+    with mock.patch('os.path.exists') as mock_exists:
+        with mock.patch('builtins.open', mock.mock_open(read_data="0")):
+            mock_exists.return_value = True
+            assert sensor.is_leak() is True
+
+    # Test when file does not exist
+    with mock.patch('os.path.exists') as mock_exists:
+        mock_exists.return_value = False
+        assert sensor.is_leak() is None
+
+def test_liquid_cooling_init():
+
+    with mock.patch('os.path.exists') as mock_exists, \
+         mock.patch('os.path.join', side_effect=lambda *args: "/".join(args)) as mock_join, \
+         mock.patch('glob.glob') as mock_glob:
+        
+        # Setup mock to simulate 4 leakage sensors
+        mock_exists.side_effect = [True, True, True, True]
+        mock_glob.return_value = [
+            "/var/run/hw-management/system/leakage1",
+            "/var/run/hw-management/system/leakage2", 
+            "/var/run/hw-management/system/leakage3",
+            "/var/run/hw-management/system/leakage4"
+        ]
+
+        liquid_cooling = LiquidCooling()
+        
+        # Verify the number of sensors initialized
+        assert liquid_cooling.leakage_sensors_num == 4
+        
+        sensors = liquid_cooling.get_leak_sensor_status()
+        assert len(sensors) == 0

src/system-health/health_checker/hardware_checker.py

@@ -1,8 +1,11 @@
 from natsort import natsorted
+from swsscommon import swsscommon
 from swsscommon.swsscommon import SonicV2Connector
 
 from .health_checker import HealthChecker
 
+EVENTS_PUBLISHER_SOURCE = "sonic-events-host"
+EVENTS_PUBLISHER_TAG = "liquid-cooling-leak"
 
 class HardwareChecker(HealthChecker):
     """
@@ -12,6 +15,7 @@ class HardwareChecker(HealthChecker):
     ASIC_TEMPERATURE_KEY = 'TEMPERATURE_INFO|ASIC'
     FAN_TABLE_NAME = 'FAN_INFO'
     PSU_TABLE_NAME = 'PSU_INFO'
+    LIQUID_COOLING_TABLE_NAME = 'LIQUID_COOLING_INFO'
 
     def __init__(self):
         HealthChecker.__init__(self)
@@ -26,6 +30,7 @@ def check(self, config):
         self._check_asic_status(config)
         self._check_fan_status(config)
         self._check_psu_status(config)
+        self._check_liquid_cooling_status(config)
 
     def _check_asic_status(self, config):
         """
@@ -283,3 +288,58 @@ def _ignore_check(cls, ignore_set, category, object_name, check_point):
         elif '{}.{}'.format(object_name, check_point) in ignore_set:
             return True
         return False
+
+    def publish_events(self, leak_sensors):
+        params = swsscommon.FieldValueMap()
+        events_handle = swsscommon.events_init_publisher(EVENTS_PUBLISHER_SOURCE)
+        for sensor in leak_sensors:
+            params["leak_sensor"] = sensor
+            swsscommon.event_publish(events_handle, EVENTS_PUBLISHER_TAG, params)
+        swsscommon.events_deinit_publisher(events_handle)
+
+
+    def _check_liquid_cooling_status(self, config):
+        """
+        Check liquid cooling status including:
+            1. Check all leakage sensors are in good state
+        :param config: Health checker configuration
+        :return:
+        """
+        if config.user_defined_checkers and 'liquid_cooling' not in config.user_defined_checkers:
+            return
+
+        keys = self._db.keys(self._db.STATE_DB, HardwareChecker.LIQUID_COOLING_TABLE_NAME + '*')
+        if not keys:
+            self.set_object_not_ok('Liquid Cooling', 'Liquid Cooling', 'Failed to get liquid cooling information')
+            return
+
+        leaking_sensors = []
+
+        for key in natsorted(keys):
+            key_list = key.split('|')
+            if len(key_list) != 2:  # error data in DB, log it and ignore
+                self.set_object_not_ok('Liquid Cooling', key, 'Invalid key for LIQUID_COOLING_INFO: {}'.format(key))
+                continue
+
+            name = key_list[1]
+            if config.ignore_devices and name in config.ignore_devices:
+                continue
+
+            data_dict = self._db.get_all(self._db.STATE_DB, key)
+            leak_status = data_dict.get('leak_status', None)
+            if leak_status is None:
+                self.set_object_not_ok('Liquid Cooling', name, 'Failed to get leakage sensor status for {}'.format(name))
+                continue
+
+            if leak_status.lower() == 'true':
+                if name not in leaking_sensors:
+                    leaking_sensors.append(name)
+                    self.set_object_not_ok('Liquid Cooling', name, 'Leakage sensor {} is leaking'.format(name))
+                    continue    
+
+            if leak_status.lower() == 'false' and name in leaking_sensors:
+                leaking_sensors.remove(name)
+
+            self.set_object_ok('Liquid Cooling', name)
+            
+        self.publish_events(leaking_sensors)

src/system-health/health_checker/service_checker.py

@@ -70,8 +70,6 @@ def __init__(self):
 
         self.load_critical_process_cache()
 
-        self.events_handle = swsscommon.events_init_publisher(EVENTS_PUBLISHER_SOURCE)
-
     def get_expected_running_containers(self, feature_table):
         """Get a set of containers that are expected to running on SONiC
 
@@ -342,7 +340,6 @@ def check(self, config):
         self.reset()
         self.check_by_monit(config)
         self.check_services(config)
-        swsscommon.events_deinit_publisher(self.events_handle)
 
     def _parse_supervisorctl_status(self, process_status):
         """Expected input:
@@ -366,9 +363,11 @@ def _parse_supervisorctl_status(self, process_status):
     def publish_events(self, container_name, critical_process_list):
         params = swsscommon.FieldValueMap()
         params["ctr_name"] = container_name
+        events_handle = swsscommon.events_init_publisher(EVENTS_PUBLISHER_SOURCE)
         for process_name in critical_process_list:
             params["process_name"] = process_name
-            swsscommon.event_publish(self.events_handle, EVENTS_PUBLISHER_TAG, params)
+            swsscommon.event_publish(events_handle, EVENTS_PUBLISHER_TAG, params)
+        swsscommon.events_deinit_publisher(events_handle)
 
     def check_process_existence(self, container_name, critical_process_list, config, feature_table):
         """Check whether the process in the specified container is running or not.

src/system-health/health_checker/utils.py

@@ -1,6 +1,5 @@
 import subprocess
 
-
 def run_command(command):
     """
     Utility function to run an shell command and return the output.

src/system-health/tests/test_system_health.py

@@ -471,6 +471,33 @@ def test_hardware_checker():
         }
     })
 
+    MockConnector.data.update({
+        'LIQUID_COOLING_INFO|liquid_cooling_1': {
+                'leak_status': 'True',
+                'leak_sensor_name': 'liquid_cooling_1'
+            },
+            'LIQUID_COOLING_INFO|liquid_cooling_2': {
+                'leak_status': 'False',
+                'leak_sensor_name': 'liquid_cooling_2'
+            },
+            'LIQUID_COOLING_INFO|liquid_cooling_3': {
+                'leak_status': 'True',
+                'leak_sensor_name': 'liquid_cooling_3'
+            },
+            'LIQUID_COOLING_INFO|liquid_cooling_4': {
+                'leak_status': 'False',
+                'leak_sensor_name': 'liquid_cooling_4'
+            },
+            'LIQUID_COOLING_INFO|liquid_cooling_5': {
+                'leak_status': 'True',
+                'leak_sensor_name': 'liquid_cooling_5'
+            },
+            'LIQUID_COOLING_INFO|liquid_cooling_6': {
+                'leak_status': 'False',
+                'leak_sensor_name': 'liquid_cooling_6'
+            }
+    })
+
     checker = HardwareChecker()
     assert checker.get_category() == 'Hardware'
     config = Config()
@@ -521,6 +548,24 @@ def test_hardware_checker():
     assert checker._info['PSU 7'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
     assert checker._info['PSU 7'][HealthChecker.INFO_FIELD_OBJECT_MSG] == 'System power exceeds threshold but power_critical_threshold is invalid'
 
+    assert 'liquid_cooling_1' in checker._info
+    assert checker._info['liquid_cooling_1'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
+
+    assert 'liquid_cooling_2' in checker._info
+    assert checker._info['liquid_cooling_2'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
+
+    assert 'liquid_cooling_3' in checker._info
+    assert checker._info['liquid_cooling_3'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
+
+    assert 'liquid_cooling_4' in checker._info
+    assert checker._info['liquid_cooling_4'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
+
+    assert 'liquid_cooling_5' in checker._info
+    assert checker._info['liquid_cooling_5'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_NOT_OK
+
+    assert 'liquid_cooling_6' in checker._info
+    assert checker._info['liquid_cooling_6'][HealthChecker.INFO_FIELD_OBJECT_STATUS] == HealthChecker.STATUS_OK
+
 
 def test_config():
     config = Config()