New host service to support gnmi full config update

Author: ganglyu

azure-pipelines.yml

@@ -56,6 +56,8 @@ stages:
         sudo dpkg -i libnl-route-3-200_*.deb
         sudo dpkg -i libnl-nf-3-200_*.deb
         sudo dpkg -i libyang_1.0.73_*.deb
+        sudo dpkg -i libyang-cpp_1.0.73_*.deb
+        sudo dpkg -i python3-yang_1.0.73_*.deb
         sudo dpkg -i libswsscommon_1.0.0_amd64.deb
         sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
       workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/

host_modules/config_engine.py

@@ -10,16 +10,52 @@ class Config(host_service.HostModule):
     """
     DBus endpoint that executes the config command
     """
-    @host_service.method(host_service.bus_name(MOD_NAME), in_signature='s', out_signature='is')
-    def reload(self, config_db_json):
+    @host_service.method(host_service.bus_name(MOD_NAME), in_signature='ss', out_signature='is')
+    def reload(self, config_db_json, caller):
 
+        if caller and len(caller.strip()):
+            # Mask the caller service, should not restart after config reload
+            cmd = ['/usr/bin/systemctl', 'mask', caller]
+            subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         cmd = ['/usr/local/bin/config', 'reload', '-y']
         if config_db_json and len(config_db_json.strip()):
             cmd.append('/dev/stdin')
             input_bytes = (config_db_json + '\n').encode('utf-8')
             result = subprocess.run(cmd, input=input_bytes, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         else:
             result = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if caller and len(caller.strip()):
+            # Unmask the caller service
+            cmd = ['/usr/bin/systemctl', 'unmask', caller]
+            subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        msg = ''
+        if result.returncode:
+            lines = result.stderr.decode().split('\n')
+            for line in lines:
+                if 'Error' in line:
+                    msg = line
+                    break
+        return result.returncode, msg
+
+    @host_service.method(host_service.bus_name(MOD_NAME), in_signature='ss', out_signature='is')
+    def reload_force(self, config_db_json, caller):
+
+        if caller and len(caller.strip()):
+            # Mask the caller service, should not restart after config reload
+            cmd = ['/usr/bin/systemctl', 'mask', caller]
+            subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        # Force config reload without system checks
+        cmd = ['/usr/local/bin/config', 'reload', '-y', '-f']
+        if config_db_json and len(config_db_json.strip()):
+            cmd.append('/dev/stdin')
+            input_bytes = (config_db_json + '\n').encode('utf-8')
+            result = subprocess.run(cmd, input=input_bytes, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        else:
+            result = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if caller and len(caller.strip()):
+            # Unmask the caller service
+            cmd = ['/usr/bin/systemctl', 'unmask', caller]
+            subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         msg = ''
         if result.returncode:
             lines = result.stderr.decode().split('\n')
@@ -45,4 +81,3 @@ def save(self, config_file):
                     msg = line
                     break
         return result.returncode, msg
-

host_modules/yang_validator.py

@@ -0,0 +1,27 @@
+"""Yang validation handler"""
+
+from host_modules import host_service
+import json
+import sonic_yang
+
+YANG_MODELS_DIR = "/usr/local/yang-models"
+MOD_NAME = 'yang'
+
+class Yang(host_service.HostModule):
+    """
+    DBus endpoint that runs yang validation
+    """
+    @host_service.method(host_service.bus_name(MOD_NAME), in_signature='s', out_signature='is')
+    def validate(self, config_db_json):
+        config = json.loads(config_db_json)
+        # Run yang validation
+        yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR)
+        yang_parser.loadYangModel()
+        try:
+            yang_parser.loadData(configdbJson=config)
+            yang_parser.validate_data_tree()
+        except sonic_yang.SonicYangException as e:
+            return -1, str(e)
+        if len(yang_parser.tablesWithOutYang):
+            return -1, "Tables without yang models: " + str(yang_parser.tablesWithOutYang)
+        return 0, ""

scripts/sonic-host-server

@@ -12,7 +12,7 @@ import dbus.service
 import dbus.mainloop.glib
 
 from gi.repository import GObject
-from host_modules import config_engine, gcu, host_service, showtech, systemd_service, file_service
+from host_modules import config_engine, gcu, host_service, showtech, systemd_service, file_service, yang_validator
 
 
 def register_dbus():
@@ -23,7 +23,8 @@ def register_dbus():
             'host_service': host_service.HostService('host_service'),
             'showtech': showtech.Showtech('showtech'),
             'systemd': systemd_service.SystemdService('systemd'),
-            'file_stat': file_service.FileService('file')
+            'file_stat': file_service.FileService('file'),
+            'yang': yang_validator.Yang('yang')
             }
     for mod_name, handler_class in mod_dict.items():
         handlers[mod_name] = handler_class

tests/host_modules/config_test.py

@@ -18,10 +18,14 @@ def test_reload(self, MockInit, MockBusName, MockSystemBus):
             mock_run.return_value = res_mock
             config_db_json = "{}"
             config_stub = config_engine.Config(config_engine.MOD_NAME)
-            ret, msg = config_stub.reload(config_db_json)
-            call_args = mock_run.call_args[0][0]
-            assert "reload" in call_args
-            assert "/dev/stdin" in call_args
+            ret, msg = config_stub.reload(config_db_json, "gnmi")
+            call_args_list = mock_run.call_args_list
+            args, _ = call_args_list[0]
+            assert ["/usr/bin/systemctl", "mask", "gnmi"] in args
+            args, _ = call_args_list[1]
+            assert ["/usr/local/bin/config", "reload", "-y", "/dev/stdin"] in args
+            args, _ = call_args_list[2]
+            assert ["/usr/bin/systemctl", "unmask", "gnmi"] in args
             assert ret == test_ret, "Return value is wrong"
             assert msg == "", "Return message is wrong"
         with mock.patch("subprocess.run") as mock_run:
@@ -33,10 +37,57 @@ def test_reload(self, MockInit, MockBusName, MockSystemBus):
             mock_run.return_value = res_mock
             config_db_json = "{}"
             config_stub = config_engine.Config(config_engine.MOD_NAME)
-            ret, msg = config_stub.reload(config_db_json)
-            call_args = mock_run.call_args[0][0]
-            assert "reload" in call_args
-            assert "/dev/stdin" in call_args
+            ret, msg = config_stub.reload(config_db_json, "gnmi")
+            call_args_list = mock_run.call_args_list
+            args, _ = call_args_list[0]
+            assert ["/usr/bin/systemctl", "mask", "gnmi"] in args
+            args, _ = call_args_list[1]
+            assert ["/usr/local/bin/config", "reload", "-y", "/dev/stdin"] in args
+            args, _ = call_args_list[2]
+            assert ["/usr/bin/systemctl", "unmask", "gnmi"] in args
+            assert ret == test_ret, "Return value is wrong"
+            assert msg == "Error: this is the test message", "Return message is wrong"
+
+    @mock.patch("dbus.SystemBus")
+    @mock.patch("dbus.service.BusName")
+    @mock.patch("dbus.service.Object.__init__")
+    def test_reload_force(self, MockInit, MockBusName, MockSystemBus):
+        with mock.patch("subprocess.run") as mock_run:
+            res_mock = mock.Mock()
+            test_ret = 0
+            test_msg = b"Error: this is the test message\nHello world\n"
+            attrs = {"returncode": test_ret, "stderr": test_msg}
+            res_mock.configure_mock(**attrs)
+            mock_run.return_value = res_mock
+            config_db_json = "{}"
+            config_stub = config_engine.Config(config_engine.MOD_NAME)
+            ret, msg = config_stub.reload_force(config_db_json, "gnmi")
+            call_args_list = mock_run.call_args_list
+            args, _ = call_args_list[0]
+            assert ["/usr/bin/systemctl", "mask", "gnmi"] in args
+            args, _ = call_args_list[1]
+            assert ["/usr/local/bin/config", "reload", "-y", "-f", "/dev/stdin"] in args
+            args, _ = call_args_list[2]
+            assert ["/usr/bin/systemctl", "unmask", "gnmi"] in args
+            assert ret == test_ret, "Return value is wrong"
+            assert msg == "", "Return message is wrong"
+        with mock.patch("subprocess.run") as mock_run:
+            res_mock = mock.Mock()
+            test_ret = 1
+            test_msg = b"Error: this is the test message\nHello world\n"
+            attrs = {"returncode": test_ret, "stderr": test_msg}
+            res_mock.configure_mock(**attrs)
+            mock_run.return_value = res_mock
+            config_db_json = "{}"
+            config_stub = config_engine.Config(config_engine.MOD_NAME)
+            ret, msg = config_stub.reload_force(config_db_json, "gnmi")
+            call_args_list = mock_run.call_args_list
+            args, _ = call_args_list[0]
+            assert ["/usr/bin/systemctl", "mask", "gnmi"] in args
+            args, _ = call_args_list[1]
+            assert ["/usr/local/bin/config", "reload", "-y", "-f", "/dev/stdin"] in args
+            args, _ = call_args_list[2]
+            assert ["/usr/bin/systemctl", "unmask", "gnmi"] in args
             assert ret == test_ret, "Return value is wrong"
             assert msg == "Error: this is the test message", "Return message is wrong"
 

tests/host_modules/yang_test.py

@@ -0,0 +1,15 @@
+import sys
+import os
+import pytest
+from unittest import mock
+from host_modules import yang_validator
+
+class TestConfigEngine(object):
+    @mock.patch("dbus.SystemBus")
+    @mock.patch("dbus.service.BusName")
+    @mock.patch("dbus.service.Object.__init__")
+    def test_reload(self, MockInit, MockBusName, MockSystemBus):
+        config_db_json = "{}"
+        yang_stub = yang_validator.Yang(yang_validator.MOD_NAME)
+        ret, _ = yang_stub.validate(config_db_json)
+        assert ret == 0, "Yang validation failed"