Merge pull request #6891 from chaen/v8.0_FEAT_globalLogLevel

fstagni · web-flow · commit 84f099579ac7 · 2023-03-09T16:23:26.000+01:00
[8.0] Allow to define global log levels
diff --git a/dirac.cfg b/dirac.cfg
@@ -1000,6 +1000,25 @@ Resources
     }
   }
   ##END
+
+  # Configuration for logging backends
+  # https://dirac.readthedocs.io/en/latest/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.html#backend-resources
+  LogBackends
+  {
+    # Configure the stdout backend
+    stdout
+    {
+      LogLevel = INFO
+    }
+    # Example for a log backend sending to message queue
+    # see https://dirac.readthedocs.io/en/latest/AdministratorGuide/ServerInstallations/centralizedLogging.html
+    mqLogs
+    {
+      MsgQueue = lhcb-mb.cern.ch::Queues::lhcb.dirac.logging
+      # Name of the plugin if not the section name
+      Plugin = messageQueue
+    }
+  }
 }
 Operations
 {
@@ -1108,6 +1127,21 @@ Operations
       # Module used for InputData resolution if not specified in the JDL
       InputDataModule = DIRAC.Core.Utilities.InputDataResolution
     }
+    Logging
+    {
+      # Default log backends and level applied to Services if
+      # it is not defined in the service specific section
+      DefaultServicesBackends = stdout
+      DefaultServicesLogLevel = INFO
+
+      # Similar options for agents
+      DefaultAgentsBackends = stdout, mqLogs
+      DefaultAgentsLogLevel = VERBOSE
+
+      # Default log level that is applied in last resort
+      DefaultLogLevel = DEBUG
+
+    }
     # Options for the pilot3
     # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.html
     Pilot
diff --git a/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py b/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py
@@ -41,7 +41,7 @@ def __init__(self, loadDefaultCFG=True):
             gLogger.debug("dirac.cfg should be at", f"{defaultCFGFile}")
             retVal = self.loadFile(defaultCFGFile)
             if not retVal["OK"]:
-                gLogger.warn(f"Can't load {defaultCFGFile} file")
+                gLogger.debug(f"Can't load {defaultCFGFile} file")
         self.sync()
 
     def getBackupDir(self):
diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py
@@ -35,7 +35,6 @@ def main():
     localCfg = Script.localCfg
     localCfg.addMandatoryEntry("/DIRAC/Setup")
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
-    localCfg.addDefaultEntry("LogLevel", "INFO")
     localCfg.addDefaultEntry("LogColor", True)
     resultDict = localCfg.loadUserData()
     if not resultDict["OK"]:
diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py
@@ -33,7 +33,6 @@ def main():
     localCfg.setConfigurationForTornado()
     localCfg.addMandatoryEntry("/DIRAC/Setup")
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
-    localCfg.addDefaultEntry("LogLevel", "INFO")
     localCfg.addDefaultEntry("LogColor", True)
     resultDict = localCfg.loadUserData()
     if not resultDict["OK"]:
diff --git a/src/DIRAC/Core/scripts/dirac_agent.py b/src/DIRAC/Core/scripts/dirac_agent.py
@@ -25,7 +25,6 @@ def main():
     localCfg.setConfigurationForAgent(agentName)
     localCfg.addMandatoryEntry("/DIRAC/Setup")
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
-    localCfg.addDefaultEntry("LogLevel", "INFO")
     localCfg.addDefaultEntry("LogColor", True)
     resultDict = localCfg.loadUserData()
     if not resultDict["OK"]:
diff --git a/src/DIRAC/Core/scripts/dirac_executor.py b/src/DIRAC/Core/scripts/dirac_executor.py
@@ -29,7 +29,6 @@ def main():
     localCfg.setConfigurationForExecutor(mainName)
     localCfg.addMandatoryEntry("/DIRAC/Setup")
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
-    localCfg.addDefaultEntry("LogLevel", "INFO")
     localCfg.addDefaultEntry("LogColor", True)
     resultDict = localCfg.loadUserData()
     if not resultDict["OK"]:
diff --git a/src/DIRAC/Core/scripts/dirac_service.py b/src/DIRAC/Core/scripts/dirac_service.py
@@ -28,7 +28,6 @@ def main():
     # localCfg.addMandatoryEntry( "HandlerPath" )
     localCfg.addMandatoryEntry("/DIRAC/Setup")
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
-    localCfg.addDefaultEntry("LogLevel", "INFO")
     localCfg.addDefaultEntry("LogColor", True)
     resultDict = localCfg.loadUserData()
     if not resultDict["OK"]:
diff --git a/src/DIRAC/FrameworkSystem/private/standardLogging/LoggingRoot.py b/src/DIRAC/FrameworkSystem/private/standardLogging/LoggingRoot.py
@@ -50,8 +50,8 @@ def __init__(self):
         for level in levels:
             logging.addLevelName(levels[level], level)
 
-        # root Logger level is set to NOTICE by default
-        self._logger.setLevel(LogLevels.NOTICE)
+        # root Logger level is set to INFO by default
+        self._logger.setLevel(LogLevels.INFO)
 
         # initialization of the UTC time
         # Actually, time.gmtime is equal to UTC time: it has its DST flag to 0 which means there is no clock advance
@@ -116,14 +116,45 @@ def initialize(self, systemName, cfgPath, forceInit=False):
                 for handler in handlersToRemove:
                     self._logger.removeHandler(handler)
 
-                levelName = gConfig.getValue(f"{cfgPath}/LogLevel", None)
+                levelName = self.__getLogLevelFromCFG(cfgPath)
                 if levelName is not None:
                     self.setLevel(levelName)
 
                 LoggingRoot.__configuredLogging = True
         finally:
             self._lockConfig.release()
 
+    def __getLogLevelFromCFG(self, cfgPath):
+        """
+        Get the logging level from the cfg, following this order:
+
+        * cfgPath/LogLevel
+        * Operations/<setup/vo>/Logging/Default<Agents/Services>LogLevel
+        * Operations/<setup/vo>/Logging/DefaultLogLevel
+
+        :param str cfgPath: configuration path
+        """
+        # We have to put the import line here to avoid a dependancy loop
+        from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
+        from DIRAC import gConfig
+
+        # Search desired log level in the component
+        logLevel = gConfig.getValue(f"{cfgPath}/LogLevel")
+        if not logLevel:
+            # get the second last string representing the component type in the configuration
+            # example : 'Agents', 'Services'
+            component = cfgPath.split("/")[-2]
+
+            operation = Operations()
+            # Search desired logLevel in the operation section according to the
+            # component type
+            logLevel = operation.getValue(f"Logging/Default{component}LogLevel")
+            if not logLevel:
+                # Search desired logLevel in the operation section
+                logLevel = operation.getValue("Logging/DefaultLogLevel")
+
+        return logLevel
+
     def __getBackendsFromCFG(self, cfgPath):
         """
         Get backends from the configuration and register them in LoggingRoot.
@@ -135,14 +166,14 @@ def __getBackendsFromCFG(self, cfgPath):
         from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
         from DIRAC import gConfig
 
-        # get the second last string representing the component type in the configuration
-        # example : 'Agents', 'Services'
-        component = cfgPath.split("/")[-2]
-        operation = Operations()
-
         # Search desired backends in the component
         backends = gConfig.getValue(f"{cfgPath}/LogBackends", [])
         if not backends:
+            # get the second last string representing the component type in the configuration
+            # example : 'Agents', 'Services'
+            component = cfgPath.split("/")[-2]
+
+            operation = Operations()
             # Search desired backends in the operation section according to the
             # component type
             backends = operation.getValue(f"Logging/Default{component}Backends", [])
