docs: Updated documentation for PilotsHistory

Author: rupozzi

docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst

@@ -10,9 +10,10 @@ Monitoring System
 Overview
 =========
 
-The Monitoring system is used to monitor various components of DIRAC. Currently, we have five monitoring types:
+The Monitoring system is used to monitor various components of DIRAC. Currently, we have several monitoring types:
 
   - WMSHistory: for monitoring the DIRAC WMS
+  - PilotsHistory: for monitoring of DIRAC pilots
   - Component Monitoring: for monitoring DIRAC components such as services, agents, etc.
   - RMS Monitoring: for monitoring the DIRAC RequestManagement System (mostly the Request Executing Agent).
   - PilotSubmission Monitoring: for monitoring the DIRAC pilot submission statistics from SiteDirector agents
@@ -122,6 +123,11 @@ You can configure the MQ in the local dirac.cfg file where the agent is running:
   Note: the JSON file already contains the index patterns needed for the visualizations. You may need to adapt the index patterns to your existing ones.
 
 
+Enable PilotsHistory monitoring
+===============================
+In order to enable PilotsHistory monitoring you need to set the flag ``monitoringEnabled = True`` in Operations/Defaults.
+
+
 Enable Component monitoring
 ===========================
 

src/DIRAC/MonitoringSystem/Client/Types/PilotsHistory.py

@@ -19,7 +19,7 @@ def __init__(self):
 
         self.keyFields = ["TaskQueueID", "GridSite", "GridType", "Status"]
 
-        self.monitoringFields = ["Pilots"]
+        self.monitoringFields = ["NumOfPilots"]
 
         self.index = "pilotshistory_index"
 
@@ -29,7 +29,7 @@ def __init__(self):
                 "GridSite": {"type": "keyword"},
                 "GridType": {"type": "keyword"},
                 "Status": {"type": "keyword"},
-                "Pilots": {"type": "long"},
+                "NumOfPilots": {"type": "long"},
             }
         )
 

src/DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -42,15 +42,15 @@ class StatesAccountingAgent(AgentModule):
 
     # PilotsHistory fields
     __pilotKeyFields = ["TaskQueueID", "GridSite", "GridType", "Status"]
-    __pilotValueFields = ["Pilots"]
+    __pilotValueFields = ["NumOfPilots"]
 
     def initialize(self):
         """Standard initialization"""
         # This agent will always loop every 15 minutes
         self.am_setOption("PollingTime", 900)
 
         self.backends = self.am_getOption("Backends", "Accounting").replace(" ", "").split(",")
-        self.monitoringEnabled = Operations().getValue("monitoringEnabled", "False")
+        self.monitoringEnabled = Operations().getValue("monitoringEnabled", False)
 
         messageQueue = self.am_getOption("MessageQueue", "dirac.wmshistory")
 
@@ -62,7 +62,7 @@ def initialize(self):
             self.datastores["Monitoring"] = MonitoringReporter(
                 monitoringType="WMSHistory", failoverQueueName=messageQueue
             )
-            self.pilotReporter = MonitoringReporter(monitoringType="PilotsHistory")
+            self.pilotReporter = MonitoringReporter(monitoringType="PilotsHistory", failoverQueueName=messageQueue)
 
         self.__jobDBFields = []
         for field in self.__summaryKeyFieldsMapping:
@@ -77,32 +77,34 @@ def execute(self):
         """Main execution method"""
 
         # PilotsHistory to Monitoring
+        self.log.info("Committing PilotsHistory to Monitoring")
         if self.monitoringEnabled:
             result = PilotAgentsDB.getSummarySnapshot(self.__pilotKeyFields)
             now = Time.dateTime()
             if not result["OK"]:
                 self.log.error(
-                    "Can't get the PilotAgentsDB summary", "%s: won't commit at this cycle" % result["Message"]
+                    "Can't get the PilotAgentsDB summary",
+                    "%s: won't commit PilotsHistory at this cycle" % result["Message"],
                 )
                 return S_ERROR()
 
             values = result["Value"][1]
             for record in values:
                 record = record[1:]
                 rD = {}
-                for iP in range(len(self.__pilotKeyFields)):
+                for iP in enumerate(self.__pilotKeyFields):
                     rD[self.__pilotKeyFields[iP]] = record[iP]
                 record = record[len(self.__pilotKeyFields) :]
-                for iP in range(len(self.__pilotValueFields)):
+                for iP in enumerate(self.__pilotValueFields):
                     rD[self.__pilotValueFields[iP]] = int(record[iP])
                 rD["timestamp"] = int(Time.toEpoch(now))
-                self.log.verbose("Adding following PilotsHistory record to Reporter: \n", rD)
+                self.log.debug("Adding following PilotsHistory record to Reporter: \n", rD)
                 self.pilotReporter.addRecord(rD)
 
-            self.log.info("Committing PilotsHistory to Monitoring")
+            self.log.info("Committing to Monitoring...")
             result = self.pilotReporter.commit()
             if not result["OK"]:
-                self.log.error("Could not commit PilotsHistory to Monitoring")
+                self.log.error("Could not commit to Monitoring")
                 return S_ERROR()
             self.log.verbose("Done committing PilotsHistory to Monitoring")