fix: Fixing discrepancies between Accounting and Monitoring

Author: rupozzi

src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py

@@ -49,6 +49,7 @@ def __init__(self, monitoringType="", failoverQueueName="dirac.monitoring"):
         self.__monitoringType = monitoringType
         self.__failoverQueueName = failoverQueueName
         self.__defaultMQProducer = None
+        self.__commitTimer = threading.Timer(5, self.commit)
 
     def __del__(self):
         if self.__defaultMQProducer is not None:
@@ -164,6 +165,18 @@ def commit(self):
             self.__documentLock.release()
         return S_OK(recordSent)
 
+    def delayedCommit(self):
+        """
+        If needed start a timer that will run the commit later
+        allowing to send more registers at once (reduces overheads).
+        """
+
+        if not self.__commitTimer.is_alive():
+            self.__commitTimer = threading.Timer(5, self.commit)
+            self.__commitTimer.start()
+
+        return S_OK()
+
     def __getProducer(self):
         """
         This method is used to get the default MQ producer or create it if needed.

src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py

@@ -36,6 +36,6 @@ def __init__(self):
             }
         )
 
-        self.dataToKeep = 86400 * 30
+        self.dataToKeep = 86400 * 30 * 14
 
         self.checkType()

src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1414,18 +1414,18 @@ def sendPilotSubmissionMonitoring(self, siteName, ceName, queueName, numTotal, n
             "SiteDirector": siteDirName,
             "Site": siteName,
             "CE": ceName,
-            "Queue": queueName,
+            "Queue": ceName + ":" + queueName,
             "Status": status,
             "NumTotal": numTotal,
             "NumSucceded": numSucceeded,
             "timestamp": int(toEpoch(dateTime())),
         }
         pilotMonitoringReporter.addRecord(pilotMonitoringData)
-        result = pilotMonitoringReporter.commit()
 
         self.log.verbose("Committing pilot submission to monitoring")
+        result = pilotMonitoringReporter.delayedCommit()
         if not result["OK"]:
-            self.log.error("Couldn't commit pilot submission to monitoring", result["Message"])
+            self.log.error("Could not commit pilot submission to monitoring", result["Message"])
             return S_ERROR()
         self.log.verbose("Done committing to monitoring")
         return S_OK()