Merge pull request #5945 from rupozzi/pilotfix

[integration] Hackaton fix: pilotsubmission discrepancies between Accounting and Monitoring

Author: fstagni

src/DIRAC/AccountingSystem/private/Plotters/PilotSubmissionPlotter.py

@@ -34,12 +34,7 @@ def _reportSubmission(self, reportRequest):
             return retVal
         dataDict, granularity = retVal["Value"]
         self.stripDataField(dataDict, 0)
-        dataDict, _ = self._divideByFactor(dataDict, granularity)
-        dataDict = self._fillWithZero(granularity, reportRequest["startTime"], reportRequest["endTime"], dataDict)
-        baseDataDict, graphDataDict, _, unit = self._findSuitableRateUnit(
-            dataDict, self._getAccumulationMaxValue(dataDict), "jobs"
-        )
-        return S_OK({"data": baseDataDict, "graphDataDict": graphDataDict, "granularity": granularity, "unit": unit})
+        return S_OK({"data": dataDict, "granularity": granularity})
 
     def _plotSubmission(self, reportRequest, plotInfo, filename):
         """
@@ -55,9 +50,15 @@ def _plotSubmission(self, reportRequest, plotInfo, filename):
             "starttime": reportRequest["startTime"],
             "endtime": reportRequest["endTime"],
             "span": plotInfo["granularity"],
-            "ylabel": plotInfo["unit"],
+            "ylabel": "Submissions",
         }
-        return self._generateTimedStackedBarPlot(filename, plotInfo["graphDataDict"], metadata)
+        plotInfo["data"] = self._fillWithZero(
+            granularity=plotInfo["granularity"],
+            startEpoch=reportRequest["startTime"],
+            endEpoch=reportRequest["endTime"],
+            dataDict=plotInfo["data"],
+        )
+        return self._generateTimedStackedBarPlot(filename, plotInfo["data"], metadata)
 
     _reportSubmissionEfficiencyName = "Submission efficiency"
 

src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py

@@ -36,6 +36,6 @@ def __init__(self):
             }
         )
 
-        self.dataToKeep = 86400 * 30
+        self.dataToKeep = 86400 * 30 * 14
 
         self.checkType()

src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1414,18 +1414,18 @@ def sendPilotSubmissionMonitoring(self, siteName, ceName, queueName, numTotal, n
             "SiteDirector": siteDirName,
             "Site": siteName,
             "CE": ceName,
-            "Queue": queueName,
+            "Queue": ceName + ":" + queueName,
             "Status": status,
             "NumTotal": numTotal,
             "NumSucceded": numSucceeded,
             "timestamp": int(toEpoch(dateTime())),
         }
         pilotMonitoringReporter.addRecord(pilotMonitoringData)
-        result = pilotMonitoringReporter.commit()
 
         self.log.verbose("Committing pilot submission to monitoring")
+        result = pilotMonitoringReporter.commit()
         if not result["OK"]:
-            self.log.error("Couldn't commit pilot submission to monitoring", result["Message"])
+            self.log.error("Could not commit pilot submission to monitoring", result["Message"])
             return S_ERROR()
         self.log.verbose("Done committing to monitoring")
         return S_OK()