fix: Fixes from reviews

Author: rupozzi

src/DIRAC/MonitoringSystem/Client/DataOperationSender.py

@@ -4,7 +4,6 @@
 
 """
 
-from email.mime import base
 import DIRAC
 from DIRAC import S_OK, gLogger
 

src/DIRAC/MonitoringSystem/private/DBUtils.py

@@ -176,3 +176,47 @@ def _accumulate(granularity, startEpoch, endEpoch, dataDict):
                     lastValue += currentDict[timeEpoch]
                 currentDict[timeEpoch] = lastValue
         return dataDict
+
+    def stripDataField(self, dataDict, fieldId):
+        """
+        Strip <fieldId> data and sum the rest as it was data from one key
+
+        :param dict dataDict: dictionary of the form::
+
+            { 'key' : { <timeEpoch1>: [1, 2, 3],
+                        <timeEpoch2>: [3, 4, 5].. } }
+
+          The dataDict is modified in this function and the return structure is:
+
+          .. code-block :: python
+
+                  dataDict : { 'key' : { <timeEpoch1>: 1,
+                                         <timeEpoch2>: 3.. } }
+
+        :param int fieldId:
+
+        :returns: list of dictionaries
+
+          .. code-block:: python
+
+                     [ { <timeEpoch1>: 2, <timeEpoch2>: 4... }
+                       { <timeEpoch1>: 3, <timeEpoch2>): 5... } ]
+
+        :rtype: python:list
+
+        """
+        remainingData = [{}]  # Hack for empty data
+        for key in dataDict:
+            for timestamp in dataDict[key]:
+                for iPos in dataDict[key][timestamp]:
+                    remainingData.append({})
+                break
+            break
+        for key in dataDict:
+            for timestamp in dataDict[key]:
+                strippedField = float(dataDict[key][timestamp].pop(fieldId))
+                for iPos, value in enumerate(dataDict[key][timestamp]):
+                    remainingData[iPos].setdefault(timestamp, 0.0)
+                    remainingData[iPos] += float(value)
+                dataDict[key][timestamp] = strippedField
+        return remainingData

src/DIRAC/MonitoringSystem/private/Plotters/BasePlotter.py

@@ -1,5 +1,5 @@
 """
-It is used to creates several plots
+It is used to create several plots
 """
 import time
 import copy
@@ -418,21 +418,23 @@ def _calculateEfficiencyDict(self, totDataDict, dataDict):
         """
         It returns a dict with efficiency calculated from each key in totDataDict and dataDict
         """
-        for (totItem, totVal), (item, val) in zip(totDataDict.items(), dataDict.items()):
+        for item, val in dataDict.items():
+            totVal = totDataDict.get(item, {})
             try:
                 dataDict[item] = {key: (float(val[key]) / float(totVal[key])) * 100 for key in val if key in totVal}
-            except ZeroDivisionError and TypeError:
+            except (ZeroDivisionError, TypeError):
+                gLogger.warn("Error in ", val)
                 gLogger.warn("Dividing by zero or using None type field. Skipping the key of this dict item...")
         return dataDict
 
     def _sumDictValues(self, dataDict):
         """
-        Sums all the values of the keys of each item in the dict.
-        Returns a dict with item-value pair and removes the keys.
+        Sums the values of each item in `dataDict`.
+        Returns the dictionary with the same keys and the values replaced by their sum.
         """
-        for item, key in dataDict.items():
+        for key, values in dataDict.items():
             sum = 0
-            for val in key:
-                sum += key[val]
-            dataDict[item] = sum
+            for val in values.values():
+                sum += val
+            dataDict[key] = sum
         return dataDict

src/DIRAC/MonitoringSystem/private/Plotters/DataOperationPlotter.py

@@ -2,7 +2,7 @@
 this class is used to define the plot using the plot attributes.
 """
 
-from DIRAC import S_OK
+from DIRAC import S_OK, S_ERROR
 from DIRAC.MonitoringSystem.Client.Types.DataOperation import DataOperation
 from DIRAC.MonitoringSystem.private.Plotters.BasePlotter import BasePlotter
 
@@ -33,11 +33,6 @@ def _reportFailedTransfers(self, reportRequest):
         return self.__reportTransfers(reportRequest, "Failed", ("Succeeded", 1))
 
     def __reportTransfers(self, reportRequest, titleType, togetherFieldsToPlot):
-        """It is used to retrieve the data from the database.
-
-        :param dict reportRequest: contains attributes used to create the plot.
-        :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length
-        """
         retVal = self._getTimedData(
             reportRequest["startTime"],
             reportRequest["endTime"],
@@ -48,7 +43,9 @@ def __reportTransfers(self, reportRequest, titleType, togetherFieldsToPlot):
         if not retVal["OK"]:
             return retVal
         dataDict, granularity = retVal["Value"]
-
+        strippedData = self.stripDataField(dataDict, togetherFieldsToPlot[1])
+        if strippedData:
+            dataDict[togetherFieldsToPlot[0]] = strippedData[0]
         dataDict, maxValue = self._divideByFactor(dataDict, granularity)
         dataDict = self._fillWithZero(granularity, reportRequest["startTime"], reportRequest["endTime"], dataDict)
         baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableRateUnit(
@@ -105,14 +102,14 @@ def _reportQuality(self, reportRequest):
         dataDict, granularity = retVal["Value"]
         totDataDict, granularity = retTotVal["Value"]
         # Check that the dicts are not empty
-        if bool(dataDict) and bool(totDataDict):
+        if dataDict and totDataDict:
             # Return the efficiency in dataDict
             effDict = self._calculateEfficiencyDict(totDataDict, dataDict)
-        return S_OK({"data": effDict, "granularity": granularity})
+            return S_OK({"data": effDict, "granularity": granularity})
+        return S_ERROR("Error in plotting the data: The dictionaries are empty!")
 
     def _plotQuality(self, reportRequest, plotInfo, filename):
-        """
-        Make 2 dimensional pilotSubmission efficiency plot
+        """Make 2 dimensional pilotSubmission efficiency plot
 
         :param dict reportRequest: Condition to select data
         :param dict plotInfo: Data for plot.

src/DIRAC/MonitoringSystem/private/Plotters/PilotSubmissionMonitoringPlotter.py

@@ -21,7 +21,7 @@ class PilotSubmissionMonitoringPlotter(BasePlotter):
     _typeName = "PilotSubmissionMonitoring"
     _typeKeyFields = PilotSubmissionMonitoring().keyFields
 
-    _reportNumberOfSubmissions = "Total Number of Submission"
+    _reportNumberOfSubmissions = "Total Number of Submissions"
 
     def _reportNumberOfSubmissions(self, reportRequest):
         """It is used to retrieve the data from the database.
@@ -75,7 +75,7 @@ def _reportNumSucceeded(self, reportRequest):
         :param dict reportRequest: contains attributes used to create the plot.
         :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length
         """
-        # Retrieve the number of succeded submissions
+        # Retrieve the number of succeeded submissions
         retVal = self._getTimedData(
             startTime=reportRequest["startTime"],
             endTime=reportRequest["endTime"],
@@ -101,7 +101,7 @@ def _reportNumSucceeded(self, reportRequest):
         totDataDict, granularity = retTotVal["Value"]
         # Check that the dicts are not empty
         if bool(dataDict) and bool(totDataDict):
-            # Retrun the efficiency in dataDict
+            # Return the efficiency in dataDict
             effDict = self._calculateEfficiencyDict(totDataDict, dataDict)
         return S_OK({"data": effDict, "granularity": granularity})