feat: use summary table and triggers (PilotAgentsDB.PilotAgents, summaries for StatesAccounting)

Author: fstagni

src/DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -43,7 +43,7 @@ class StatesAccountingAgent(AgentModule):
     __renameFieldsMapping = {"JobType": "JobSplitType"}
 
     # PilotsHistory fields
-    __pilotsMapping = ["GridSite", "GridType", "Status", "NumOfPilots"]
+    __pilotsMapping = ["GridSite", "GridType", "Status", "VO", "NumOfPilots"]
 
     def initialize(self):
         """Standard initialization"""
@@ -85,7 +85,8 @@ def execute(self):
         # PilotsHistory to Monitoring
         if "Monitoring" in self.pilotMonitoringOption:
             self.log.info("Committing PilotsHistory to Monitoring")
-            result = PilotAgentsDB().getSummarySnapshot()
+            sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, GridType, Status, VO;"
+            result = PilotAgentsDB()._query(sql)
             now = datetime.datetime.utcnow()
             if not result["OK"]:
                 self.log.error(

src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py

@@ -943,15 +943,12 @@ def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems):
     def getPilotMonitorSelectors(self):
         """Get distinct values for the Pilot Monitor page selectors"""
 
-        paramNames = ["VO", "GridType", "Status", "DestinationSite", "GridSite"]
-
         resultDict = {}
-        for param in paramNames:
-            result = self.getDistinctAttributeValues("PilotAgents", param)
-            if result["OK"]:
-                resultDict[param] = result["Value"]
-            else:
+        for param in ["VO", "Status", "DestinationSite", "GridSite"]:
+            result = self.getDistinctAttributeValues("PilotsHistorySummary", param)
+            if not result["OK"]:
                 resultDict = []
+            resultDict[param] = result["Value"]    
 
         return S_OK(resultDict)
 
@@ -1048,18 +1045,6 @@ def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems):
 
         return S_OK(resultDict)
 
-    def getSummarySnapshot(self, requestedFields=False):
-        """Get the summary snapshot for a given combination"""
-        if not requestedFields:
-            requestedFields = ["GridSite", "GridType", "Status"]
-        valueFields = ["COUNT(PilotID)"]
-        defString = ", ".join(requestedFields)
-        valueString = ", ".join(valueFields)
-        result = self._query(f"SELECT {defString}, {valueString} FROM PilotAgents GROUP BY {defString}")
-        if not result["OK"]:
-            return result
-        return S_OK(((requestedFields + valueFields), result["Value"]))
-
 
 class PivotedPilotSummaryTable:
     """

src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql

@@ -1,5 +1,3 @@
--- $Header: /tmp/libdirac/tmp.stZoy15380/dirac/DIRAC3/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql,v 1.20 2009/08/26 09:39:53 rgracian Exp $
-
 -- ------------------------------------------------------------------------------
 --
 --  Schema definition for the PilotAgentsDB database - containing the Pilots status
@@ -48,7 +46,6 @@ CREATE TABLE `PilotAgents` (
   KEY `Statuskey` (`GridSite`,`DestinationSite`,`Status`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
-
 DROP TABLE IF EXISTS `JobToPilotMapping`;
 CREATE TABLE `JobToPilotMapping` (
   `PilotID` INT(11) UNSIGNED NOT NULL,
@@ -65,3 +62,77 @@ CREATE TABLE `PilotOutput` (
   `StdError` MEDIUMTEXT,
   PRIMARY KEY (`PilotID`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+-- ------------------------------------------------------------------------------
+-- summary table and triggers
+-- ------------------------------------------------------------------------------
+
+-- summary for PilotsHistory
+
+DROP TABLE IF EXISTS `PilotsHistorySummary`;
+CREATE TABLE `PilotsHistorySummary` (
+  `GridSite` VARCHAR(128),
+  `DestinationSite` VARCHAR(128),
+  `Status` VARCHAR(32),
+  `VO` VARCHAR(128),
+  `PilotCount` INT,
+  PRIMARY KEY (`GridSite`,`DestinationSite`,`Status`, `VO`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+-- now the triggers
+
+DELIMITER //
+
+CREATE TRIGGER trg_PilotAgents_insert
+AFTER INSERT ON PilotAgents
+FOR EACH ROW
+BEGIN
+  INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount)
+  VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+  ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+END;
+//
+
+CREATE TRIGGER trg_PilotAgents_delete
+AFTER DELETE ON PilotAgents
+FOR EACH ROW
+BEGIN
+  UPDATE PilotsHistorySummary
+  SET PilotCount = PilotCount - 1
+  WHERE GridSite = OLD.GridSite
+    AND DestinationSite = OLD.DestinationSite
+    AND Status = OLD.Status
+    AND VO = OLD.VO;
+
+  -- Optional cleanup (remove zero rows)
+  DELETE FROM PilotsHistorySummary WHERE PilotCount = 0;
+END;
+//
+
+CREATE TRIGGER trg_PilotAgents_update_status
+AFTER UPDATE ON PilotAgents
+FOR EACH ROW
+BEGIN
+  IF OLD.Status != NEW.Status THEN
+
+    -- Decrease count from old status
+    UPDATE PilotsHistorySummary
+    SET PilotCount = PilotCount - 1
+    WHERE GridSite = OLD.GridSite
+      AND DestinationSite = OLD.DestinationSite
+      AND Status = OLD.Status
+      AND VO = OLD.VO;
+
+    -- Delete row if count drops to zero
+    DELETE FROM PilotsHistorySummary WHERE PilotCount = 0;
+
+    -- Increase count for new status
+    INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount)
+    VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+    ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+
+  END IF;
+END;
+//