feat: use summary tables and triggers (PilotAgentsDB.PilotAgents, summaries for StatesAccounting)

Author: fstagni

src/DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -43,7 +43,7 @@ class StatesAccountingAgent(AgentModule):
     __renameFieldsMapping = {"JobType": "JobSplitType"}
 
     # PilotsHistory fields
-    __pilotsMapping = ["GridSite", "GridType", "Status", "NumOfPilots"]
+    __pilotsMapping = ["GridSite", "GridType", "Status", "VO", "NumOfPilots"]
 
     def initialize(self):
         """Standard initialization"""
@@ -85,7 +85,8 @@ def execute(self):
         # PilotsHistory to Monitoring
         if "Monitoring" in self.pilotMonitoringOption:
             self.log.info("Committing PilotsHistory to Monitoring")
-            result = PilotAgentsDB().getSummarySnapshot()
+            sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, GridType, Status, VO;"
+            result = PilotAgentsDB()._query(sql)
             now = datetime.datetime.utcnow()
             if not result["OK"]:
                 self.log.error(

src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py

@@ -1049,18 +1049,6 @@ def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems):
 
         return S_OK(resultDict)
 
-    def getSummarySnapshot(self, requestedFields=False):
-        """Get the summary snapshot for a given combination"""
-        if not requestedFields:
-            requestedFields = ["GridSite", "GridType", "Status"]
-        valueFields = ["COUNT(PilotID)"]
-        defString = ", ".join(requestedFields)
-        valueString = ", ".join(valueFields)
-        result = self._query(f"SELECT {defString}, {valueString} FROM PilotAgents GROUP BY {defString}")
-        if not result["OK"]:
-            return result
-        return S_OK(((requestedFields + valueFields), result["Value"]))
-
 
 class PivotedPilotSummaryTable:
     """

src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql

@@ -150,3 +150,83 @@ END;
 //
 
 DELIMITER ;
+
+
+-- summary for PilotsHistory
+
+DROP TABLE IF EXISTS `PilotsHistorySummary`;
+CREATE TABLE IF NOT EXISTS `PilotsHistorySummary` (
+  `GridSite` VARCHAR(128),
+  `GridType` VARCHAR(32),
+  `Status` VARCHAR(32),
+  `VO` VARCHAR(128),
+  `PilotCount` INT,
+  PRIMARY KEY (`GridSite`,`GridType`,`Status`, `VO`),
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+DELIMITER //
+
+CREATE TRIGGER trg_PilotAgents_insert
+AFTER INSERT ON PilotAgents
+FOR EACH ROW
+BEGIN
+  INSERT INTO PilotsHistorySummary (GridSite, GridType, Status, VO, PilotCount)
+  VALUES (NEW.GridSite, NEW.GridType, NEW.Status, NEW.VO, 1)
+  ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+END;
+//
+
+DELIMITER ;
+
+DELIMITER //
+
+CREATE TRIGGER trg_PilotAgents_delete
+AFTER DELETE ON PilotAgents
+FOR EACH ROW
+BEGIN
+  UPDATE PilotsHistorySummary
+  SET PilotCount = PilotCount - 1
+  WHERE GridSite = OLD.GridSite
+    AND GridType = OLD.GridType
+    AND Status = OLD.Status;
+    AND VO = OLD.VO;
+
+  -- Optional cleanup (remove zero rows)
+  DELETE FROM PilotsHistorySummary
+  WHERE PilotCount = 0;
+END;
+//
+
+DELIMITER ;
+
+DELIMITER //
+
+CREATE TRIGGER trg_PilotAgents_update_status
+AFTER UPDATE ON PilotAgents
+FOR EACH ROW
+BEGIN
+  IF OLD.Status != NEW.Status THEN
+
+    -- Decrease count from old status
+    UPDATE PilotsHistorySummary
+    SET PilotCount = PilotCount - 1
+    WHERE GridSite = OLD.GridSite
+      AND GridType = OLD.GridType
+      AND Status = OLD.Status;
+      AND VO = OLD.VO;
+
+    -- Delete row if count drops to zero
+    DELETE FROM PilotsHistorySummary
+    WHERE PilotCount = 0;
+
+    -- Increase count for new status
+    INSERT INTO PilotsHistorySummary (GridSite, GridType, Status, VO, PilotCount)
+    VALUES (NEW.GridSite, NEW.GridType, NEW.Status, NEW.VO 1)
+    ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
+
+  END IF;
+END;
+//
+
+DELIMITER ;