fix: restored use of GridType

Author: fstagni

src/DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -45,7 +45,7 @@ class StatesAccountingAgent(AgentModule):
     __renameFieldsMapping = {"JobType": "JobSplitType"}
 
     # PilotsHistory fields
-    __pilotsMapping = ["GridSite", "GridType", "Status", "VO", "NumOfPilots"]
+    __pilotsMapping = ["GridSite", "ComputingElement", "GridType", "Status", "VO", "NumOfPilots"]
 
     def initialize(self):
         """Standard initialization"""
@@ -89,7 +89,7 @@ def execute(self):
         # PilotsHistory to Monitoring
         if "Monitoring" in self.pilotMonitoringOption:
             self.log.info("Committing PilotsHistory to Monitoring")
-            sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, DestinationSite, Status, VO;"
+            sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, ComputingElement, GridType, Status, VO;"
             result = PilotAgentsDB()._query(sql)
             now = datetime.datetime.utcnow()
             if not result["OK"]:

src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql

@@ -74,10 +74,11 @@ DROP TABLE IF EXISTS `PilotsHistorySummary`;
 CREATE TABLE `PilotsHistorySummary` (
   `GridSite` VARCHAR(128),
   `ComputingElement` VARCHAR(128),
+  `GridType` VARCHAR(128),
   `Status` VARCHAR(32),
   `VO` VARCHAR(64),
   `PilotCount` INT,
-  PRIMARY KEY (`GridSite`,`ComputingElement`,`Status`, `VO`)
+  PRIMARY KEY (`GridSite`,`ComputingElement`,`GridType`,`Status`, `VO`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 
@@ -89,8 +90,8 @@ CREATE TRIGGER trg_PilotAgents_insert
 AFTER INSERT ON PilotAgents
 FOR EACH ROW
 BEGIN
-  INSERT INTO PilotsHistorySummary (GridSite, ComputingElement, Status, VO, PilotCount)
-  VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+  INSERT INTO PilotsHistorySummary (GridSite, ComputingElement, GridType, Status, VO, PilotCount)
+  VALUES (NEW.GridSite, NEW.DestinationSite, NEW.GridType, NEW.Status, NEW.VO, 1)
   ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
 END;
 //
@@ -103,6 +104,7 @@ BEGIN
   SET PilotCount = PilotCount - 1
   WHERE GridSite = OLD.GridSite
     AND ComputingElement = OLD.DestinationSite
+    AND GridType = OLD.GridType
     AND Status = OLD.Status
     AND VO = OLD.VO;
 
@@ -111,6 +113,7 @@ BEGIN
   WHERE PilotCount = 0
     AND GridSite = OLD.GridSite
     AND ComputingElement = OLD.DestinationSite
+    AND GridType = OLD.GridType
     AND Status = OLD.Status
     AND VO = OLD.VO;
 END;
@@ -127,15 +130,16 @@ BEGIN
     SET PilotCount = PilotCount - 1
     WHERE GridSite = OLD.GridSite
       AND ComputingElement = OLD.DestinationSite
+      AND GridType = OLD.GridType
       AND Status = OLD.Status
       AND VO = OLD.VO;
 
     -- Delete row if count drops to zero
     DELETE FROM PilotsHistorySummary WHERE PilotCount = 0;
 
     -- Increase count for new status
-    INSERT INTO PilotsHistorySummary (GridSite, ComputingElement, Status, VO, PilotCount)
-    VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1)
+    INSERT INTO PilotsHistorySummary (GridSite, ComputingElement, GridType, Status, VO, PilotCount)
+    VALUES (NEW.GridSite, NEW.DestinationSite, NEW.GridType, NEW.Status, NEW.VO, 1)
     ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1;
 
   END IF;

tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py

@@ -249,16 +249,16 @@ def test_summarySnapshot():
         sql = f"INSERT INTO PilotAgents (InitialJobID, CurrentJobID, PilotJobReference, PilotStamp, DestinationSite, Queue, GridSite, VO, GridType, BenchMark, SubmissionTime, LastUpdateTime, Status, StatusReason, AccountingSent) VALUES ({placeholders})"
         res = paDB._updatemany(sql, processed_data)
         assert res["OK"], res["Message"]
-    sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, DestinationSite, Status, VO;"
+    sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, ComputingElement, GridType, Status, VO;"
     result = PilotAgentsDB()._query(sql)
     assert result["OK"], result["Message"]
     values = result["Value"][1]
-    assert len(values) == 5, "Expected 5 record in the summary"
+    assert len(values) == 6, "Expected 6 record in the summary"
     # Check it corresponds to the basic "GROUP BY" query
-    sql = "SELECT GridSite, DestinationSite, Status, VO, COUNT(*) FROM PilotAgents GROUP BY GridSite, DestinationSite, Status, VO ORDER BY GridSite, DestinationSite, Status, VO;"
+    sql = "SELECT GridSite, DestinationSite, GridType, Status, VO, COUNT(*) FROM PilotAgents GROUP BY GridSite, DestinationSite, GridType, Status, VO ORDER BY GridSite, DestinationSite, GridType, Status, VO;"
     result_grouped = PilotAgentsDB()._query(sql)
     assert result_grouped["OK"], result_grouped["Message"]
-    sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, DestinationSite, Status, VO;"
+    sql = "SELECT * FROM PilotsHistorySummary ORDER BY GridSite, ComputingElement, GridType, Status, VO;"
     result_summary = PilotAgentsDB()._query(sql)
     assert result_summary["OK"], result_summary["Message"]
     assert result_grouped["Value"] == result_summary["Value"], "Summary and grouped query results differ"
@@ -274,10 +274,10 @@ def test_summarySnapshot():
         res = paDB._update(sql)
         assert res["OK"], res["Message"]
     # Check it corresponds to the basic "GROUP BY" query
-    sql = "SELECT GridSite, DestinationSite, Status, VO, COUNT(*) FROM PilotAgents GROUP BY GridSite, DestinationSite, Status, VO ORDER BY GridSite, DestinationSite, Status, VO;"
+    sql = "SELECT GridSite, DestinationSite, GridType, Status, VO, COUNT(*) FROM PilotAgents GROUP BY GridSite, DestinationSite, GridType, Status, VO ORDER BY GridSite, DestinationSite, GridType, Status, VO;"
     result_grouped = PilotAgentsDB()._query(sql)
     assert result_grouped["OK"], result_grouped["Message"]
-    sql = "select * FROM PilotsHistorySummary ORDER BY GridSite, DestinationSite, Status, VO;"
+    sql = "select * FROM PilotsHistorySummary ORDER BY GridSite, ComputingElement, GridType, Status, VO;"
     result_summary = PilotAgentsDB()._query(sql)
     assert result_summary["OK"], result_summary["Message"]
     assert result_grouped["Value"] == result_summary["Value"], "Summary and grouped query results differ"