refactor: moved RPC calls to WebAppHandler

Author: fstagni

src/DIRAC/MonitoringSystem/Service/WebAppHandler.py

@@ -333,6 +333,74 @@ def export_getSiteSummarySelectors(cls):
 
         return S_OK(resultDict)
 
+    types_getApplicationStates = []
+
+    @classmethod
+    def export_getApplicationStates(cls, condDict=None, older=None, newer=None):
+        """Return Distinct Values of ApplicationStatus job Attribute in WMS"""
+        return cls.jobDB.getDistinctJobAttributes("ApplicationStatus", condDict, older, newer)
+
+    types_getJobTypes = []
+
+    @classmethod
+    def export_getJobTypes(cls, condDict=None, older=None, newer=None):
+        """Return Distinct Values of JobType job Attribute in WMS"""
+        return cls.jobDB.getDistinctJobAttributes("JobType", condDict, older, newer)
+
+    types_getOwners = []
+
+    @classmethod
+    def export_getOwners(cls, condDict=None, older=None, newer=None):
+        """
+        Return Distinct Values of Owner job Attribute in WMS
+        """
+        return cls.jobDB.getDistinctJobAttributes("Owner", condDict, older, newer)
+
+    types_getOwnerGroup = []
+
+    @classmethod
+    def export_getOwnerGroup(cls):
+        """
+        Return Distinct Values of OwnerGroup from the JobDB
+        """
+        return cls.jobDB.getDistinctJobAttributes("OwnerGroup")
+
+    types_getJobGroups = []
+
+    @classmethod
+    def export_getJobGroups(cls, condDict=None, older=None, cutDate=None):
+        """
+        Return Distinct Values of ProductionId job Attribute in WMS
+        """
+        return cls.jobDB.getDistinctJobAttributes("JobGroup", condDict, older, newer=cutDate)
+
+    types_getSites = []
+
+    @classmethod
+    def export_getSites(cls, condDict=None, older=None, newer=None):
+        """
+        Return Distinct Values of Site job Attribute in WMS
+        """
+        return cls.jobDB.getDistinctJobAttributes("Site", condDict, older, newer)
+
+    types_getStates = []
+
+    @classmethod
+    def export_getStates(cls, condDict=None, older=None, newer=None):
+        """
+        Return Distinct Values of Status job Attribute in WMS
+        """
+        return cls.jobDB.getDistinctJobAttributes("Status", condDict, older, newer)
+
+    types_getMinorStates = []
+
+    @classmethod
+    def export_getMinorStates(cls, condDict=None, older=None, newer=None):
+        """
+        Return Distinct Values of Minor Status job Attribute in WMS
+        """
+        return cls.jobDB.getDistinctJobAttributes("MinorStatus", condDict, older, newer)
+
     ##############################################################################
     # Transformations
     ##############################################################################

src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -16,14 +16,12 @@
 
 # # from DIRAC
 from DIRAC import S_ERROR, S_OK
-from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities.DErrno import cmpError
 from DIRAC.Core.Utilities.List import breakListIntoChunks
 from DIRAC.Core.Utilities.Proxy import executeWithUserProxy
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
-from DIRAC.DataManagementSystem.Client.DataManager import DataManager
 from DIRAC.RequestManagementSystem.Client.File import File
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
@@ -36,6 +34,7 @@
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
+from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 
 # # agent's name
 AGENT_NAME = "Transformation/TransformationCleaningAgent"
@@ -65,6 +64,8 @@ def __init__(self, *args, **kwargs):
         self.reqClient = None
         # # file catalog client
         self.metadataClient = None
+        # # JobDB
+        self.jobDB = None
 
         # # transformations types
         self.transformationTypes = None
@@ -127,6 +128,8 @@ def initialize(self):
         self.metadataClient = FileCatalogClient()
         # # job monitoring client
         self.jobMonitoringClient = JobMonitoringClient()
+        # # job DB
+        self.jobDB = JobDB()
 
         return S_OK()
 
@@ -224,7 +227,7 @@ def finalize(self):
         So, we should just clean from time to time.
         What I added here is done only when the agent finalize, and it's quite light-ish operation anyway.
         """
-        res = self.jobMonitoringClient.getJobGroups(None, datetime.utcnow() - timedelta(days=365))
+        res = self.jobDB.getDistinctJobAttributes("JobGroup", None, datetime.utcnow() - timedelta(days=365))
         if not res["OK"]:
             self.log.error("Failed to get job groups", res["Message"])
             return res

src/DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py

@@ -50,82 +50,6 @@ def getJobsAttributes(cls, *args, **kwargs):
             return res
         return S_OK(strToIntDict(res["Value"]))
 
-    ##############################################################################
-    types_getApplicationStates = []
-
-    @classmethod
-    def export_getApplicationStates(cls, condDict=None, older=None, newer=None):
-        """Return Distinct Values of ApplicationStatus job Attribute in WMS"""
-        return cls.jobDB.getDistinctJobAttributes("ApplicationStatus", condDict, older, newer)
-
-    ##############################################################################
-    types_getJobTypes = []
-
-    @classmethod
-    def export_getJobTypes(cls, condDict=None, older=None, newer=None):
-        """Return Distinct Values of JobType job Attribute in WMS"""
-        return cls.jobDB.getDistinctJobAttributes("JobType", condDict, older, newer)
-
-    ##############################################################################
-    types_getOwners = []
-
-    @classmethod
-    def export_getOwners(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Owner job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Owner", condDict, older, newer)
-
-    ##############################################################################
-    types_getOwnerGroup = []
-
-    @classmethod
-    def export_getOwnerGroup(cls):
-        """
-        Return Distinct Values of OwnerGroup from the JobDB
-        """
-        return cls.jobDB.getDistinctJobAttributes("OwnerGroup")
-
-    ##############################################################################
-    types_getJobGroups = []
-
-    @classmethod
-    def export_getJobGroups(cls, condDict=None, older=None, cutDate=None):
-        """
-        Return Distinct Values of ProductionId job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("JobGroup", condDict, older, newer=cutDate)
-
-    ##############################################################################
-    types_getSites = []
-
-    @classmethod
-    def export_getSites(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Site job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Site", condDict, older, newer)
-
-    ##############################################################################
-    types_getStates = []
-
-    @classmethod
-    def export_getStates(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Status job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Status", condDict, older, newer)
-
-    ##############################################################################
-    types_getMinorStates = []
-
-    @classmethod
-    def export_getMinorStates(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Minor Status job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("MinorStatus", condDict, older, newer)
-
     ##############################################################################
     types_getJobs = []
 

tests/Integration/FutureClient/WorkloadManagement/Test_JobMonitoring.py

@@ -12,12 +12,6 @@
 TEST_JOB_IDS = [TEST_JOBS] + TEST_JOBS + [str(x) for x in TEST_JOBS]
 
 
-def test_getApplicationStates():
-    # JobMonitoringClient().getApplicationStates(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getApplicationStates
-    pytest.skip()
-
-
 def test_getCounters():
     # JobMonitoringClient().getCounters(attrList: list, attrDict = None, cutDate = )
     method = JobMonitoringClient().getCounters
@@ -42,12 +36,6 @@ def test_getJobAttributes():
     pytest.skip()
 
 
-def test_getJobGroups():
-    # JobMonitoringClient().getJobGroups(condDict = None, older = None, cutDate = None)
-    method = JobMonitoringClient().getJobGroups
-    pytest.skip()
-
-
 def test_getJobHeartBeatData():
     # JobMonitoringClient().getJobHeartBeatData(jobID: int)
     method = JobMonitoringClient().getJobHeartBeatData
@@ -90,12 +78,6 @@ def test_getJobSummary():
     pytest.skip()
 
 
-def test_getJobTypes():
-    # JobMonitoringClient().getJobTypes(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getJobTypes
-    pytest.skip()
-
-
 def test_getJobs():
     # JobMonitoringClient().getJobs(attrDict = None, cutDate = None)
     method = JobMonitoringClient().getJobs
@@ -141,33 +123,3 @@ def test_getJobsSummary():
     # JobMonitoringClient().getJobsSummary(jobIDs: list)
     method = JobMonitoringClient().getJobsSummary
     pytest.skip()
-
-
-def test_getMinorStates():
-    # JobMonitoringClient().getMinorStates(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getMinorStates
-    pytest.skip()
-
-
-def test_getOwnerGroup():
-    # JobMonitoringClient().getOwnerGroup()
-    method = JobMonitoringClient().getOwnerGroup
-    pytest.skip()
-
-
-def test_getOwners():
-    # JobMonitoringClient().getOwners(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getOwners
-    pytest.skip()
-
-
-def test_getSites():
-    # JobMonitoringClient().getSites(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getSites
-    pytest.skip()
-
-
-def test_getStates():
-    # JobMonitoringClient().getStates(condDict = None, older = None, newer = None)
-    method = JobMonitoringClient().getStates
-    pytest.skip()

tests/Integration/Monitoring/Test_WebAppClient.py

@@ -1,5 +1,7 @@
 # pylint: disable=invalid-name, missing-docstring
 
+import datetime
+
 import DIRAC
 
 DIRAC.initialize()  # Initialize configuration
@@ -8,10 +10,45 @@
 from DIRAC.MonitoringSystem.Client.WebAppClient import WebAppClient
 
 
-def test_WMSAdministratorClient():
+def test_WebAppClient():
     res = WebAppClient().getSiteSummaryWeb({}, [], 0, 100)
     assert res["OK"], res["Message"]
     assert res["Value"]["TotalRecords"] in [0, 1, 2, 34]
 
     res = WebAppClient().getSiteSummarySelectors()
     assert res["OK"], res["Message"]
+
+    res = WebAppClient().getApplicationStates()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getJobTypes()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getOwners()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getOwnerGroup()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getJobGroups()
+    assert res["OK"], res["Message"]
+    resJG_empty = res["Value"]
+
+    res = WebAppClient().getJobGroups(None, datetime.datetime.utcnow())
+    assert res["OK"], res["Message"]
+    resJG_olderThanNow = res["Value"]
+    assert resJG_empty == resJG_olderThanNow
+
+    res = WebAppClient().getJobGroups(None, datetime.datetime.utcnow() - datetime.timedelta(days=365))
+    assert res["OK"], res["Message"]
+    resJG_olderThanOneYear = res["Value"]
+    assert set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow))
+
+    res = WebAppClient().getSites()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getStates()
+    assert res["OK"], res["Message"]
+
+    res = WebAppClient().getMinorStates()
+    assert res["OK"], res["Message"]

tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py

@@ -387,38 +387,6 @@ def test_JobStateUpdateAndJobMonitoringMultiple(lfn: str) -> None:
             jobIDs.append(jobID)
 
     try:
-        res = jobMonitoringClient.getSites()
-        assert res["OK"], res["Message"]
-        assert set(res["Value"]) <= {"ANY", "DIRAC.Jenkins.ch", "Site"}
-
-        res = jobMonitoringClient.getJobTypes()
-        assert res["OK"], res["Message"]
-
-        res = jobMonitoringClient.getApplicationStates()
-        assert res["OK"], res["Message"]
-        assert "Unknown" in res["Value"]
-
-        res = jobMonitoringClient.getOwners()
-        assert res["OK"], res["Message"]
-        res = jobMonitoringClient.getOwnerGroup()
-        assert res["OK"], res["Message"]
-        res = jobMonitoringClient.getJobGroups()
-        assert res["OK"], res["Message"]
-        resJG_empty = res["Value"]
-        res = jobMonitoringClient.getJobGroups(None, datetime.datetime.utcnow())
-        assert res["OK"], res["Message"]
-        resJG_olderThanNow = res["Value"]
-        assert resJG_empty == resJG_olderThanNow
-        res = jobMonitoringClient.getJobGroups(None, datetime.datetime.utcnow() - datetime.timedelta(days=365))
-        assert res["OK"], res["Message"]
-        resJG_olderThanOneYear = res["Value"]
-        assert set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow))
-
-        res = jobMonitoringClient.getStates()
-        assert res["OK"], res["Message"]
-        res = jobMonitoringClient.getMinorStates()
-        assert res["OK"], res["Message"]
-
         res = jobMonitoringClient.getJobs()
         assert res["OK"], res["Message"]
         assert {str(x) for x in jobIDs} <= set(res["Value"])
@@ -445,7 +413,6 @@ def test_JobStateUpdateAndJobMonitoringMultiple(lfn: str) -> None:
         res = jobMonitoringClient.getJobSummary(int(jobID))
         assert res["OK"], res["Message"]
         assert res["Value"]["Status"] in (JobStatus.CHECKING, JobStatus.WAITING)
-        assert res["Value"]["MinorStatus"] == "MinorStatus"
 
         res = jobStateUpdateClient.setJobStatusBulk(
             jobID,