Merge pull request #5901 from TaykYoku/8.0_fixOAuth-5

[integration] move ProxyManager to HTTPs

Author: fstagni

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/MyProxyRenewalAgent/index.rst

@@ -39,4 +39,3 @@ Agents associated with Framework System:
    :maxdepth: 2
 
    CAUpdateAgent/index
-   MyProxyRenewalAgent/index

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/index.rst

@@ -17,10 +17,7 @@
     ("DIRAC.DataManagementSystem.Agent.FTS3Agent", {}),
     ("DIRAC.FrameworkSystem.Agent.CAUpdateAgent", {}),
     ("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent", {}),
-    (
-        "DIRAC.FrameworkSystem.Agent.MyProxyRenewalAgent",
-        {"IgnoreOptions": ["MinValidity", "ValidityPeriod", "MinimumLifeTime", "RenewedLifeTime"]},
-    ),
+    ("DIRAC.FrameworkSystem.Agent.ProxyRenewalAgent", {}),
     ("DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent", {}),
     (
         "DIRAC.RequestManagementSystem.Agent.RequestExecutingAgent",

src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py

@@ -1,4 +1,8 @@
-"""DIRAC server has various passive components listening to incoming client requests and reacting accordingly by serving requested information, such as **services** or **APIs**. This module is basic for each of these components and describes the basic concept of access to them.
+"""DIRAC server has various passive components listening to incoming client requests
+   and reacting accordingly by serving requested information,
+   such as **services** or **APIs**.
+
+   This module is basic for each of these components and describes the basic concept of access to them.
 """
 import time
 import inspect
@@ -145,7 +149,8 @@ def _runActions(self, reqObj):
 
 
 class BaseRequestHandler(RequestHandler):
-    """This class primarily describes the process of processing an incoming request and the methods of authentication and authorization.
+    """This class primarily describes the process of processing an incoming request
+    and the methods of authentication and authorization.
 
     Each HTTP request is served by a new instance of this class.
 
@@ -168,7 +173,8 @@ class BaseRequestHandler(RequestHandler):
         {TornadoService, TornadoREST} -> BaseRequestHandler;
         BaseRequestHandler -> RequestHandler [label="  inherit", fontsize=8];
 
-    In order to create a class that inherits from ``BaseRequestHandler``, first you need to determine what HTTP methods need to be supported.
+    In order to create a class that inherits from ``BaseRequestHandler``,
+    first you need to determine what HTTP methods need to be supported.
     Override the class variable ``SUPPORTED_METHODS`` by writing down the necessary methods there.
     Note that by default all HTTP methods are supported.
 
@@ -182,8 +188,10 @@ class BaseRequestHandler(RequestHandler):
         - ``auth_<method name>`` describes authorization rules for a single method and has higher priority than ``DEFAULT_AUTHORIZATION``
         - ``METHOD_PREFIX`` helps in finding the target method, see the :py:meth:`_getMethod` methods, where described how exactly.
 
-    It is worth noting that DIRAC supports several ways to authorize the request and they are all descriptive in ``DEFAULT_AUTHENTICATION``.
-    Authorization schema is associated with ``_authz<SHEMA SHORT NAME>`` method and will be applied alternately as they are defined in the variable until one of them is successfully executed.
+    It is worth noting that DIRAC supports several ways to authorize
+    the request and they are all descriptive in ``DEFAULT_AUTHENTICATION``.
+    Authorization schema is associated with ``_authz<SHEMA SHORT NAME>``
+    method and will be applied alternately as they are defined in the variable until one of them is successfully executed.
     If no authorization method completes successfully, access will be denied.
     The following authorization schemas are supported by default:
 
@@ -250,7 +258,8 @@ def _authzMYAUTH(self):
         - authentication request using one of the available algorithms called ``DEFAULT_AUTHENTICATION``, see :py:meth:`_gatherPeerCredentials` for more details.
         - and finally authorizing the request to access the component, see :py:meth:`authQuery <DIRAC.Core.DISET.AuthManager.AuthManager.authQuery>` for more details.
 
-    If all goes well, then a method is executed, the name of which coincides with the name of the request method (e.g.: :py:meth:`get`) which does:
+    If all goes well, then a method is executed,
+    the name of which coincides with the name of the request method (e.g.: :py:meth:`get`) which does:
 
         - execute the target method in an executor a separate thread.
         - defines the arguments of the target method, see :py:meth:`_getMethodArgs`.
@@ -279,7 +288,8 @@ def _authzMYAUTH(self):
     # Below are variables that the developer can OVERWRITE as needed
 
     # System name with which this component is associated.
-    # Developer can overwrite this if your handler is outside the DIRAC system package (src/DIRAC/XXXSystem/<path to your handler>)
+    # Developer can overwrite this
+    # if your handler is outside the DIRAC system package (src/DIRAC/XXXSystem/<path to your handler>)
     SYSTEM_NAME = None
     COMPONENT_NAME = None
 
@@ -295,7 +305,9 @@ def _authzMYAUTH(self):
     # Prefix of the target methods names if need to use a special prefix. By default its "export_".
     METHOD_PREFIX = "export_"
 
-    # What authorization type to use. This definition refers to the type of authentication, ie which algorithm will be used to verify the incoming request and obtain user credentials.
+    # What authorization type to use.
+    # This definition refers to the type of authentication,
+    # ie which algorithm will be used to verify the incoming request and obtain user credentials.
     # These algorithms will be applied in the same order as in the list.
     #  SSL - add to list to enable certificate reading
     #  JWT - add to list to enable reading Bearer token
@@ -697,7 +709,8 @@ def _gatherPeerCredentials(self, grants: list = None) -> dict:
           - reading Bearer token, see :py:meth`_authzJWT`.
           - authentication as visitor, that is, without verification, see :py:meth`_authzVISITOR`.
 
-        To add your own authentication type, create a `_authzYourGrantType` method that should return ``S_OK(dict)`` in case of successful authorization.
+        To add your own authentication type, create a `_authzYourGrantType` method that should return ``S_OK(dict)``
+        in case of successful authorization.
 
         :param grants: grants to use
 
@@ -724,7 +737,7 @@ def _gatherPeerCredentials(self, grants: list = None) -> dict:
         raise Exception("; ".join(err))
 
     def _authzSSL(self):
-        """Load client certchain in DIRAC and extract informations.
+        """Load client certchain in DIRAC and extract information.
 
         :return: S_OK(dict)/S_ERROR()
         """
@@ -755,6 +768,22 @@ def _authzSSL(self):
 
         credDict = res["Value"]
 
+        credDict["x509Chain"] = peerChain
+        res = peerChain.isProxy()
+        if not res["OK"]:
+            return res
+        credDict["isProxy"] = res["Value"]
+
+        if credDict["isProxy"]:
+            credDict["DN"] = credDict["identity"]
+        else:
+            credDict["DN"] = credDict["subject"]
+
+        res = peerChain.isLimitedProxy()
+        if not res["OK"]:
+            return res
+        credDict["isLimitedProxy"] = res["Value"]
+
         # We check if client sends extra credentials...
         if "extraCredentials" in self.request.arguments:
             extraCred = self.get_argument("extraCredentials")
@@ -763,7 +792,7 @@ def _authzSSL(self):
         return S_OK(credDict)
 
     def _authzJWT(self, accessToken=None):
-        """Load token claims in DIRAC and extract informations.
+        """Load token claims in DIRAC and extract information.
 
         :param str accessToken: access_token
 

src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py

@@ -0,0 +1,85 @@
+"""  ProxyRenewalAgent keeps the proxy repository clean.
+
+    .. literalinclude:: ../ConfigTemplate.cfg
+      :start-after: ##BEGIN ProxyRenewalAgent
+      :end-before: ##END
+      :dedent: 2
+      :caption: ProxyRenewalAgent options
+"""
+import concurrent.futures
+
+from DIRAC import S_OK
+
+from DIRAC.Core.Base.AgentModule import AgentModule
+from DIRAC.FrameworkSystem.DB.ProxyDB import ProxyDB
+
+DEFAULT_MAIL_FROM = "[email protected]"
+
+
+class ProxyRenewalAgent(AgentModule):
+    def initialize(self):
+
+        requiredLifeTime = self.am_getOption("MinimumLifeTime", 3600)
+        renewedLifeTime = self.am_getOption("RenewedLifeTime", 54000)
+        mailFrom = self.am_getOption("MailFrom", DEFAULT_MAIL_FROM)
+        self.useMyProxy = self.am_getOption("UseMyProxy", False)
+        self.proxyDB = ProxyDB(useMyProxy=self.useMyProxy, mailFrom=mailFrom)
+
+        self.log.info(f"Minimum Life time      : {requiredLifeTime}")
+        self.log.info(f"Life time on renew     : {renewedLifeTime}")
+        if self.useMyProxy:
+            self.log.info(f"MyProxy server         : {self.proxyDB.getMyProxyServer()}")
+            self.log.info(f"MyProxy max proxy time : {self.proxyDB.getMyProxyMaxLifeTime()}")
+
+        return S_OK()
+
+    def __renewProxyForCredentials(self, userDN, userGroup):
+        lifeTime = self.am_getOption("RenewedLifeTime", 54000)
+        self.log.info(f"Renewing for {userDN}@{userGroup} {lifeTime} secs")
+        res = self.proxyDB.renewFromMyProxy(userDN, userGroup, lifeTime=lifeTime)
+        if not res["OK"]:
+            self.log.error("Failed to renew proxy", f"for {userDN}@{userGroup} : {res['Message']}")
+        else:
+            self.log.info(f"Renewed proxy for {userDN}@{userGroup}")
+
+    def execute(self):
+        """The main agent execution method"""
+        self.log.verbose("Purging expired requests")
+        res = self.proxyDB.purgeExpiredRequests()
+        if not res["OK"]:
+            self.log.error(res["Message"])
+        else:
+            self.log.info(f"Purged {res['Value']} requests")
+
+        self.log.verbose("Purging expired tokens")
+        res = self.proxyDB.purgeExpiredTokens()
+        if not res["OK"]:
+            self.log.error(res["Message"])
+        else:
+            self.log.info(f"Purged {res['Value']} tokens")
+
+        self.log.verbose("Purging expired proxies")
+        res = self.proxyDB.purgeExpiredProxies()
+        if not res["OK"]:
+            self.log.error(res["Message"])
+        else:
+            self.log.info(f"Purged {res['Value']} proxies")
+
+        self.log.verbose("Purging logs")
+        res = self.proxyDB.purgeLogs()
+        if not res["OK"]:
+            self.log.error(res["Message"])
+
+        if self.useMyProxy:
+            res = self.proxyDB.getCredentialsAboutToExpire(self.am_getOption("MinimumLifeTime", 3600))
+            if not res["OK"]:
+                return res
+            data = res["Value"]
+            self.log.info(f"Renewing {len(data)} proxies...")
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                futures = []
+                for record in data:
+                    userDN = record[0]
+                    userGroup = record[1]
+                    futures.append(executor.submit(self.__renewProxyForCredentials, userDN, userGroup))
+            return S_OK()