refactor: increase productivity, improve parsing of annotations, docs

fix: implement minor fixes

Author: TaykYoku

docs/source/DeveloperGuide/APIs/index.rst

@@ -41,18 +41,8 @@ You need to choose the system for which you want to write an API, then create a
 The example described is likely to be sufficient for most writing cases. But here are some additional features, see :py:class:`~DIRAC.Core.Tornado.Server.private.BaseRequestHandler`:
 
   - ``USE_AUTHZ_GRANTS`` set the list and order of steps to authorize the request. For example, set ``USE_AUTHZ_GRANTS = ["JWT"]`` to allow access to your endpoint only with a valid access token.
-  - ``AUTH_PROPS`` set the authorization requirements. For example, ``AUTH_PROPS = ['authenticated']`` will allow access only to authenticated users.
+  - ``DEFAULT_AUTHORIZATION`` set the authorization requirements. For example, ``DEFAULT_AUTHORIZATION = ['authenticated']`` will allow access only to authenticated users.
   - in addition to standard S_OK/S_ERROR you can return text, whether the dictionary for example or nothing, the result will be sent with a 200 status.
-  - ``path_<my_method>`` will allow you to consider the path parameters as positional arguments of the target method. For example:
-
-    .. code-block:: python
-
-        # It's allow make request like a GET /user/contacts/Bob
-        path_user = ["(contacts|IDs)", "([A-z%0-9-_]+)"]
-
-        def web_user(self, option:str, name:str):
-            return Registry.getUserOption(name, option)
-
   - If your API is complex enough and may include, for example, redirection or additional headers, you can use :py:class:`~DIRAC.Core.Tornado.Server.private.BaseRequestHandler.TornadoResponse`
     to add all these necessary things, which is thread-safe because TornadoResponse will call your actions outside the thread in which this method is executed:
 

docs/source/_static/Systems/Core/BaseRequestHandler.png

@@ -9,32 +9,11 @@
 
 __RCSID__ = "$Id$"
 
-import inspect
 from tornado.web import url as TornadoURL, RequestHandler
 
 from DIRAC import gConfig, gLogger, S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client import PathFinder
 from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader
-from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
-
-
-def urlFinder(module):
-    """
-    Tries to guess the url from module name.
-    The URL would be of the form ``/System/Component`` (e.g. ``DataManagement/FileCatalog``)
-    We search something which looks like ``<...>.<component>System.<...>.<service>Handler``
-
-    :param module: full module name (e.g. "DIRAC.something.something")
-
-    :returns: the deduced URL or None
-    """
-    sections = module.split(".")
-    for section in sections:
-        # This condition is a bit long
-        # We search something which look like <...>.<component>System.<...>.<service>Handler
-        # If find we return /<component>/<service>
-        if section.endswith("System") and sections[-1].endswith("Handler"):
-            return "/".join(["", section[: -len("System")], sections[-1][: -len("Handler")]])
 
 
 class HandlerManager(object):
@@ -46,7 +25,7 @@ class HandlerManager(object):
 
     Each of the Handler will have one associated route to it:
 
-    * Directly specified as ``LOCATION`` in the handler module
+    * Directly specified as ``DEFAULT_LOCATION`` in the handler module
     * automatically deduced from the module name, of the form
       ``System/Component`` (e.g. ``DataManagement/FileCatalog``)
     """
@@ -64,59 +43,8 @@ def __init__(self, services, endpoints):
             If ``True``, loads all endpoints from CS
         :type endpoints: bool or list
         """
-        self.loader = None
         self.__handlers = {}
-        self.__services = services
-        self.__endpoints = endpoints
-        self.__objectLoader = ObjectLoader()
-
-    def __addHandler(self, handlerPath, handler, urls=None, port=None):
-        """
-        Function which add handler to list of known handlers
-
-        :param str handlerPath: module name, e.g.: `Framework/Auth`
-        :param object handler: handler class
-        :param list urls: request path
-        :param int port: port
-
-        :return: S_OK()/S_ERROR()
-        """
-        # First of all check if we can find route
-        # If urls is not given, try to discover it
-        if urls is None:
-            # FIRST TRY: Url is hardcoded
-            try:
-                urls = handler.LOCATION
-            # SECOND TRY: URL can be deduced from path
-            except AttributeError:
-                gLogger.debug("No location defined for %s try to get it from path" % handlerPath)
-                urls = urlFinder(handlerPath)
-
-        if not urls:
-            gLogger.warn("URL not found for %s" % (handlerPath))
-            return S_ERROR("URL not found for %s" % (handlerPath))
-
-        for url in urls if isinstance(urls, (list, tuple)) else [urls]:
-            # We add "/" if missing at begin, e.g. we found "Framework/Service"
-            # URL can't be relative in Tornado
-            if url and not url.startswith("/"):
-                url = "/%s" % url
-
-            # Some new handler
-            if handlerPath not in self.__handlers:
-                gLogger.debug("Add new handler %s with port %s" % (handlerPath, port))
-                self.__handlers[handlerPath] = {"URLs": [], "Port": port}
-
-            # Check if URL already loaded
-            if (url, handler) in self.__handlers[handlerPath]["URLs"]:
-                gLogger.debug("URL: %s already loaded for %s " % (url, handlerPath))
-                continue
-
-            # Finally add the URL to handlers
-            gLogger.info("Add new URL %s to %s handler" % (url, handlerPath))
-            self.__handlers[handlerPath]["URLs"].append((url, handler))
-
-        return S_OK()
+        self.instances = dict(Service=services, API=endpoints)
 
     def discoverHandlers(self, handlerInstance):
         """
@@ -160,130 +88,91 @@ def discoverHandlers(self, handlerInstance):
                     pass
         return urls
 
-    def loadServicesHandlers(self, services=None):
-        """
-        Load a list of handler from list of service using DIRAC moduleLoader
+    def __load(self, instances, componentType, pathFinder):
+        """Load a list of handler from list of given instances using DIRAC moduleLoader
         Use :py:class:`DIRAC.Core.Base.private.ModuleLoader`
 
+        :return: S_OK()/S_ERROR()
+        """
+        # list of instances, e.g. ['Framework/Hello', 'Configuration/Server']
+        if isinstance(instances, str):
+            # make sure we have a list
+            instances = [instances]
+
+        instances = self.instances[componentType] if instances is None else instances if instances else []
+
+        # `True` means automatically view the configuration
+        if instances is True:
+            instances = self.discoverHandlers(f"{componentType}s")
+        if not instances:
+            return S_OK()
+
+        # Extract ports
+        ports, instances = self.__extractPorts(instances)
+
+        loader = ModuleLoader(componentType, pathFinder, RequestHandler, moduleSuffix="Handler")
+
+        # Use DIRAC system to load: search in CS if path is given and if not defined
+        # it search in place it should be (e.g. in DIRAC/FrameworkSystem/< component type >)
+        result = loader.loadModules(instances)
+        if result["OK"]:
+            for module in loader.getModules().values():
+                handler = module["classObj"]
+                fullComponentName = module["modName"]
+
+                # Define the system and component name as the attributes of the handler that belongs to them
+                handler.SYSTEM_NAME, handler.COMPONENT_NAME = fullComponentName.split("/")
+
+                gLogger.info(f"Found new handler {fullComponentName}")
+
+                # at this stage we run the basic handler initialization
+                # see DIRAC.Core.Tornado.Server.private.BaseRequestHandler for more details
+                # this method should return a list of routes associated with the handler, it is a regular expressions
+                # see https://www.tornadoweb.org/en/stable/routing.html#tornado.routing.URLSpec, ``pattern`` argument.
+                urls = handler._BaseRequestHandler__pre_initialize()
+
+                # First of all check if we can find route
+                if not urls:
+                    gLogger.warn(f"URL not found for {fullComponentName}")
+                    return S_ERROR(f"URL not found for {fullComponentName}")
+
+                # Add new handler routes
+                self.__handlers[fullComponentName] = dict(URLs=list(set(urls)), Port=ports.get(fullComponentName))
+
+        return result
+
+    def loadServicesHandlers(self, services=None):
+        """Load services
+
         :param services: List of service handlers to load. Default value set at initialization
             If ``True``, loads all services from CS
         :type services: bool or list
 
         :return: S_OK()/S_ERROR()
         """
-        # list of services, e.g. ['Framework/Hello', 'Configuration/Server']
-        if isinstance(services, str):
-            services = [services]
-        # list of services
-        self.__services = self.__services if services is None else services if services else []
-
-        if self.__services is True:
-            self.__services = self.discoverHandlers("Services")
-
-        if self.__services:
-            # Extract ports
-            ports, self.__services = self.__extractPorts(self.__services)
-
-            self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler")
-
-            # Use DIRAC system to load: search in CS if path is given and if not defined
-            # it search in place it should be (e.g. in DIRAC/FrameworkSystem/Service)
-            load = self.loader.loadModules(self.__services)
-            if not load["OK"]:
-                return load
-            for module in self.loader.getModules().values():
-                url = module["loadName"]
-
-                # URL can be like https://domain:port/service/name or just service/name
-                # Here we just want the service name, for tornado
-                serviceTuple = url.replace("https://", "").split("/")[-2:]
-                url = "%s/%s" % (serviceTuple[0], serviceTuple[1])
-                self.__addHandler(module["loadName"], module["classObj"], url, ports.get(module["modName"]))
-        return S_OK()
-
-    def __extractPorts(self, serviceURIs):
-        """Extract ports from serviceURIs
-
-        :param list serviceURIs: list of uri that can contain port, .e.g:: System/Service:port
-
-        :return: (dict, list)
-        """
-        portMapping = {}
-        newURLs = []
-        for _url in serviceURIs:
-            if ":" in _url:
-                urlTuple = _url.split(":")
-                if urlTuple[0] not in portMapping:
-                    portMapping[urlTuple[0]] = urlTuple[1]
-                newURLs.append(urlTuple[0])
-            else:
-                newURLs.append(_url)
-        return (portMapping, newURLs)
+        return self.__load(services, "Service", PathFinder.getServiceSection)
 
     def loadEndpointsHandlers(self, endpoints=None):
-        """
-        Load a list of handler from list of endpoints using DIRAC moduleLoader
-        Use :py:class:`DIRAC.Core.Base.private.ModuleLoader`
+        """Load endpoints
 
         :param endpoints: List of endpoint handlers to load. Default value set at initialization
             If ``True``, loads all endpoints from CS
         :type endpoints: bool or list
 
         :return: S_OK()/S_ERROR()
         """
-        # list of endpoints, e.g. ['Framework/Auth', ...]
-        if isinstance(endpoints, str):
-            endpoints = [endpoints]
-        # list of endpoints. If __endpoints is ``True`` then list of endpoints will dicover from CS
-        self.__endpoints = self.__endpoints if endpoints is None else endpoints if endpoints else []
-
-        if self.__endpoints is True:
-            self.__endpoints = self.discoverHandlers("APIs")
-
-        if self.__endpoints:
-            # Extract ports
-            ports, self.__endpoints = self.__extractPorts(self.__endpoints)
-
-            self.loader = ModuleLoader("API", PathFinder.getAPISection, RequestHandler, moduleSuffix="Handler")
-
-            # Use DIRAC system to load: search in CS if path is given and if not defined
-            # it search in place it should be (e.g. in DIRAC/FrameworkSystem/API)
-            load = self.loader.loadModules(self.__endpoints)
-            if not load["OK"]:
-                return load
-            for module in self.loader.getModules().values():
-                handler = module["classObj"]
-                if not handler.LOCATION:
-                    handler.LOCATION = urlFinder(module["loadName"])
-                urls = []
-                # Look for methods that are exported
-                for mName, mObj in inspect.getmembers(handler):
-                    if inspect.isroutine(mObj) and mName.find(handler.METHOD_PREFIX) == 0:
-                        methodName = mName[len(handler.METHOD_PREFIX) :]
-                        args = getattr(handler, "path_%s" % methodName, [])
-                        gLogger.debug(
-                            " - Route %s/%s ->  %s %s" % (handler.LOCATION, methodName, module["loadName"], mName)
-                        )
-                        url = "%s%s" % (handler.LOCATION, "" if methodName == "index" else ("/%s" % methodName))
-                        if args:
-                            url += r"[\/]?%s" % "/".join(args)
-                        urls.append(url)
-                        gLogger.debug("  * %s" % url)
-                self.__addHandler(module["loadName"], handler, urls, ports.get(module["modName"]))
-        return S_OK()
+        return self.__load(endpoints, "API", PathFinder.getAPISection)
 
     def getHandlersURLs(self):
         """
         Get all handler for usage in Tornado, as a list of tornado.web.url
-        If there is no handler found before, it try to find them
 
         :returns: a list of URL (not the string with "https://..." but the tornado object)
                   see http://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec
         """
         urls = []
-        for handlerData in self.__handlers.values():
-            for url in handlerData["URLs"]:
-                urls.append(TornadoURL(*url))
+        for handler in self.__handlers:
+            urls += self.__handlers[handler]["URLs"]
         return urls
 
     def getHandlersDict(self):
@@ -295,3 +184,22 @@ def getHandlersDict(self):
                   and port as value for 'Port' key
         """
         return self.__handlers
+
+    def __extractPorts(self, serviceURIs: list) -> tuple:
+        """Extract ports from serviceURIs
+
+        :param list serviceURIs: list of uri that can contain port, .e.g:: System/Service:port
+
+        :return: (dict, list)
+        """
+        portMapping = {}
+        newURLs = []
+        for _url in serviceURIs:
+            if ":" in _url:
+                urlTuple = _url.split(":")
+                if urlTuple[0] not in portMapping:
+                    portMapping[urlTuple[0]] = urlTuple[1]
+                newURLs.append(urlTuple[0])
+            else:
+                newURLs.append(_url)
+        return (portMapping, newURLs)