Merge branch 'main' into add-metakg-endpoint

Author: NikkiBytes

src/handlers/api.py

@@ -17,6 +17,7 @@
 from controller.exceptions import ControllerError, NotFoundError
 from pipeline import MetaKGQueryPipeline
 from utils.downloader import DownloadError, download_async
+from utils.http_error import SmartAPIHTTPError
 from utils.metakg.biolink_helpers import get_expanded_values
 from utils.metakg.cytoscape_formatter import CytoscapeDataFormatter
 from utils.metakg.export import edges2graphml
@@ -68,7 +69,7 @@ def get(self):
                 # raising HTTPError will cause headers to be emptied
                 self.finish()
             else:
-                raise HTTPError(403)
+                raise HTTPError(status_code=403)
 
 
 class LoginHandler(AuthHandler):
@@ -113,7 +114,7 @@ class ValidateHandler(BaseHandler):
 
     async def get(self):
         if self.request.body:
-            raise HTTPError(400, reason="GET takes no request body.")
+            raise HTTPError(status_code=400, reason="GET takes no request body.")
 
         raw = await self.download(self.args.url)
         self.validate(raw)
@@ -139,9 +140,8 @@ def validate(self, raw):
             smartapi = SmartAPI(SmartAPI.VALIDATION_ONLY)
             smartapi.raw = raw
             smartapi.validate()
-
         except (ControllerError, AssertionError) as err:
-            raise HTTPError(400, reason=str(err))
+            raise SmartAPIHTTPError(400, reason=str(err))
         else:
             self.finish({"success": True, "details": f"valid SmartAPI ({smartapi.version}) metadata."})
 
@@ -165,19 +165,19 @@ async def post(self):
         """
 
         if SmartAPI.find(self.args.url, "url"):
-            raise HTTPError(409)
+            raise HTTPError(status_code=409)
 
         try:
             file = await download_async(self.args.url)
         except DownloadError as err:
-            raise HTTPError(400, reason=str(err)) from err
+            raise HTTPError(status_code=400, reason=str(err)) from err
 
         try:
             smartapi = SmartAPI(self.args.url)
             smartapi.raw = file.raw
             smartapi.validate()
         except (ControllerError, AssertionError) as err:
-            raise HTTPError(400, reason=str(err)) from err
+            raise HTTPError(status_code=400, reason=str(err)) from err
 
         if self.args.dryrun:
             raise Finish({"success": True, "details": f"[Dryrun] Valid {smartapi.version} Metadata"})
@@ -187,7 +187,7 @@ async def post(self):
             smartapi.refresh(file)  # populate webdoc meta
             _id = smartapi.save()
         except ControllerError as err:
-            raise HTTPError(400, reason=str(err)) from err
+            raise HTTPError(status_code=400, reason=str(err)) from err
         else:
             self.finish({"success": True, "_id": _id})
             await self._notify(smartapi)
@@ -252,21 +252,21 @@ async def put(self, _id):
         try:
             smartapi = SmartAPI.get(_id)
         except NotFoundError:
-            raise HTTPError(404)
+            raise HTTPError(status_code=404)
 
         if smartapi.username != self.current_user["login"]:
-            raise HTTPError(403)
+            raise HTTPError(status_code=403)
 
         if self.args.slug is not None:
             if self.args.slug in {"api"}:  # reserved
-                raise HTTPError(400, reason="slug is reserved")
+                raise HTTPError(status_code=400, reason="slug is reserved")
 
             try:  # update slug
                 smartapi.slug = self.args.slug or None
                 smartapi.save()
 
             except (ControllerError, ValueError) as err:
-                raise HTTPError(400, reason=str(err)) from err
+                raise HTTPError(status_code=400, reason=str(err)) from err
 
             self.finish({"success": True})
 
@@ -292,15 +292,15 @@ def delete(self, _id):
         try:
             smartapi = SmartAPI.get(_id)
         except NotFoundError:
-            raise HTTPError(404)
+            raise HTTPError(status_code=404)
 
         if smartapi.username != self.current_user["login"]:
-            raise HTTPError(403)
+            raise HTTPError(status_code=403)
 
         try:
             _id = smartapi.delete()
         except ControllerError as err:
-            raise HTTPError(400, reason=str(err)) from err
+            raise HTTPError(status_code=400, reason=str(err)) from err
 
         self.finish({"success": True, "_id": _id})
 
@@ -346,41 +346,41 @@ class UptimeHandler(BaseHandler):
     @github_authenticated
     def get(self):
         if self.request.body:
-            raise HTTPError(400, reason="GET takes no request body.")
+            raise HTTPError(status_code=400, reason="GET takes no request body.")
 
         if self.args.id:
             try:
                 smartapi = SmartAPI.get(self.args.id)
                 if smartapi.username != self.current_user["login"]:
-                    raise HTTPError(403)
+                    raise HTTPError(status_code=403)
                 status = smartapi.check()
                 smartapi.save()
             except NotFoundError:
-                raise HTTPError(404)
+                raise HTTPError(status_code=404)
             except (ControllerError, AssertionError) as err:
-                raise HTTPError(400, reason=str(err))
+                raise HTTPError(status_code=400, reason=str(err))
             else:
                 self.finish({"success": True, "details": status})
         else:
-            raise HTTPError(400, reason="Missing required parameter: id")
+            raise HTTPError(status_code=400, reason="Missing required parameter: id")
 
     @github_authenticated
     def post(self):
         if self.args.id:
             try:
                 smartapi = SmartAPI.get(self.args.id)
                 if smartapi.username != self.current_user["login"]:
-                    raise HTTPError(403)
+                    raise HTTPError(status_code=403)
                 status = smartapi.check()
                 smartapi.save()
             except NotFoundError:
-                raise HTTPError(404)
+                raise HTTPError(status_code=404)
             except (ControllerError, AssertionError) as err:
-                raise HTTPError(400, reason=str(err))
+                raise HTTPError(status_code=400, reason=str(err))
             else:
                 self.finish({"success": True, "details": status})
         else:
-            raise HTTPError(400, reason="Missing required form field: id")
+            raise HTTPError(status_code=400, reason="Missing required form field: id")
 
 
 class MetaKGHandlerMixin:
@@ -600,14 +600,14 @@ def write(self, chunk):
 
 class MetaKGPathFinderHandler(QueryHandler):
     """
-    A handler for querying paths in a knowledge graph using MetaKGPathFinder.
+    A handler for querying paths in a knowledge graph using the custom MetaKGPathFinder module.
 
     Attributes:
     - name: Unique identifier for this handler.
     - kwargs: Configuration for GET request parameters.
 
-    The primary GET method accepts 'subject', 'object', and 'cutoff' parameters, then retrieves
-    and returns paths in JSON format between the specified entities up to the given 'cutoff' length.
+    The primary GET method accepts the required 'subject', 'object', and 'cutoff'(default=3) parameters, then retrieves
+    and returns paths in JSON format between the specified nodes up to the given 'cutoff' length.
     """
 
     name = "metakgpathfinder"
@@ -672,6 +672,11 @@ def setup_pathfinder_rawquery(self, expanded_fields):
 
     @capture_exceptions
     async def get(self, *args, **kwargs):
+
+        # Check if subject and object are the same - not allowed
+        if self.args.subject == self.args.object:
+            raise ValueError("Subject and object must be different.")
+
         query_data = {"q": self.args.q}
 
         # Initialize with the original subject and object, and setup for expansion
@@ -715,6 +720,10 @@ async def get(self, *args, **kwargs):
             bte=self.args.bte
         )
 
+        # # Error check path results
+        if "error" in paths_with_edges:
+            raise HTTPError(400, reason=str(paths_with_edges["error"]))
+
         # Check if rawquery parameter is true -- respond with correct output
         if self.args.rawquery:
             raw_query_output = self.setup_pathfinder_rawquery(expanded_fields)

src/index.py

@@ -1,6 +1,7 @@
 """ SmartAPI Entry Point """
 
 import logging
+from os.path import exists
 
 from threading import Thread
 
@@ -20,7 +21,8 @@ def run_routine():
 
 class WebAppHandler(RequestHandler):
     def get(self):
-        self.render("../web-app/dist/index.html")
+        if exists("../web-app/dist/index.html"):
+            self.render("../web-app/dist/index.html")
 
 
 if __name__ == "__main__":

src/utils/downloader.py

@@ -121,7 +121,10 @@ def get_raw(self):
 
 def download(url, timeout=5, raise_error=True):
     try:
-        response = requests.get(url, timeout=timeout)
+        headers = {
+            'User-Agent': 'SmartAPI' # UA required by GitHub
+        }
+        response = requests.get(url, headers=headers, timeout=timeout)
         if raise_error:
             response.raise_for_status()
         result = RequestsParser(response)
@@ -151,7 +154,10 @@ async def download_async(url, timeout=20, raise_error=True):
 
 
 def download_mapping(url):
-    response = requests.get(url)
+    headers = {
+        'User-Agent': 'SmartAPI' # UA required by GitHub
+    }
+    response = requests.get(url, headers=headers, timeout=60)
     response.raise_for_status()
 
     return decoder.to_dict(stream=response.content, ext=file_extension(url), ctype=response.headers.get("Content-Type"))

src/utils/http_error.py

@@ -0,0 +1,39 @@
+from typing import Optional, Any
+from tornado.web import HTTPError
+import re
+
+
+class SmartAPIHTTPError(HTTPError):
+    """An extended HTTPError class with additional details and message sanitization.
+
+    Adds the following enhancements:
+    - A `details` parameter for including extra context about the error.
+    - A `clean_error_message` method for sanitizing log messages and details.
+
+    :arg str details: Additional information about the error.
+    """
+
+    def __init__(
+        self,
+        status_code: int = 500,
+        log_message: Optional[str] = None,
+        *args: Any,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(status_code, log_message, *args, **kwargs)
+        if self.reason:
+            self.reason = self.clean_error_message(self.reason)
+        if self.log_message:
+            self.log_message = self.clean_error_message(self.log_message)
+
+    @staticmethod
+    def clean_error_message(message: str) -> str:
+        """
+        Sanitizes an error message by replacing newlines, tabs, and reducing multiple spaces.
+
+        :param message: The error message to sanitize.
+        :return: A cleaned and sanitized version of the message.
+        """
+        message = message.replace("\n", " ")  # Replace actual newlines with spaces
+        message = re.sub(r'\s+', ' ', message)  # Normalize spaces
+        return message.strip()

src/utils/metakg/path_finder.py

@@ -1,12 +1,8 @@
-import logging
-
 import networkx as nx
+
 from controller.metakg import MetaKG
 from model import ConsolidatedMetaKGDoc
 
-logger = logging.basicConfig(level=logging.INFO, filename="missing_bte.log")
-
-
 class MetaKGPathFinder:
     def __init__(self, query_data=None, expanded_fields=None):
         """
@@ -104,43 +100,51 @@ def get_paths(self, cutoff=2, api_details=False, predicate_filter=None, bte=Fals
             If True, includes full details of the 'api' in the result.
         - predicate_filter: list (default=None)
             A list of predicates to filter the results by.
-
+        - bte: bool (default=False)
+            If True, includes BTE information in the result.
         Returns:
         - all_paths_with_edges: list of dict
             A list containing paths and their edge information for all subject-object pairs.
         """
 
         all_paths_with_edges = []
 
-        # Predicate Filter Setup
         predicate_filter_set = set(predicate_filter) if predicate_filter else None
+
         if 'predicate' in self.expanded_fields and self.expanded_fields['predicate']:
             predicate_filter_set.update(self.expanded_fields['predicate'])
 
-        # Graph iteration over subject-object pairs
-        for subject in self.expanded_fields["subject"]:
-            for object in self.expanded_fields["object"]:
-                try:
-                    # Check if a path exists between the subject and object
-                    if nx.has_path(self.G, subject, object):
-                        raw_paths = nx.all_simple_paths(self.G, source=subject, target=object, cutoff=cutoff)
-                        for path in raw_paths:
-                            paths_data = {"path": path, "edges": []}
-                            edge_added = False
-                            for i in range(len(path) - 1):
-                                source_node = path[i]
-                                target_node = path[i + 1]
-                                edge_key = f"{source_node}-{target_node}"
-                                edge_data = self.predicates.get(edge_key, [])
-
-                                for data in edge_data:
-                                    if predicate_filter_set and data["predicate"] not in predicate_filter_set:
-                                        continue
-                                    paths_data = self.build_edge_results(paths_data, data, api_details, source_node, target_node, bte)
-                                    edge_added = True
-                            if edge_added:
-                                all_paths_with_edges.append(paths_data)
-                except Exception:
-                    continue
-
-        return all_paths_with_edges
+        try:
+            # Graph iteration over subject-object pairs
+            for subject in self.expanded_fields["subject"]:
+                for object in self.expanded_fields["object"]:
+                    if subject not in self.G:
+                        return { "error": f"Subject node {subject} is not found in the MetaKG" }
+                    if object not in self.G:
+                        return { "error": f"Object node {object} is not found in the MetaKG" }
+                    try:
+                        # Check if a path exists between the subject and object
+                        if nx.has_path(self.G, subject, object):
+                            raw_paths = nx.all_simple_paths(self.G, source=subject, target=object, cutoff=cutoff)
+                            for path in raw_paths:
+                                paths_data = {"path": path, "edges": []}
+                                edge_added = False
+                                for i in range(len(path) - 1):
+                                    source_node = path[i]
+                                    target_node = path[i + 1]
+                                    edge_key = f"{source_node}-{target_node}"
+                                    edge_data = self.predicates.get(edge_key, [])
+
+                                    for data in edge_data:
+                                        if predicate_filter_set and data["predicate"] not in predicate_filter_set:
+                                            continue
+                                        paths_data = self.build_edge_results(paths_data, data, api_details, source_node, target_node, bte)
+                                        edge_added = True
+                                if edge_added:
+                                    all_paths_with_edges.append(paths_data)
+                    except nx.exception.NodeNotFound as node_err:
+                        return { "error": node_err }
+            return all_paths_with_edges
+
+        except Exception as e:
+            return  { "error": e }

src/utils/monitor.py

@@ -295,10 +295,10 @@ def make_api_call(self):
                 if "example" in _param:
                     # parameter in path
                     if _param["in"] == "path":
-                        url = url.replace("{" + _param["name"] + "}", _param["example"])
+                        url = str(url).replace("{" + _param["name"] + "}", str(_param["example"]))
                     # parameter in query
                     elif _param["in"] == "query":
-                        params = {_param["name"]: _param["example"]}
+                        params[_param["name"]] = _param["example"]
 
                 elif "required" in _param and _param["required"] is True:
                     paramsRequired = True  # pylint: disable=invalid-name