unify logging of exceptions in API

Author: fynnos

backend/src/common/exception_handler.py

@@ -0,0 +1,17 @@
+from fastapi import Request
+from fastapi.responses import PlainTextResponse
+from loguru import logger
+
+exception_handlers = []
+
+
+def exception_handler(http_status_code: int):
+    def decorator(exception_class):
+        def handle_exception(req: Request, exc: Exception):
+            logger.exception(exc)
+            return PlainTextResponse(str(exc), status_code=http_status_code)
+
+        exception_handlers.append((exception_class, handle_exception))
+        return exception_class
+
+    return decorator

backend/src/core/auth/authz_user.py

@@ -1,16 +1,18 @@
 from typing import NoReturn
 
-from fastapi import Depends, Request
+from fastapi import Depends, Request, status
 from loguru import logger
 from sqlalchemy.orm import Session
 
 from common.crud_enum import Crud
 from common.dependencies import get_current_user, get_db_session
+from common.exception_handler import exception_handler
 from core.user.user_orm import UserORM
 from repos.db.crud_base import NoSuchElementError
 from repos.db.orm_base import ORMBase
 
 
+@exception_handler(status.HTTP_403_FORBIDDEN)
 class ForbiddenError(Exception):
     def __init__(self):
         super().__init__("User is not authorized")

backend/src/core/auth/validation.py

@@ -1,14 +1,16 @@
 from typing import Sequence
 
-from fastapi import Depends
+from fastapi import Depends, status
 from sqlalchemy.orm import Session
 
 from common.crud_enum import Crud
 from common.dependencies import get_db_session
+from common.exception_handler import exception_handler
 from repos.db.crud_base import NoSuchElementError
 from repos.db.orm_base import ORMBase
 
 
+@exception_handler(status.HTTP_400_BAD_REQUEST)
 class InvalidError(Exception):
     def __init__(self, note=None):
         note_description = f": {note}" if note is not None else ""

backend/src/core/doc/source_document_crud.py

@@ -1,8 +1,10 @@
+from fastapi import status
 from fastapi.encoders import jsonable_encoder
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
 
+from common.exception_handler import exception_handler
 from common.sdoc_status_enum import SDocStatus
 from core.annotation.annotation_document_orm import AnnotationDocumentORM
 from core.doc.folder_crud import crud_folder
@@ -22,6 +24,7 @@
 from systems.event_system.events import source_document_deleted
 
 
+@exception_handler(status.HTTP_500_INTERNAL_SERVER_ERROR)
 class SourceDocumentPreprocessingUnfinishedError(Exception):
     def __init__(self, sdoc_id: int):
         super().__init__(f"SourceDocument {sdoc_id} is still getting preprocessed!")

backend/src/main.py

@@ -3,7 +3,6 @@
 
 import os
 from contextlib import asynccontextmanager
-from http import HTTPStatus
 
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -17,8 +16,7 @@
 from starlette.middleware.sessions import SessionMiddleware
 from uvicorn.main import run
 
-from modules.crawler.crawler_exceptions import NoDataToCrawlError
-from repos.elastic.elastic_crud_base import NoSuchObjectInElasticSearchError
+from common.exception_handler import exception_handlers
 from repos.elastic.elastic_repo import ElasticSearchRepo
 from repos.llm_repo import LLMRepo
 from utils.import_utils import import_by_suffix
@@ -40,23 +38,10 @@
     startup(reset_data=RESET_DATA, sql_echo=False)
     os.environ["STARTUP_DONE"] = "1"
 
+from rq.exceptions import NoSuchJobError
+
 from config import conf
-from core.auth.authz_user import ForbiddenError
-from core.auth.validation import InvalidError
-from core.doc.source_document_crud import SourceDocumentPreprocessingUnfinishedError
-from modules.eximport.export_service import (
-    ExportJobPreparationError,
-    NoSuchExportJobError,
-)
-from modules.eximport.import_service import ImportJobPreparationError
-from modules.eximport.no_data_export_error import NoDataToExportError
-from repos.db.crud_base import NoSuchElementError
-from repos.filesystem_repo import (
-    FileAlreadyExistsInFilesystemError,
-    FileNotFoundInFilesystemError,
-    FilesystemRepo,
-    SourceDocumentNotFoundInFilesystemError,
-)
+from repos.filesystem_repo import FilesystemRepo
 
 # import all jobs dynamically
 import_by_suffix("_job.py")
@@ -126,100 +111,32 @@ def custom_openapi():
 app.add_middleware(SessionMiddleware, secret_key=conf.api.auth.session.secret)
 
 
-# add custom exception handlers
-# TODO Flo: find a better place for this! (and Exceptions in general. move into own file)
-@app.exception_handler(NoSuchElementError)
-async def no_such_element_error_handler(_, exc: NoSuchElementError):
-    return PlainTextResponse(str(exc), status_code=404)
-
-
-@app.exception_handler(NoDataToCrawlError)
-async def no_data_to_crawl_handler(_, exc: NoDataToCrawlError):
-    return PlainTextResponse(str(exc), status_code=400)
-
-
-@app.exception_handler(NoDataToExportError)
-async def no_data_to_export_handler(_, exc: NoDataToExportError):
-    return PlainTextResponse(str(exc), status_code=404)
-
-
-@app.exception_handler(NoSuchExportJobError)
-async def no_such_export_job_handler(_, exc: NoSuchExportJobError):
-    return PlainTextResponse(str(exc), status_code=404)
-
-
-@app.exception_handler(ExportJobPreparationError)
-async def export_job_preparation_error_handler(_, exc: ExportJobPreparationError):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(ImportJobPreparationError)
-async def import_job_preparation_error_handler(_, exc: ImportJobPreparationError):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(NoSuchObjectInElasticSearchError)
-async def no_such_object_in_es_error_handler(_, exc: NoSuchObjectInElasticSearchError):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(SourceDocumentNotFoundInFilesystemError)
-async def source_document_not_found_in_filesystem_error_handler(
-    _, exc: SourceDocumentNotFoundInFilesystemError
-):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(SourceDocumentPreprocessingUnfinishedError)
-async def source_document_preprocessing_unfinished_error_handler(
-    _, exc: SourceDocumentPreprocessingUnfinishedError
-):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(FileNotFoundInFilesystemError)
-async def file_not_found_in_filesystem_error_handler(
-    _, exc: FileNotFoundInFilesystemError
-):
-    return PlainTextResponse(str(exc), status_code=500)
-
-
-@app.exception_handler(FileAlreadyExistsInFilesystemError)
-async def file_already_exists_in_filesystem_error_handler(
-    _, exc: FileAlreadyExistsInFilesystemError
-):
-    return PlainTextResponse(str(exc), status_code=406)
+# import & register all endpoints dynamically
+endpoint_modules = import_by_suffix("_endpoint.py")
+endpoint_modules.sort(key=lambda x: x.__name__.split(".")[-1])
+for em in endpoint_modules:
+    app.include_router(em.router)
 
 
 @app.exception_handler(IntegrityError)
 async def integrity_error_handler(_, exc: IntegrityError):
+    logger.exception(exc)
     if isinstance(exc.orig, UniqueViolation):
         msg = str(exc.orig.pgerror).split("\n")[1]
         return PlainTextResponse(msg, status_code=409)
     else:
         return PlainTextResponse(str(exc), status_code=500)
 
 
-@app.exception_handler(ForbiddenError)
-def forbidden_error_handler(_, exc: ForbiddenError):
-    return PlainTextResponse(str(exc), status_code=403)
-
-
-@app.exception_handler(InvalidError)
-def invalid_error_handler(_, exc: InvalidError):
-    return PlainTextResponse(str(exc), status_code=HTTPStatus.BAD_REQUEST)
-
+@app.exception_handler(NoSuchJobError)
+async def no_such_job_error_handler(_, exc: NoSuchJobError):
+    logger.exception(exc)
+    return PlainTextResponse(str(exc), status_code=404)
 
-# import & register all exception handlers dynamically
-exception_hanlders = import_by_suffix("_exception_handler.py")
-for eh in exception_hanlders:
-    eh.register_exception_handlers(app)
 
-# import & register all endpoints dynamically
-endpoint_modules = import_by_suffix("_endpoint.py")
-endpoint_modules.sort(key=lambda x: x.__name__.split(".")[-1])
-for em in endpoint_modules:
-    app.include_router(em.router)
+# register all exception handlers in fastAPI
+for ex_class, handler_func in exception_handlers:
+    app.add_exception_handler(ex_class, handler_func)
 
 
 def main() -> None:

backend/src/modules/crawler/crawler_exceptions.py

@@ -2,27 +2,29 @@
 import subprocess
 from pathlib import Path
 
+from fastapi import status
 from loguru import logger
 from pydantic import Field
 
+from common.exception_handler import exception_handler
 from common.job_type import JobType
 from core.project.project_crud import crud_project
-from modules.crawler.crawler_exceptions import NoDataToCrawlError
 from modules.doc_processing.doc_processing_dto import ProcessingJobInput
 from repos.db.sql_repo import SQLRepo
 from repos.filesystem_repo import FilesystemRepo
-from systems.job_system.job_dto import (
-    EndpointGeneration,
-    Job,
-    JobOutputBase,
-    JobTiming,
-)
+from systems.job_system.job_dto import EndpointGeneration, Job, JobOutputBase, JobTiming
 from systems.job_system.job_register_decorator import register_job
 
 sqlr = SQLRepo()
 fsr: FilesystemRepo = FilesystemRepo()
 
 
+@exception_handler(status.HTTP_400_BAD_REQUEST)
+class NoDataToCrawlError(Exception):
+    def __init__(self, what_msg: str):
+        super().__init__(what_msg)
+
+
 class CrawlerJobInput(ProcessingJobInput):
     project_id: int = Field(
         description="The ID of the Project to import the crawled data."

backend/src/modules/crawler/crawler_job.py

@@ -10,7 +10,7 @@
     BBoxAnnotationExportCollection,
     BBoxAnnotationExportSchema,
 )
-from modules.eximport.no_data_export_error import NoDataToExportError
+from modules.eximport.export_exceptions import NoDataToExportError
 from repos.filesystem_repo import FilesystemRepo
 
 

backend/src/modules/eximport/bbox_annotations/export_bbox_annotations.py

@@ -10,7 +10,7 @@
     CodeExportCollection,
     CodeExportSchema,
 )
-from modules.eximport.no_data_export_error import NoDataToExportError
+from modules.eximport.export_exceptions import NoDataToExportError
 from repos.filesystem_repo import FilesystemRepo
 
 

backend/src/modules/eximport/codes/export_codes.py

@@ -6,15 +6,9 @@
 from loguru import logger
 from sqlalchemy.orm import Session
 
-from modules.concept_over_time_analysis.cota_crud import (
-    crud_cota,
-)
-from modules.concept_over_time_analysis.cota_dto import (
-    COTARead,
-)
-from modules.concept_over_time_analysis.cota_orm import (
-    ConceptOverTimeAnalysisORM,
-)
+from modules.concept_over_time_analysis.cota_crud import crud_cota
+from modules.concept_over_time_analysis.cota_dto import COTARead
+from modules.concept_over_time_analysis.cota_orm import ConceptOverTimeAnalysisORM
 from modules.eximport.cota.cota_export_schema import (
     COTAExportCollection,
     COTAExportSchema,
@@ -24,7 +18,7 @@
     transform_timeline_settings_for_export,
     transform_training_settings_for_export,
 )
-from modules.eximport.no_data_export_error import NoDataToExportError
+from modules.eximport.export_exceptions import NoDataToExportError
 from repos.filesystem_repo import FilesystemRepo