🔧 Update (codebase): Improve custom exceptions usage

Author: jpcadena

app/api/api_v1/router/user.py

@@ -18,7 +18,7 @@
     ServiceException,
 )
 from app.crud.user import UserRepository, get_user_repository
-from app.schemas.user import User, UserCreate, UsersResponse, UserUpdate
+from app.schemas.user import User, UserCreate, UserUpdate, UsersResponse
 
 logger: logging.Logger = logging.getLogger(__name__)
 router: APIRouter = APIRouter(prefix="/user", tags=["user"])

pipeline/engineering/transformation.py

@@ -3,6 +3,7 @@
 """
 
 from pipeline.core.decorators import benchmark, with_logging
+from pipeline.exceptions.exceptions import DataQualityException
 from pipeline.models.weather import Weather
 from pipeline.schemas.api.weather import APIWeather, CurrentWeather
 from pipeline.schemas.files.weather import CSVWeather
@@ -21,6 +22,14 @@ def transform_data(csv_weather: CSVWeather, api_weather: APIWeather) -> Weather:
     :rtype: Weather
     """
     current_weather: CurrentWeather = api_weather.current
+    if not (
+        csv_weather.min_temp <= current_weather.temp <= csv_weather.max_temp
+    ):
+        raise DataQualityException(
+            f"Current temperature {current_weather.temp} is outside the range"
+            f" of min_temp {csv_weather.min_temp} and max_temp"
+            f" {csv_weather.max_temp}"
+        )
     weather: Weather = Weather(
         date=csv_weather.date,
         min_temp=csv_weather.min_temp,

pipeline/models/base/base.py

@@ -6,11 +6,9 @@
 
 from sqlalchemy.orm import DeclarativeBase
 
-from pipeline.models.weather import Weather
-
 
 class Base(DeclarativeBase):
     pass
 
 
-U = TypeVar("U", bound="Weather")
+U = TypeVar("U", bound="Weather")  # type: ignore

pipeline/models/weather.py

@@ -103,15 +103,17 @@ class Weather(AuditMixin, BaseWithID):
             name="weather_min_max_temp_check",
         ),
         CheckConstraint(
-            "temp_9am BETWEEN min_temp AND max_temp",
+            f"temp_9am {settings.LOWEST_TEMP} AND {settings.HIGHEST_TEMP}",
             name="weather_temp_9am_range_check",
         ),
         CheckConstraint(
-            "temp_3pm BETWEEN min_temp AND max_temp",
+            f"temp_3pm BETWEEN {settings.LOWEST_TEMP} AND"
+            f" {settings.HIGHEST_TEMP}",
             name="weather_temp_3pm_range_check",
         ),
         CheckConstraint(
-            "current_temp BETWEEN min_temp AND max_temp",
+            f"current_temp BETWEEN {settings.LOWEST_TEMP} AND"
+            f" {settings.HIGHEST_TEMP}",
             name="weather_current_temp_range_check",
         ),
     )

pipeline/repository/base.py

@@ -3,12 +3,12 @@
 """
 
 import logging
-from typing import Any
 
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
 
 from pipeline.core.decorators import benchmark
+from pipeline.exceptions.exceptions import DatabaseException
 from pipeline.models.base.base import U
 
 logger: logging.Logger = logging.getLogger(__name__)
@@ -28,21 +28,21 @@ def __init__(
     @benchmark
     def handle_sql_exception(
         self,
-        arg0: Any,
-        exc: Any,
+        message: str,
+        exc: SQLAlchemyError,
     ) -> None:
         """
         Handle the exception raised from SQLAlchemy database operation
-        :param arg0: Any argument to add on the exception
-        :type arg0: Any
+        :param message: Custom message for the exception
+        :type message: str
         :param exc: The exception raised
-        :type exc: Any
+        :type exc: SQLAlchemyError
         :return: None
         :rtype: NoneType
         """
         self.session.rollback()
-        logger.error(f"{arg0}{exc}")
-        raise
+        logger.error(f"{message}{exc}")
+        raise DatabaseException(f"{message}{exc}")
 
     @benchmark
     def add(

pipeline/repository/weather.py

@@ -4,10 +4,14 @@
 
 import logging
 
-from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 from sqlalchemy.orm import Session
 
 from pipeline.core.decorators import with_logging
+from pipeline.exceptions.exceptions import (
+    DataQualityException,
+    DatabaseException,
+)
 from pipeline.models.weather import Weather
 from pipeline.repository.base import BaseRepository
 
@@ -38,5 +42,13 @@ def handle_weather(self, weather: Weather) -> None:
                 self.update(Weather(**existing_weather.__dict__))
             else:
                 self.add(weather)
+        except IntegrityError as exc:
+            logger.error(f"Integrity error while handling weather data: {exc}")
+            raise DataQualityException(
+                f"Data quality issue: {str(exc)}"
+            ) from exc
         except SQLAlchemyError as exc:
             self.handle_sql_exception("Failed to handle weather data: ", exc)
+            raise DatabaseException(
+                f"Database operation failed: {str(exc)}"
+            ) from exc

pipeline/services/external/api/api.py

@@ -16,7 +16,9 @@
 from pipeline.core.decorators import benchmark, with_logging
 from pipeline.exceptions.exceptions import (
     APIValidationError,
+    ConnectionException,
     NoAPIResponseException,
+    RateLimitExceededException,
 )
 from pipeline.schemas.api.weather import T
 
@@ -131,6 +133,22 @@ def _api_call(
             if hasattr(model_instance, "meta") and model_instance.meta is None:
                 logger.warning("Expected pagination data missing in response")
             return model_instance
+        except requests.exceptions.HTTPError as exc:
+            if exc.response is not None and exc.response.status_code == 429:
+                raise RateLimitExceededException(
+                    "Rate limit exceeded. Please try again later."
+                ) from exc
+            raise APIValidationError(
+                f"HTTP error occurred: {exc.response.status_code}"
+            ) from exc
+        except requests.exceptions.ConnectionError as exc:
+            raise ConnectionException(
+                "A connection error occurred. Please check your network."
+            ) from exc
+        except requests.exceptions.Timeout as exc:
+            raise ConnectionException(
+                "The request timed out. Please try again later."
+            ) from exc
         except requests.exceptions.RequestException as exc:
             if exc.response is None:
                 raise NoAPIResponseException(

pipeline/services/external/api/weather.py

@@ -3,12 +3,16 @@
  package.
 """
 
+import logging
+
 from pydantic_extra_types.coordinate import Latitude, Longitude
 
 from pipeline.core.decorators import with_logging
 from pipeline.schemas.api.weather import APIWeather
 from pipeline.services.external.api.api import ApiService
 
+logger: logging.Logger = logging.getLogger(__name__)
+
 
 class WeatherApiService(ApiService):
     """

pyproject.toml

@@ -173,6 +173,8 @@ select = [
 ignore = [
     "E501", # line too long, handled by black
     "B008", # do not perform function calls in argument defaults
+    "I001",
+    "F821",
 ]
 fixable = [
     "A",