Merge pull request #59 from HeyLittleJohn/logger-fix

PATCH: more fixes to logging so it can be used as a common function outside of the curator package

Author: jhirschibar

curator/data_pipeline/QuotePool.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 import queue
 import traceback
 from typing import (
@@ -24,7 +25,7 @@
     TaskID,
 )
 
-from curator.proj_constants import log
+log = logging.getLogger(__name__)
 
 
 class QuoteScheduler(RoundRobin):

curator/data_pipeline/download.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 from aiomultiprocess import Pool
@@ -22,9 +23,11 @@
 from db_tools.queries import lookup_multi_ticker_ids
 from db_tools.utils import OptionTicker
 
-from curator.proj_constants import POLYGON_BASE_URL, log
+from curator.proj_constants import POLYGON_BASE_URL
 from curator.utils import pool_kwarg_config
 
+log = logging.getLogger(__name__)
+
 planned_exceptions = (
     InvalidArgs,
     ProjClientConnectionError,

curator/data_pipeline/orchestrator.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 from data_pipeline.download import (
@@ -20,7 +21,7 @@
 from db_tools.utils import generate_o_ticker_lookup, pull_tickers_from_db
 from pandas import DataFrame
 
-from curator.proj_constants import log
+log = logging.getLogger(__name__)
 
 
 async def import_all(tickers: list, start_date: datetime, end_date: datetime, months_hist: int):

curator/data_pipeline/path_runner.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from abc import ABC, abstractmethod
 from datetime import datetime
@@ -14,7 +15,7 @@
 )
 from db_tools.utils import OptionTicker
 
-from curator.proj_constants import BASE_DOWNLOAD_PATH, POSTGRES_BATCH_MAX, log
+from curator.proj_constants import BASE_DOWNLOAD_PATH, POSTGRES_BATCH_MAX, logger_setup
 from curator.utils import (
     clean_o_ticker,
     months_ago,
@@ -23,6 +24,8 @@
     timestamp_to_datetime,
 )
 
+log = logging.getLogger(__name__)
+
 
 class PathRunner(ABC):
     """Base class for runner that will traverse the directory structure and retrieve/clean raw data.

curator/data_pipeline/polygon_utils.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 import os
 from abc import ABC, abstractmethod
 from datetime import date, datetime
@@ -17,7 +18,7 @@
 from dateutil.relativedelta import relativedelta
 from db_tools.utils import OptionTicker
 
-from curator.proj_constants import BASE_DOWNLOAD_PATH, POLYGON_API_KEY, POLYGON_BASE_URL, log
+from curator.proj_constants import BASE_DOWNLOAD_PATH, POLYGON_API_KEY, POLYGON_BASE_URL
 from curator.utils import (
     extract_underlying_from_o_ticker,
     first_weekday_of_month,
@@ -28,6 +29,8 @@
     write_api_data_to_file,
 )
 
+log = logging.getLogger(__name__)
+
 
 class Timespans(Enum):
     second = "second"

curator/data_pipeline/uploader.py

@@ -103,5 +103,6 @@ async def upload_options_quotes(ticker: str):
 
 
 if __name__ == "__main__":
+    log.info("more success!")
     failed_paths = asyncio.run(upload_options_quotes(ticker="QQQ"))
     print(f"failed paths: {failed_paths}")

curator/proj_constants.py

@@ -9,20 +9,22 @@
 from pathlib import Path
 
 import pandas_market_calendars as mcal
-import sentry_sdk
 import uvloop
-from sentry_sdk import capture_exception
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
 
-SENTRY_URL = os.environ.get("SENTRY_URL")
-sentry_sdk.init(
-    dsn=SENTRY_URL,
-    traces_sample_rate=0.1,
-)
-
 ENVIRONMENT = os.environ.get("ENVIRONMENT")
 DEBUG = False
+SENTRY_URL = os.environ.get("SENTRY_URL", None)
+
+if SENTRY_URL:
+    import sentry_sdk
+    from sentry_sdk import capture_exception
+
+    sentry_sdk.init(
+        dsn=SENTRY_URL,
+        traces_sample_rate=0.1,
+    )
 
 
 # NOTE: use this function if pass variables to env via docker .env file. Otherwise use .pgpass
@@ -64,11 +66,12 @@ def db_uri_maker() -> str:
 
 POOL_DEFAULT_KWARGS = {
     "processes": CPUS,
-    "exception_handler": capture_exception,
     "loop_initializer": uvloop.new_event_loop,
     "childconcurrency": int(MAX_CONCURRENT_REQUESTS / CPUS),
     "queuecount": CPUS,
 }
+if SENTRY_URL:
+    POOL_DEFAULT_KWARGS["exception_handler"] = capture_exception
 
 async_engine = create_async_engine(
     POSTGRES_DATABASE_URL,
@@ -92,7 +95,7 @@ def db_uri_maker() -> str:
 )
 
 
-def logger_setup(project_name: str, debug=False):
+def logger_setup(project_name: str, debug=False, name=__name__) -> Logger:
     root_logger: Logger = logging.getLogger()
     log: Logger = logging.getLogger(__name__)
     log_formatter = logging.Formatter(
@@ -144,6 +147,7 @@ def filter(self, record):
         + datetime.now().strftime("%Y-%m-%d")
         + ".log"
     )
+    os.makedirs(os.path.dirname(log_path), exist_ok=True)
     file_handler = FileHandler(log_path)
     file_handler.setFormatter(log_formatter)
 
@@ -152,7 +156,7 @@ def filter(self, record):
     return log
 
 
-log = logger_setup("curator", DEBUG)
+log = logger_setup("curator", debug=DEBUG)
 
 # market calendar
 o_cal = mcal.get_calendar("CBOE_Equity_Options")

curator/utils.py

@@ -1,6 +1,7 @@
 import functools
 import inspect
 import json
+import logging
 import os
 from datetime import datetime
 from json import JSONDecodeError
@@ -116,8 +117,9 @@ async def _session_work(session: AsyncSession, args, kwargs):
 
     @functools.wraps(func)
     async def wrapper_events(*args, **kwargs):
+        wrap_log = logging.getLogger(__name__)
         func_mod_and_name = f"{func.__module__}.{func.__name__}"
-        log.info(f"Starting {func_mod_and_name}")
+        wrap_log.info(f"Starting {func_mod_and_name}")
         session_passed = False
         for arg in list(args) + list(kwargs.values()):
             if issubclass(type(arg), AsyncSession):
@@ -138,10 +140,10 @@ async def wrapper_events(*args, **kwargs):
                 finally:
                     await session.close()
 
-            log.info(f"Finished {func_mod_and_name}")
+            wrap_log.info(f"Finished {func_mod_and_name}")
             return func_return
         except Exception as e:
-            log.exception(e)
+            wrap_log.exception(e)
             raise
 
     return wrapper_events