refactor: updated package to utilize uv, updated lock and pyproject.toml

Author: jhirschibar

.flake8

@@ -3,14 +3,6 @@ UID="$(shell id -u)"
 DOCKER_BUILDKIT=1
 BUILDKIT_PROGRESS=plain
 
-.PHONY: check 
-
-check:
-	poetry install
-	poetry run isort curator/
-	poetry run black curator/
-	poetry run flake8 curator/
-
 DESCRIPTION="DB Update"
 update_db:
 	alembic revision --autogenerate -m $(DESCRIPTION)

Makefile

@@ -10,15 +10,16 @@ We can then use the policy and the bot to identify options spread positions that
 
 ## Getting Started
 
-This is a python project, and will utilize pyenv, poetry, pre-commit, and will be greatly benefitted by an Nvidia GPU for training. This project will build on the default `dev` branch and push to `prod` via "Squash Merge" PR only. At which time, we will cut a new release / tag for the project.
+This is a python project, and will utilize uv to manage dependencies. This project will build on the default `dev` branch and push to `prod` via "Squash Merge" PR only. At which time, we will cut a new release / tag for the project.
 
-### PyEnv
+### UV
 
-Install [pyenv](https://github.com/pyenv/pyenv) and use it to create a virtualenv for this project with a version of python that is >=3.11
+Install [uv](https://docs.astral.sh/uv/getting-started/installation/) and use it to create a venv for this project with a version of python that is >=3.11
 
-### Poetry
+```bash
+uv venv --python 3.11
+```
 
-We use [Poetry](https://python-poetry.org/) for dependency management. Once poetry is installed on your machine and you've cloned this repo, go into the project root `curator` directory and run `poetry install` to install all dependencies indicated in the pyTOML.
 
 ### Data Sources
 

README.md

@@ -259,19 +259,18 @@ def clean_o_ticker_progress(self):
 
                 elif (
                     len(
-                        (
-                            self.o_ticker_queue_progress.get(otkr, set())
-                            | self.o_ticker_skip_tids.get(otkr, set())
-                        )
+                        (self.o_ticker_queue_progress.get(otkr, set()) | self.o_ticker_skip_tids.get(otkr, set()))
                         - self.tid_result_progress
                     )
                     == 0
                 ):
                     self.completely_processed_otkrs.append(otkr)
-                    log.info(f"all processed for {otkr}!! \
+                    log.info(
+                        f"all processed for {otkr}!! \
 ({len(self.o_ticker_queue_progress.get(otkr, []))} processed, \
 {total_tids - len(self.o_ticker_queue_progress.get(otkr, []))} will be skipped, \
-{total_tids} expected)")
+{total_tids} expected)"
+                    )
 
 
 class QuotePool(Pool):

curator/data_pipeline/QuotePool.py

@@ -1,15 +1,16 @@
 from datetime import datetime
 
 from aiomultiprocess import Pool
-from data_pipeline.exceptions import (
+
+from curator.data_pipeline.exceptions import (
     InvalidArgs,
     ProjBaseException,
     ProjClientConnectionError,
     ProjClientResponseError,
     ProjIndexError,
     ProjTimeoutError,
 )
-from data_pipeline.polygon_utils import (
+from curator.data_pipeline.polygon_utils import (
     CurrentContractSnapshot,
     HistoricalOptionsPrices,
     HistoricalQuotes,
@@ -18,10 +19,9 @@
     PolygonPaginator,
     StockMetaData,
 )
-from data_pipeline.QuotePool import QuotePool
-from db_tools.queries import lookup_multi_ticker_ids
-from db_tools.utils import OptionTicker
-
+from curator.data_pipeline.QuotePool import QuotePool
+from curator.db_tools.queries import lookup_multi_ticker_ids
+from curator.db_tools.utils import OptionTicker
 from curator.proj_constants import POLYGON_BASE_URL, log
 from curator.utils import pool_kwarg_config
 
@@ -142,7 +142,7 @@ async def download_options_quotes(ticker: str, o_tickers: list[OptionTicker], mo
     for i in range(0, len(batch_o_tickers), BATCH_SIZE_OTICKERS):
         small_batch = batch_o_tickers[i : i + BATCH_SIZE_OTICKERS]
 
-        log.info(f"downloading quotes for {i+BATCH_SIZE_OTICKERS}/{len(batch_o_tickers)} batch of o_tickers")
+        log.info(f"downloading quotes for {i + BATCH_SIZE_OTICKERS}/{len(batch_o_tickers)} batch of o_tickers")
 
         await api_quote_downloader(
             paginator=op_quotes,

curator/data_pipeline/download.py

@@ -2,20 +2,18 @@
 from datetime import datetime
 
 import typer
-from data_pipeline.orchestrator import (
+
+from curator.data_pipeline.orchestrator import (
     import_all,
     import_partial,
     remove_tickers_from_universe,
 )
-
 from curator.utils import months_ago
 
 DEFAULT_MONTHS_HIST = 24
 DEFAULT_START_DATE = months_ago(months=DEFAULT_MONTHS_HIST)
 
-app = typer.Typer(
-    help="CLI for adding stocks to the data pull process and for refreshing stock/options pricing data"
-)
+app = typer.Typer(help="CLI for adding stocks to the data pull process and for refreshing stock/options pricing data")
 
 
 def validate_partial(ctx, param, value: list[int]):

curator/data_pipeline/main.py

@@ -1,25 +1,25 @@
 from datetime import datetime
 
-from data_pipeline.download import (
+from pandas import DataFrame
+
+from curator.data_pipeline.download import (
     download_options_contracts,
     download_options_prices,
     download_options_quotes,
     download_options_snapshots,
     download_stock_metadata,
     download_stock_prices,
 )
-from data_pipeline.uploader import (
+from curator.data_pipeline.uploader import (
     upload_options_contracts,
     upload_options_prices,
     upload_options_quotes,
     upload_options_snapshots,
     upload_stock_metadata,
     upload_stock_prices,
 )
-from db_tools.queries import delete_stock_ticker, latest_date_per_ticker
-from db_tools.utils import generate_o_ticker_lookup, pull_tickers_from_db
-from pandas import DataFrame
-
+from curator.db_tools.queries import delete_stock_ticker, latest_date_per_ticker
+from curator.db_tools.utils import generate_o_ticker_lookup, pull_tickers_from_db
 from curator.proj_constants import log
 
 
@@ -79,9 +79,7 @@ async def import_all(tickers: list, start_date: datetime, end_date: datetime, mo
     await upload_stock_prices(ticker_lookup)
 
 
-async def import_partial(
-    partial: list[int], tickers: list, start_date: datetime, end_date: datetime, months_hist: int
-):
+async def import_partial(partial: list[int], tickers: list, start_date: datetime, end_date: datetime, months_hist: int):
     """This will download, clean, and upload data for the components specified in `partial`
     This is meant to be used on an adhoc basis to fill in data gaps or backfill changes
     """
@@ -128,9 +126,7 @@ async def import_partial(
         for ticker in final_tickers:
             ticker_counter += 1
             log.info(f"downloading quotes for {ticker}  ({ticker_counter}/{len(final_tickers)})")
-            await download_options_quotes(
-                ticker=ticker, o_tickers=list(o_tickers.values()), months_hist=months_hist
-            )
+            await download_options_quotes(ticker=ticker, o_tickers=list(o_tickers.values()), months_hist=months_hist)
             temp_paths = await upload_options_quotes(ticker)
             failed_paths.append(temp_paths)
         log.info(f"failed to parse these paths: {failed_paths}")
@@ -145,9 +141,7 @@ async def remove_tickers_from_universe(tickers: list[str]):
         log.info(f"ticker {ticker} successfully deleted")
 
 
-async def refresh_import(
-    tickers: list, start_date: datetime, end_date: datetime, months_hist: int, partial: list[int]
-):
+async def refresh_import(tickers: list, start_date: datetime, end_date: datetime, months_hist: int, partial: list[int]):
     """refreshes all or partial components for the given tickers
     start date and end date is set to today, start date from the most recent going back to 24 months at most"""
     all_ = True if len(tickers) == 0 else False

curator/data_pipeline/orchestrator.py

@@ -4,16 +4,15 @@
 from json import JSONDecodeError
 from typing import Any, Generator
 
-from db_tools.queries import (
+from curator.db_tools.queries import (
     update_options_prices,
     update_options_quotes,
     update_options_snapshot,
     update_options_tickers,
     update_stock_metadata,
     update_stock_prices,
 )
-from db_tools.utils import OptionTicker
-
+from curator.db_tools.utils import OptionTicker
 from curator.proj_constants import BASE_DOWNLOAD_PATH, POSTGRES_BATCH_MAX, log
 from curator.utils import (
     clean_o_ticker,
@@ -136,10 +135,7 @@ def generate_path_args(self) -> list[tuple[str]]:
         if self.all_:
             return [(self._determine_most_recent_file(self.base_directory),)]
         else:
-            return [
-                (self._determine_most_recent_file(f"{self.base_directory}/{ticker}"),)
-                for ticker in self.tickers
-            ]
+            return [(self._determine_most_recent_file(f"{self.base_directory}/{ticker}"),) for ticker in self.tickers]
 
     def clean_data(self, results: list[dict], ticker_data: tuple = ()):
         clean_results = []

curator/data_pipeline/path_runner.py

@@ -13,10 +13,10 @@
     ClientResponseError,
     ServerDisconnectedError,
 )
-from data_pipeline.exceptions import ProjAPIError, ProjAPIOverload
 from dateutil.relativedelta import relativedelta
-from db_tools.utils import OptionTicker
 
+from curator.data_pipeline.exceptions import ProjAPIError, ProjAPIOverload
+from curator.db_tools.utils import OptionTicker
 from curator.proj_constants import BASE_DOWNLOAD_PATH, POLYGON_API_KEY, POLYGON_BASE_URL, log
 from curator.utils import (
     extract_underlying_from_o_ticker,
@@ -102,9 +102,7 @@ async def _execute_request(self, session: ClientSession, url: str, payload: dict
             json_response = await response.json() if status_code == 200 else {}
             return (status_code, json_response)
 
-    async def _query_all(
-        self, session: ClientSession, url: str, payload: dict = {}, limit: bool = False
-    ) -> list[dict]:
+    async def _query_all(self, session: ClientSession, url: str, payload: dict = {}, limit: bool = False) -> list[dict]:
         """Query the API until all results have been returned
         Args:
             session: aiohttp ClientSession
@@ -238,9 +236,7 @@ def generate_request_args(self, args_data: list[str] = []):
         Returns:
         urls: list(tuple), each tuple contains the url, the payload for the request and an empty string"""
         if not self.all_:
-            urls = [
-                (self.url_base, dict(self.payload, **{"ticker": ticker}), ticker) for ticker in self.tickers
-            ]
+            urls = [(self.url_base, dict(self.payload, **{"ticker": ticker}), ticker) for ticker in self.tickers]
         else:
             urls = [(self.url_base, self.payload, "")]
         return urls
@@ -510,18 +506,14 @@ def __init__(self, o_ticker_lookup: dict[str, int], months_hist: int = 24):
         self.start_date, self.close_date = self._determine_start_end_dates(
             string_to_date("2060-01-01")
         )  # NOTE: magic number meant to always trigger the newest date (today)
-        self.dates = trading_days_in_range(
-            str(self.start_date), str(self.close_date), count=False, cal_type="o_cal"
-        )
+        self.dates = trading_days_in_range(str(self.start_date), str(self.close_date), count=False, cal_type="o_cal")
         self.dates_stamps = self._prepare_timestamps(self.dates)
         self.o_ticker_lookup = o_ticker_lookup
 
     def _construct_url(self, o_ticker: str) -> str:
         return f"/v3/quotes/{o_ticker}"
 
-    def generate_request_args(
-        self, args_data: list[OptionTicker]
-    ) -> tuple[list[tuple[str, dict]], dict[str, int]]:
+    def generate_request_args(self, args_data: list[OptionTicker]) -> tuple[list[tuple[str, dict]], dict[str, int]]:
         """Generate the urls to query the options quotes endpoint.
         Inputs should be OptionTickers. We then generate the date ranges.
         To prepare the args, we make the timestamp pairs (1 hour wide) and query for the oldest quote in each window.
@@ -563,14 +555,14 @@ def _prepare_timestamps(dates: pd.DataFrame) -> list[int]:
         dates = dates.tz_localize("US/Eastern")
         for i in range(9):
             if i >= 7:
-                dates[f"{i+9}_oclock"] = dates.index + pd.Timedelta(hours=i + 9)
+                dates[f"{i + 9}_oclock"] = dates.index + pd.Timedelta(hours=i + 9)
             else:
-                dates[f"{i+9}_oclock"] = dates.index + pd.Timedelta(hours=i + 9, minutes=30)
-            dates[f"{i+9}_oclock"] = dates[f"{i+9}_oclock"].dt.strftime("%Y-%m-%dT%H:%M:%S%z")
-            dates[f"{i+9}_oclock"] = (
-                dates[f"{i+9}_oclock"].astype(str).str.slice(stop=-2)
+                dates[f"{i + 9}_oclock"] = dates.index + pd.Timedelta(hours=i + 9, minutes=30)
+            dates[f"{i + 9}_oclock"] = dates[f"{i + 9}_oclock"].dt.strftime("%Y-%m-%dT%H:%M:%S%z")
+            dates[f"{i + 9}_oclock"] = (
+                dates[f"{i + 9}_oclock"].astype(str).str.slice(stop=-2)
                 + ":"
-                + dates[f"{i+9}_oclock"].astype(str).str.slice(start=-2)
+                + dates[f"{i + 9}_oclock"].astype(str).str.slice(start=-2)
             )
         dates.drop(columns=["market_open", "market_close"], inplace=True)
         dates = pd.DataFrame({"timestamp.gte": dates.values.flatten()})
@@ -618,11 +610,7 @@ async def download_data(self, o_ticker: str, payload: dict, session: ClientSessi
     def lookup_date_timestamps_from_record(self, timestamp: int) -> list[int]:
         date = timestamp_to_datetime(timestamp, msec_units=False, nano_sec=True)
         date = str(date.date())
-        return (
-            self.dates_stamps["nanosecond.gte"]
-            .loc[self.dates_stamps["timestamp.gte"].str.contains(date)]
-            .to_list()
-        )
+        return self.dates_stamps["nanosecond.gte"].loc[self.dates_stamps["timestamp.gte"].str.contains(date)].to_list()
 
     # TODO: finish this algorithm
     def search_for_timestamps(self, data: list[dict]) -> list[dict]:

curator/data_pipeline/polygon_utils.py

@@ -2,7 +2,8 @@
 from typing import Any
 
 from aiomultiprocess import Pool
-from data_pipeline.path_runner import (
+
+from curator.data_pipeline.path_runner import (
     MetaDataRunner,
     OptionsContractsRunner,
     OptionsPricesRunner,
@@ -11,7 +12,6 @@
     PathRunner,
     StockPricesRunner,
 )
-
 from curator.proj_constants import CPUS, log
 from curator.utils import pool_kwarg_config
 
@@ -31,9 +31,7 @@ async def etl_pool_uploader(runner: PathRunner, pool_kwargs: dict = {}, path_inp
     """
 
     log.info(f"generating the path args to be uploaded -- {runner.runner_type}")
-    path_args = (
-        runner.generate_path_args() if not path_input_args else runner.generate_path_args(path_input_args)
-    )
+    path_args = runner.generate_path_args() if not path_input_args else runner.generate_path_args(path_input_args)
 
     log.info(
         f"uploading data to the database -- Starting Process Pool -- Upload Function: {runner.upload_func.__qualname__}"