general refactors

Author: leftmove

.gitignore

@@ -1,3 +1,5 @@
 .DS_Store
 .vscode
+.idea
+.fleet
 .gitignore

backend/.gitignore

@@ -1,4 +1,5 @@
 whale-project.code-workspace
+.fleet
 
 start.sh
 example.json

backend/pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
     "sentry-sdk[celery]<3.0.0,>=2.0.1",
     "meilisearch<1.0.0,>=0.31.4",
     "jinja2>=3.1.5",
+    "python-multipart>=0.0.20",
 ]
 name = "wallstreetlocal"
 version = "2.0.0"

backend/routers/filer.py

@@ -207,7 +207,11 @@ def create_historical(cik, company, stamp):
                         if change_query is None:
                             continue
                         database.edit_filing(
-                            {**filer_query, "access_number": access_number},
+                            {
+                                **filer_query,
+                                "access_number": access_number,
+                                "stocks": {"$exists": True},
+                            },
                             {"$set": {change_query: change_stock}},
                         )
 

backend/routers/filing.py

@@ -167,7 +167,7 @@ async def record_filing_csv(cik: str, access_number: str, headers: str = None):
 @cache(2)
 async def filings_info(cik: str):
     pipeline = [
-        {"$match": {"cik": cik, "form": {"$in": database.holding_forms}}},
+        {"$match": {"cik": cik, "form": {"$in": database.holding_forms}}, "stocks": {"$exists": True}},
         {"$project": {"cik": 0, "stocks": 0, "_id": 0}},
     ]
     cursor = database.search_filings(pipeline)

backend/routers/lib/analysis.py

@@ -683,7 +683,9 @@ def analyze_changes(cik, prev_access, current_access):
 def analyze_filings(cik, filings, last_report):
     stock_cache = {}
     filings_map = dict(zip([f["access_number"] for f in filings], filings))
-    filings_sorted = sorted([f for f in filings], key=lambda d: d.get("report_date", 0))
+    filings_sorted = sorted(
+        [f for f in filings], key=lambda d: d.get("report_date", 0), reverse=True
+    )
     for filing in filings:
         access_number = filing.get("access_number", "")
         filing_stocks = filing.get("stocks")
@@ -873,6 +875,7 @@ def sort_pipeline(
     stock_structure: str = "array",
     collection_search=database.search_filers,
     match_query={},
+    project=[],
     additional_one: list = [],
     additional_two: list = [],
 ):
@@ -905,6 +908,12 @@ def sort_pipeline(
         ]
     )
 
+    if project:
+        new_project = {}
+        for p in project:
+            new_project[p] = 0
+        pipeline.append({"$project": {"_id": 0, **new_project}})
+
     if additional_two:
         pipeline.extend(additional_two)
 

backend/routers/lib/web.py

@@ -464,7 +464,9 @@ def scrape_stocks(cik, data, filing, last_report, empty=False):
 
 def process_stocks(cik, filings):
     filings_list = sorted(
-        [f for f in filter(None, filings)], key=lambda d: d.get("report_date", 0)
+        [f for f in filter(None, filings)],
+        key=lambda d: d.get("report_date", 0),
+        reverse=True,
     )
     last_report = filings_list[-1]["access_number"]
     for document in filings_list:

backend/routers/stocks.py

@@ -88,6 +88,7 @@ async def stock_filing(
     sold: bool,
     reverse: bool,
     unavailable: bool,
+    projections = ["changes"]
 ):
     filer = database.find_filer(cik, {"_id": 1})
     if not filer:
@@ -104,6 +105,7 @@ async def stock_filing(
             unavailable,
             stock_structure="dict",
             collection_search=database.search_filings,
+            project=projections,
             match_query={
                 "access_number": access_number,
                 "stocks": {"$exists": True},

backend/uv.lock

@@ -7,6 +7,7 @@ example.json
 .env
 
 .vscode
+.fleet
 
 node_modules
 .pnp
@@ -42,6 +43,6 @@ docker-compose.dev.yml
 docker-compose.dev.yaml
 
 docker-proj.zip
-
-# Sentry Config File
-.env.sentry-build-plugin
+
+# Sentry Config File
+.env.sentry-build-plugin