top holdings and other feats

Author: leftmove

backend/routers/filer.py

@@ -62,6 +62,7 @@ class HTTPError(BaseModel):
 # 0 : Filer is done being built, logs are no longer kept.
 
 # Different Filer Stages
+# 5 : Filer is broken, and has no query data. Will never be updated unless manually.
 # 4 : Filer was non-existent before and is being fully built from the ground up.
 # 3 : Filer still building, but estimation time has been calculated.
 # 2 : Filer's newest filing is built, but older filings are still being updated and sorted. Estimation time is calculated.
@@ -79,12 +80,21 @@ def create_recent(cik, company, stamp):
         last_report = company["last_report"]
         recent_filing = database.find_filing(cik, last_report)
 
-        for access_number, filing_stocks in web.process_stocks(cik, [recent_filing]):
-            recent_filing["stocks"] = filing_stocks
-            database.edit_filing(
-                {**filer_query, "access_number": access_number},
-                {"$set": {"stocks": filing_stocks}},
-            )
+        try:
+            for access_number, filing_stocks in web.process_stocks(
+                cik, [recent_filing]
+            ):
+                recent_filing["stocks"] = filing_stocks
+                database.edit_filing(
+                    {**filer_query, "access_number": access_number},
+                    {"$set": {"stocks": filing_stocks}},
+                )
+        except (
+            IndexError
+        ) as e:  # Filer is broken, no forms found. Special case for first query.
+            break_filer(cik)
+            report_error(cik, e)
+            raise HTTPException(404, detail="Forms not found for filer.")
 
         database.add_log(cik, "Queried Filer Recent Stocks", company_name, cik)
     except Exception as e:
@@ -353,6 +363,15 @@ async def rollback_filer(
     return {"description": "Filer rollback started."}
 
 
+async def break_filer(cik: str):
+    document_reports = web.check_forms(cik)
+    if len(document_reports) == 0:
+        database.edit_status(cik, 5)
+        database.add_log(cik, "Filer has no filings.", "Error", cik)
+    else:
+        report_error(cik, Exception("Filer is broken but has forms."))
+
+
 @cache(24)
 @router.get("/search", tags=["filers"], status_code=200)
 async def search_filers(q: str, limit: int = 4):
@@ -475,6 +494,8 @@ async def filer_info(cik: str):
         raise HTTPException(404, detail="Filer not found.")
 
     status = database.find_log(cik, {"status": 1, "_id": 0})
+    if status is None:
+        raise HTTPException(404, detail="Filer log not found.")
     filer["status"] = status["status"]
 
     return {"description": "Found filer.", "filer": filer}

backend/routers/filing.py

@@ -180,12 +180,22 @@ async def filings_info(cik: str):
 
 
 @router.get("/info", status_code=200)
-async def filing_info(cik: str, access_number: str):
+async def filing_info(cik: str, access_number: str, include_filer: bool = False):
     filing = database.find_filing(cik, access_number, {"_id": 0, "cik": 0, "stocks": 0})
     if filing is None:
         raise HTTPException(detail="Filing not found.", status_code=404)
 
+    filer = database.find_filer(cik, {"_id": 0, "stocks": 0}) if include_filer else None
+    if filer is None and include_filer:
+        raise HTTPException(404, detail="Filer not found.")
+
     status = database.find_log(cik, {"status": 1, "_id": 0})
+    if status is None:
+        raise HTTPException(404, detail="Filer log not found.")
     filing["status"] = status["status"]
 
-    return {"description": "Filing found.", "filing": filing}
+    return {
+        "description": "Filing found.",
+        "filing": filing,
+        **({"filer": filer} if include_filer else {}),
+    }

backend/routers/lib/analysis.py

@@ -143,22 +143,22 @@ def serialize_global(local_stock, global_stock):
     )
     portfolio_percentage_str = (
         "{:.2f}".format(round(portfolio_percentage, 4))
-        if portfolio_percentage != "N/A"
+        if portfolio_percentage != "N/A" and type(portfolio_percentage) == float
         else "N/A"
     )
     ownership_percentage_str = (
         "{:.2f}".format(round(ownership_percentage, 4))
-        if ownership_percentage != "N/A"
+        if ownership_percentage != "N/A" and type(ownership_percentage) == float
         else "N/A"
     )
     gain_value_str = (
         "{:.2f}".format(round(gain_value, 2))
-        if update and buy_timeseries != "N/A"
+        if update and buy_timeseries != "N/A" and type(gain_value) == float
         else "N/A"
     )
     gain_percent_str = (
         "{:.2f}".format(round(gain_percent, 2))
-        if update and buy_timeseries != "N/A"
+        if update and buy_timeseries != "N/A" and type(gain_percent) == float
         else "N/A"
     )
 
@@ -303,13 +303,21 @@ def serialize_local(
 
 
 def analyze_total(cik, stocks, access_number):
-    market_values = []
+    market_map = []
     for key in stocks:
         stock = stocks[key]
         value = stock.get("market_value", 0)
-        market_values.append(value)
-
+        market_map.append({"cusip": key, "market_value": value})
+
+    market_values = [stock["market_value"] for stock in market_map]
+    top_holdings = [
+        stock["cusip"]
+        for stock in sorted(market_map, key=lambda x: x["market_value"], reverse=True)[
+            :5
+        ][: min(5, len(market_map))]
+    ]
     total = sum(market_values)
+
     database.edit_filing(
         {
             "cik": cik,
@@ -319,6 +327,7 @@ def analyze_total(cik, stocks, access_number):
         {
             "$set": {
                 "market_value": total,
+                "top_holdings": top_holdings,
             }
         },
     )
@@ -621,14 +630,27 @@ def sort_pipeline(
     reverse: bool,
     unavailable: bool,
     additional: list = [],
+    stock_structure: str = "array",
     collection_search=database.search_filers,
+    match_query={},
 ):
     if limit < 0:
         raise ValueError
 
     pipeline = [
-        {"$match": {"cik": cik}},
+        {
+            "$match": {"cik": cik, **match_query},
+        },
     ]
+
+    if stock_structure == "dict":
+        pipeline.extend(
+            [
+                {"$set": {"stocks": {"$objectToArray": "$stocks"}}},
+                {"$set": {"stocks": "$stocks.v"}},
+            ]
+        )
+
     if additional:
         pipeline.extend(additional)
 
@@ -646,15 +668,18 @@ def sort_pipeline(
     if unavailable is False:
         sort_query = f"${sort}"
 
+    pipeline.append({"$project": {"_id": 1}})
     cursor = collection_search(pipeline)
     results = [result for result in cursor]
     if not cursor or not results:
         raise LookupError
     count = len(results)
 
+    pipeline.pop(-1)
     pipeline.append(
         {"$sort": {sort: 1 if reverse else -1, "_id": 1}},
     )
+
     if unavailable is False:
         sort_stage = pipeline.pop(-1)
         pipeline.extend(

backend/routers/lib/database.py

@@ -134,6 +134,18 @@ def search_filers(pipeline):
     return cursor
 
 
+@retry_on_rate_limit()
+def search_filings(pipeline):
+    cursor = filings.aggregate(pipeline)
+    return cursor
+
+
+@retry_on_rate_limit()
+def search_filings(pipeline):
+    cursor = filings.aggregate(pipeline)
+    return cursor
+
+
 @retry_on_rate_limit()
 def search_filer(cik, project={"_id": 0}):
     cursor = main.aggregate(pipeline=[{"$match": {"cik": cik}}, {"$project": project}])
@@ -197,12 +209,6 @@ def map_filings(cik, key="access_number", project={"_id": 0}, form_type="13F-HR"
     return results_dict
 
 
-@retry_on_rate_limit()
-def search_filings(pipeline):
-    cursor = filings.aggregate(pipeline)
-    return cursor
-
-
 @retry_on_rate_limit()
 def add_filings(filing_list):
     filings.insert_many(filing_list)

backend/routers/stocks.py

@@ -40,8 +40,8 @@ async def query_stocks(cik: str, background: BackgroundTasks):
 
 
 @cache
-@router.get("/info", tags=["stocks", "filers"], status_code=200)
-async def stock_info(
+@router.get("/filer", tags=["stocks", "filers"], status_code=200)
+async def stock_filer(
     cik: str,
     limit: int,
     offset: int,
@@ -77,6 +77,57 @@ async def stock_info(
     )
 
 
+@cache
+@router.get("/filing", tags=["stocks", "filings"], status_code=200)
+async def stock_filing(
+    cik: str,
+    access_number: str,
+    limit: int,
+    offset: int,
+    sort: str,
+    sold: bool,
+    reverse: bool,
+    unavailable: bool,
+):
+    filer = database.find_filer(cik, {"_id": 1})
+    if not filer:
+        raise HTTPException(detail="Filer not found.", status_code=404)
+
+    try:
+        pipeline, count = analysis.sort_pipeline(
+            cik,
+            limit,
+            offset,
+            sort,
+            sold,
+            reverse,
+            unavailable,
+            stock_structure="dict",
+            collection_search=database.search_filings,
+            match_query={
+                "access_number": access_number,
+                "stocks": {"$exists": True},
+            },
+        )
+        cursor = database.search_filings(pipeline)
+    except LookupError as e:
+        errors.report_error(cik, e)
+        raise HTTPException(detail="No results found.", status_code=422)
+    except Exception as e:
+        errors.report_error(cik, e)
+        cursor = []
+        count = 0
+
+    try:
+        stock_list = [result for result in cursor]
+    except KeyError:
+        raise HTTPException(detail="Error while searching.", status_code=500)
+
+    return BrowserCachedResponse(
+        content={"stocks": stock_list, "count": count}, cache_hours=cache_time
+    )
+
+
 @cache(4)
 @router.get("/timeseries", tags=["stocks", "filers"], status_code=200)
 async def stock_timeseries(cik: str, time: float):

backend/static/statistics.json

@@ -1,12 +1,12 @@
 {
       "latest": {
-            "count": 1699,
-            "total": 1271220.7323975563,
-            "average": 748.2170290744887
+            "count": 1718,
+            "total": 1273623.8951702118,
+            "average": 741.3410332771896
       },
       "historical": {
-            "count": 1684,
-            "total": 5897062.192974329,
-            "average": 3501.818404379055
+            "count": 1695,
+            "total": 5920188.0643363,
+            "average": 3492.73632114236
       }
 }

frontend/components/Analysis/Analysis.jsx

@@ -33,7 +33,9 @@ const Analysis = ({ text = "Analysis", icon = "folder", children }) => {
             styles["analysis-dummy"],
             open ? styles["dummy-expand"] : styles["dummy-contract"],
           ].join(" ")}
-        ></div>
+        >
+          <span className={styles["dummy-text"]}>(Tools and Downloads)</span>
+        </div>
         <span
           className={[styles["analysis-tip"], fontLight.className].join(" ")}
         >

frontend/components/Analysis/Analysis.module.css

@@ -18,9 +18,9 @@
   align-items: center;
   cursor: pointer;
 
-  width: 10%;
+  width: 19%;
   border-radius: 5px;
-  padding: 10px;
+  padding: 5px;
   opacity: 1;
   margin: auto;
   margin-top: 10px;
@@ -43,6 +43,17 @@
   height: 20px;
   width: 1%;
   transition: width 0.5s ease;
+  position: relative;
+}
+
+.dummy-text {
+  display: none;
+  position: absolute;
+  transition: opacity 0.3s ease;
+}
+
+.analysis-expanded .dummy-text {
+  opacity: 0;
 }
 
 .dummy-expand {