filing progress

Author: leftmove

backend/routers/filing.py

@@ -39,68 +39,73 @@ class HTTPError(BaseModel):
 @router.get(
     "/query",
     tags=["filing"],
-    status_code=201,
+    status_code=200,
 )
-async def query_filer(
+async def query_filing(
     cik: str, access_number: str, background: BackgroundTasks = BackgroundTasks
 ):
     filing = database.find_filing(cik, access_number, {"_id": 1})
+
     if not filing:
-        try:
-            sec_data = sec_filer_search(cik)
-        except Exception as e:
-            logging.error(e)
-            raise HTTPException(404, detail="CIK not found.")
+
+        update, last_report = web.check_new(cik)
+        company = database.find_filer(cik)
+
+        if not update:
+            return update_filing(company, last_report, background)
+
+        if not company:
+            raise HTTPException(404, detail="Filer not found.")
+
+        filings = database.find_filings(cik, {"access_number": 1})
+        access_numbers = [filing["access_number"] for filing in filings]
+
+        if access_number not in access_numbers:
+            raise HTTPException(404, detail="Filing not found. Invalid access number.")
 
         if production_environment:
-            worker.create_filer.delay(cik, sec_data)
+            worker.repair_filer.delay(cik)
         else:
-            background.add_task(filer.create_filer, cik, sec_data)
+            background.add_task(filer.repair_filer, cik)
 
-        return {"status": "Filing creation started."}
-    else:
+        raise HTTPException(201, detail="Filing found but not queried. Repair started.")
 
-        if filing["form"] not in database.holding_forms:
-            raise HTTPException(404, detail="Filing type not supported.")
+    else:
 
-        return update_filing({"cik": cik})
+        return {"description": "Filing already queried."}
 
 
 # Exact functionality as `update_filer`, so changes need to be synced
 
 
-def update_filing(filing, background: BackgroundTasks = BackgroundTasks):
-    cik = filing["cik"]
+def update_filing(
+    company, last_report: str, background: BackgroundTasks = BackgroundTasks
+):
+    cik = company["cik"]
     time = datetime.now().timestamp()
 
     operation = database.find_log(cik)
     if operation is None:
-        raise HTTPException(404, detail="Filing log not found.")
+        raise HTTPException(404, detail="Filer log not found.")
     elif (
         production_environment and operation["status"] == 2 or operation["status"] == 1
     ):
         raise HTTPException(  # @IgnoreException
-            302, detail="Filing is partially building."
+            302, detail="Filer is partially building."
         )
     elif operation["status"] >= 2:
-        raise HTTPException(409, detail="Filing still building.")
-
-    update, last_report = web.check_new(cik)
-    if not update:
-        raise HTTPException(
-            200, detail="Filing is already up to date."
-        )  # @IgnoreException
+        raise HTTPException(409, detail="Filer still building.")
 
     database.edit_status(cik, 1)
     database.edit_filer({"cik": cik}, {"$set": {"last_report": last_report}})
 
     stamp = {"name": company["name"], "start": time}
     if production_environment:
-        worker.create_historical.delay(cik, company, stamp)
+        worker.create_recent.delay(cik, company, stamp)
     else:
-        background.add_task(create_historical, cik, company, stamp)
+        background.add_task(filer.create_recent, cik, company, stamp)
 
-    return {"description": "Filer update started."}
+    return {"description": "Filing update started."}
 
 
 @cache(24)
@@ -148,6 +153,9 @@ async def record_filing_csv(cik: str, access_number: str, headers: str = None):
         file_name = f"wallstreetlocal-{cik}-{access_number}.csv"
 
     filing = database.find_filing(cik, access_number)
+    if filing is None:
+        raise HTTPException(404, detail="Filing not found.")
+
     stock_dict = filing["stocks"]
     stock_list = [stock_dict[cusip] for cusip in stock_dict]
 
@@ -157,15 +165,27 @@ async def record_filing_csv(cik: str, access_number: str, headers: str = None):
     )
 
 
-@router.get("/info", status_code=200)
-async def query_filings(cik: str):
+@router.get("/filer", status_code=200)
+async def filings_info(cik: str):
     pipeline = [
-        {"$match": {"cik": cik, "form": "13F-HR"}},
-        {"$project": {"stocks": 0, "_id": 0}},
+        {"$match": {"cik": cik, "form": {"$in": database.holding_forms}}},
+        {"$project": {"cik": 0, "stocks": 0, "_id": 0}},
     ]
     cursor = database.search_filings(pipeline)
     if not cursor:
         raise HTTPException(detail="Filer not found.", status_code=404)
     filings = [result for result in cursor]
 
     return {"filings": filings}
+
+
+@router.get("/info", status_code=200)
+async def filing_info(cik: str, access_number: str):
+    filing = database.find_filing(cik, access_number, {"_id": 0, "cik": 0, "stocks": 0})
+    if filing is None:
+        raise HTTPException(detail="Filing not found.", status_code=404)
+
+    status = database.find_log(cik, {"status": 1, "_id": 0})
+    filing["status"] = status["status"]
+
+    return {"description": "Filing found.", "filing": filing}

backend/routers/general.py

@@ -31,13 +31,13 @@
 @cache(24)
 @router.get("/", status_code=200)
 async def info():
-    return {"message": "Hello World!"}
+    return {"description": "Hello World!"}
 
 
 @cache
 @router.get("/undefined", status_code=200)
 async def info_undefined():
-    return {"message": "Hello World!"}
+    return {"description": "Hello World!"}
 
 
 @cache(4)
@@ -69,13 +69,13 @@ async def health():
     if health_passed < 0.8:
         raise HTTPException(status_code=500, detail="The server doesn't seem healthy.")
 
-    return {"message": "The server is healthy."}
+    return {"description": "The server is healthy."}
 
 
 # @router.get("/error", include_in_schema=False)
 # async def trigger_error():
 #     1 / 0
-#     return {"message": "This will never be reached."}
+#     return {"description": "This will never be reached."}
 
 
 @router.get("/task-error", include_in_schema=False)
@@ -86,7 +86,7 @@ async def task_error(password: str):
 
     worker.delay_error.delay()
 
-    return {"message": "Task error triggered."}
+    return {"description": "Task error triggered."}
 
 
 # Terrible code

backend/routers/lib/analysis.py

@@ -699,8 +699,7 @@ def create_json(content, file_name):
             filer_json = json.load(f)
             if (datetime.now().timestamp() - filer_json["updated"]) > 60 * 60 * 3:
                 raise ValueError
-    except Exception as e:
-        errors.report_error(file_name, e)
+    except FileNotFoundError:
         with open(file_path, "w") as r:
             json.dump(content, r, indent=6)
 
@@ -765,8 +764,7 @@ def create_csv(content, file_name, headers=None):
                     expire_time = 60 * 60 * 24 * 3
                     cache.set_key(file_path, "bababooey", expire_time)
                     raise ValueError
-    except Exception as e:
-        errors.report_error(file_name, e)
+    except FileNotFoundError:
         stock_list = create_dataframe(content, headers)
         with open(file_path, "w") as f:
             writer = csv.writer(f)
@@ -938,11 +936,7 @@ def analyze_allocation(cik):
     allocation_statistic = {
         "filings": allocation_access,
     }
-    database.add_statistic(
-        cik,
-        "allocation",
-        allocation_statistic,
-    )
+    database.add_statistic(cik, "allocation", allocation_statistic, completion)
 
     return allocation_list
 

backend/routers/lib/database.py

@@ -183,7 +183,7 @@ def find_filing(cik, access_number, project={"_id": 0}, form_type="13F-HR"):
 
 
 @retry_on_rate_limit()
-def find_filings(cik, project={"_id": 0}, form_type="13F-HR"):
+def find_filings(cik, project={"_id": 0}, form_type={"$in": holding_forms}):
     cursor = filings.find({"cik": cik, "form": form_type}, project)
     results = [result for result in cursor]
     return results

backend/routers/stocks.py

@@ -59,6 +59,9 @@ async def stock_info(
             cik, limit, offset, sort, sold, reverse, unavailable
         )
         cursor = database.search_filers(pipeline)
+    except LookupError as e:
+        errors.report_error(cik, e)
+        raise HTTPException(detail="No results found.", status_code=422)
     except Exception as e:
         errors.report_error(cik, e)
         cursor = []

backend/static/statistics.json

@@ -1,12 +1,12 @@
 {
       "latest": {
-            "count": 1690,
-            "total": 1255375.0261363983,
-            "average": 742.8254592523067
+            "count": 1699,
+            "total": 1271220.7323975563,
+            "average": 748.2170290744887
       },
       "historical": {
-            "count": 1675,
-            "total": 5818102.588788748,
-            "average": 3473.494082858954
+            "count": 1684,
+            "total": 5897062.192974329,
+            "average": 3501.818404379055
       }
 }

frontend/components/Explorer/Explorer.jsx

@@ -75,7 +75,7 @@ const Explorer = () => {
       })
       .catch((e) => console.error(e));
   const { isLoading: loading, error } = useSWR(
-    cik ? [server + "/filing/info", cik] : null,
+    cik ? [server + "/filing/filer", cik] : null,
     ([url, cik]) => filingFetcher(url, cik),
     {
       revalidateOnFocus: false,

frontend/components/Filer/Info.jsx

@@ -11,25 +11,22 @@ import Explorer from "components/Explorer/Explorer";
 import Header from "components/Header/Header";
 
 const convertTitle = (d) => {
-    if (d) {
-        d = d.replace(
-            /(^\w|\s\w)(\S*)/g,
-            (_, m1, m2) => {
-                if (/[a-z][A-Z]|[A-Z][a-z]/.test(m1 + m2)) {
-                    return m1 + m2; // Return the original word
-                } else {
-                    return m1.toUpperCase() + m2.toLowerCase(); // Convert to title case
-                }
-            }
-        );
-        ["LLC", "LP", "L.P.", "LLP", "N.A."].forEach((word) => {
-            d = d.replace(
-                word.at(0).toUpperCase() + word.toLowerCase().slice(1),
-                word
-            );
-        });
-    }
-    return d;
+  if (d) {
+    d = d.replace(/(^\w|\s\w)(\S*)/g, (_, m1, m2) => {
+      if (/[a-z][A-Z]|[A-Z][a-z]/.test(m1 + m2)) {
+        return m1 + m2; // Return the original word
+      } else {
+        return m1.toUpperCase() + m2.toLowerCase(); // Convert to title case
+      }
+    });
+    ["LLC", "LP", "L.P.", "LLP", "N.A."].forEach((word) => {
+      d = d.replace(
+        word.at(0).toUpperCase() + word.toLowerCase().slice(1),
+        word
+      );
+    });
+  }
+  return d;
 };
 
 const Info = (props) => {

frontend/components/Filer/Redirect.jsx

@@ -14,7 +14,7 @@ const Redirect = (props) => {
   const { ellipsis } = useEllipsis();
   useEffect(() => {
     setTimeout(() => {
-      router.replace("/filers/" + cik + "?continuous=true");
+      router.replace("/filers/" + cik + "/overview?continuous=true");
     }, props.delay || wait);
   }, [cik]);
 

frontend/components/Filing/Info.jsx

@@ -0,0 +1,35 @@
+"use server";
+
+import styles from "@/styles/Filer.module.css";
+
+import Head from "next/head";
+
+import Tabs from "components/Tabs/Tabs";
+import Index from "components/Index/Index";
+import Charts from "components/Charts/Charts";
+import Explorer from "components/Explorer/Explorer";
+import Header from "components/Header/Header";
+import { convertTitle } from "components/Filer/Info";
+
+const Info = (props) => {
+  const cik = props.cik || null;
+  const tab = props.tab || "recent";
+  const title = `Filers - ${cik}`; // Fallback for header title
+
+  return (
+    <>
+      <Head>
+        <title>{title}</title>
+      </Head>
+      <Header cik={cik} tab={tab} />
+      <Tabs />
+      <div className={styles.data}>
+        {tab === "filings" ? <Explorer /> : null}
+        {/* {tab === "charts" ? <Charts /> : null} */}
+        {tab === "stocks" ? <Index /> : null}
+      </div>
+    </>
+  );
+};
+
+export default Info;