Merge pull request #50 from drift-labs:goldhaxx/DPE-3459/fix-risk-dashboard-loading-issues

Goldhaxx/DPE-3459/fix-risk-dashboard-loading-issues

Author: goldhaxx

backend/api/__init__.py

@@ -8,12 +8,12 @@
     market_recommender_api,
     metadata,
     open_interest_api,
-    pnl,
+    pnl_api,
     positions,
     price_shock,
     snapshot,
     ucache,
     user_retention_explorer_api,
     user_retention_summary_api,
-    vaults,
+    vaults_api,
 )

backend/api/pnl.py renamed to backend/api/pnl_api.py

@@ -9,6 +9,7 @@
 @router.get("/top_pnl")
 def get_top_pnl(request: BackendRequest, limit: int = 1000):
     vat: Vat = request.state.backend_state.vat
+    last_oracle_slot = getattr(request.state.backend_state, "last_oracle_slot", 0)
 
     pnl_data = []
     for user in vat.users.values():
@@ -32,4 +33,4 @@ def get_top_pnl(request: BackendRequest, limit: int = 1000):
             continue
 
     pnl_data.sort(key=lambda x: x["total_pnl"], reverse=True)
-    return pnl_data[:limit]
+    return {"pnl": pnl_data[:limit], "slot": last_oracle_slot}

backend/api/vaults.py renamed to backend/api/vaults_api.py

@@ -20,6 +20,7 @@ async def get_vault_data(request: BackendRequest):
 
     client = await create_vault_client()
     analytics = await client.calculate_analytics()
+    last_oracle_slot = getattr(request.state.backend_state, "last_oracle_slot", 0)
 
     vaults = []
     print(analytics["vaults"][0])
@@ -60,5 +61,6 @@ async def get_vault_data(request: BackendRequest):
         vault_depositors[vault_info["pubkey"]] = serializable_depositors
 
     return {
-        "data": {"analytics": serializable_analytics, "depositors": vault_depositors}
+        "data": {"analytics": serializable_analytics, "depositors": vault_depositors},
+        "slot": last_oracle_slot,
     }

backend/app.py

@@ -17,12 +17,12 @@
     market_recommender_api,
     metadata,
     open_interest_api,
-    pnl,
+    pnl_api,
     positions,
     price_shock,
     snapshot,
     ucache,
-    vaults,
+    vaults_api,
     high_leverage_api,
     user_retention_summary_api,
     user_retention_explorer_api,
@@ -88,8 +88,8 @@ async def lifespan(app: FastAPI):
 app.include_router(snapshot.router, prefix="/api/snapshot", tags=["snapshot"])
 app.include_router(ucache.router, prefix="/api/ucache", tags=["ucache"])
 app.include_router(deposits_api.router, prefix="/api/deposits", tags=["deposits"])
-app.include_router(pnl.router, prefix="/api/pnl", tags=["pnl"])
-app.include_router(vaults.router, prefix="/api/vaults", tags=["vaults"])
+app.include_router(pnl_api.router, prefix="/api/pnl", tags=["pnl"])
+app.include_router(vaults_api.router, prefix="/api/vaults", tags=["vaults"])
 app.include_router(positions.router, prefix="/api/positions", tags=["positions"])
 app.include_router(market_recommender_api.router, prefix="/api/market-recommender", tags=["market-recommender"])
 app.include_router(open_interest_api.router, prefix="/api/open-interest", tags=["open-interest"])

src/main.py

@@ -13,12 +13,12 @@
 from page.deposits_page import deposits_page
 from page.health import health_page
 from page.liquidation_curves_page import liquidation_curves_page
-from page.market_inspector import market_inspector_page
+from page.market_inspector_page import market_inspector_page
 from page.orderbook import orderbook_page
-from page.pnl import pnl_page
+from page.pnl_page import pnl_page
 from page.price_shock import price_shock_cached_page
 from page.swap import show as swap_page
-from page.vaults import vaults_page
+from page.vaults_page import vaults_page
 from page.welcome import welcome_page
 from page.market_recommender_page import market_recommender_page
 from page.open_interest_page import open_interest_page

src/page/market_inspector.py renamed to src/page/market_inspector_page.py

@@ -6,7 +6,7 @@
 import inspect
 import pandas as pd
 
-from anchorpy import Wallet
+from anchorpy.provider import Wallet
 from dotenv import load_dotenv
 from driftpy.drift_client import DriftClient
 from driftpy.market_map.market_map import MarketMap
@@ -30,7 +30,7 @@ def format_pubkey(pubkey: str) -> str:
     """Truncate pubkey for display."""
     return f"{str(pubkey)[:10]}...{str(pubkey)[-10:]}"
 
-def format_number(number: int, decimals=6) -> str:
+def format_number(number: int | float, decimals=6) -> str:
     """Format large numbers for better readability."""
     return f"{number / (10 ** decimals):,.6f}"
 
@@ -146,7 +146,7 @@ def display_attribute(market_data, attr_path: str, debug_mode: bool = False):
 
     # If it's a Pubkey or similar
     if hasattr(val, '__class__') and val.__class__.__name__ == 'Pubkey':
-        return f"{attr_path}: {format_pubkey(val)}"
+        return f"{attr_path}: {format_pubkey(str(val))}"
 
     # If it's an int that might represent a 'price' or 'reserve'
     if isinstance(val, int):
@@ -201,7 +201,7 @@ def format_complex_object(attr_name, obj):
                 if isinstance(attr_val, (int, float)) and any(x in attr_name.lower() for x in ["price", "amount", "balance"]):
                     formatted_value = format_number(attr_val, 6)
                 elif hasattr(attr_val, '__class__') and attr_val.__class__.__name__ == 'Pubkey':
-                    formatted_value = format_pubkey(attr_val)
+                    formatted_value = format_pubkey(str(attr_val))
                 else:
                     formatted_value = str(attr_val)
                 result.append(f"  • {attr_name}: {formatted_value}")
@@ -228,7 +228,7 @@ def format_complex_object(attr_name, obj):
         if isinstance(value, (int, float)) and any(x in name.lower() for x in ["price", "amount", "balance"]):
             formatted_value = format_number(value, 6)
         elif hasattr(value, '__class__') and value.__class__.__name__ == 'Pubkey':
-            formatted_value = format_pubkey(value)
+            formatted_value = format_pubkey(str(value))
         elif (hasattr(value, '__class__')
               and not isinstance(value, (str, int, float, bool, list, dict))
               and hasattr(value, '__dict__')):
@@ -278,7 +278,7 @@ async def _fetch_market_maps():
     perp_market_map = MarketMap(
         MarketMapConfig(
             drift_client.program,
-            MarketType.Perp(),
+            MarketType.Perp(),  # type: ignore
             WebsocketConfig(resub_timeout_ms=10000),
             connection,
         )
@@ -289,7 +289,7 @@ async def _fetch_market_maps():
     spot_market_map = MarketMap(
         MarketMapConfig(
             drift_client.program,
-            MarketType.Spot(),
+            MarketType.Spot(),  # type: ignore
             WebsocketConfig(resub_timeout_ms=10000),
             connection,
         )

src/page/pnl.py

@@ -0,0 +1,83 @@
+import pandas as pd
+import streamlit as st
+import time
+
+from lib.api import fetch_api_data
+from src.utils import get_current_slot
+
+RETRY_DELAY_SECONDS = 5
+
+
+def is_processing(result):
+    """Checks if the API result indicates backend processing."""
+    return isinstance(result, dict) and result.get("result") == "processing"
+
+
+def has_error(result):
+    """Checks if the API result indicates an error."""
+    return result is None or (
+        isinstance(result, dict) and result.get("result") == "error"
+    ) or ("pnl" not in result if isinstance(result, dict) else False)
+
+
+def pnl_page():
+    st.title("Top PnL by User (All Time)")
+    response = fetch_api_data("pnl", "top_pnl", retry=False)
+
+    if is_processing(response):
+        st.info(
+            f"Backend is processing PnL data. Auto-refreshing in {RETRY_DELAY_SECONDS} seconds..."
+        )
+        with st.spinner("Please wait..."):
+            time.sleep(RETRY_DELAY_SECONDS)
+        st.rerun()
+        return
+
+    if has_error(response):
+        error_msg = (
+            response["message"]
+            if isinstance(response, dict) and "message" in response
+            else "Could not connect or fetch PnL data."
+        )
+        st.error(f"Failed to fetch PnL data: {error_msg}")
+        if st.button("Retry"):
+            st.rerun()
+        return
+
+    pnl_data = response["pnl"]
+    if not pnl_data:
+        st.info("No PnL data available.")
+        return
+
+    slot = response.get("slot", 0)
+    current_slot = get_current_slot()
+    if slot > 0 and current_slot > 0:
+        slot_age = current_slot - slot
+        st.info(f"Data from slot {slot}, which is {slot_age} slots old.")
+
+    df = pd.DataFrame(pnl_data)
+    for col in ["realized_pnl", "unrealized_pnl", "total_pnl"]:
+        df[col] = df[col].map("${:,.2f}".format)
+
+    csv = df.to_csv(index=False)
+    st.download_button(
+        "Download PnL Data CSV", csv, "top_pnl.csv", "text/csv", key="download-pnl"
+    )
+    st.dataframe(
+        df,
+        height=650,
+        column_config={
+            "authority": st.column_config.TextColumn(
+                "Authority",
+                help="Authority address",
+            ),
+            "user_key": st.column_config.TextColumn(
+                "User Account",
+                help="User account address",
+            ),
+            "realized_pnl": st.column_config.NumberColumn("All Time Realized PnL"),
+            "unrealized_pnl": st.column_config.NumberColumn("Unrealized PnL"),
+            "total_pnl": st.column_config.NumberColumn("Total PnL"),
+        },
+        hide_index=True,
+    )

src/page/pnl_page.py

@@ -1,7 +1,23 @@
 import pandas as pd
 import streamlit as st
+import time
 
 from lib.api import fetch_api_data
+from src.utils import get_current_slot
+
+RETRY_DELAY_SECONDS = 10
+
+
+def is_processing(result):
+    """Checks if the API result indicates backend processing."""
+    return isinstance(result, dict) and result.get("result") == "processing"
+
+
+def has_error(result):
+    """Checks if the API result indicates an error."""
+    return result is None or (
+        isinstance(result, dict) and result.get("result") == "error"
+    ) or ("data" not in result if isinstance(result, dict) else False)
 
 
 def vaults_page():
@@ -11,15 +27,37 @@ def vaults_page():
         "This page may be out of date up to 30 minutes."
     )
 
-    with st.spinner("Loading vault data..."):
-        response = fetch_api_data("vaults", "data", retry=True, max_wait_time=60)
-        if not response or "data" not in response:
-            st.error("Failed to load vault data")
-            return
+    response = fetch_api_data("vaults", "data", retry=False)
 
-        data = response["data"]
-        analytics = data["analytics"]
-        all_depositors = data["depositors"]
+    if is_processing(response):
+        st.info(
+            f"Backend is processing vault data. Auto-refreshing in {RETRY_DELAY_SECONDS} seconds..."
+        )
+        with st.spinner("Please wait..."):
+            time.sleep(RETRY_DELAY_SECONDS)
+        st.rerun()
+        return
+
+    if has_error(response):
+        error_msg = "Invalid response from backend."
+        if isinstance(response, dict) and "message" in response:
+            error_msg = response["message"]
+        elif has_error(response):
+            error_msg = "Could not connect or fetch vault data."
+        st.error(f"Failed to load vault data: {error_msg}")
+        if st.button("Retry"):
+            st.rerun()
+        return
+
+    slot = response.get("slot", 0)
+    current_slot = get_current_slot()
+    if slot > 0 and current_slot > 0:
+        slot_age = current_slot - slot
+        st.info(f"Data from slot {slot}, which is {slot_age} slots old.")
+
+    data = response["data"]
+    analytics = data["analytics"]
+    all_depositors = data["depositors"]
 
     unique_depositors = set()
     for depositors in all_depositors.values():
@@ -60,10 +98,21 @@ def vaults_page():
 
     st.subheader("Vault Depositor Details")
 
+    all_vaults_df_indexed = all_vaults_df.set_index("pubkey")
+
+    def format_vault_option(pubkey: str) -> str:
+        try:
+            vault_info = all_vaults_df_indexed.loc[pubkey]
+            name = vault_info["name"]
+            net_deposits = vault_info["true_net_deposits"]
+            return f"{name} (${net_deposits:,.2f} USD) - {pubkey[:4]}...{pubkey[-4:]}"
+        except KeyError:
+            return f"Unknown Vault - {pubkey[:4]}...{pubkey[-4:]}"
+
     selected_vault = st.selectbox(
         "Select Vault",
-        [vault["pubkey"] for vault in all_vaults_df.to_dict(orient="records")],
-        format_func=lambda x: f"{all_vaults_df[all_vaults_df['pubkey'] == x]['name'].values[0]} (${all_vaults_df[all_vaults_df['pubkey'] == x]['true_net_deposits'].values[0]:,.2f} USD) - {x[:4]}...{x[-4:]}",
+        all_vaults_df["pubkey"].tolist(),
+        format_func=format_vault_option,
     )
 
     if selected_vault:

src/page/vaults.py renamed to src/page/vaults_page.py

@@ -35,7 +35,7 @@ def welcome_page():
         label="💰 **Deposits** - Track total deposits across all the protocol",
     )
     st.page_link(
-        StreamlitPage("page/pnl.py", url_path="pnl"),
+        StreamlitPage("page/pnl_page.py", url_path="pnl"),
         label="💸 **PnL** - Track top trader PnLs",
     )