Merge pull request #46 from drift-labs:goldhaxx/DATA-77/fix-health-page-errors

goldhaxx · web-flow · commit f0f928e09263 · 2025-06-20T14:58:14.000-07:00
fixed health page loading issues, refactor health page endpoints and improve logging.
diff --git a/backend/api/health.py b/backend/api/health.py
@@ -120,13 +120,13 @@ def get_account_health_distribution(request: BackendRequest):
 
 
 @router.get("/largest_perp_positions")
-def get_largest_perp_positions(request: BackendRequest, number_of_positions: int, market_index: int = None):
+def get_largest_perp_positions(request: BackendRequest, market_index: int = None):
     """
     Get the largest perp positions by notional value across all users or for a specific market if market_index is provided.
     """
     vat: Vat = request.state.backend_state.vat
     logger.info(
-        f"==> [largest_perp_positions] Called with number_of_positions={number_of_positions}, "
+        f"==> [largest_perp_positions] Called with "
         f"market_index={market_index}, current_pickle={request.state.backend_state.current_pickle_path}"
     )
     try:
@@ -168,21 +168,17 @@ def get_largest_perp_positions(request: BackendRequest, number_of_positions: int
                         position.base_asset_amount / BASE_PRECISION,  # Keep original sign for display
                     ))
 
-    # Sort all positions by value (descending) and take top N
-    positions = sorted(all_positions, key=lambda x: x[0], reverse=True)[:number_of_positions]
+    # Sort all positions by value (descending)
+    positions = sorted(all_positions, key=lambda x: x[0], reverse=True)
 
+    # Add summary logging
+    total_value_returned = sum(pos[0] for pos in positions)
     logger.info(
         f"==> [largest_perp_positions] Stats => total_checked={total_positions_checked}, "
-        f"positions_meeting_criteria={positions_meeting_criteria}, positions_returned={len(positions)}"
+        f"positions_meeting_criteria={positions_meeting_criteria}, positions_returned={len(positions)}, "
+        f"total_notional_value=${total_value_returned:,.2f}"
     )
 
-    # Log each position with value and sign
-    for idx, (value, pubkey, market_idx, amt) in enumerate(positions, 1):
-        logger.info(
-            f"==> [largest_perp_positions] Position {idx}: Market Index={market_idx}, "
-            f"Value=${value:,.2f}, Base Asset Amount={amt:,.2f}, Public Key={pubkey}"
-        )
-
     data = {
         "Market Index": [pos[2] for pos in positions],
         "Value": [f"${pos[0]:,.2f}" for pos in positions],
@@ -194,17 +190,16 @@ def get_largest_perp_positions(request: BackendRequest, number_of_positions: int
 
 
 @router.get("/most_levered_perp_positions_above_1m")
-def get_most_levered_perp_positions_above_1m(request: BackendRequest, number_of_positions: int = 10, market_index: int = None):
+def get_most_levered_perp_positions_above_1m(request: BackendRequest, market_index: int = None):
     """
     Get the most leveraged perpetual positions with value above $1 million.
 
     This endpoint calculates the leverage of each perpetual position with a value
-    over $1 million and returns the most leveraged positions, limited by number_of_positions.
+    over $1 million and returns the most leveraged positions.
     Results can be filtered by market_index if provided.
 
     Args:
         request: The backend request object
-        number_of_positions: Maximum number of positions to return (default: 10)
         market_index: Optional market index to filter by
 
     Returns:
@@ -216,7 +211,7 @@ def get_most_levered_perp_positions_above_1m(request: BackendRequest, number_of_
         - Public Key (list[str]): The public keys of the position holders
     """
     vat: Vat = request.state.backend_state.vat
-    top_positions: list[tuple[float, str, int, float, float]] = []
+    all_positions: list[tuple[float, str, int, float, float]] = []
 
     for user in vat.users.values():
         try:
@@ -242,26 +237,21 @@ def get_most_levered_perp_positions_above_1m(request: BackendRequest, number_of_
                         ) * market_price_ui
                         leverage = base_asset_value / total_collateral
                         if base_asset_value > 1_000_000:
-                            heap_item = (
+                            item = (
                                 to_financial(base_asset_value),
                                 user.user_public_key,
                                 position.market_index,
                                 position.base_asset_amount / BASE_PRECISION,
                                 leverage,
                             )
-
-                            if len(top_positions) < number_of_positions:
-                                heapq.heappush(top_positions, heap_item)
-                            else:
-                                heapq.heappushpop(top_positions, heap_item)
+                            all_positions.append(item)
 
     positions = sorted(
-        top_positions,
+        all_positions,
         key=lambda x: x[4],
+        reverse=True
     )
 
-    positions.reverse()
-
     data = {
         "Market Index": [pos[2] for pos in positions],
         "Value": [f"${pos[0]:,.2f}" for pos in positions],
@@ -274,17 +264,16 @@ def get_most_levered_perp_positions_above_1m(request: BackendRequest, number_of_
 
 
 @router.get("/largest_spot_borrows")
-def get_largest_spot_borrows(request: BackendRequest, number_of_positions: int = 10, market_index: int = None):
+def get_largest_spot_borrows(request: BackendRequest, market_index: int = None):
     """
     Get the largest spot borrowing positions by value.
 
     This endpoint retrieves the largest spot borrowing positions across all users,
-    calculated based on the current market prices. Results can be limited by 
-    number_of_positions and filtered by market_index if provided.
+    calculated based on the current market prices. Results can be filtered by 
+    market_index if provided.
 
     Args:
         request: The backend request object
-        number_of_positions: Maximum number of positions to return (default: 10)
         market_index: Optional market index to filter by
 
     Returns:
@@ -295,7 +284,7 @@ def get_largest_spot_borrows(request: BackendRequest, number_of_positions: int =
         - Public Key (list[str]): The public keys of the borrowers
     """
     vat: Vat = request.state.backend_state.vat
-    top_borrows: list[tuple[float, str, int, float]] = []
+    all_borrows: list[tuple[float, str, int, float]] = []
 
     for user in vat.users.values():
         for position in user.get_user_account().spot_positions:
@@ -312,24 +301,15 @@ def get_largest_spot_borrows(request: BackendRequest, number_of_positions: int =
                     borrow_value = (
                         position.scaled_balance / SPOT_BALANCE_PRECISION
                     ) * market_price_ui
-                    heap_item = (
+                    item = (
                         to_financial(borrow_value),
                         user.user_public_key,
                         position.market_index,
                         position.scaled_balance / SPOT_BALANCE_PRECISION,
                     )
+                    all_borrows.append(item)
 
-                    if len(top_borrows) < number_of_positions:
-                        heapq.heappush(top_borrows, heap_item)
-                    else:
-                        heapq.heappushpop(top_borrows, heap_item)
-
-    borrows = sorted(
-        (value, pubkey, market_idx, amt)
-        for value, pubkey, market_idx, amt in top_borrows
-    )
-
-    borrows.reverse()
+    borrows = sorted(all_borrows, key=lambda x: x[0], reverse=True)
 
     data = {
         "Market Index": [pos[2] for pos in borrows],
@@ -342,17 +322,16 @@ def get_largest_spot_borrows(request: BackendRequest, number_of_positions: int =
 
 
 @router.get("/most_levered_spot_borrows_above_1m")
-def get_most_levered_spot_borrows_above_1m(request: BackendRequest, number_of_positions: int = 10, market_index: int = None):
+def get_most_levered_spot_borrows_above_1m(request: BackendRequest, market_index: int = None):
     """
     Get the most leveraged spot borrowing positions with value above $750,000.
 
     This endpoint calculates the leverage of each spot borrowing position with a value
-    over $750,000 and returns the most leveraged positions, limited by number_of_positions.
+    over $750,000 and returns the most leveraged positions.
     Results can be filtered by market_index if provided.
 
     Args:
         request: The backend request object
-        number_of_positions: Maximum number of positions to return (default: 10)
         market_index: Optional market index to filter by
 
     Returns:
@@ -365,7 +344,7 @@ def get_most_levered_spot_borrows_above_1m(request: BackendRequest, number_of_po
         - Error (list[str]): Error details if any (empty string if no error)
     """
     vat: Vat = request.state.backend_state.vat
-    top_borrows: list[tuple[float, str, int, float, float, str]] = []  # Added error field
+    all_borrows: list[tuple[float, str, int, float, float, str]] = []  # Added error field
     error_positions = []  # Track positions with errors for logging
 
     for user in vat.users.values():
@@ -406,17 +385,16 @@ def get_most_levered_spot_borrows_above_1m(request: BackendRequest, number_of_po
                         "error": position_error
                     })
                     
-                    # If we don't have enough items yet, add this one with error
-                    if len(top_borrows) < number_of_positions:
-                        heap_item = (
-                            borrow_value,  # Will be sorted last due to 0 value
-                            user.user_public_key,
-                            position.market_index,
-                            scaled_balance,
-                            leverage,
-                            position_error,
-                        )
-                        heapq.heappush(top_borrows, heap_item)
+                    # Add this one with error
+                    item = (
+                        borrow_value,  # Will be sorted last due to 0 value
+                        user.user_public_key,
+                        position.market_index,
+                        scaled_balance,
+                        leverage,
+                        position_error,
+                    )
+                    all_borrows.append(item)
                 else:
                     try:
                         market_price_ui = market_price.price / PRICE_PRECISION
@@ -432,19 +410,15 @@ def get_most_levered_spot_borrows_above_1m(request: BackendRequest, number_of_po
                                 position_error = "Zero collateral"
                         
                         if borrow_value > 750_000:
-                            heap_item = (
+                            item = (
                                 to_financial(borrow_value),
                                 user.user_public_key,
                                 position.market_index,
                                 position.scaled_balance / SPOT_BALANCE_PRECISION,
                                 leverage,
                                 position_error,  # Empty string if no error
                             )
-
-                            if len(top_borrows) < number_of_positions:
-                                heapq.heappush(top_borrows, heap_item)
-                            else:
-                                heapq.heappushpop(top_borrows, heap_item)
+                            all_borrows.append(item)
                     except Exception as e:
                         calc_error = f"Calculation error: {str(e)}"
                         position_error = calc_error if not position_error else f"{position_error}; {calc_error}"
@@ -465,7 +439,7 @@ def get_most_levered_spot_borrows_above_1m(request: BackendRequest, number_of_po
         logger.warning(f"Found {len(error_positions)} positions with errors: {error_positions}")
 
     positions = sorted(
-        top_borrows,
+        all_borrows,
         key=lambda x: x[4] if not x[5] else float('-inf'),  # Sort error positions first
         reverse=True,
     )
diff --git a/backend/middleware/cache_middleware.py b/backend/middleware/cache_middleware.py
@@ -310,78 +310,23 @@ def _serve_cached_response(self, cache_file: str, cache_status: str, request_pat
 
     def _log_perp_positions_details(self, data, cache_status, cache_file, request_path):
          """Helper method to log details specifically for the largest_perp_positions endpoint."""
-         # Direct print statements guaranteed to show in the terminal
-         print(f"\n======== CACHED POSITIONS ({cache_status}) ========")
-         print(f"Cache file: {cache_file}")
-
          positions_returned = 0
-         # Handle potential variations in response structure (list vs dict)
-         if isinstance(data, list) and len(data) > 0 and isinstance(data[0], dict):
+         if isinstance(data, list):
              positions_returned = len(data)
-             print(f"Total positions returned (list format): {positions_returned}")
-             for idx, pos in enumerate(data):
-                 position_num = idx + 1
-                 print(f"Position {position_num}: {pos}")
-
          elif isinstance(data, dict) and "Market Index" in data:
              positions_returned = len(data.get("Market Index", []))
-             print(f"Total positions returned (dict format): {positions_returned}")
-
-             if positions_returned > 0:
-                 market_indices = data.get("Market Index", [])
-                 values = data.get("Value", [])
-                 base_amounts = data.get("Base Asset Amount", [])
-                 public_keys = data.get("Public Key", [])
-
-                 # Log positions in the correct order (1 to N)
-                 for idx in range(positions_returned):
-                     market_idx = market_indices[idx] if idx < len(market_indices) else "N/A"
-                     value = values[idx] if idx < len(values) else "N/A"
-                     base_amt = base_amounts[idx] if idx < len(base_amounts) else "N/A"
-                     pub_key = public_keys[idx] if idx < len(public_keys) else "N/A"
-
-                     # Positions are already sorted, so we can log them as is
-                     position_num = idx + 1
-                     print(f"Position {position_num}:")
-                     print(f"   Market Index: {market_idx}")
-                     print(f"   Value: {value}")
-                     print(f"   Base Asset Amount: {base_amt}")
-                     print(f"   Public Key: {pub_key}")
-             else:
-                 print("No positions found in cached dictionary response")
-         else:
-             print(f"Warning: Unrecognized format for position data or empty data. Data: {data}")
-
-         print("======== END CACHED POSITIONS ========\n")
-
-         # Also log to standard logger for Loki
-         log = logging.getLogger("backend.api.health") # Use appropriate logger name if different
-         log.info(f"[CACHED {cache_status}] BEGIN POSITION DETAILS ({request_path})----------------------------------------")
-         if isinstance(data, list) and positions_returned > 0:
-             for idx, pos in enumerate(data):
-                 log.info(f"Position {idx + 1}: {pos}")
-         elif isinstance(data, dict) and positions_returned > 0:
-             market_indices = data.get("Market Index", [])
-             values = data.get("Value", [])
-             base_amounts = data.get("Base Asset Amount", [])
-             public_keys = data.get("Public Key", [])
-             for idx in range(positions_returned):
-                 market_idx = market_indices[idx] if idx < len(market_indices) else "N/A"
-                 value = values[idx] if idx < len(values) else "N/A"
-                 base_amt = base_amounts[idx] if idx < len(base_amounts) else "N/A"
-                 pub_key = public_keys[idx] if idx < len(public_keys) else "N/A"
-
-                 position_num = idx + 1
-                 log.info(
-                     f"Position {position_num}:\n"
-                     f"   Market Index: {market_idx}\n"
-                     f"   Value: {value}\n"
-                     f"   Base Asset Amount: {base_amt}\n"
-                     f"   Public Key: {pub_key}"
-                 )
-         else:
-             log.info("No positions logged.")
-         log.info(f"[CACHED {cache_status}] END POSITION DETAILS ({request_path}) ----------------------------------------")
+
+         summary_message = f"[{cache_status}] ({request_path}): Found {positions_returned} cached positions. Details suppressed to reduce log noise."
+         
+         # Log summary to console
+         print(f"\n======== CACHED POSITIONS SUMMARY ({cache_status}) ========")
+         print(summary_message)
+         print(f"Cache file: {cache_file}")
+         print("======== END CACHED POSITIONS SUMMARY ========\n")
+         
+         # Log summary to standard logger
+         log = logging.getLogger("backend.api.health")
+         log.info(summary_message)
 
 
     async def _fetch_and_cache(
diff --git a/requirements.txt b/requirements.txt
@@ -12,7 +12,7 @@ uvicorn==0.34.0
 requests==2.32.3
 plotly==6.0.0
 anchorpy==0.21.0
-driftpy>=0.8.56
+driftpy==0.8.57
 ccxt==4.2.17
 rich>=10.14.0
 aiofiles==24.1.0
diff --git a/src/page/health.py b/src/page/health.py
