Enhance Market Recommender and Introduce Caching Mechanism

- Updated the market recommender logic to utilize the CoinGecko API more effectively, including new constants for ignored symbols and improved data handling.
- Implemented a caching mechanism for API calls using a TTL cache, significantly reducing redundant requests and improving performance.
- Enhanced logging throughout the market data fetching process, providing clearer insights into cache hits, misses, and data processing.
- Added new utility functions for cleaning market names and filtering out ignored symbols, ensuring more accurate data representation.
- Introduced a new test suite for the caching functionality, validating the effectiveness of the caching mechanism and ensuring consistent results across API calls.
- Refined the Streamlit frontend to improve user experience with additional filtering options and clearer display of market recommendations.

These changes aim to optimize the market recommender's performance and reliability while enhancing the overall user experience.

Author: goldhaxx

backend/api/market_recommender.py

@@ -1,13 +1,22 @@
 # This is the new version of the market recommender which makes use of the coingecko API via the /backend/utils/coingecko_api.py utility script.
 
+import os
+import time
 import logging
-from typing import Dict, List
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Tuple, Any
 from fastapi import APIRouter
-from backend.utils.coingecko_api import fetch_all_coingecko_market_data, fetch_all_coingecko_volumes
 from driftpy.pickle.vat import Vat
 from backend.state import BackendRequest
-import os
-from datetime import datetime, timedelta
+from backend.utils.cache_utils import ttl_cache
+from backend.utils.coingecko_api import (
+    fetch_all_coingecko_market_data,
+    fetch_all_coingecko_volumes
+)
+from backend.utils.drift_data_api import (
+    fetch_drift_data_api_data,
+    clean_market_name
+)
 
 # --- Constants ---
 PRICE_PRECISION = 1e6  # Add price precision constant
@@ -16,7 +25,32 @@
 }
 
 # Symbols to ignore
-IGNORE_SYMBOLS = ['USDT', 'USDC']
+IGNORED_SYMBOLS = ['USDT', 'USDC', 'USDE', 'USDS', 'SUSDS']
+
+# Basket market prefixes to strip when normalizing symbols
+BASKET_MARKET_PREFIXES = {
+    '1M': '',  # e.g., 1MPEPE -> PEPE
+    '1K': '',  # e.g., 1KWEN -> WEN
+}
+
+# Drift perp markets to ignore
+IGNORED_DRIFT_PERP_MARKETS = [
+    {"index": 39, "name": "REPUBLICAN-POPULAR-AND-WIN-BET"},
+    {"index": 38, "name": "FED-CUT-50-SEPT-2024-BET"},
+    {"index": 50, "name": "LNDO-WIN-F1-24-US-GP"},
+    {"index": 37, "name": "KAMALA-POPULAR-VOTE-2024-BET"},
+    {"index": 49, "name": "VRSTPN-WIN-F1-24-DRVRS-CHMP"},
+    {"index": 68, "name": "NBAFINALS25-BOS-BET"},
+    {"index": 67, "name": "NBAFINALS25-OKC-BET"},
+    {"index": 40, "name": "BREAKPOINT-IGGYERIC-BET"},
+    {"index": 48, "name": "WLF-5B-1W-BET"},
+    {"index": 58, "name": "SUPERBOWL-LIX-CHIEFS-BET"},
+    {"index": 57, "name": "SUPERBOWL-LIX-LIONS-BET"},
+    {"index": 43, "name": "LANDO-F1-SGP-WIN-BET"},
+    {"index": 41, "name": "DEMOCRATS-WIN-MICHIGAN-BET"},
+    {"index": 46, "name": "WARWICK-FIGHT-WIN-BET"},
+    {"index": 36, "name": "TRUMP-WIN-2024-BET"}
+]
 
 # Symbol conversions
 SYMBOL_CONVERSIONS = {
@@ -47,30 +81,34 @@ async def get_market_data(request: BackendRequest, number_of_tokens: int = 2):
     """
     return main(request.state.backend_state.vat, number_of_tokens)
 
+@ttl_cache(ttl_seconds=43200)  # Cache for 12 hours
 def fetch_coingecko_market_data(number_of_tokens: int = 2) -> List[Dict]:
     """
     Fetches market data for top N coins from CoinGecko API.
+    Responses are cached for 12 hours.
     
     Args:
         number_of_tokens (int, optional): Number of tokens to fetch data for. Defaults to 2.
     
     Returns:
         List of dictionaries containing market data for each coin
     """
-    logger.info(f"Fetching market data for top {number_of_tokens} coins from CoinGecko...")
+    logger.info(f"🔍 Attempting to fetch market data for top {number_of_tokens} coins...")
 
     try:
-        # Fetch market data
+        # Fetch market data (this call is now cached)
+        logger.info("📊 Fetching CoinGecko market data...")
         market_data = fetch_all_coingecko_market_data(number_of_tokens)
 
         if not market_data:
-            logger.error("Failed to fetch market data from CoinGecko")
+            logger.error("❌ Failed to fetch market data from CoinGecko")
             return []
 
         # Get list of coin IDs for volume fetching
         coin_ids = list(market_data.keys())
+        logger.info(f"📈 Found {len(coin_ids)} coins, fetching volume data...")
 
-        # Fetch 30-day volume data
+        # Fetch 30-day volume data (this call is now cached)
         volume_data = fetch_all_coingecko_volumes(coin_ids)
 
         # Format response data
@@ -97,16 +135,16 @@ def fetch_coingecko_market_data(number_of_tokens: int = 2) -> List[Dict]:
                     }
                 }
                 formatted_data.append(formatted_entry)
-                logger.info(f"Processed market data for {formatted_entry['symbol']}")
+                logger.debug(f"✅ Processed market data for {formatted_entry['symbol']}")
             except Exception as e:
-                logger.error(f"Error formatting data for coin {coin_id}: {e}")
+                logger.error(f"❌ Error formatting data for coin {coin_id}: {e}")
                 continue
 
-        logger.info(f"Successfully processed market data for {len(formatted_data)} coins")
+        logger.info(f"✨ Successfully processed market data for {len(formatted_data)} coins")
         return formatted_data
 
     except Exception as e:
-        logger.error(f"Error in fetch_coingecko_market_data: {e}", exc_info=True)
+        logger.error(f"❌ Error in fetch_coingecko_market_data: {e}", exc_info=True)
         return []
 
 def fetch_driftpy_data(vat: Vat) -> Dict:
@@ -173,17 +211,22 @@ def fetch_driftpy_data(vat: Vat) -> Dict:
             try:
                 # Get market name and clean it
                 market_name = bytes(market.data.name).decode('utf-8').strip('\x00').strip()
+                # Check if this market should be ignored
+                if any(ignored_market["index"] == market.data.market_index for ignored_market in IGNORED_DRIFT_PERP_MARKETS):
+                    logger.info(f"Skipping ignored market: {market_name} (index: {market.data.market_index})")
+                    continue
+
                 # Preserve original case for API calls
                 original_market_name = market_name
 
-                # Extract symbol from market name (e.g., "BTC-PERP" -> "BTC")
-                # Keep original case for API calls, but use uppercase for dictionary keys
+                # Extract symbol from market name and handle basket markets
+                # e.g., "1MPEPE-PERP" -> "PEPE", "1KWEN-PERP" -> "WEN"
                 symbol_parts = market_name.split('-')
                 if len(symbol_parts) > 0:
-                    # Use uppercase for normalized symbol (dictionary keys)
-                    normalized_symbol = symbol_parts[0].upper().strip()
+                    # Clean the symbol part (removes basket prefixes)
+                    normalized_symbol = clean_market_name(symbol_parts[0]).upper()
                 else:
-                    normalized_symbol = market_name.upper().strip()
+                    normalized_symbol = clean_market_name(market_name).upper()
 
                 # Get oracle price from perp_oracles
                 market_price = vat.perp_oracles.get(market.data.market_index)
@@ -382,13 +425,13 @@ def fetch_api_page(url: str, retries: int = 5) -> Dict:
 
         return {"success": False, "records": [], "meta": {"totalRecords": 0}}
 
-    def fetch_market_trades(symbol: str, start_date: datetime, end_date: datetime = None) -> List[Dict]:
+    def fetch_market_trades(market_name: str, start_date: datetime, end_date: datetime = None) -> List[Dict]:
         """
-        Fetch market candle data for a symbol using the candles endpoint.
-        This replaces the inefficient day-by-day trade fetching with a single API call.
+        Fetch market candle data for a market using the candles endpoint.
+        Uses the original market name (e.g., "1MPEPE-PERP") for API calls.
         
         Args:
-            symbol: Market symbol (e.g., "SOL-PERP")
+            market_name: Original market name (e.g., "1MPEPE-PERP", "SOL-PERP")
             start_date: Start date for candle data
             end_date: End date for candle data (defaults to current time)
             
@@ -398,8 +441,8 @@ def fetch_market_trades(symbol: str, start_date: datetime, end_date: datetime =
         if end_date is None:
             end_date = datetime.now()
 
-        # Log the start of fetching for this symbol
-        logger.info(f"Starting to fetch candle data for {symbol} from {start_date} to {end_date}")
+        # Log the start of fetching for this market
+        logger.info(f"Starting to fetch candle data for {market_name} from {start_date} to {end_date}")
 
         # Convert dates to Unix timestamps (seconds)
         # NOTE: For the Drift API, startTs should be the more recent timestamp (end_date)
@@ -416,7 +459,7 @@ def fetch_market_trades(symbol: str, start_date: datetime, end_date: datetime =
 
         # Use the candles endpoint with daily resolution (D)
         # The API expects: startTs = most recent, endTs = oldest (reverse of what might be expected)
-        url = f"{DRIFT_DATA_API_BASE}/market/{symbol}/candles/D?startTs={start_ts}&endTs={end_ts}&limit={min(days, 31)}"
+        url = f"{DRIFT_DATA_API_BASE}/market/{market_name}/candles/D?startTs={start_ts}&endTs={end_ts}&limit={min(days, 31)}"
 
         logger.info(f"Requesting candles with startTs={start_ts} (now), endTs={end_ts} (past)")
 
@@ -425,19 +468,19 @@ def fetch_market_trades(symbol: str, start_date: datetime, end_date: datetime =
 
         if data.get("success") and "records" in data:
             records_count = len(data["records"])
-            logger.info(f"Successfully fetched {records_count} candles for {symbol}")
+            logger.info(f"Successfully fetched {records_count} candles for {market_name}")
             return data["records"]
         else:
-            logger.warning(f"Failed to fetch candle data for {symbol}")
+            logger.warning(f"Failed to fetch candle data for {market_name}")
             return []
 
-    def calculate_market_volume(symbol: str) -> dict:
+    def calculate_market_volume(market_name: str) -> dict:
         """
         Calculate total trading volume for a market over the past 30 days using candle data.
-        Returns both quote volume (in USD) and base volume (in token units).
+        Uses the original market name for API calls.
         
         Args:
-            symbol: Market symbol (e.g., "SOL-PERP")
+            market_name: Original market name (e.g., "1MPEPE-PERP", "SOL-PERP")
             
         Returns:
             dict: Dictionary with quote_volume and base_volume
@@ -446,7 +489,7 @@ def calculate_market_volume(symbol: str) -> dict:
         start_date = end_date - timedelta(days=DAYS_TO_CONSIDER)
 
         try:
-            candles = fetch_market_trades(symbol, start_date, end_date)
+            candles = fetch_market_trades(market_name, start_date, end_date)
 
             total_quote_volume = 0.0
             total_base_volume = 0.0
@@ -462,7 +505,7 @@ def calculate_market_volume(symbol: str) -> dict:
                 except (ValueError, TypeError):
                     continue
 
-            logger.info(f"Calculated {DAYS_TO_CONSIDER}-day volumes for {symbol}: " 
+            logger.info(f"Calculated {DAYS_TO_CONSIDER}-day volumes for {market_name}: " 
                         f"${total_quote_volume:,.2f} (quote), {total_base_volume:,.2f} (base)")
 
             return {
@@ -471,7 +514,7 @@ def calculate_market_volume(symbol: str) -> dict:
             }
 
         except Exception as e:
-            logger.error(f"Error calculating volume for {symbol}: {str(e)}")
+            logger.error(f"Error calculating volume for {market_name}: {str(e)}")
             return {
                 "quote_volume": 0.0,
                 "base_volume": 0.0
@@ -486,7 +529,10 @@ def calculate_market_volume(symbol: str) -> dict:
         logger.info(f"Processing {len(discovered_markets)} discovered markets")
 
         for symbol, market_info in discovered_markets.items():
-            drift_markets[symbol] = {
+            # Clean the symbol to handle basket markets
+            normalized_symbol = clean_market_name(symbol).upper()
+            
+            drift_markets[normalized_symbol] = {
                 "drift_is_listed_spot": market_info.get("drift_is_listed_spot", "false"),
                 "drift_is_listed_perp": market_info.get("drift_is_listed_perp", "false"),
                 "drift_perp_markets": {},
@@ -502,11 +548,17 @@ def calculate_market_volume(symbol: str) -> dict:
             total_quote_volume = 0.0
             total_base_volume = 0.0
             for perp_market_name in market_info.get("drift_perp_markets", {}):
+                # Check if this market should be ignored
+                market_index = next((market["index"] for market in IGNORED_DRIFT_PERP_MARKETS if market["name"] == perp_market_name), None)
+                if market_index is not None:
+                    logger.info(f"Skipping ignored perp market: {perp_market_name}")
+                    continue
+
                 # Use original market name (case-preserved) for API call
                 logger.info(f"Calculating volume for perp market: {perp_market_name} (preserving case)")
                 volume_data = calculate_market_volume(perp_market_name)  # Returns dict with quote and base volumes
 
-                drift_markets[symbol]["drift_perp_markets"][perp_market_name] = {
+                drift_markets[normalized_symbol]["drift_perp_markets"][perp_market_name] = {
                     "drift_perp_oracle_price": market_info["drift_perp_markets"][perp_market_name].get("drift_perp_oracle_price", 0.0),
                     "drift_perp_quote_volume_30d": volume_data["quote_volume"],
                     "drift_perp_base_volume_30d": volume_data["base_volume"],
@@ -522,7 +574,7 @@ def calculate_market_volume(symbol: str) -> dict:
                 logger.info(f"Calculating volume for spot market: {spot_market_name} (preserving case)")
                 volume_data = calculate_market_volume(spot_market_name)  # Returns dict with quote and base volumes
 
-                drift_markets[symbol]["drift_spot_markets"][spot_market_name] = {
+                drift_markets[normalized_symbol]["drift_spot_markets"][spot_market_name] = {
                     "drift_spot_oracle_price": market_info["drift_spot_markets"][spot_market_name].get("drift_spot_oracle_price", 0.0),
                     "drift_spot_quote_volume_30d": volume_data["quote_volume"],
                     "drift_spot_base_volume_30d": volume_data["base_volume"],
@@ -532,13 +584,13 @@ def calculate_market_volume(symbol: str) -> dict:
                 total_base_volume += volume_data["base_volume"]
 
             # Update total volumes
-            drift_markets[symbol]["drift_total_quote_volume_30d"] = total_quote_volume
-            drift_markets[symbol]["drift_total_base_volume_30d"] = total_base_volume
+            drift_markets[normalized_symbol]["drift_total_quote_volume_30d"] = total_quote_volume
+            drift_markets[normalized_symbol]["drift_total_base_volume_30d"] = total_base_volume
 
             # Update OI and funding rate if perp markets exist
             if market_info.get("drift_is_listed_perp") == "true":
-                drift_markets[symbol]["drift_open_interest"] = total_quote_volume * 0.75  # Example calculation
-                drift_markets[symbol]["drift_funding_rate_1h"] = market_info.get("drift_funding_rate_1h", 0.0)
+                drift_markets[normalized_symbol]["drift_open_interest"] = total_quote_volume * 0.75  # Example calculation
+                drift_markets[normalized_symbol]["drift_funding_rate_1h"] = market_info.get("drift_funding_rate_1h", 0.0)
 
     except Exception as e:
         logger.error(f"Error in fetch_drift_data_api_data: {e}")
@@ -818,8 +870,11 @@ def process_drift_markets(scored_data: List[Dict], drift_data: Dict) -> Dict:
         if not symbol:
             continue
 
+        # Clean and normalize the symbol to handle basket markets
+        normalized_symbol = clean_market_name(symbol).upper()
+            
         # Initialize default structure for the symbol
-        processed_drift_data[symbol] = {
+        processed_drift_data[normalized_symbol] = {
             "drift_is_listed_perp": symbol_drift_data.get('drift_is_listed_perp', 'false'),
             "drift_is_listed_spot": symbol_drift_data.get('drift_is_listed_spot', 'false'),
             "drift_perp_markets": symbol_drift_data.get('drift_perp_markets', {}),
@@ -836,7 +891,7 @@ def process_drift_markets(scored_data: List[Dict], drift_data: Dict) -> Dict:
         total_base_volume = 0.0
 
         # Sum volumes from perp markets
-        for market_name, market_data in processed_drift_data[symbol].get("drift_perp_markets", {}).items():
+        for market_name, market_data in processed_drift_data[normalized_symbol].get("drift_perp_markets", {}).items():
             total_quote_volume += market_data.get("drift_perp_quote_volume_30d", 0.0)
             total_base_volume += market_data.get("drift_perp_base_volume_30d", 0.0)
             # Clean up redundant/old fields if they exist
@@ -846,24 +901,23 @@ def process_drift_markets(scored_data: List[Dict], drift_data: Dict) -> Dict:
                  del market_data["drift_perp_oi"]
 
         # Sum volumes from spot markets
-        for market_name, market_data in processed_drift_data[symbol].get("drift_spot_markets", {}).items():
+        for market_name, market_data in processed_drift_data[normalized_symbol].get("drift_spot_markets", {}).items():
             total_quote_volume += market_data.get("drift_spot_quote_volume_30d", 0.0)
             total_base_volume += market_data.get("drift_spot_base_volume_30d", 0.0)
              # Clean up redundant/old fields if they exist
             if "drift_spot_volume_30d" in market_data:
                 del market_data["drift_spot_volume_30d"]
 
         # Update total volumes at the symbol level
-        processed_drift_data[symbol]["drift_total_quote_volume_30d"] = total_quote_volume
-        processed_drift_data[symbol]["drift_total_base_volume_30d"] = total_base_volume
+        processed_drift_data[normalized_symbol]["drift_total_quote_volume_30d"] = total_quote_volume
+        processed_drift_data[normalized_symbol]["drift_total_base_volume_30d"] = total_base_volume
 
         # Ensure OI and funding rate are correctly placed (usually associated with perps)
         # If no perps listed, these should likely be 0 or handled appropriately
-        if processed_drift_data[symbol]["drift_is_listed_perp"] == 'false':
-             processed_drift_data[symbol]["drift_open_interest"] = 0.0
-             processed_drift_data[symbol]["drift_funding_rate_1h"] = 0.0
-             processed_drift_data[symbol]["drift_max_leverage"] = 0.0 # Max leverage applies to perps
-
+        if processed_drift_data[normalized_symbol]["drift_is_listed_perp"] == 'false':
+             processed_drift_data[normalized_symbol]["drift_open_interest"] = 0.0
+             processed_drift_data[normalized_symbol]["drift_funding_rate_1h"] = 0.0
+             processed_drift_data[normalized_symbol]["drift_max_leverage"] = 0.0 # Max leverage applies to perps
 
     return processed_drift_data