Merge pull request #40 from drift-labs:goldhaxx/DATA-73/add-filter-for-each-isolated-pool-on-price-shock-page

goldhaxx/DATA-73/add-filter-for-each-isolated-pool-on-price-shock-page

resolved some package dependency issues

Author: goldhaxx

.gitignore

@@ -44,3 +44,5 @@ supervisord.pid
 # Exchange data
 exchange_data/*
 list_recommender.log
+
+.testvenv

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.3
+aiohttp>=3.9.1,<4.0.0
 fastapi==0.115.8
 gunicorn==23.0.0
 matplotlib==3.10.0

src/page/market_inspector.py

@@ -340,12 +340,35 @@ def market_inspector_page():
         available_attrs = get_perp_market_attributes()
 
     # 3) Let user select which market
-    selected_market = st.selectbox(
+    market_indices = [m.data.market_index for m in markets]
+
+    if not market_indices:
+        st.warning(f"No {market_type_choice} markets available to display.")
+        # Ensure selected_attrs is cleared if no markets are available for the chosen type
+        if "selected_attrs" in st.session_state: # Reset selected attributes if market type changes or no markets
+            st.session_state.selected_attrs = []
+        return # Stop further processing for this render if no markets
+
+    market_lookup = {m.data.market_index: m for m in markets}
+
+    # Use a unique key for the selectbox based on market_type_choice
+    # to ensure it resets if the options change significantly (e.g., switching market types)
+    selectbox_key = f"market_selectbox_{market_type_choice}"
+
+    selected_market_index = st.selectbox(
         "Select Market:",
-        markets,
-        format_func=display_name,
+        market_indices,
+        format_func=lambda index: display_name(market_lookup[index]),
+        key=selectbox_key
     )
 
+    selected_market = market_lookup.get(selected_market_index) # Reassign selected_market
+
+    if not selected_market:
+        # This case should ideally not be hit if market_indices is not empty and selectbox works as expected
+        st.error("Failed to retrieve selected market data. Please ensure a market is selected or try refreshing.")
+        return
+
     # 4) Let user pick which attributes to show (multi-select)
     st.write("Select which attributes you would like to see:")
 
@@ -378,18 +401,44 @@ def on_attribute_selection_change():
     # 5) Display results
     if not selected_attrs:
         st.info("Please select at least one attribute to display.")
-        with st.expander("All markets"):
+        with st.expander("All markets (Serialized Data)"): # Clarified title
             st.write("Perp markets:")
-            perp_markets = sorted(perp_market_map.values(), key=lambda m: m.data.market_index)
-            perps = pd.concat([pd.DataFrame(serialize_perp_market(x.data)).T for x in perp_markets], axis=1)
-            perps.columns = [m.data.market_index for m in perp_markets]
-            st.write(perps)
+            perp_markets_list = sorted(perp_market_map.values(), key=lambda m: m.data.market_index)
+            if perp_markets_list:
+                try:
+                    perp_data_for_df = [serialize_perp_market(m.data) for m in perp_markets_list]
+                    if perp_data_for_df: # Ensure list is not empty before concat
+                        df_perp = pd.concat(perp_data_for_df, axis=0).reset_index(drop=True)
+                        if 'market_index' in df_perp.columns:
+                            # Ensure market_index is the first column for better readability
+                            df_perp = df_perp[['market_index'] + [col for col in df_perp.columns if col != 'market_index']]
+                        st.dataframe(df_perp)
+                    else:
+                        st.write("No data to display for perp markets.")                        
+                except Exception as e:
+                    st.error(f"Error displaying perp markets table: {e}")
+                    st.caption("Raw data might contain non-serializable fields or other issues.")
+            else:
+                st.write("No perp markets found.")
 
             st.write("Spot markets:")
-            spot_markets = sorted(spot_market_map.values(), key=lambda m: m.data.market_index)
-            spots = pd.concat([pd.DataFrame(serialize_spot_market(x.data)).T for x in spot_markets], axis=1)
-            spots.columns = [m.data.market_index for m in spot_markets]
-            st.write(spots)
+            spot_markets_list = sorted(spot_market_map.values(), key=lambda m: m.data.market_index)
+            if spot_markets_list:
+                try:
+                    spot_data_for_df = [serialize_spot_market(m.data) for m in spot_markets_list]
+                    if spot_data_for_df: # Ensure list is not empty before concat
+                        df_spot = pd.concat(spot_data_for_df, axis=0).reset_index(drop=True)
+                        if 'market_index' in df_spot.columns:
+                            # Ensure market_index is the first column for better readability
+                            df_spot = df_spot[['market_index'] + [col for col in df_spot.columns if col != 'market_index']]
+                        st.dataframe(df_spot)
+                    else:
+                        st.write("No data to display for spot markets.")
+                except Exception as e:
+                    st.error(f"Error displaying spot markets table: {e}")
+                    st.caption("Raw data might contain non-serializable fields or other issues.")
+            else:
+                st.write("No spot markets found.")
         return
 
     st.write(f"**Market Index:** {selected_market.data.market_index}")
@@ -440,8 +489,42 @@ def on_attribute_selection_change():
             # Wrap in code block
             st.markdown(f"```\n{formatted_line}\n```")
 
-    with st.expander("All markets"):
+    with st.expander("All markets (Serialized Data)"): # Clarified title and reused logic
         st.write("Perp markets:")
-        st.write(pd.DataFrame(sorted(perp_market_map.values(), key=lambda m: m.data.market_index)))
+        # Use a different variable name to avoid conflicts if any part of the script is re-run in a weird way
+        perp_markets_list_bottom = sorted(perp_market_map.values(), key=lambda m: m.data.market_index)
+        if perp_markets_list_bottom:
+            try:
+                perp_data_for_df_bottom = [serialize_perp_market(m.data) for m in perp_markets_list_bottom]
+                if perp_data_for_df_bottom: # Ensure list is not empty before concat
+                    df_perp_bottom = pd.concat(perp_data_for_df_bottom, axis=0).reset_index(drop=True)
+                    if 'market_index' in df_perp_bottom.columns:
+                        # Ensure market_index is the first column
+                        df_perp_bottom = df_perp_bottom[['market_index'] + [col for col in df_perp_bottom.columns if col != 'market_index']]
+                    st.dataframe(df_perp_bottom)
+                else:
+                    st.write("No data to display for perp markets.")
+            except Exception as e:
+                st.error(f"Error displaying perp markets table (bottom): {e}")
+                st.caption("Raw data might contain non-serializable fields or other issues.")
+        else:
+            st.write("No perp markets found.")
+
         st.write("Spot markets:")
-        st.write(pd.DataFrame(sorted(spot_market_map.values(), key=lambda m: m.data.market_index)))
+        spot_markets_list_bottom = sorted(spot_market_map.values(), key=lambda m: m.data.market_index)
+        if spot_markets_list_bottom:
+            try:
+                spot_data_for_df_bottom = [serialize_spot_market(m.data) for m in spot_markets_list_bottom]
+                if spot_data_for_df_bottom: # Ensure list is not empty before concat
+                    df_spot_bottom = pd.concat(spot_data_for_df_bottom, axis=0).reset_index(drop=True)
+                    if 'market_index' in df_spot_bottom.columns:
+                        # Ensure market_index is the first column
+                        df_spot_bottom = df_spot_bottom[['market_index'] + [col for col in df_spot_bottom.columns if col != 'market_index']]
+                    st.dataframe(df_spot_bottom)
+                else:
+                    st.write("No data to display for spot markets.")
+            except Exception as e:
+                st.error(f"Error displaying spot markets table (bottom): {e}")
+                st.caption("Raw data might contain non-serializable fields or other issues.")
+        else:
+            st.write("No spot markets found.")

src/utils.py

@@ -1,7 +1,8 @@
 import datetime
 import os
-from typing import Optional
+from typing import Optional, Any, Union
 import pandas as pd
+import copy # Added for deepcopy
 
 import requests
 from driftpy.decode.utils import decode_name
@@ -19,10 +20,37 @@
     WebsocketConfig as UserMapWebsocketConfig,
 )
 from driftpy.user_map.userstats_map import UserStatsMap
+from solders.pubkey import Pubkey # Import Pubkey
 
 from dotenv import load_dotenv
 load_dotenv()
 
+# Helper function to stringify sumtypes, Pubkeys, and other problematic types
+def _stringify_value(value: Any) -> Any:
+    if hasattr(value, 'kind') and isinstance(value.kind, str):
+        # Common pattern for driftpy sumtypes (e.g., MarketType, OracleSource)
+        return value.kind
+    elif (hasattr(value, '__class__') and
+          hasattr(value.__class__, '__module__') and
+          'sumtypes' in value.__class__.__module__):
+        # Another pattern for sumtypes (e.g., MarketStatus)
+        return value.__class__.__name__
+    elif isinstance(value, Pubkey):
+        return str(value)
+    elif isinstance(value, list):
+        return [_stringify_value(item) for item in value]
+    elif isinstance(value, dict):
+        return {k: _stringify_value(v) for k, v in value.items()}
+    # Add other specific type checks if needed, e.g., for specific complex objects
+    # that are not sumtypes but still cause issues with Arrow.
+    return value
+
+def _prepare_for_serialization(obj_dict: dict) -> dict:
+    prepared_dict = {}
+    for key, value in obj_dict.items():
+        prepared_dict[key] = _stringify_value(value)
+    return prepared_dict
+
 def to_financial(num):
     num_str = str(num)
     decimal_pos = num_str.find(".")
@@ -260,61 +288,94 @@ def human_amm_df(df):
 
 
 def serialize_perp_market(market: PerpMarketAccount):
-
-    market_df = pd.json_normalize(market.__dict__).drop(['amm', 'insurance_claim', 'pnl_pool'],axis=1).pipe(human_market_df)
-    market_df['pubkey'] = str(market.pubkey)
-    market_df['name'] = decode_name(market.name)
+    # Prepare market data by stringifying sumtypes and Pubkeys
+    market_dict_prepared = _prepare_for_serialization(copy.deepcopy(market.__dict__))
+    amm_dict_prepared = _prepare_for_serialization(copy.deepcopy(market.amm.__dict__))
+    hist_oracle_data_prepared = _prepare_for_serialization(copy.deepcopy(market.amm.historical_oracle_data.__dict__))
+    fee_pool_prepared = _prepare_for_serialization(copy.deepcopy(market.amm.fee_pool.__dict__))
+    insurance_claim_prepared = _prepare_for_serialization(copy.deepcopy(market.insurance_claim.__dict__))
+    pnl_pool_prepared = _prepare_for_serialization(copy.deepcopy(market.pnl_pool.__dict__))
+
+    market_df = pd.json_normalize(market_dict_prepared).drop(['amm', 'insurance_claim', 'pnl_pool'],axis=1, errors='ignore').pipe(human_market_df)
+    # 'name' is bytes, decode_name handles it; 'pubkey' is already stringified by _prepare_for_serialization if it was a Pubkey object
+    if 'name' in market_df.columns and market_dict_prepared.get('name'): # Check if name exists before decoding
+         market_df['name'] = decode_name(market_dict_prepared['name']) # Use original bytes for decode_name
     market_df.columns = ['market.'+col for col in market_df.columns]
 
-    amm_df= pd.json_normalize(market.amm.__dict__).drop(['historical_oracle_data', 'fee_pool'],axis=1).pipe(human_amm_df)
+    amm_df= pd.json_normalize(amm_dict_prepared).drop(['historical_oracle_data', 'fee_pool'],axis=1, errors='ignore').pipe(human_amm_df)
     amm_df.columns = ['market.amm.'+col for col in amm_df.columns]
 
-    amm_hist_oracle_df= pd.json_normalize(market.amm.historical_oracle_data.__dict__).pipe(human_amm_df)
+    amm_hist_oracle_df= pd.json_normalize(hist_oracle_data_prepared).pipe(human_amm_df)
     amm_hist_oracle_df.columns = ['market.amm.historical_oracle_data.'+col for col in amm_hist_oracle_df.columns]
 
-    market_amm_pool_df = pd.json_normalize(market.amm.fee_pool.__dict__).pipe(human_amm_df)
+    market_amm_pool_df = pd.json_normalize(fee_pool_prepared).pipe(human_amm_df)
     market_amm_pool_df.columns = ['market.amm.fee_pool.'+col for col in market_amm_pool_df.columns]
 
-    market_if_df = pd.json_normalize(market.insurance_claim.__dict__).pipe(human_market_df)
+    market_if_df = pd.json_normalize(insurance_claim_prepared).pipe(human_market_df)
     market_if_df.columns = ['market.insurance_claim.'+col for col in market_if_df.columns]
 
-    market_pool_df = pd.json_normalize(market.pnl_pool.__dict__).pipe(human_amm_df)
+    market_pool_df = pd.json_normalize(pnl_pool_prepared).pipe(human_amm_df)
     market_pool_df.columns = ['market.pnl_pool.'+col for col in market_pool_df.columns]
 
     result_df = pd.concat([market_df, amm_df, amm_hist_oracle_df, market_amm_pool_df, market_if_df, market_pool_df],axis=1)
+    
+    # Final conversion of object columns to string for Arrow compatibility
+    for col in result_df.columns:
+        if result_df[col].dtype == 'object':
+            try:
+                result_df[col] = result_df[col].astype(str)
+            except Exception:
+                # Fallback if astype(str) fails for any reason on a column
+                result_df[col] = result_df[col].apply(lambda x: str(x) if pd.notnull(x) else x)
     return result_df
 
 
 def serialize_spot_market(spot_market: SpotMarketAccount):
-    spot_market_df = pd.json_normalize(spot_market.__dict__).drop([
+    # Prepare spot_market data
+    spot_market_dict_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.__dict__))
+    insurance_fund_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.insurance_fund.__dict__))
+    hist_oracle_data_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.historical_oracle_data.__dict__))
+    hist_index_data_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.historical_index_data.__dict__))
+    revenue_pool_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.revenue_pool.__dict__))
+    spot_fee_pool_prepared = _prepare_for_serialization(copy.deepcopy(spot_market.spot_fee_pool.__dict__))
+
+    spot_market_df = pd.json_normalize(spot_market_dict_prepared).drop([
         'historical_oracle_data', 'historical_index_data',
-        'insurance_fund', # todo
+        'insurance_fund', 
         'spot_fee_pool', 'revenue_pool'
-        ], axis=1).pipe(human_amm_df)
-    spot_market_df['name'] = decode_name(spot_market.name)
-    spot_market_df['pubkey'] = str(spot_market.pubkey)
-    spot_market_df['oracle'] = str(spot_market.oracle)
-    spot_market_df['mint'] = str(spot_market.mint)
-    spot_market_df['vault'] = str(spot_market.vault)
+        ], axis=1, errors='ignore').pipe(human_amm_df) # Note: using human_amm_df as per original
+    
+    # 'name' is bytes, decode_name handles it. Other Pubkey fields are stringified.
+    if 'name' in spot_market_df.columns and spot_market_dict_prepared.get('name'):
+        spot_market_df['name'] = decode_name(spot_market_dict_prepared['name']) # Use original bytes
 
     spot_market_df.columns = ['spot_market.'+col for col in spot_market_df.columns]
 
-    if_df= pd.json_normalize(spot_market.insurance_fund.__dict__).pipe(human_amm_df)
+    if_df= pd.json_normalize(insurance_fund_prepared).pipe(human_amm_df)
     if_df.columns = ['spot_market.insurance_fund.'+col for col in if_df.columns]
 
-    hist_oracle_df= pd.json_normalize(spot_market.historical_oracle_data.__dict__).pipe(human_amm_df)
+    hist_oracle_df= pd.json_normalize(hist_oracle_data_prepared).pipe(human_amm_df)
     hist_oracle_df.columns = ['spot_market.historical_oracle_data.'+col for col in hist_oracle_df.columns]
 
-    hist_index_df= pd.json_normalize(spot_market.historical_index_data.__dict__).pipe(human_amm_df)
+    hist_index_df= pd.json_normalize(hist_index_data_prepared).pipe(human_amm_df)
     hist_index_df.columns = ['spot_market.historical_index_data.'+col for col in hist_index_df.columns]
 
 
-    market_pool_df = pd.json_normalize(spot_market.revenue_pool.__dict__).pipe(human_amm_df)
+    market_pool_df = pd.json_normalize(revenue_pool_prepared).pipe(human_amm_df)
     market_pool_df.columns = ['spot_market.revenue_pool.'+col for col in market_pool_df.columns]
 
 
-    market_fee_df = pd.json_normalize(spot_market.spot_fee_pool.__dict__).pipe(human_amm_df)
+    market_fee_df = pd.json_normalize(spot_fee_pool_prepared).pipe(human_amm_df)
     market_fee_df.columns = ['spot_market.spot_fee_pool.'+col for col in market_fee_df.columns]
 
     result_df = pd.concat([spot_market_df, if_df, hist_oracle_df, hist_index_df, market_pool_df, market_fee_df],axis=1)
+    
+    # Final conversion of object columns to string for Arrow compatibility
+    for col in result_df.columns:
+        if result_df[col].dtype == 'object':
+            try:
+                result_df[col] = result_df[col].astype(str)
+            except Exception:
+                # Fallback if astype(str) fails for any reason on a column
+                result_df[col] = result_df[col].apply(lambda x: str(x) if pd.notnull(x) else x)
     return result_df