Skip to content

Commit 26673ea

Browse files
goldhaxxgoldhaxx
committed
Update user retention configuration and introduce new API endpoint for retention summary:
- Increased the Gunicorn timeout from 300 to 1200 seconds to accommodate longer processing times. - Renamed the user retention API router from `user_retention_api` to `user_retention_summary_api` for clarity. - Added a new API endpoint in `user_retention_summary_api.py` to fetch and process user retention data, providing a summary of new traders and their retention rates across different markets. - Updated the frontend to reflect the new user retention summary page, enhancing user experience with improved data presentation and filtering options. - Introduced a new JSON file `markets.json` to manage market data dynamically, facilitating easier updates and maintenance.
1 parent 4e440f0 commit 26673ea

File tree

6 files changed

+184
-71
lines changed

6 files changed

+184
-71
lines changed

‎backend/api/user_retention_api.py‎ renamed to ‎backend/api/user_retention_summary_api.py‎

Lines changed: 59 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,35 @@
1111
from fastapi import APIRouter, HTTPException
1212
from pydantic import BaseModel
1313
import logging
14+
import json
1415

1516
import boto3
1617

18+
def load_markets_from_json(file_path: str) -> Dict[str, Dict[str, Any]]:
19+
"""Loads market data from a JSON file and formats it for the API."""
20+
try:
21+
with open(file_path, 'r') as f:
22+
markets_data = json.load(f)
23+
24+
formatted_markets = {}
25+
for market in markets_data:
26+
formatted_markets[market["marketName"]] = {
27+
"index": market["marketIndex"],
28+
"launch_ts": market["launchTs"],
29+
"category": market["category"]
30+
}
31+
logger.info(f"Successfully loaded and formatted {len(formatted_markets)} markets from {file_path}")
32+
return formatted_markets
33+
except FileNotFoundError:
34+
logger.error(f"Market file not found at {file_path}. API will not have market data.")
35+
return {}
36+
except json.JSONDecodeError:
37+
logger.error(f"Error decoding JSON from {file_path}.")
38+
return {}
39+
except Exception as e:
40+
logger.error(f"An unexpected error occurred while loading markets: {e}")
41+
return {}
42+
1743
def log_current_identity():
1844
try:
1945
sts = boto3.client("sts")
@@ -33,15 +59,9 @@ def log_current_identity():
3359

3460
# ───────────────────────── 1. CONFIG (adapted from hype_market_retention.py) ───────────────────────── #
3561

36-
HYPE_MARKETS: Dict[str, Dict[str, int]] = {
37-
"WIF-PERP": {"index": 23, "launch_ts": 1706219971000},
38-
"POPCAT-PERP":{"index": 34, "launch_ts": 1720013054000},
39-
"HYPE-PERP": {"index": 59, "launch_ts": 1733374800000}, # Example, adjust as needed
40-
"KAITO-PERP": {"index": 69, "launch_ts": 1739545901000}, # Example, adjust as needed
41-
"FARTCOIN-PERP": {"index": 71, "launch_ts": 1743086746000}, # Example, adjust as needed
42-
"TRUMP-PERP": {"index": 64, "launch_ts": 1737219250000}, # Example, adjust as needed
43-
"LAUNCHCOIN-PERP": {"index": 74, "launch_ts": 1747318237000} # Example, adjust as needed
44-
}
62+
# Load markets from the JSON file instead of using a hardcoded dictionary.
63+
# The path is relative to the root of the project where the application is run.
64+
ALL_MARKETS = load_markets_from_json("shared/markets.json")
4565

4666
NEW_TRADER_WINDOW_DAYS: int = 7
4767
RETENTION_WINDOWS_DAYS: List[int] = [14, 28] # Affects Pydantic model if changed
@@ -61,6 +81,7 @@ def log_current_identity():
6181

6282
class RetentionSummaryItem(BaseModel):
6383
market: str
84+
category: List[str]
6485
new_traders: int
6586
new_traders_list: Optional[List[str]] = None
6687
retained_users_14d: Optional[int] = None
@@ -153,9 +174,9 @@ async def fetch_and_process_retention_data() -> pd.DataFrame:
153174
log_current_identity()
154175

155176
# 3A. discover "new traders" per market
156-
logger.info("Scanning for new traders across all hype markets...")
177+
logger.info("Scanning for new traders across all markets...")
157178
new_traders_frames: List[pd.DataFrame] = []
158-
for mkt, cfg in HYPE_MARKETS.items():
179+
for mkt, cfg in ALL_MARKETS.items():
159180
logger.info(f"• Scanning new traders for {mkt}…")
160181
q = sql_new_traders(cfg["index"], cfg["launch_ts"])
161182
# logger.debug(f"Generated SQL for {mkt} new traders:\\n{q}")
@@ -188,7 +209,7 @@ async def fetch_and_process_retention_data() -> pd.DataFrame:
188209
# 3B. retention look-ups
189210
retention_records: List[Dict[str, object]] = []
190211
if not new_traders.empty:
191-
for mkt, cfg in HYPE_MARKETS.items():
212+
for mkt, cfg in ALL_MARKETS.items():
192213
mkt_traders = new_traders[new_traders.market == mkt].trader.tolist()
193214
if not mkt_traders:
194215
logger.info(f"No new traders for market {mkt}, skipping retention lookup.")
@@ -264,7 +285,7 @@ async def fetch_and_process_retention_data() -> pd.DataFrame:
264285
if retention.empty and new_traders.empty:
265286
logger.info("Both new_traders and retention are empty. Returning empty summary.")
266287
# Ensure all expected columns exist, even if with no data, matching Pydantic model
267-
cols = ['market', 'new_traders', 'new_traders_list']
288+
cols = ['market', 'category', 'new_traders', 'new_traders_list']
268289
for win_days in RETENTION_WINDOWS_DAYS:
269290
cols.append(f'retained_users_{win_days}d')
270291
cols.append(f'retention_ratio_{win_days}d')
@@ -294,9 +315,13 @@ async def fetch_and_process_retention_data() -> pd.DataFrame:
294315
else: # No new traders found at all
295316
final_summary_counts = pd.DataFrame({'market': pd.Series(dtype='str'), 'new_traders': pd.Series(dtype='int')})
296317

297-
# Ensure all HYPE_MARKETS are present in final_summary, even if they had 0 new traders
298-
all_market_names_df = pd.DataFrame({'market': list(HYPE_MARKETS.keys())})
299-
final_summary = pd.merge(all_market_names_df, final_summary_counts, on='market', how='left').fillna({'new_traders': 0})
318+
# Ensure all ALL_MARKETS are present in final_summary, even if they had 0 new traders
319+
all_markets_df = pd.DataFrame([
320+
{'market': name, 'category': config['category']}
321+
for name, config in ALL_MARKETS.items()
322+
])
323+
324+
final_summary = pd.merge(all_markets_df, final_summary_counts, on='market', how='left').fillna({'new_traders': 0})
300325
final_summary['new_traders'] = final_summary['new_traders'].astype(int)
301326

302327
# Merge new traders lists
@@ -350,14 +375,19 @@ async def fetch_and_process_retention_data() -> pd.DataFrame:
350375
except Exception as e:
351376
logger.error(f"Error in fetch_and_process_retention_data: {e}", exc_info=True)
352377
# For Pydantic model compatibility, return DataFrame with expected columns in case of error
353-
cols = ['market', 'new_traders', 'new_traders_list']
378+
cols = ['market', 'category', 'new_traders', 'new_traders_list']
354379
for w in RETENTION_WINDOWS_DAYS:
355380
cols.extend([f'retained_users_{w}d', f'retention_ratio_{w}d', f'retained_users_{w}d_list'])
356381
empty_df_on_error = pd.DataFrame(columns=cols)
357382

358383
error_data = []
359-
for market_name in HYPE_MARKETS.keys():
360-
record: Dict[str, Any] = {'market': market_name, 'new_traders': 0, 'new_traders_list': []}
384+
for market_name, config in ALL_MARKETS.items():
385+
record: Dict[str, Any] = {
386+
'market': market_name,
387+
'category': config.get('category', []),
388+
'new_traders': 0,
389+
'new_traders_list': []
390+
}
361391
for w in RETENTION_WINDOWS_DAYS:
362392
record[f'retained_users_{w}d'] = 0
363393
record[f'retention_ratio_{w}d'] = 0.0
@@ -388,14 +418,19 @@ async def get_user_retention_summary():
388418
logger.info("Received request for /summary endpoint.")
389419
summary_df = await fetch_and_process_retention_data()
390420

391-
if summary_df.empty and not HYPE_MARKETS: # No markets configured
421+
if summary_df.empty and not ALL_MARKETS: # No markets configured
392422
return []
393-
if summary_df.empty and HYPE_MARKETS: # Markets configured but no data (e.g. no new users at all)
423+
if summary_df.empty and ALL_MARKETS: # Markets configured but no data (e.g. no new users at all)
394424
# Construct a list of RetentionSummaryItem with 0 values for all configured markets
395425
# This ensures the frontend gets a consistent structure.
396426
results = []
397-
for market_name in HYPE_MARKETS.keys():
398-
item_data: Dict[str, Any] = {"market": market_name, "new_traders": 0, "new_traders_list": []}
427+
for market_name, config in ALL_MARKETS.items():
428+
item_data: Dict[str, Any] = {
429+
"market": market_name,
430+
"category": config.get('category', []),
431+
"new_traders": 0,
432+
"new_traders_list": []
433+
}
399434
for win_day in RETENTION_WINDOWS_DAYS:
400435
item_data[f"retained_users_{win_day}d"] = 0
401436
item_data[f"retention_ratio_{win_day}d"] = 0.0
@@ -415,9 +450,4 @@ async def get_user_retention_summary():
415450
raise http_exc
416451
except Exception as e:
417452
logger.error(f"Unhandled error in /summary endpoint: {e}", exc_info=True)
418-
raise HTTPException(status_code=500, detail="An internal server error occurred.")
419-
420-
# To run this API locally (example):
421-
# Ensure FastAPI and Uvicorn are installed: pip install fastapi uvicorn
422-
# Save this file as user_retention_api.py
423-
# Run with: uvicorn user_retention_api:router --reload --port 8001 (assuming this router is added to a FastAPI app)
453+
raise HTTPException(status_code=500, detail="An internal server error occurred.")

‎backend/app.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
ucache,
2525
vaults,
2626
high_leverage_api,
27-
user_retention_api,
27+
user_retention_summary_api,
2828
)
2929
from backend.middleware.cache_middleware import CacheMiddleware
3030
from backend.middleware.readiness import ReadinessMiddleware
@@ -93,7 +93,7 @@ async def lifespan(app: FastAPI):
9393
app.include_router(market_recommender_api.router, prefix="/api/market-recommender", tags=["market-recommender"])
9494
app.include_router(open_interest_api.router, prefix="/api/open-interest", tags=["open-interest"])
9595
app.include_router(high_leverage_api.router, prefix="/api/high-leverage", tags=["high-leverage"])
96-
app.include_router(user_retention_api.router, prefix="/api/user-retention", tags=["user-retention"])
96+
app.include_router(user_retention_summary_api.router, prefix="/api/user-retention-summary", tags=["user-retention-summary"])
9797
# NOTE: All other routes should be in /api/* within the /api folder. Routes outside of /api are not exposed in k8s
9898
@app.get("/")
9999
async def root():

‎gunicorn_config.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
workers = 2
22
worker_class = "uvicorn.workers.UvicornWorker"
33
bind = "0.0.0.0:8000"
4-
timeout = 300
4+
timeout = 1200
55
keepalive = 65
66
max_requests = 1000
77
max_requests_jitter = 50
@@ -11,4 +11,4 @@
1111

1212
# Restart workers that die unexpectedly
1313
worker_exit_on_restart = True
14-
worker_restart_delay = 20
14+
worker_restart_delay = 20

‎shared/markets.json‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
[
2+
{"marketIndex": 0, "marketName": "SOL-PERP", "category": ["L1", "Infra", "Solana"], "launchTs": 1667560505000},
3+
{"marketIndex": 1, "marketName": "BTC-PERP", "category": ["L1", "Payment"], "launchTs": 1670347281000},
4+
{"marketIndex": 2, "marketName": "ETH-PERP", "category": ["L1", "Infra"], "launchTs": 1670347281000},
5+
{"marketIndex": 3, "marketName": "APT-PERP", "category": ["L1", "Infra"], "launchTs": 1675802661000},
6+
{"marketIndex": 4, "marketName": "1MBONK-PERP", "category": ["Meme", "Solana"], "launchTs": 1677690149000},
7+
{"marketIndex": 5, "marketName": "POL-PERP", "category": ["L2", "Infra"], "launchTs": 1677690149000},
8+
{"marketIndex": 6, "marketName": "ARB-PERP", "category": ["L2", "Infra"], "launchTs": 1679501812000},
9+
{"marketIndex": 7, "marketName": "DOGE-PERP", "category": ["Meme", "Dog"], "launchTs": 1680808053000},
10+
{"marketIndex": 8, "marketName": "BNB-PERP", "category": ["Exchange"], "launchTs": 1680808053000},
11+
{"marketIndex": 9, "marketName": "SUI-PERP", "category": ["L1"], "launchTs": 1683125906000},
12+
{"marketIndex": 10, "marketName": "1MPEPE-PERP", "category": ["Meme"], "launchTs": 1683781239000},
13+
{"marketIndex": 11, "marketName": "OP-PERP", "category": ["L2", "Infra"], "launchTs": 1686091480000},
14+
{"marketIndex": 12, "marketName": "RENDER-PERP", "category": ["Infra", "Solana"], "launchTs": 1687201081000},
15+
{"marketIndex": 13, "marketName": "XRP-PERP", "category": ["Payments"], "launchTs": 1689270550000},
16+
{"marketIndex": 14, "marketName": "HNT-PERP", "category": ["IoT", "Solana"], "launchTs": 1692294955000},
17+
{"marketIndex": 15, "marketName": "INJ-PERP", "category": ["L1", "Exchange"], "launchTs": 1698074659000},
18+
{"marketIndex": 16, "marketName": "LINK-PERP", "category": ["Oracle"], "launchTs": 1698074659000},
19+
{"marketIndex": 17, "marketName": "RLB-PERP", "category": ["Exchange"], "launchTs": 1699265968000},
20+
{"marketIndex": 18, "marketName": "PYTH-PERP", "category": ["Oracle", "Solana"], "launchTs": 1700542800000},
21+
{"marketIndex": 19, "marketName": "TIA-PERP", "category": ["Data"], "launchTs": 1701880540000},
22+
{"marketIndex": 20, "marketName": "JTO-PERP", "category": ["MEV", "Solana"], "launchTs": 1701967240000},
23+
{"marketIndex": 21, "marketName": "SEI-PERP", "category": ["L1"], "launchTs": 1703173331000},
24+
{"marketIndex": 22, "marketName": "AVAX-PERP", "category": ["Rollup", "Infra"], "launchTs": 1704209558000},
25+
{"marketIndex": 23, "marketName": "WIF-PERP", "category": ["Meme", "Dog", "Solana"], "launchTs": 1706219971000},
26+
{"marketIndex": 24, "marketName": "JUP-PERP", "category": ["Exchange", "Infra", "Solana"], "launchTs": 1706713201000},
27+
{"marketIndex": 25, "marketName": "DYM-PERP", "category": ["Rollup", "Infra"], "launchTs": 1708448765000},
28+
{"marketIndex": 26, "marketName": "TAO-PERP", "category": ["AI", "Infra"], "launchTs": 1709136669000},
29+
{"marketIndex": 27, "marketName": "W-PERP", "category": ["Bridge"], "launchTs": 1710418343000},
30+
{"marketIndex": 28, "marketName": "KMNO-PERP", "category": ["Lending", "Solana"], "launchTs": 1712240681000},
31+
{"marketIndex": 29, "marketName": "TNSR-PERP", "category": ["NFT", "Solana"], "launchTs": 1712593532000},
32+
{"marketIndex": 30, "marketName": "DRIFT-PERP", "category": ["DEX", "Solana"], "launchTs": 1716595200000},
33+
{"marketIndex": 31, "marketName": "CLOUD-PERP", "category": ["LST", "Solana"], "launchTs": 1717597648000},
34+
{"marketIndex": 32, "marketName": "IO-PERP", "category": ["DePIN", "Solana"], "launchTs": 1718021389000},
35+
{"marketIndex": 33, "marketName": "ZEX-PERP", "category": ["DEX", "Solana"], "launchTs": 1719415157000},
36+
{"marketIndex": 34, "marketName": "POPCAT-PERP", "category": ["Meme", "Solana"], "launchTs": 1720013054000},
37+
{"marketIndex": 35, "marketName": "1KWEN-PERP", "category": ["Solana", "Meme"], "launchTs": 1720633344000},
38+
{"marketIndex": 36, "marketName": "TRUMP-WIN-2024-BET", "category": ["Prediction", "Election"], "launchTs": 1723996800000},
39+
{"marketIndex": 37, "marketName": "KAMALA-POPULAR-VOTE-2024-BET", "category": ["Prediction", "Election"], "launchTs": 1723996800000},
40+
{"marketIndex": 38, "marketName": "FED-CUT-50-SEPT-2024-BET", "category": ["Prediction", "Election"], "launchTs": 1724250126000},
41+
{"marketIndex": 39, "marketName": "REPUBLICAN-POPULAR-AND-WIN-BET", "category": ["Prediction", "Election"], "launchTs": 1724250126000},
42+
{"marketIndex": 40, "marketName": "BREAKPOINT-IGGYERIC-BET", "category": ["Prediction", "Solana"], "launchTs": 1724250126000},
43+
{"marketIndex": 41, "marketName": "DEMOCRATS-WIN-MICHIGAN-BET", "category": ["Prediction", "Election"], "launchTs": 1725551484000},
44+
{"marketIndex": 42, "marketName": "TON-PERP", "category": ["L1"], "launchTs": 1725551484000},
45+
{"marketIndex": 43, "marketName": "LANDO-F1-SGP-WIN-BET", "category": ["Prediction", "Sports"], "launchTs": 1726646453000},
46+
{"marketIndex": 44, "marketName": "MOTHER-PERP", "category": ["Solana", "Meme"], "launchTs": 1727291859000},
47+
{"marketIndex": 45, "marketName": "MOODENG-PERP", "category": ["Solana", "Meme"], "launchTs": 1727965864000},
48+
{"marketIndex": 46, "marketName": "WARWICK-FIGHT-WIN-BET", "category": ["Prediction", "Sport"], "launchTs": 1727965864000},
49+
{"marketIndex": 47, "marketName": "DBR-PERP", "category": ["Bridge"], "launchTs": 1728574493000},
50+
{"marketIndex": 48, "marketName": "WLF-5B-1W-BET", "category": ["Prediction"], "launchTs": 1728574493000},
51+
{"marketIndex": 49, "marketName": "VRSTPN-WIN-F1-24-DRVRS-CHMP-BET", "category": ["Prediction", "Sport"], "launchTs": 1729209600000},
52+
{"marketIndex": 50, "marketName": "LNDO-WIN-F1-24-US-GP-BET", "category": ["Prediction", "Sport"], "launchTs": 1729209600000},
53+
{"marketIndex": 51, "marketName": "1KMEW-PERP", "category": ["Meme"], "launchTs": 1729702915000},
54+
{"marketIndex": 52, "marketName": "MICHI-PERP", "category": ["Meme"], "launchTs": 1730402722000},
55+
{"marketIndex": 53, "marketName": "GOAT-PERP", "category": ["Meme"], "launchTs": 1731443152000},
56+
{"marketIndex": 54, "marketName": "FWOG-PERP", "category": ["Meme"], "launchTs": 1731443152000},
57+
{"marketIndex": 55, "marketName": "PNUT-PERP", "category": ["Meme"], "launchTs": 1731443152000},
58+
{"marketIndex": 56, "marketName": "RAY-PERP", "category": ["DEX"], "launchTs": 1732721897000},
59+
{"marketIndex": 57, "marketName": "SUPERBOWL-LIX-LIONS-BET", "category": ["Prediction", "Sport"], "launchTs": 1732721897000},
60+
{"marketIndex": 58, "marketName": "SUPERBOWL-LIX-CHIEFS-BET", "category": ["Prediction", "Sport"], "launchTs": 1732721897000},
61+
{"marketIndex": 59, "marketName": "HYPE-PERP", "category": ["DEX"], "launchTs": 1733374800000},
62+
{"marketIndex": 60, "marketName": "LTC-PERP", "category": ["Payment"], "launchTs": 1733374800000},
63+
{"marketIndex": 61, "marketName": "ME-PERP", "category": ["DEX"], "launchTs": 1733839936000},
64+
{"marketIndex": 62, "marketName": "PENGU-PERP", "category": ["Meme"], "launchTs": 1734444000000},
65+
{"marketIndex": 63, "marketName": "AI16Z-PERP", "category": ["AI"], "launchTs": 1736384970000},
66+
{"marketIndex": 64, "marketName": "TRUMP-PERP", "category": ["Meme"], "launchTs": 1737219250000},
67+
{"marketIndex": 65, "marketName": "MELANIA-PERP", "category": ["Meme"], "launchTs": 1737360280000},
68+
{"marketIndex": 66, "marketName": "BERA-PERP", "category": ["L1", "EVM"], "launchTs": 1738850177000},
69+
{"marketIndex": 67, "marketName": "NBAFINALS25-OKC-BET", "category": ["Prediction", "Sport"], "launchTs": 1739463226000},
70+
{"marketIndex": 68, "marketName": "NBAFINALS25-BOS-BET", "category": ["Prediction", "Sport"], "launchTs": 1739463226000},
71+
{"marketIndex": 69, "marketName": "KAITO-PERP", "category": ["AI"], "launchTs": 1739545901000},
72+
{"marketIndex": 70, "marketName": "IP-PERP", "category": ["L1"], "launchTs": 1740150623000},
73+
{"marketIndex": 71, "marketName": "FARTCOIN-PERP", "category": ["Meme"], "launchTs": 1743086746000},
74+
{"marketIndex": 72, "marketName": "ADA-PERP", "category": ["L1"], "launchTs": 1743708559000},
75+
{"marketIndex": 73, "marketName": "PAXG-PERP", "category": ["RWA"], "launchTs": 1744402932000},
76+
{"marketIndex": 74, "marketName": "LAUNCHCOIN-PERP", "category": ["Meme"], "launchTs": 1747318237000}
77+
]

‎src/main.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from page.market_recommender_page import market_recommender_page
2424
from page.open_interest_page import open_interest_page
2525
from page.high_leverage_page import high_leverage_page
26-
from page.user_retention_page import user_retention_page
26+
from page.user_retention_summary_page import user_retention_summary_page
2727

2828
load_dotenv()
2929

@@ -151,9 +151,9 @@ def apply_custom_css(css):
151151
icon="⚡",
152152
),
153153
st.Page(
154-
needs_backend(user_retention_page),
155-
url_path="user-retention",
156-
title="User Retention",
154+
needs_backend(user_retention_summary_page),
155+
url_path="user-retention-summary",
156+
title="User Retention Summary",
157157
icon="👥",
158158
),
159159
]

0 commit comments

Comments
 (0)