diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 06abbcb34..9aeb96f04 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -58,7 +58,8 @@ def __init__( else: self.multi_id = str(uuid.uuid4()) - self.model: Optional[Aimodel] = None + self.model_UP: Optional[Aimodel] = None + self.model_DOWN: Optional[Aimodel] = None @property def predict_feed(self) -> ArgFeed: @@ -95,9 +96,9 @@ def run(self): def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ppss, pdr_ss, st = self.ppss, self.ppss.predictoor_ss, self.st transform = pdr_ss.aimodel_data_ss.transform - stake_amt = pdr_ss.stake_amount.amt_eth others_stake = pdr_ss.others_stake.amt_eth revenue = pdr_ss.revenue.amt_eth + model_conf_thr = self.ppss.trader_ss.sim_confidence_threshold testshift = ppss.sim_ss.test_n - test_i - 1 # eg [99, 98, .., 2, 1, 0] data_f = AimodelDataFactory(pdr_ss) # type: ignore[arg-type] @@ -105,37 +106,70 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): train_feeds = self.predict_train_feedset.train_on # X, ycont, and x_df are all expressed in % change wrt prev candle - X, ytran, yraw, x_df, _ = data_f.create_xy( + p = predict_feed + predict_feed_close = p + predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe) + predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe) + _, _, yraw_close, _, _ = data_f.create_xy( mergedohlcv_df, testshift, - predict_feed, + predict_feed_close, train_feeds, ) - colnames = list(x_df.columns) - - st_, fin = 0, X.shape[0] - 1 - X_train, X_test = X[st_:fin, :], X[fin : fin + 1, :] - ytran_train, _ = ytran[st_:fin], ytran[fin : fin + 1] + X_UP, ytran_UP, yraw_high, x_df_high, _ = data_f.create_xy( + mergedohlcv_df, + testshift, + predict_feed_high, + train_feeds, + ) + X_DOWN, ytran_DOWN, yraw_low, _, _ = data_f.create_xy( + mergedohlcv_df, + testshift, + predict_feed_low, + train_feeds, + ) + colnames_high = list(x_df_high.columns) - cur_high, cur_low = data_f.get_highlow(mergedohlcv_df, predict_feed, testshift) + cur_close, next_close = yraw_close[-2], yraw_close[-1] + cur_high, next_high = yraw_high[-2], yraw_high[-1] + cur_low, next_low = yraw_low[-2], yraw_low[-1] - cur_close = yraw[-2] - next_close = yraw[-1] + st_, fin = 0, X_UP.shape[0] - 1 + X_train_UP, X_test_UP = X_UP[st_:fin, :], X_UP[fin : fin + 1, :] + ytran_train_UP, _ = ytran_UP[st_:fin], ytran_UP[fin : fin + 1] + X_train_DOWN, X_test_DOWN = X_DOWN[st_:fin, :], X_DOWN[fin : fin + 1, :] + ytran_train_DOWN, _ = ytran_DOWN[st_:fin], ytran_DOWN[fin : fin + 1] + percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml if transform == "None": - y_thr = cur_close + y_thr_UP = cur_close * (1 + percent_change_needed) + y_thr_DOWN = cur_close * (1 - percent_change_needed) else: # transform = "RelDiff" - y_thr = 0.0 - ytrue = data_f.ycont_to_ytrue(ytran, y_thr) + y_thr_UP = +np.std(yraw_close) * percent_change_needed + y_thr_DOWN = -np.std(yraw_close) * percent_change_needed + ytrue_UP = data_f.ycont_to_ytrue(ytran_UP, y_thr_UP) + ytrue_DOWN = data_f.ycont_to_ytrue(ytran_DOWN, y_thr_DOWN) - ytrue_train, _ = ytrue[st_:fin], ytrue[fin : fin + 1] + ytrue_train_UP, _ = ytrue_UP[st_:fin], ytrue_UP[fin : fin + 1] + ytrue_train_DOWN, _ = ytrue_DOWN[st_:fin], ytrue_DOWN[fin : fin + 1] if ( - self.model is None + self.model_UP is None or self.st.iter_number % pdr_ss.aimodel_ss.train_every_n_epochs == 0 ): model_f = AimodelFactory(pdr_ss.aimodel_ss) - self.model = model_f.build(X_train, ytrue_train, ytran_train, y_thr) + self.model_UP = model_f.build( + X_train_UP, + ytrue_train_UP, + ytran_train_UP, + y_thr_UP, + ) + self.model_DOWN = model_f.build( + X_train_DOWN, + ytrue_train_DOWN, + ytran_train_DOWN, + y_thr_DOWN, + ) # current time recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) @@ -143,23 +177,52 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ut = UnixTimeMs(recent_ut - testshift * timeframe.ms) # predict price direction - prob_up: float = self.model.predict_ptrue(X_test)[0] # in [0.0, 1.0] - prob_down: float = 1.0 - prob_up - conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] - conf_down = (prob_down - 0.5) * 2.0 # to range [0,1] - conf_threshold = self.ppss.trader_ss.sim_confidence_threshold - pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold - pred_down: bool = prob_up < 0.5 and conf_down > conf_threshold - st.probs_up.append(prob_up) + prob_up_UP = self.model_UP.predict_ptrue(X_test_UP)[0] + prob_up_DOWN = self.model_DOWN.predict_ptrue(X_test_DOWN)[0] + prob_down_DOWN = 1.0 - prob_up_DOWN + + models_in_conflict = (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \ + (prob_up_UP < 0.5 and prob_down_DOWN < 0.5) + if models_in_conflict: + conf_up = conf_down = 0.0 + pred_up = pred_down = False + prob_up_MERGED = 0.5 + elif prob_up_UP >= prob_down_DOWN: + conf_up = (prob_up_UP - 0.5) * 2.0 # to range [0,1] + conf_down = 0.0 + pred_up = conf_up > model_conf_thr + pred_down = False + prob_up_MERGED = prob_up_UP + else: # prob_down_DOWN > prob_up_UP + conf_up = 0.0 + conf_down = (prob_down_DOWN - 0.5) * 2.0 + pred_up = False + pred_down = conf_down > model_conf_thr + prob_up_MERGED = 1.0 - prob_down_DOWN + + st.probs_up_UP.append(prob_up_UP) + st.ytrues_hat_UP.append(prob_up_UP > 0.5) + st.probs_up_DOWN.append(prob_up_DOWN) + st.ytrues_hat_DOWN.append(prob_up_DOWN > 0.5) + st.probs_up_MERGED.append(prob_up_MERGED) # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 - stake_up = stake_amt * prob_up - stake_down = stake_amt * (1.0 - prob_up) + max_stake_amt = pdr_ss.stake_amount.amt_eth + if models_in_conflict or not (pred_up or pred_down): + stake_up = stake_down = 0 + elif prob_up_UP >= prob_down_DOWN: + stake_amt = max_stake_amt * conf_up + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - prob_up_MERGED) + else: # prob_down_DOWN > prob_up_UP + stake_amt = max_stake_amt * conf_down + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - prob_up_MERGED) acct_up_profit -= stake_up acct_down_profit -= stake_down - profit = self.trader.trade_iter( + trader_profit = self.trader.trade_iter( cur_close, pred_up, pred_down, @@ -169,43 +232,46 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): cur_low, ) - st.trader_profits_USD.append(profit) - - # observe true price - true_up = next_close > cur_close - st.ytrues.append(true_up) - - # update classifier metrics - n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat)) - n_trials = len(st.ytrues) - acc_est = n_correct / n_trials - acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) - (precision, recall, f1, _) = precision_recall_fscore_support( - st.ytrues, - st.ytrues_hat, - average="binary", - zero_division=0.0, - ) - if min(st.ytrues) == max(st.ytrues): - loss = 3.0 + st.trader_profits_USD.append(trader_profit) + + # observe true price change + true_up_close = next_close > cur_close + + if transform == "None": + true_up_UP = next_high > y_thr_UP + true_up_DOWN = next_low > y_thr_DOWN else: - loss = log_loss(st.ytrues, st.probs_up) + raise NotImplementedError("build me") + st.ytrues_UP.append(true_up_UP) + st.ytrues_DOWN.append(true_up_DOWN) + + # update classifier performances + perf_UP = get_perf(st.ytrues_UP, st.ytrues_hat_UP, st.probs_up_UP) + perf_DOWN = get_perf(st.ytrues_DOWN, st.ytrues_hat_DOWN, st.probs_up_DOWN) + perf_merged = merge_tups(perf_UP, perf_DOWN) + (acc_est, acc_l, acc_u, precision, recall, f1, loss) = perf_merged + yerr = 0.0 - if self.model.do_regr: - pred_ycont = self.model.predict_ycont(X_test)[0] + if self.model_UP.do_regr: + pred_ycont_UP = self.model_UP.predict_ycont(X_test_UP)[0] + pred_ycont_DOWN = self.model_DOWN.predict_ycont(X_test_DOWN)[0] if transform == "None": - pred_next_close = pred_ycont + pred_next_high = pred_ycont_UP + pred_next_low = pred_ycont_DOWN else: # transform = "RelDiff" relchange = pred_ycont - pred_next_close = cur_close + relchange * cur_close - yerr = next_close - pred_next_close + pred_next_high = cur_high + relchange * cur_high + pred_next_low = cur_low + relchange * cur_low + yerr_UP = next_high - pred_next_high + yerr_DOWN = next_low - pred_next_low + yerr = np.mean([yerr_UP, yerr_DOWN]) st.aim.update(acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr) # track predictoor profit - tot_stake = others_stake + stake_amt + tot_stake = others_stake + stake_up + stake_down others_stake_correct = others_stake * pdr_ss.others_accuracy - if true_up: + if true_up_close: tot_stake_correct = others_stake_correct + stake_up percent_to_me = stake_up / tot_stake_correct acct_up_profit += (revenue + tot_stake) * percent_to_me @@ -221,16 +287,16 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): save_state, is_final_state = self.save_state(test_i, self.ppss.sim_ss.test_n) if save_state: - colnames = [shift_one_earlier(colname) for colname in colnames] - most_recent_x = X[-1, :] + cs_ = [shift_one_earlier(c_) for c_ in colnames_high] + most_recent_x = X_UP[-1, :] slicing_x = most_recent_x # plot about the most recent x d = AimodelPlotdata( - self.model, - X_train, - ytrue_train, - ytran_train, - y_thr, - colnames, + self.model_UP, + X_train_UP, + ytrue_train_UP, + ytran_train_UP, + y_thr_UP, + cs_, slicing_x, ) self.st.iter_number = test_i @@ -257,3 +323,27 @@ def save_state(self, i: int, N: int): return False, False return True, False + +def get_perf(ytrues, ytrues_hat, probs_up) -> tuple: + """Get classifier performances: accuracy, precision/recall/f1, log loss""" + n_correct = sum(np.array(ytrues) == np.array(ytrues_hat)) + n_trials = len(ytrues) + acc_est = n_correct / n_trials + acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) + + (precision, recall, f1, _) = precision_recall_fscore_support( + ytrues, + ytrues_hat, + average="binary", + zero_division=0.0, + ) + + if min(ytrues) == max(ytrues): + loss = 3.0 # magic number + else: + loss = log_loss(ytrues, probs_up) + + return (acc_est, acc_l, acc_u, precision, recall, f1, loss) + +def merge_tups(tup1, tup2): + return (np.mean([val1, val2]) for val1, val2 in zip(tup1, tup2)) diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index ac23d7d1a..94fc25e6f 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -20,8 +20,9 @@ def __init__(self, ppss, st, test_i, ut, acct_up_profit, acct_down_profit): self.acct_up_profit = acct_up_profit self.acct_down_profit = acct_down_profit - self.n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat)) - self.n_trials = len(st.ytrues) + # TODO: account for DOWN too + self.n_correct = sum(np.array(st.ytrues_UP) == np.array(st.ytrues_hat_UP)) + self.n_trials = len(st.ytrues_UP) for key, item in st.recent_metrics(extras=["prob_up"]).items(): setattr(self, key, item) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 17e9f0607..4e6a760e9 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -159,7 +159,7 @@ def plot_trader_profit_vs_time(self): @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.probs_up + x = self.st.probs_up_MERGED y = self.st.pdr_profits_OCEAN fig = go.Figure( go.Scatter( @@ -179,7 +179,7 @@ def plot_pdr_profit_vs_ptrue(self): @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.probs_up + x = self.st.probs_up_MERGED y = self.st.trader_profits_USD fig = go.Figure( go.Scatter( diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index d01d10e62..400f825ad 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -74,9 +74,18 @@ def __init__(self): def init_loop_attributes(self): # 'i' is iteration number i - # base data - self.ytrues: List[bool] = [] # [i] : was-truly-up - self.probs_up: List[float] = [] # [i] : predicted-prob-up + # base data for UP classifier + self.ytrues_UP: List[bool] = [] # [i] : true value + self.ytrues_hat_UP: List[bool] = [] # [i] : model pred. value + self.probs_up_UP: List[float] = [] # [i] : model's pred. prob. + + # base data for DOWN classifier + self.ytrues_DOWN: List[bool] = [] # [i] : true value + self.ytrues_hat_DOWN: List[bool] = [] # [i] : model pred. value + self.probs_up_DOWN: List[float] = [] # [i] : model's pred. prob. + + # merged values + self.probs_up_MERGED: List[float] = [] # [i] : merged pred. prob. # aimodel metrics self.aim = AimodelMetrics() @@ -105,14 +114,10 @@ def recent_metrics( ) if extras and "prob_up" in extras: - rm["prob_up"] = self.probs_up[-1] + rm["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN return rm - @property - def ytrues_hat(self) -> List[bool]: - return [p > 0.5 for p in self.probs_up] - @property def n_correct(self) -> int: return sum((p > 0.5) == t for p, t in zip(self.probs_up, self.ytrues))