From d0fe232628d8b3d37173f11a86b2701896f9b44e Mon Sep 17 00:00:00 2001 From: trentmc Date: Sat, 22 Jun 2024 08:52:57 +0200 Subject: [PATCH 01/50] Fix #1278:[Sim] Experiment: Reframe to classify 'will price go up *anytime* in next 5 min?', etc --- pdr_backend/sim/sim_engine.py | 97 +++++++++++++++++++++++------------ ppss.yaml | 4 +- 2 files changed, 66 insertions(+), 35 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 06abbcb34..c78f47a6e 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -58,7 +58,8 @@ def __init__( else: self.multi_id = str(uuid.uuid4()) - self.model: Optional[Aimodel] = None + self.model_high: Optional[Aimodel] = None + self.model_low: Optional[Aimodel] = None @property def predict_feed(self) -> ArgFeed: @@ -105,37 +106,64 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): train_feeds = self.predict_train_feedset.train_on # X, ycont, and x_df are all expressed in % change wrt prev candle - X, ytran, yraw, x_df, _ = data_f.create_xy( + p = predict_feed + predict_feed_close = p + predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe) + predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe) + _, _, yraw_close, x_df_close, _ = data_f.create_xy( mergedohlcv_df, testshift, - predict_feed, + predict_feed_close, train_feeds, ) - colnames = list(x_df.columns) - - st_, fin = 0, X.shape[0] - 1 - X_train, X_test = X[st_:fin, :], X[fin : fin + 1, :] - ytran_train, _ = ytran[st_:fin], ytran[fin : fin + 1] - - cur_high, cur_low = data_f.get_highlow(mergedohlcv_df, predict_feed, testshift) - - cur_close = yraw[-2] - next_close = yraw[-1] - + X_high, ytran_high, yraw_high, _, _ = data_f.create_xy( + mergedohlcv_df, + testshift, + predict_feed_high, + train_feeds, + ) + X_low, ytran_low, yraw_low, _, _ = data_f.create_xy( + mergedohlcv_df, + testshift, + predict_feed_low, + train_feeds, + ) + colnames = list(x_df_close.columns) + + cur_high, cur_low = data_f.get_highlow(mergedohlcv_df, p, testshift) + cur_close = yraw_close[-2] + next_close = yraw_close[-1] + + st_, fin = 0, X_high.shape[0] - 1 + X_train_high, X_test_high = X_high[st_:fin, :], X_high[fin : fin + 1, :] + ytran_train_high, _ = ytran_high[st_:fin], ytran_high[fin : fin + 1] + X_train_low, X_test_low = X_low[st_:fin, :], X_low[fin : fin + 1, :] + ytran_train_low, _ = ytran_low[st_:fin], ytran_low[fin : fin + 1] + + percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml if transform == "None": - y_thr = cur_close + y_thr_high = cur_close * (1 + percent_change_needed) + y_thr_low = cur_close * (1 - percent_change_needed) else: # transform = "RelDiff" - y_thr = 0.0 - ytrue = data_f.ycont_to_ytrue(ytran, y_thr) + y_thr_high = + np.std(yraw_close) * percent_change_needed + y_thr_high = - np.std(yraw_close) * percent_change_needed + ytrue_high = data_f.ycont_to_ytrue(ytran_high, y_thr_high) + ytrue_low = data_f.ycont_to_ytrue(ytran_low, y_thr_low) - ytrue_train, _ = ytrue[st_:fin], ytrue[fin : fin + 1] + ytrue_train_high, _ = ytrue_high[st_:fin], ytrue_high[fin : fin + 1] + ytrue_train_low, _ = ytrue_low[st_:fin], ytrue_low[fin : fin + 1] if ( - self.model is None + self.model_high is None or self.st.iter_number % pdr_ss.aimodel_ss.train_every_n_epochs == 0 ): model_f = AimodelFactory(pdr_ss.aimodel_ss) - self.model = model_f.build(X_train, ytrue_train, ytran_train, y_thr) + self.model_high = model_f.build( + X_train_high, ytrue_train_high, ytran_train_high, y_thr_high, + ) + self.model_low = model_f.build( + X_train_low, ytrue_train_low, ytran_train_low, y_thr_low, + ) # current time recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) @@ -143,13 +171,16 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ut = UnixTimeMs(recent_ut - testshift * timeframe.ms) # predict price direction - prob_up: float = self.model.predict_ptrue(X_test)[0] # in [0.0, 1.0] - prob_down: float = 1.0 - prob_up + prob_up: float = self.model_high.predict_ptrue(X_test_high)[0] + prob_down: float = 1.0 - self.model_low.predict_ptrue(X_test_low)[0] conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] conf_down = (prob_down - 0.5) * 2.0 # to range [0,1] conf_threshold = self.ppss.trader_ss.sim_confidence_threshold pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold - pred_down: bool = prob_up < 0.5 and conf_down > conf_threshold + pred_down: bool = prob_down > 0.5 and conf_down > conf_threshold + if pred_up and pred_down: # predictions conflict, so reset them + prob_up = prob_down = 0.0 + pred_up = pred_down = False st.probs_up.append(prob_up) # predictoor: (simulate) submit predictions with stake @@ -191,8 +222,8 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): else: loss = log_loss(st.ytrues, st.probs_up) yerr = 0.0 - if self.model.do_regr: - pred_ycont = self.model.predict_ycont(X_test)[0] + if self.model_high.do_regr: + pred_ycont = self.model_high.predict_ycont(X_test)[0] if transform == "None": pred_next_close = pred_ycont else: # transform = "RelDiff" @@ -221,16 +252,16 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): save_state, is_final_state = self.save_state(test_i, self.ppss.sim_ss.test_n) if save_state: - colnames = [shift_one_earlier(colname) for colname in colnames] - most_recent_x = X[-1, :] + colnames1 = [shift_one_earlier(colname) for colname in colnames] + most_recent_x = X_high[-1, :] slicing_x = most_recent_x # plot about the most recent x d = AimodelPlotdata( - self.model, - X_train, - ytrue_train, - ytran_train, - y_thr, - colnames, + self.model_high, + X_train_high, + ytrue_train_high, + ytran_train_high, + y_thr_high, + colnames1, slicing_x, ) self.st.iter_number = test_i diff --git a/ppss.yaml b/ppss.yaml index 8fc04d396..2e0a6190f 100644 --- a/ppss.yaml +++ b/ppss.yaml @@ -53,8 +53,8 @@ trader_ss: buy_amt: 1000 USD # buy this amount in each epoch fee_percent: 0.0 # simulated % fee confidence_threshold: 0.02 # skip trade if confidence < this - stop_loss_percent: 1.0 # sell if price drops by this %, . 1.0 = disabled - take_profit_percent: 100.0 # sell if price rises by this %, . 100.0 = disabled + stop_loss_percent: 0.02 # sell if price drops by this %, . 1.0 = disabled + take_profit_percent: 0.02 # sell if price rises by this %, . 100.0 = disabled bot_only: min_buffer: 30 # in s. only trade if there's > this time left From 0eb7bc99622173b7c3b28ffdedd1cdbb3bbf7430 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sat, 22 Jun 2024 09:17:15 +0200 Subject: [PATCH 02/50] tweaks --- pdr_backend/sim/sim_engine.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index c78f47a6e..ee13331f2 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -96,7 +96,6 @@ def run(self): def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ppss, pdr_ss, st = self.ppss, self.ppss.predictoor_ss, self.st transform = pdr_ss.aimodel_data_ss.transform - stake_amt = pdr_ss.stake_amount.amt_eth others_stake = pdr_ss.others_stake.amt_eth revenue = pdr_ss.revenue.amt_eth @@ -173,24 +172,26 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # predict price direction prob_up: float = self.model_high.predict_ptrue(X_test_high)[0] prob_down: float = 1.0 - self.model_low.predict_ptrue(X_test_low)[0] + if (prob_up + prob_down) > 1.0: # ensure predictions don't conflict + prob_up = prob_up / (prob_up + prob_down) + prob_down = 1.0 - prob_up conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] conf_down = (prob_down - 0.5) * 2.0 # to range [0,1] conf_threshold = self.ppss.trader_ss.sim_confidence_threshold pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold pred_down: bool = prob_down > 0.5 and conf_down > conf_threshold - if pred_up and pred_down: # predictions conflict, so reset them - prob_up = prob_down = 0.0 - pred_up = pred_down = False st.probs_up.append(prob_up) # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 - stake_up = stake_amt * prob_up - stake_down = stake_amt * (1.0 - prob_up) + max_stake_amt = pdr_ss.stake_amount.amt_eth + assert (prob_up + prob_down) <= 1.0 + stake_up = max_stake_amt * prob_up + stake_down = max_stake_amt * prob_down acct_up_profit -= stake_up acct_down_profit -= stake_down - profit = self.trader.trade_iter( + trader_profit = self.trader.trade_iter( cur_close, pred_up, pred_down, @@ -200,11 +201,12 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): cur_low, ) - st.trader_profits_USD.append(profit) + st.trader_profits_USD.append(trader_profit) # observe true price true_up = next_close > cur_close - st.ytrues.append(true_up) + true_up_high = next_close > cur_close * (1 + percent_change_needed) + st.ytrues.append(true_up_high) # update classifier metrics n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat)) @@ -234,7 +236,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): st.aim.update(acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr) # track predictoor profit - tot_stake = others_stake + stake_amt + tot_stake = others_stake + stake_up + stake_down others_stake_correct = others_stake * pdr_ss.others_accuracy if true_up: tot_stake_correct = others_stake_correct + stake_up From f4b177504db639f7d7e398a3d7bd09d09af8a5e4 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sat, 22 Jun 2024 10:18:37 +0200 Subject: [PATCH 03/50] wip --- pdr_backend/sim/sim_engine.py | 46 ++++++++++++++++------------------- pdr_backend/sim/sim_state.py | 5 +--- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index ee13331f2..b711e44ce 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -109,29 +109,20 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): predict_feed_close = p predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe) predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe) - _, _, yraw_close, x_df_close, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_close, - train_feeds, + _, _, yraw_close, _, _ = data_f.create_xy( + mergedohlcv_df, testshift, predict_feed_close, train_feeds, ) - X_high, ytran_high, yraw_high, _, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_high, - train_feeds, + X_high, ytran_high, yraw_high, x_df_high, _ = data_f.create_xy( + mergedohlcv_df, testshift, predict_feed_high, train_feeds, ) X_low, ytran_low, yraw_low, _, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_low, - train_feeds, + mergedohlcv_df, testshift, predict_feed_low, train_feeds, ) - colnames = list(x_df_close.columns) - - cur_high, cur_low = data_f.get_highlow(mergedohlcv_df, p, testshift) - cur_close = yraw_close[-2] - next_close = yraw_close[-1] + colnames_high = list(x_df_high.columns) + + cur_close, next_close = yraw_close[-2], yraw_close[-1] + cur_high, next_high = yraw_high[-2], yraw_high[-1] + cur_low, next_low = yraw_low[-2], yraw_low[-1] st_, fin = 0, X_high.shape[0] - 1 X_train_high, X_test_high = X_high[st_:fin, :], X_high[fin : fin + 1, :] @@ -181,6 +172,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold pred_down: bool = prob_down > 0.5 and conf_down > conf_threshold st.probs_up.append(prob_up) + st.ytrues_hat.append(prob_up > 0.5) # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 @@ -203,9 +195,13 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): st.trader_profits_USD.append(trader_profit) - # observe true price - true_up = next_close > cur_close - true_up_high = next_close > cur_close * (1 + percent_change_needed) + # observe true price change + true_up_close = next_close > cur_close + + if transform == "None": + true_up_high = next_high > y_thr_high + else: + raise NotImplementedError("build me") st.ytrues.append(true_up_high) # update classifier metrics @@ -238,7 +234,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # track predictoor profit tot_stake = others_stake + stake_up + stake_down others_stake_correct = others_stake * pdr_ss.others_accuracy - if true_up: + if true_up_close: tot_stake_correct = others_stake_correct + stake_up percent_to_me = stake_up / tot_stake_correct acct_up_profit += (revenue + tot_stake) * percent_to_me @@ -254,7 +250,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): save_state, is_final_state = self.save_state(test_i, self.ppss.sim_ss.test_n) if save_state: - colnames1 = [shift_one_earlier(colname) for colname in colnames] + cs_ = [shift_one_earlier(c_) for c_ in colnames_high] most_recent_x = X_high[-1, :] slicing_x = most_recent_x # plot about the most recent x d = AimodelPlotdata( @@ -263,7 +259,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ytrue_train_high, ytran_train_high, y_thr_high, - colnames1, + cs_, slicing_x, ) self.st.iter_number = test_i diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index d01d10e62..e135d1584 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -76,6 +76,7 @@ def init_loop_attributes(self): # base data self.ytrues: List[bool] = [] # [i] : was-truly-up + self.ytrues_hat: List[bool] = [] # [i] : was-predicted-up self.probs_up: List[float] = [] # [i] : predicted-prob-up # aimodel metrics @@ -109,10 +110,6 @@ def recent_metrics( return rm - @property - def ytrues_hat(self) -> List[bool]: - return [p > 0.5 for p in self.probs_up] - @property def n_correct(self) -> int: return sum((p > 0.5) == t for p, t in zip(self.probs_up, self.ytrues)) From 24a739ebdda2c86ef1c7e78ff02f522a5608c181 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sat, 22 Jun 2024 10:21:42 +0200 Subject: [PATCH 04/50] make linters happy --- pdr_backend/sim/sim_engine.py | 39 ++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index b711e44ce..2047a9fc8 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -110,19 +110,28 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe) predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe) _, _, yraw_close, _, _ = data_f.create_xy( - mergedohlcv_df, testshift, predict_feed_close, train_feeds, + mergedohlcv_df, + testshift, + predict_feed_close, + train_feeds, ) X_high, ytran_high, yraw_high, x_df_high, _ = data_f.create_xy( - mergedohlcv_df, testshift, predict_feed_high, train_feeds, + mergedohlcv_df, + testshift, + predict_feed_high, + train_feeds, ) X_low, ytran_low, yraw_low, _, _ = data_f.create_xy( - mergedohlcv_df, testshift, predict_feed_low, train_feeds, + mergedohlcv_df, + testshift, + predict_feed_low, + train_feeds, ) colnames_high = list(x_df_high.columns) cur_close, next_close = yraw_close[-2], yraw_close[-1] cur_high, next_high = yraw_high[-2], yraw_high[-1] - cur_low, next_low = yraw_low[-2], yraw_low[-1] + cur_low, _ = yraw_low[-2], yraw_low[-1] st_, fin = 0, X_high.shape[0] - 1 X_train_high, X_test_high = X_high[st_:fin, :], X_high[fin : fin + 1, :] @@ -130,13 +139,13 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): X_train_low, X_test_low = X_low[st_:fin, :], X_low[fin : fin + 1, :] ytran_train_low, _ = ytran_low[st_:fin], ytran_low[fin : fin + 1] - percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml + percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml if transform == "None": y_thr_high = cur_close * (1 + percent_change_needed) y_thr_low = cur_close * (1 - percent_change_needed) else: # transform = "RelDiff" - y_thr_high = + np.std(yraw_close) * percent_change_needed - y_thr_high = - np.std(yraw_close) * percent_change_needed + y_thr_high = +np.std(yraw_close) * percent_change_needed + y_thr_low = -np.std(yraw_close) * percent_change_needed ytrue_high = data_f.ycont_to_ytrue(ytran_high, y_thr_high) ytrue_low = data_f.ycont_to_ytrue(ytran_low, y_thr_low) @@ -149,10 +158,16 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ): model_f = AimodelFactory(pdr_ss.aimodel_ss) self.model_high = model_f.build( - X_train_high, ytrue_train_high, ytran_train_high, y_thr_high, + X_train_high, + ytrue_train_high, + ytran_train_high, + y_thr_high, ) self.model_low = model_f.build( - X_train_low, ytrue_train_low, ytran_train_low, y_thr_low, + X_train_low, + ytrue_train_low, + ytran_train_low, + y_thr_low, ) # current time @@ -162,8 +177,8 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # predict price direction prob_up: float = self.model_high.predict_ptrue(X_test_high)[0] - prob_down: float = 1.0 - self.model_low.predict_ptrue(X_test_low)[0] - if (prob_up + prob_down) > 1.0: # ensure predictions don't conflict + prob_down: float = 1.0 - self.model_low.predict_ptrue(X_test_low)[0] # type: ignore + if (prob_up + prob_down) > 1.0: # ensure predictions don't conflict prob_up = prob_up / (prob_up + prob_down) prob_down = 1.0 - prob_up conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] @@ -221,7 +236,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): loss = log_loss(st.ytrues, st.probs_up) yerr = 0.0 if self.model_high.do_regr: - pred_ycont = self.model_high.predict_ycont(X_test)[0] + pred_ycont = self.model_high.predict_ycont(X_test_high)[0] if transform == "None": pred_next_close = pred_ycont else: # transform = "RelDiff" From b288253387d8d5908287c9bc0e91cd89e0fe9298 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sat, 22 Jun 2024 10:32:29 +0200 Subject: [PATCH 05/50] undo --- ppss.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ppss.yaml b/ppss.yaml index 2e0a6190f..8fc04d396 100644 --- a/ppss.yaml +++ b/ppss.yaml @@ -53,8 +53,8 @@ trader_ss: buy_amt: 1000 USD # buy this amount in each epoch fee_percent: 0.0 # simulated % fee confidence_threshold: 0.02 # skip trade if confidence < this - stop_loss_percent: 0.02 # sell if price drops by this %, . 1.0 = disabled - take_profit_percent: 0.02 # sell if price rises by this %, . 100.0 = disabled + stop_loss_percent: 1.0 # sell if price drops by this %, . 1.0 = disabled + take_profit_percent: 100.0 # sell if price rises by this %, . 100.0 = disabled bot_only: min_buffer: 30 # in s. only trade if there's > this time left From e475ee2ab51191d4fff5079354d02988d8a8b76d Mon Sep 17 00:00:00 2001 From: trentmc Date: Sun, 23 Jun 2024 12:29:40 +0200 Subject: [PATCH 06/50] Make UP and DOWN classifier models (and related data) much more explicit. Especially, change from labels of high/low to UP/DOWN, to make things explicit. I think I fixed a bug or two --- pdr_backend/sim/sim_engine.py | 152 +++++++++++++++++++-------------- pdr_backend/sim/sim_logger.py | 5 +- pdr_backend/sim/sim_plotter.py | 4 +- pdr_backend/sim/sim_state.py | 15 ++-- 4 files changed, 105 insertions(+), 71 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 2047a9fc8..cb59113e8 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -58,8 +58,8 @@ def __init__( else: self.multi_id = str(uuid.uuid4()) - self.model_high: Optional[Aimodel] = None - self.model_low: Optional[Aimodel] = None + self.model_UP: Optional[Aimodel] = None + self.model_DOWN: Optional[Aimodel] = None @property def predict_feed(self) -> ArgFeed: @@ -115,13 +115,13 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): predict_feed_close, train_feeds, ) - X_high, ytran_high, yraw_high, x_df_high, _ = data_f.create_xy( + X_UP, ytran_UP, yraw_high, x_df_high, _ = data_f.create_xy( mergedohlcv_df, testshift, predict_feed_high, train_feeds, ) - X_low, ytran_low, yraw_low, _, _ = data_f.create_xy( + X_DOWN, ytran_DOWN, yraw_low, _, _ = data_f.create_xy( mergedohlcv_df, testshift, predict_feed_low, @@ -131,43 +131,43 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): cur_close, next_close = yraw_close[-2], yraw_close[-1] cur_high, next_high = yraw_high[-2], yraw_high[-1] - cur_low, _ = yraw_low[-2], yraw_low[-1] + cur_low, next_low = yraw_low[-2], yraw_low[-1] - st_, fin = 0, X_high.shape[0] - 1 - X_train_high, X_test_high = X_high[st_:fin, :], X_high[fin : fin + 1, :] - ytran_train_high, _ = ytran_high[st_:fin], ytran_high[fin : fin + 1] - X_train_low, X_test_low = X_low[st_:fin, :], X_low[fin : fin + 1, :] - ytran_train_low, _ = ytran_low[st_:fin], ytran_low[fin : fin + 1] + st_, fin = 0, X_UP.shape[0] - 1 + X_train_UP, X_test_UP = X_UP[st_:fin, :], X_UP[fin : fin + 1, :] + ytran_train_UP, _ = ytran_UP[st_:fin], ytran_UP[fin : fin + 1] + X_train_DOWN, X_test_DOWN = X_DOWN[st_:fin, :], X_DOWN[fin : fin + 1, :] + ytran_train_DOWN, _ = ytran_DOWN[st_:fin], ytran_DOWN[fin : fin + 1] percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml if transform == "None": - y_thr_high = cur_close * (1 + percent_change_needed) - y_thr_low = cur_close * (1 - percent_change_needed) + y_thr_UP = cur_close * (1 + percent_change_needed) + y_thr_DOWN = cur_close * (1 - percent_change_needed) else: # transform = "RelDiff" - y_thr_high = +np.std(yraw_close) * percent_change_needed - y_thr_low = -np.std(yraw_close) * percent_change_needed - ytrue_high = data_f.ycont_to_ytrue(ytran_high, y_thr_high) - ytrue_low = data_f.ycont_to_ytrue(ytran_low, y_thr_low) + y_thr_UP = +np.std(yraw_close) * percent_change_needed + y_thr_DOWN = -np.std(yraw_close) * percent_change_needed + ytrue_UP = data_f.ycont_to_ytrue(ytran_UP, y_thr_UP) + ytrue_DOWN = data_f.ycont_to_ytrue(ytran_DOWN, y_thr_DOWN) - ytrue_train_high, _ = ytrue_high[st_:fin], ytrue_high[fin : fin + 1] - ytrue_train_low, _ = ytrue_low[st_:fin], ytrue_low[fin : fin + 1] + ytrue_train_UP, _ = ytrue_UP[st_:fin], ytrue_UP[fin : fin + 1] + ytrue_train_DOWN, _ = ytrue_DOWN[st_:fin], ytrue_DOWN[fin : fin + 1] if ( - self.model_high is None + self.model_UP is None or self.st.iter_number % pdr_ss.aimodel_ss.train_every_n_epochs == 0 ): model_f = AimodelFactory(pdr_ss.aimodel_ss) - self.model_high = model_f.build( - X_train_high, - ytrue_train_high, - ytran_train_high, - y_thr_high, + self.model_UP = model_f.build( + X_train_UP, + ytrue_train_UP, + ytran_train_UP, + y_thr_UP, ) - self.model_low = model_f.build( - X_train_low, - ytrue_train_low, - ytran_train_low, - y_thr_low, + self.model_DOWN = model_f.build( + X_train_DOWN, + ytrue_train_DOWN, + ytran_train_DOWN, + y_thr_DOWN, ) # current time @@ -176,18 +176,24 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): ut = UnixTimeMs(recent_ut - testshift * timeframe.ms) # predict price direction - prob_up: float = self.model_high.predict_ptrue(X_test_high)[0] - prob_down: float = 1.0 - self.model_low.predict_ptrue(X_test_low)[0] # type: ignore + prob_up_UP = self.model_UP.predict_ptrue(X_test_UP)[0] + prob_up_DOWN = self.model_DOWN.predict_ptrue(X_test_DOWN)[0] + + prob_up, prob_down = prob_up_UP, 1.0 - prob_up_DOWN if (prob_up + prob_down) > 1.0: # ensure predictions don't conflict prob_up = prob_up / (prob_up + prob_down) prob_down = 1.0 - prob_up + conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] conf_down = (prob_down - 0.5) * 2.0 # to range [0,1] conf_threshold = self.ppss.trader_ss.sim_confidence_threshold pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold pred_down: bool = prob_down > 0.5 and conf_down > conf_threshold - st.probs_up.append(prob_up) - st.ytrues_hat.append(prob_up > 0.5) + + st.probs_up_UP.append(prob_up_UP) + st.probs_up_DOWN.append(prob_up_DOWN) + st.ytrues_hat_UP.append(prob_up_UP > 0.5) + st.ytrues_hat_DOWN.append(prob_up_DOWN > 0.5) # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 @@ -214,35 +220,33 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): true_up_close = next_close > cur_close if transform == "None": - true_up_high = next_high > y_thr_high + true_up_UP = next_high > y_thr_UP + true_up_DOWN = next_low > y_thr_DOWN else: raise NotImplementedError("build me") - st.ytrues.append(true_up_high) - - # update classifier metrics - n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat)) - n_trials = len(st.ytrues) - acc_est = n_correct / n_trials - acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) - (precision, recall, f1, _) = precision_recall_fscore_support( - st.ytrues, - st.ytrues_hat, - average="binary", - zero_division=0.0, - ) - if min(st.ytrues) == max(st.ytrues): - loss = 3.0 - else: - loss = log_loss(st.ytrues, st.probs_up) + st.ytrues_UP.append(true_up_UP) + st.ytrues_DOWN.append(true_up_DOWN) + + # update classifier performances + perf_UP = get_perf(st.ytrues_UP, st.ytrues_hat_UP, st.probs_up_UP) + perf_DOWN = get_perf(st.ytrues_DOWN, st.ytrues_hat_DOWN, st.probs_up_DOWN) + perf_merged = merge_tups(perf_UP, perf_DOWN) + (acc_est, acc_l, acc_u, precision, recall, f1, loss) = perf_merged + yerr = 0.0 - if self.model_high.do_regr: - pred_ycont = self.model_high.predict_ycont(X_test_high)[0] + if self.model_UP.do_regr: + pred_ycont_UP = self.model_UP.predict_ycont(X_test_UP)[0] + pred_ycont_DOWN = self.model_DOWN.predict_ycont(X_test_DOWN)[0] if transform == "None": - pred_next_close = pred_ycont + pred_next_high = pred_ycont_UP + pred_next_low = pred_ycont_DOWN else: # transform = "RelDiff" relchange = pred_ycont - pred_next_close = cur_close + relchange * cur_close - yerr = next_close - pred_next_close + pred_next_high = cur_high + relchange * cur_high + pred_next_low = cur_low + relchange * cur_low + yerr_UP = next_high - pred_next_high + yerr_DOWN = next_low - pred_next_low + yerr = np.mean([yerr_UP, yerr_DOWN]) st.aim.update(acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr) @@ -266,14 +270,14 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): if save_state: cs_ = [shift_one_earlier(c_) for c_ in colnames_high] - most_recent_x = X_high[-1, :] + most_recent_x = X_UP[-1, :] slicing_x = most_recent_x # plot about the most recent x d = AimodelPlotdata( - self.model_high, - X_train_high, - ytrue_train_high, - ytran_train_high, - y_thr_high, + self.model_UP, + X_train_UP, + ytrue_train_UP, + ytran_train_UP, + y_thr_UP, cs_, slicing_x, ) @@ -301,3 +305,27 @@ def save_state(self, i: int, N: int): return False, False return True, False + +def get_perf(ytrues, ytrues_hat, probs_up) -> tuple: + """Get classifier performances: accuracy, precision/recall/f1, log loss""" + n_correct = sum(np.array(ytrues) == np.array(ytrues_hat)) + n_trials = len(ytrues) + acc_est = n_correct / n_trials + acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) + + (precision, recall, f1, _) = precision_recall_fscore_support( + ytrues, + ytrues_hat, + average="binary", + zero_division=0.0, + ) + + if min(ytrues) == max(ytrues): + loss = 3.0 # magic number + else: + loss = log_loss(ytrues, probs_up) + + return (acc_est, acc_l, acc_u, precision, recall, f1, loss) + +def merge_tups(tup1, tup2): + return (np.mean([val1, val2]) for val1, val2 in zip(tup1, tup2)) diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index ac23d7d1a..94fc25e6f 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -20,8 +20,9 @@ def __init__(self, ppss, st, test_i, ut, acct_up_profit, acct_down_profit): self.acct_up_profit = acct_up_profit self.acct_down_profit = acct_down_profit - self.n_correct = sum(np.array(st.ytrues) == np.array(st.ytrues_hat)) - self.n_trials = len(st.ytrues) + # TODO: account for DOWN too + self.n_correct = sum(np.array(st.ytrues_UP) == np.array(st.ytrues_hat_UP)) + self.n_trials = len(st.ytrues_UP) for key, item in st.recent_metrics(extras=["prob_up"]).items(): setattr(self, key, item) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 17e9f0607..42738b819 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -159,7 +159,7 @@ def plot_trader_profit_vs_time(self): @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.probs_up + x = self.st.probs_up_UP # FIXME: account for DOWN y = self.st.pdr_profits_OCEAN fig = go.Figure( go.Scatter( @@ -179,7 +179,7 @@ def plot_pdr_profit_vs_ptrue(self): @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.probs_up + x = self.st.probs_up_UP # FIXME: account for DOWN y = self.st.trader_profits_USD fig = go.Figure( go.Scatter( diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index e135d1584..167e88213 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -74,10 +74,15 @@ def __init__(self): def init_loop_attributes(self): # 'i' is iteration number i - # base data - self.ytrues: List[bool] = [] # [i] : was-truly-up - self.ytrues_hat: List[bool] = [] # [i] : was-predicted-up - self.probs_up: List[float] = [] # [i] : predicted-prob-up + # base data for UP classifier + self.ytrues_UP: List[bool] = [] # [i] : true value + self.ytrues_hat_UP: List[bool] = [] # [i] : model pred. value + self.probs_up_UP: List[float] = [] # [i] : model's pred. probability + + # base data for DOWN classifier + self.ytrues_DOWN: List[bool] = [] # [i] : true value + self.ytrues_hat_DOWN: List[bool] = [] # [i] : model pred. value + self.probs_up_DOWN: List[float] = [] # [i] : model's pred. probability # aimodel metrics self.aim = AimodelMetrics() @@ -106,7 +111,7 @@ def recent_metrics( ) if extras and "prob_up" in extras: - rm["prob_up"] = self.probs_up[-1] + rm["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN return rm From e2f9ae2408d577b9356c1c6ad31a07b3484bc28f Mon Sep 17 00:00:00 2001 From: trentmc Date: Sun, 23 Jun 2024 13:37:34 +0200 Subject: [PATCH 07/50] Improve predictoor logic: only stake when models are confident enough --- pdr_backend/sim/sim_engine.py | 48 ++++++++++++++++++++++------------ pdr_backend/sim/sim_plotter.py | 4 +-- pdr_backend/sim/sim_state.py | 7 +++-- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index cb59113e8..06958765d 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -98,6 +98,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): transform = pdr_ss.aimodel_data_ss.transform others_stake = pdr_ss.others_stake.amt_eth revenue = pdr_ss.revenue.amt_eth + model_conf_thr = self.ppss.trader_ss.sim_confidence_threshold testshift = ppss.sim_ss.test_n - test_i - 1 # eg [99, 98, .., 2, 1, 0] data_f = AimodelDataFactory(pdr_ss) # type: ignore[arg-type] @@ -178,29 +179,44 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # predict price direction prob_up_UP = self.model_UP.predict_ptrue(X_test_UP)[0] prob_up_DOWN = self.model_DOWN.predict_ptrue(X_test_DOWN)[0] - - prob_up, prob_down = prob_up_UP, 1.0 - prob_up_DOWN - if (prob_up + prob_down) > 1.0: # ensure predictions don't conflict - prob_up = prob_up / (prob_up + prob_down) - prob_down = 1.0 - prob_up - - conf_up = (prob_up - 0.5) * 2.0 # to range [0,1] - conf_down = (prob_down - 0.5) * 2.0 # to range [0,1] - conf_threshold = self.ppss.trader_ss.sim_confidence_threshold - pred_up: bool = prob_up > 0.5 and conf_up > conf_threshold - pred_down: bool = prob_down > 0.5 and conf_down > conf_threshold - + prob_down_DOWN = 1.0 - prob_up_DOWN + + models_in_conflict = (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \ + (prob_up_UP < 0.5 and prob_down_DOWN < 0.5) + if models_in_conflict: + conf_up = conf_down = 0.0 + pred_up = pred_down = False + prob_up_MERGED = 0.5 + elif prob_up_UP >= prob_down_DOWN: + conf_up = (prob_up_UP - 0.5) * 2.0 # to range [0,1] + conf_down = 0.0 + pred_up = conf_up > model_conf_thr + pred_down = False + prob_up_MERGED = prob_up_UP + else: # prob_down_DOWN > prob_up_UP + conf_up = 0.0 + conf_down = (prob_down_DOWN - 0.5) * 2.0 + pred_up = False + pred_down = conf_down > model_conf_thr + prob_up_MERGED = 1.0 - prob_down_DOWN + st.probs_up_UP.append(prob_up_UP) - st.probs_up_DOWN.append(prob_up_DOWN) st.ytrues_hat_UP.append(prob_up_UP > 0.5) + st.probs_up_DOWN.append(prob_up_DOWN) st.ytrues_hat_DOWN.append(prob_up_DOWN > 0.5) + st.probs_up_MERGED.append(prob_up_MERGED) # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 max_stake_amt = pdr_ss.stake_amount.amt_eth - assert (prob_up + prob_down) <= 1.0 - stake_up = max_stake_amt * prob_up - stake_down = max_stake_amt * prob_down + if models_in_conflict or pred_up or pred_down: + stake_up = stake_down = 0 + elif prob_up_UP >= prob_down_DOWN: + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - prob_up_MERGED) + else: # prob_down_DOWN > prob_up_UP + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - prob_up_MERGED) acct_up_profit -= stake_up acct_down_profit -= stake_down diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 42738b819..4e6a760e9 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -159,7 +159,7 @@ def plot_trader_profit_vs_time(self): @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.probs_up_UP # FIXME: account for DOWN + x = self.st.probs_up_MERGED y = self.st.pdr_profits_OCEAN fig = go.Figure( go.Scatter( @@ -179,7 +179,7 @@ def plot_pdr_profit_vs_ptrue(self): @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.probs_up_UP # FIXME: account for DOWN + x = self.st.probs_up_MERGED y = self.st.trader_profits_USD fig = go.Figure( go.Scatter( diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 167e88213..400f825ad 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -77,12 +77,15 @@ def init_loop_attributes(self): # base data for UP classifier self.ytrues_UP: List[bool] = [] # [i] : true value self.ytrues_hat_UP: List[bool] = [] # [i] : model pred. value - self.probs_up_UP: List[float] = [] # [i] : model's pred. probability + self.probs_up_UP: List[float] = [] # [i] : model's pred. prob. # base data for DOWN classifier self.ytrues_DOWN: List[bool] = [] # [i] : true value self.ytrues_hat_DOWN: List[bool] = [] # [i] : model pred. value - self.probs_up_DOWN: List[float] = [] # [i] : model's pred. probability + self.probs_up_DOWN: List[float] = [] # [i] : model's pred. prob. + + # merged values + self.probs_up_MERGED: List[float] = [] # [i] : merged pred. prob. # aimodel metrics self.aim = AimodelMetrics() From fbaa8c31bfdbc80741aa76186206e46bace978d1 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sun, 23 Jun 2024 13:44:21 +0200 Subject: [PATCH 08/50] bug fix --- pdr_backend/sim/sim_engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 06958765d..329eb9106 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -212,9 +212,11 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): if models_in_conflict or pred_up or pred_down: stake_up = stake_down = 0 elif prob_up_UP >= prob_down_DOWN: + stake_amt = max_stake_amt * conf_up stake_up = stake_amt * prob_up_MERGED stake_down = stake_amt * (1.0 - prob_up_MERGED) else: # prob_down_DOWN > prob_up_UP + stake_amt = max_stake_amt * conf_down stake_up = stake_amt * prob_up_MERGED stake_down = stake_amt * (1.0 - prob_up_MERGED) acct_up_profit -= stake_up From a03244ed1a625c8ee40fe75abcc99f6e52cda1a6 Mon Sep 17 00:00:00 2001 From: trentmc Date: Sun, 23 Jun 2024 13:48:24 +0200 Subject: [PATCH 09/50] bug fix 2 --- pdr_backend/sim/sim_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 329eb9106..9aeb96f04 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -209,7 +209,7 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # predictoor: (simulate) submit predictions with stake acct_up_profit = acct_down_profit = 0.0 max_stake_amt = pdr_ss.stake_amount.amt_eth - if models_in_conflict or pred_up or pred_down: + if models_in_conflict or not (pred_up or pred_down): stake_up = stake_down = 0 elif prob_up_UP >= prob_down_DOWN: stake_amt = max_stake_amt * conf_up From dbe344c95505d13676f2f69855ce73e393055b4e Mon Sep 17 00:00:00 2001 From: trizin <25263018+trizin@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:30:09 +0300 Subject: [PATCH 10/50] use high/low values for X --- pdr_backend/sim/sim_engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 9aeb96f04..7b63b781f 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -120,13 +120,13 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): mergedohlcv_df, testshift, predict_feed_high, - train_feeds, + [predict_feed_high], ) X_DOWN, ytran_DOWN, yraw_low, _, _ = data_f.create_xy( mergedohlcv_df, testshift, predict_feed_low, - train_feeds, + [predict_feed_low], ) colnames_high = list(x_df_high.columns) From d3ea15569eb2ec55f2dc58a01c7c81d2dd7b4399 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 26 Jun 2024 13:30:37 +0200 Subject: [PATCH 11/50] Big refactor: mostly done first cut of writing (incl tests), but haven't run tests yet. Lots to go. --- pdr_backend/aimodel/aimodel_data_factory.py | 4 + pdr_backend/cli/arg_feed.py | 16 + pdr_backend/ppss/aimodel_data_ss.py | 9 + pdr_backend/ppss/lake_ss.py | 17 +- pdr_backend/ppss/test/test_aimodel_data_ss.py | 22 ++ pdr_backend/ppss/test/test_lake_ss.py | 14 + pdr_backend/sim/constants.py | 8 + pdr_backend/sim/sim_engine.py | 359 ++++++------------ pdr_backend/sim/sim_logger.py | 31 +- pdr_backend/sim/sim_model.py | 22 ++ pdr_backend/sim/sim_model_data.py | 42 ++ pdr_backend/sim/sim_model_data_factory.py | 82 ++++ pdr_backend/sim/sim_model_factory.py | 31 ++ pdr_backend/sim/sim_model_prediction.py | 89 +++++ pdr_backend/sim/sim_plotter.py | 167 ++------ pdr_backend/sim/sim_predictoor.py | 30 ++ pdr_backend/sim/sim_state.py | 214 ++++++++--- pdr_backend/sim/sim_trader.py | 21 +- pdr_backend/sim/test/resources.py | 17 + pdr_backend/sim/test/test_sim_constants.py | 19 + pdr_backend/sim/test/test_sim_engine.py | 44 ++- pdr_backend/sim/test/test_sim_model.py | 12 + pdr_backend/sim/test/test_sim_model_data.py | 51 +++ .../sim/test/test_sim_model_data_factory.py | 87 +++++ .../sim/test/test_sim_model_factory.py | 48 +++ .../sim/test/test_sim_model_prediction.py | 118 ++++++ pdr_backend/sim/test/test_sim_predictoor.py | 46 +++ pdr_backend/sim/test/test_sim_trader.py | 2 - ppss.yaml | 1 + 29 files changed, 1140 insertions(+), 483 deletions(-) create mode 100644 pdr_backend/sim/constants.py create mode 100644 pdr_backend/sim/sim_model.py create mode 100644 pdr_backend/sim/sim_model_data.py create mode 100644 pdr_backend/sim/sim_model_data_factory.py create mode 100644 pdr_backend/sim/sim_model_factory.py create mode 100644 pdr_backend/sim/sim_model_prediction.py create mode 100644 pdr_backend/sim/sim_predictoor.py create mode 100644 pdr_backend/sim/test/resources.py create mode 100644 pdr_backend/sim/test/test_sim_constants.py create mode 100644 pdr_backend/sim/test/test_sim_model.py create mode 100644 pdr_backend/sim/test/test_sim_model_data.py create mode 100644 pdr_backend/sim/test/test_sim_model_data_factory.py create mode 100644 pdr_backend/sim/test/test_sim_model_factory.py create mode 100644 pdr_backend/sim/test/test_sim_model_prediction.py create mode 100644 pdr_backend/sim/test/test_sim_predictoor.py diff --git a/pdr_backend/aimodel/aimodel_data_factory.py b/pdr_backend/aimodel/aimodel_data_factory.py index c328f8c69..70d259cda 100644 --- a/pdr_backend/aimodel/aimodel_data_factory.py +++ b/pdr_backend/aimodel/aimodel_data_factory.py @@ -70,6 +70,10 @@ def ycont_to_ytrue(ycont: np.ndarray, y_thr: float) -> np.ndarray: ybool = np.array([ycont_val >= y_thr for ycont_val in ycont]) return ybool + @staticmethod + def testshift(self, test_n: int, test_i: int) -> int: + return test_n - test_i - 1 + def create_xy( self, mergedohlcv_df: pl.DataFrame, diff --git a/pdr_backend/cli/arg_feed.py b/pdr_backend/cli/arg_feed.py index ffd398e1b..ed7daac70 100644 --- a/pdr_backend/cli/arg_feed.py +++ b/pdr_backend/cli/arg_feed.py @@ -89,6 +89,22 @@ def from_str(feed_str: str, do_verify: bool = True) -> "ArgFeed": feed = feeds[0] return feed + def variant_close(self) -> "ArgFeed": + return self._variant_signal("close") + + def variant_high(self) -> "ArgFeed": + return self._variant_signal("high") + + def variant_low(self) -> "ArgFeed": + return self._variant_signal("low") + + def _variant_signal(self, signal_str: "str") -> "ArgFeed": + return ArgFeed( + self.exchange, + signal_str, + self.pair, + self.timeframe, + ) @enforce_types def _unpack_feeds_str(feeds_str: str) -> List[ArgFeed]: diff --git a/pdr_backend/ppss/aimodel_data_ss.py b/pdr_backend/ppss/aimodel_data_ss.py index 07c6d3b71..9b71c58d8 100644 --- a/pdr_backend/ppss/aimodel_data_ss.py +++ b/pdr_backend/ppss/aimodel_data_ss.py @@ -24,6 +24,8 @@ def __init__(self, d: dict): raise ValueError(self.max_n_train) if not 0 < self.autoregressive_n < np.inf: raise ValueError(self.autoregressive_n) + if not 0 < self.class_thr <= 1.0: + raise ValueError(self.class_thr) if self.transform not in TRANSFORM_OPTIONS: raise ValueError(self.transform) @@ -44,6 +46,11 @@ def autoregressive_n(self) -> int: For diff=2, add to model_inputs (z[t-1]-z[t-2]) - (z[t-2]-z[t-3]), .. """ return self.d["autoregressive_n"] + + @property + def class_thr(self) -> float: + """eg 0.05 = 5%. UP class needs > this, DOWN < this""" + return self.d["class_thr1"] @property def transform(self) -> int: @@ -59,12 +66,14 @@ def transform(self) -> int: def aimodel_data_ss_test_dict( max_n_train: Optional[int] = None, autoregressive_n: Optional[int] = None, + class_thr: Optional[float] = None, transform: Optional[str] = None, ) -> dict: """Use this function's return dict 'd' to construct AimodelDataSS(d)""" d = { "max_n_train": 7 if max_n_train is None else max_n_train, "autoregressive_n": 3 if autoregressive_n is None else autoregressive_n, + "class_thr": 0.002 if class_thr is None else class_thr, "transform": transform or "None", } return d diff --git a/pdr_backend/ppss/lake_ss.py b/pdr_backend/ppss/lake_ss.py index 88ffc8949..824463831 100644 --- a/pdr_backend/ppss/lake_ss.py +++ b/pdr_backend/ppss/lake_ss.py @@ -95,14 +95,19 @@ def __str__(self) -> str: @enforce_types -def lake_ss_test_dict(lake_dir: str, feeds: Optional[list] = None): +def lake_ss_test_dict( + lake_dir: str, + feeds: Optional[list] = None, + st_timestr: Optional[str] = None, + fin_timestr: Optional[str] = None, + timeframe: Optional[str] = None, +): """Use this function's return dict 'd' to construct LakeSS(d)""" - feeds = feeds or ["binance BTC/USDT c 5m"] d = { - "feeds": feeds, + "feeds": feeds or ["binance BTC/USDT c 5m"], "lake_dir": lake_dir, - "st_timestr": "2023-06-18", - "fin_timestr": "2023-06-30", - "timeframe": "5m", + "st_timestr": st_timestr or "2023-06-18", + "fin_timestr": fin_timestr or "2023-06-30", + "timeframe": timeframe or "5m", } return d diff --git a/pdr_backend/ppss/test/test_aimodel_data_ss.py b/pdr_backend/ppss/test/test_aimodel_data_ss.py index 531bf1b34..6223c44f0 100644 --- a/pdr_backend/ppss/test/test_aimodel_data_ss.py +++ b/pdr_backend/ppss/test/test_aimodel_data_ss.py @@ -15,6 +15,7 @@ def test_aimodel_data_ss__default_values(): assert ss.max_n_train == d["max_n_train"] == 7 assert ss.autoregressive_n == d["autoregressive_n"] == 3 + assert ss.class_thr == d["class_thr"] == 0.002 assert ss.transform == d["transform"] == "None" # str @@ -32,6 +33,15 @@ def test_aimodel_data_ss__nondefault_values(): ss = AimodelDataSS(aimodel_data_ss_test_dict(autoregressive_n=13)) assert ss.autoregressive_n == 13 + + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.06)) + assert ss.class_thr = 0.06 + + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.0)) + assert ss.class_thr = 0.0 + + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1.0)) + assert ss.class_thr = 1.0 ss = AimodelDataSS(aimodel_data_ss_test_dict(transform="RelDiff")) assert ss.transform == "RelDiff" @@ -45,6 +55,18 @@ def test_aimodel_data_ss__bad_inputs(): with pytest.raises(TypeError): AimodelDataSS(aimodel_data_ss_test_dict(max_n_train=3.1)) + with pytest.raises(ValueError): + AimodelDataSS(aimodel_data_ss_test_dict(class_thr=-0.1)) + + with pytest.raises(ValueError): + AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1.1)) + + with pytest.raises(TypeError): # floats only, for simplicity + AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0)) + + with pytest.raises(TypeError): # floats only, for simplicity + AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1)) + with pytest.raises(ValueError): AimodelDataSS(aimodel_data_ss_test_dict(autoregressive_n=0)) diff --git a/pdr_backend/ppss/test/test_lake_ss.py b/pdr_backend/ppss/test/test_lake_ss.py index f4e7cca72..73cc889b8 100644 --- a/pdr_backend/ppss/test/test_lake_ss.py +++ b/pdr_backend/ppss/test/test_lake_ss.py @@ -125,3 +125,17 @@ def test_lake_ss_test_dict_2_specify_feeds(tmpdir): d = lake_ss_test_dict(lake_dir, feeds) assert d["lake_dir"] == lake_dir assert d["feeds"] == feeds + + +@enforce_types +def test_lake_ss_test_dict_3_nondefault_time_settings(tmpdir): + lake_dir = os.path.join(tmpdir, "lake_data") + d = lake_ss_test_dict( + lake_dir, + st_timestr="2023-01-20", + fin_timestr="2023-01-21", + timeframe="1h", + ) + assert d["st_timestr"] == "2023-01-20" + assert d["fin_timesetr"] == "2023-01-21" + assert d["timeframe"] == "1h" diff --git a/pdr_backend/sim/constants.py b/pdr_backend/sim/constants.py new file mode 100644 index 000000000..16004ae17 --- /dev/null +++ b/pdr_backend/sim/constants.py @@ -0,0 +1,8 @@ +from enum import Enum + +class Dirn(Enum): + UP = 1 + DOWN = 2 + +UP = Dirn.UP +DOWN = Dirn.DOWN diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 7b63b781f..61faad92a 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -18,10 +18,13 @@ from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset from pdr_backend.lake.ohlcv_data_factory import OhlcvDataFactory from pdr_backend.ppss.ppss import PPSS +from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_logger import SimLogLine +from pdr_backend.sim.sim_model import SimModel from pdr_backend.sim.sim_plotter import SimPlotter -from pdr_backend.sim.sim_trader import SimTrader +from pdr_backend.sim.sim_predictoor import SimPredictoor from pdr_backend.sim.sim_state import SimState +from pdr_backend.sim.sim_trader import SimTrader from pdr_backend.util.strutil import shift_one_earlier from pdr_backend.util.time_types import UnixTimeMs @@ -37,9 +40,11 @@ def __init__( predict_train_feedset: PredictTrainFeedset, multi_id: Optional[str] = None, ): + predict_feed = predict_train_feedset.predict + assert predict_feed.signal == "close", \ + "only operates on close predictions" + self.predict_train_feedset = predict_train_feedset - assert isinstance(self.predict_feed, ArgFeed) - assert self.predict_feed.signal == "close", "only operates on close predictions" self.ppss = ppss # can be disabled by calling disable_realtime_state() @@ -47,7 +52,8 @@ def __init__( self.st = SimState() - self.trader = SimTrader(ppss, self.predict_feed) + self.pdr = SimPredictoor(ppss.predictoor_ss) + self.trader = SimTrader(ppss, predict_feed) self.sim_plotter = SimPlotter() @@ -57,13 +63,36 @@ def __init__( self.multi_id = multi_id else: self.multi_id = str(uuid.uuid4()) - - self.model_UP: Optional[Aimodel] = None - self.model_DOWN: Optional[Aimodel] = None + + assert self.transform == "None" @property def predict_feed(self) -> ArgFeed: - return self.predict_train_feedset.predict + return self.predict_train_feedset.predict_feed + + @property + def timeframe(self) -> ArgTimeframe: + return self.predict_feed.timeframe + + @property + def pdr_ss(self) -> PredictorSS: + return self.ppss.predictoor_ss + + @property + def aimodel_ss_ss(self) -> AimodelSS: + return self.pdr_ss.aimodel_ss + + @property + def transform(self) -> str: + return self.aimodel_ss.transform + + @property + def others_stake(self) -> float: + return self.others_stale.amt_eth + + @property + def revenue(self) -> float: + return self.pdr_ss.revenue.amt_eth @enforce_types def _init_loop_attributes(self): @@ -75,239 +104,125 @@ def _init_loop_attributes(self): logger.addHandler(fh) self.st.init_loop_attributes() + logger.info("Initialize plot data.") self.sim_plotter.init_state(self.multi_id) @enforce_types def run(self): logger.info("Start run") + + # initialize self._init_loop_attributes() - # main loop! + # ohclv data f = OhlcvDataFactory(self.ppss.lake_ss) mergedohlcv_df = f.get_mergedohlcv_df() + + # main loop! for test_i in range(self.ppss.sim_ss.test_n): self.run_one_iter(test_i, mergedohlcv_df) + # done logger.info("Done all iters.") # pylint: disable=too-many-statements# pylint: disable=too-many-statements @enforce_types def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): - ppss, pdr_ss, st = self.ppss, self.ppss.predictoor_ss, self.st - transform = pdr_ss.aimodel_data_ss.transform - others_stake = pdr_ss.others_stake.amt_eth - revenue = pdr_ss.revenue.amt_eth - model_conf_thr = self.ppss.trader_ss.sim_confidence_threshold - - testshift = ppss.sim_ss.test_n - test_i - 1 # eg [99, 98, .., 2, 1, 0] - data_f = AimodelDataFactory(pdr_ss) # type: ignore[arg-type] - predict_feed = self.predict_train_feedset.predict - train_feeds = self.predict_train_feedset.train_on - - # X, ycont, and x_df are all expressed in % change wrt prev candle - p = predict_feed - predict_feed_close = p - predict_feed_high = ArgFeed(p.exchange, "high", p.pair, p.timeframe) - predict_feed_low = ArgFeed(p.exchange, "low", p.pair, p.timeframe) - _, _, yraw_close, _, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_close, - train_feeds, - ) - X_UP, ytran_UP, yraw_high, x_df_high, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_high, - [predict_feed_high], - ) - X_DOWN, ytran_DOWN, yraw_low, _, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - predict_feed_low, - [predict_feed_low], - ) - colnames_high = list(x_df_high.columns) + # build model + st = self.st + data_f = SimModelDataFactory(self.ppss) + model_d: SimModelData = data_f.build(iter_i, mergdohlcv_df) + model_f = SimModelFactory(self.aimodel_ss) + if model_f.do_build(st.sim_model, test_i): + st.sim_model = model_f.train(test_i, model_d) - cur_close, next_close = yraw_close[-2], yraw_close[-1] - cur_high, next_high = yraw_high[-2], yraw_high[-1] - cur_low, next_low = yraw_low[-2], yraw_low[-1] - - st_, fin = 0, X_UP.shape[0] - 1 - X_train_UP, X_test_UP = X_UP[st_:fin, :], X_UP[fin : fin + 1, :] - ytran_train_UP, _ = ytran_UP[st_:fin], ytran_UP[fin : fin + 1] - X_train_DOWN, X_test_DOWN = X_DOWN[st_:fin, :], X_DOWN[fin : fin + 1, :] - ytran_train_DOWN, _ = ytran_DOWN[st_:fin], ytran_DOWN[fin : fin + 1] - - percent_change_needed = 0.002 # magic number. TODO: move to ppss.yaml - if transform == "None": - y_thr_UP = cur_close * (1 + percent_change_needed) - y_thr_DOWN = cur_close * (1 - percent_change_needed) - else: # transform = "RelDiff" - y_thr_UP = +np.std(yraw_close) * percent_change_needed - y_thr_DOWN = -np.std(yraw_close) * percent_change_needed - ytrue_UP = data_f.ycont_to_ytrue(ytran_UP, y_thr_UP) - ytrue_DOWN = data_f.ycont_to_ytrue(ytran_DOWN, y_thr_DOWN) - - ytrue_train_UP, _ = ytrue_UP[st_:fin], ytrue_UP[fin : fin + 1] - ytrue_train_DOWN, _ = ytrue_DOWN[st_:fin], ytrue_DOWN[fin : fin + 1] - - if ( - self.model_UP is None - or self.st.iter_number % pdr_ss.aimodel_ss.train_every_n_epochs == 0 - ): - model_f = AimodelFactory(pdr_ss.aimodel_ss) - self.model_UP = model_f.build( - X_train_UP, - ytrue_train_UP, - ytran_train_UP, - y_thr_UP, - ) - self.model_DOWN = model_f.build( - X_train_DOWN, - ytrue_train_DOWN, - ytran_train_DOWN, - y_thr_DOWN, - ) - - # current time - recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) - timeframe: ArgTimeframe = predict_feed.timeframe # type: ignore - ut = UnixTimeMs(recent_ut - testshift * timeframe.ms) - - # predict price direction - prob_up_UP = self.model_UP.predict_ptrue(X_test_UP)[0] - prob_up_DOWN = self.model_DOWN.predict_ptrue(X_test_DOWN)[0] - prob_down_DOWN = 1.0 - prob_up_DOWN - - models_in_conflict = (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \ - (prob_up_UP < 0.5 and prob_down_DOWN < 0.5) - if models_in_conflict: - conf_up = conf_down = 0.0 - pred_up = pred_down = False - prob_up_MERGED = 0.5 - elif prob_up_UP >= prob_down_DOWN: - conf_up = (prob_up_UP - 0.5) * 2.0 # to range [0,1] - conf_down = 0.0 - pred_up = conf_up > model_conf_thr - pred_down = False - prob_up_MERGED = prob_up_UP - else: # prob_down_DOWN > prob_up_UP - conf_up = 0.0 - conf_down = (prob_down_DOWN - 0.5) * 2.0 - pred_up = False - pred_down = conf_down > model_conf_thr - prob_up_MERGED = 1.0 - prob_down_DOWN - - st.probs_up_UP.append(prob_up_UP) - st.ytrues_hat_UP.append(prob_up_UP > 0.5) - st.probs_up_DOWN.append(prob_up_DOWN) - st.ytrues_hat_DOWN.append(prob_up_DOWN > 0.5) - st.probs_up_MERGED.append(prob_up_MERGED) - - # predictoor: (simulate) submit predictions with stake - acct_up_profit = acct_down_profit = 0.0 - max_stake_amt = pdr_ss.stake_amount.amt_eth - if models_in_conflict or not (pred_up or pred_down): - stake_up = stake_down = 0 - elif prob_up_UP >= prob_down_DOWN: - stake_amt = max_stake_amt * conf_up - stake_up = stake_amt * prob_up_MERGED - stake_down = stake_amt * (1.0 - prob_up_MERGED) - else: # prob_down_DOWN > prob_up_UP - stake_amt = max_stake_amt * conf_down - stake_up = stake_amt * prob_up_MERGED - stake_down = stake_amt * (1.0 - prob_up_MERGED) - acct_up_profit -= stake_up - acct_down_profit -= stake_down + # make prediction + sim_model_p: SimModelPrediction = self.model.predict_next() + # predictoor takes action (stake) + stake_up, stake_down = self.pdr.predict_iter(sim_model_p) + + # trader takes action (trade) trader_profit = self.trader.trade_iter( - cur_close, - pred_up, - pred_down, - conf_up, - conf_down, - cur_high, - cur_low, + cur_close, cur_high, cur_low, sim_model_p, ) - st.trader_profits_USD.append(trader_profit) - # observe true price change true_up_close = next_close > cur_close + true_up_UP = next_high > y_thr_UP # did next high go > prev close+% ? + true_up_DOWN = next_low < y_thr_DOWN # did next low go < prev close-% ? + + # update state - classifier data + st.classif_base.update(true_up_UP, true_up_DOWN, sim_model_p) + st.classif_metrics.update(st.classif_base) + + # update predictoor profit + st.profit.update_pdr_profit( + others_stake, pdr_ss.others_stake_accuracy, + stake_up, stake_down, true_up_close) + + # update trader profit + self.profits.update_trader_profit(trader_profit) + + # log + ut = self._calc_ut(mergedohlcv_df) + SimLogLine(ppss, st, test_i, ut).log() + + # plot + do_save_state, is_final_state = self._do_save_state(test_i) + if do_save_state: + d_UP = self._aimodel_plotdata_1dirn(UP) + d_DOWN = self._aimodel_plotdata_1dirn(DOWN) + d = {UP: d_UP, DOWN: d_DOWN} + st.iter_number = test_i + self.sim_plotter.save_state(st, d, is_final_state) - if transform == "None": - true_up_UP = next_high > y_thr_UP - true_up_DOWN = next_low > y_thr_DOWN - else: - raise NotImplementedError("build me") - st.ytrues_UP.append(true_up_UP) - st.ytrues_DOWN.append(true_up_DOWN) - - # update classifier performances - perf_UP = get_perf(st.ytrues_UP, st.ytrues_hat_UP, st.probs_up_UP) - perf_DOWN = get_perf(st.ytrues_DOWN, st.ytrues_hat_DOWN, st.probs_up_DOWN) - perf_merged = merge_tups(perf_UP, perf_DOWN) - (acc_est, acc_l, acc_u, precision, recall, f1, loss) = perf_merged - - yerr = 0.0 - if self.model_UP.do_regr: - pred_ycont_UP = self.model_UP.predict_ycont(X_test_UP)[0] - pred_ycont_DOWN = self.model_DOWN.predict_ycont(X_test_DOWN)[0] - if transform == "None": - pred_next_high = pred_ycont_UP - pred_next_low = pred_ycont_DOWN - else: # transform = "RelDiff" - relchange = pred_ycont - pred_next_high = cur_high + relchange * cur_high - pred_next_low = cur_low + relchange * cur_low - yerr_UP = next_high - pred_next_high - yerr_DOWN = next_low - pred_next_low - yerr = np.mean([yerr_UP, yerr_DOWN]) - - st.aim.update(acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr) - - # track predictoor profit - tot_stake = others_stake + stake_up + stake_down - others_stake_correct = others_stake * pdr_ss.others_accuracy - if true_up_close: - tot_stake_correct = others_stake_correct + stake_up - percent_to_me = stake_up / tot_stake_correct - acct_up_profit += (revenue + tot_stake) * percent_to_me - else: - tot_stake_correct = others_stake_correct + stake_down - percent_to_me = stake_down / tot_stake_correct - acct_down_profit += (revenue + tot_stake) * percent_to_me - pdr_profit_OCEAN = acct_up_profit + acct_down_profit - st.pdr_profits_OCEAN.append(pdr_profit_OCEAN) - - SimLogLine(ppss, st, test_i, ut, acct_up_profit, acct_down_profit).log_line() - - save_state, is_final_state = self.save_state(test_i, self.ppss.sim_ss.test_n) - - if save_state: - cs_ = [shift_one_earlier(c_) for c_ in colnames_high] - most_recent_x = X_UP[-1, :] - slicing_x = most_recent_x # plot about the most recent x - d = AimodelPlotdata( - self.model_UP, - X_train_UP, - ytrue_train_UP, - ytran_train_UP, - y_thr_UP, - cs_, - slicing_x, - ) - self.st.iter_number = test_i - self.sim_plotter.save_state(self.st, d, is_final_state) + @enforce_types + def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: + st = self.st + model, model_data = st.sim_model[dirn], st.sim_model_data[dirn] + colnames = model_data.colnames + colnames = [shift_one_earlier(c) for c in colnames] + most_recent_x = model_data.X[-1, :] + slicing_x = most_recent_x + d = AimodelPlotdata( + model, + model_data.X_train, + model_data.ytrue_train, + None, + None, + colnames, + slicing_x, + ) + return d + @enforce_types + def _recent_close(self, mergedohlcv_df, testshift: int) -> Tuple[float, float]: + """@return -- (cur_close, next_close)""" + p = self.predict_feed_trainset.predict_feed + _, _, yraw_close, _, _ = data_f.create_xy( + mergedohlcv_df, + testshift, + p.variant_close(), + [p.variant_close()], + ) + cur_close, next_close = yraw_close[-2], yraw_close[-1] + return (cur_close, next_close) + + @enforce_types + def _calc_ut(self, mergedohlcv_df) -> UnixTimeMs: + recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) + ut = UnixTimeMs(recent_ut - testshift * self.timeframe.ms) + return ut + + @enforce_types def disable_realtime_state(self): self.do_state_updates = False @enforce_types - def save_state(self, i: int, N: int): - "Save state on this iteration Y/N?" + def _do_save_state(self, i: int) -> Tuple[bool, bool]: + """For this iteration i, (a) save state? (b) is it final iteration?""" if self.ppss.sim_ss.is_final_iter(i): return True, True @@ -318,32 +233,10 @@ def save_state(self, i: int, N: int): # don't save first 5 iters -> not interesting # then save the next 5 -> "stuff's happening!" # then save every 5th iter, to balance "stuff's happening" w/ speed + N = self.ppss.sim_ss.test_n do_update = i >= 5 and (i < 10 or i % 5 == 0 or (i + 1) == N) if not do_update: return False, False return True, False -def get_perf(ytrues, ytrues_hat, probs_up) -> tuple: - """Get classifier performances: accuracy, precision/recall/f1, log loss""" - n_correct = sum(np.array(ytrues) == np.array(ytrues_hat)) - n_trials = len(ytrues) - acc_est = n_correct / n_trials - acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) - - (precision, recall, f1, _) = precision_recall_fscore_support( - ytrues, - ytrues_hat, - average="binary", - zero_division=0.0, - ) - - if min(ytrues) == max(ytrues): - loss = 3.0 # magic number - else: - loss = log_loss(ytrues, probs_up) - - return (acc_est, acc_l, acc_u, precision, recall, f1, loss) - -def merge_tups(tup1, tup2): - return (np.mean([val1, val2]) for val1, val2 in zip(tup1, tup2)) diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index 94fc25e6f..ae1845553 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -11,47 +11,24 @@ @enforce_types # pylint: disable=too-many-instance-attributes class SimLogLine: - def __init__(self, ppss, st, test_i, ut, acct_up_profit, acct_down_profit): + def __init__(self, ppss, st, test_i, ut): self.st = st self.test_n = ppss.sim_ss.test_n self.test_i = test_i self.ut = ut - self.acct_up_profit = acct_up_profit - self.acct_down_profit = acct_down_profit - # TODO: account for DOWN too - self.n_correct = sum(np.array(st.ytrues_UP) == np.array(st.ytrues_hat_UP)) - self.n_trials = len(st.ytrues_UP) - - for key, item in st.recent_metrics(extras=["prob_up"]).items(): - setattr(self, key, item) - - # unused for now, but supports future configuration from ppss - self.format = "compact" - - def log_line(self): + def log(self): s = f"Iter #{self.test_i+1}/{self.test_n}" s += f" ut={self.ut}" s += f" dt={self.ut.to_timestr()[:-7]}" s += " â•‘" - s += f" prob_up={self.prob_up:.3f}" - s += " pdr_profit=" - s += f"{compactSmallNum(self.acct_up_profit)} up" - s += f" + {compactSmallNum(self.acct_down_profit)} down" - s += f" = {compactSmallNum(self.pdr_profit_OCEAN)} OCEAN" + s += f"pdr_profit = {compactSmallNum(self.st.pdr_profits_OCEAN[-1])} OCEAN" s += f" (cumul {compactSmallNum(sum(self.st.pdr_profits_OCEAN))} OCEAN)" s += " â•‘" - s += f" Acc={self.n_correct:4d}/{self.n_trials:4d} " - s += f"= {self.acc_est*100:6.2f}% [{self.acc_l*100:5.1f}%, {self.acc_u*100:5.1f}%]" - s += f" prcsn={self.precision:.3f} recall={self.recall:.3f}" - s += f" f1={self.f1:.3f}" - s += f" loss={self.loss:.3f}" - s += " â•‘" - - s += f" tdr_profit=${self.trader_profit_USD:6.2f}" + s += f" tdr_profit=${self.st.trader_profits_USD[-1]:6.2f}" s += f" (cumul ${sum(self.st.trader_profits_USD):6.2f})" logger.info(s) diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py new file mode 100644 index 000000000..4273a54cb --- /dev/null +++ b/pdr_backend/sim/sim_model.py @@ -0,0 +1,22 @@ +from enforce_typing import enforce_types + +from pdr_backend.aimodel.aimodel import Aimodel +from pdr_backend.sim.constants import UP, DOWN +from pdr_backend.sim.sim_model_data import SimModelData +from pdr_backend.sim.sim_model_prediction import SimModelPrediction + +@enforce_types +class SimModel(dict): + + def __init__(self, ppss: PPSS, model_UP: Aimodel, model_DOWN: Aimodel): + self.ppss: PPSS = ppss + self[UP] = model_UP + self[DOWN] = model_DOWN + + def predict_next(self, d: SimModelData) -> SimModelPrediction: + conf_thr = self.ppss.trader_ss.sim_confidence_threshold + prob_up_UP = self[UP].predict_ptrue(d[UP].X_test)[0] + prob_up_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] + + return SimModelPrediction(conf_thr, prob_up_UP, prob_up_DOWN) + diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py new file mode 100644 index 000000000..9bd3331f8 --- /dev/null +++ b/pdr_backend/sim/sim_model_data.py @@ -0,0 +1,42 @@ +from enforce_typing import enforce_types +import numpy as np + +from pdr_backend.sim.constants import UP, DOWN + +@enforce_types +class SimModelData(dict): + def __init__( + self, + data_UP: SimModelData1Dir, + data_DOWN: SimModelData1Dir, + ): + self[UP] = data_UP + self[DOWN] = data_DOWN + +class SimModelData1Dir: + + @enforce_types + def __init__(self, X: np.ndarray, ytrue: np.ndarray): + self.X: np.ndarray = X + self.ytrue: np.ndarray = ytrue + + @property + def st(self) -> int: + return 0 + + @property + def fin(self) -> int: + return self.X.shape[0] - 1 + + @property + def X_train(self) -> np.ndarray: + return self.X[self.st:self.fin, :] + + @property + def X_test(self) -> np.ndarray: + return self.X[self.fin : self.fin + 1, :] + + @property + def ytrue_train(self) -> List[bool]: + return self.ytrue[self.st:self.fin] + diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py new file mode 100644 index 000000000..9304336a5 --- /dev/null +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -0,0 +1,82 @@ +from enforce_typing import enforce_types +import polars as pl + +from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory +from pdr_backend.cli.predict_train_feedset import PredictTrainFeedset +from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS +from pdr_backend.ppss.ppss import PPSS +from pdr_backend.ppss.predictoor_ss import PredictoorSS + +class SimModelDataFactory: + @enforce_types + def __init__(self, ppss: PPSS, predict_train_feedset: PredictTrainFeedset): + self.ppss = ppss + self.predict_train_feedset = predict_train_feedset + + @property + def pdr_ss(self) -> PredictoorSS: + return self.ppss.predictoor_ss + + @property + def aimodel_data_ss(self) -> AimodelDataSS: + return self.pdr_ss.aimodel_data_ss + + @property + def class_thr(self) -> float: + class_thr = self.aimodel_data_ss.class_thr + + @property + def aimodel_data_factory(self) -> AimodelDataFactory: + return AimodelDataFactory(self.pdr_ss) + + @enforce_types + def testshift(self, test_i: int) -> int: + test_n = self.ppss.sim_ss.test_n + data_f = self.aimodel_data_factory + return data_f.testshift(test_n, test_i) + + @enforce_types + def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: + """Construct sim model data""" + testshift = self.testshift(test_i) # eg [99, 98, .., 2, 1, 0] + + p = self.predict_train_feedset.predict_feed + _, _, y_close, _, _ = data_f.create_xy( + mergedohlcv_df, testshift, p.variant_close(), [p.variant_close()], + ) + X_high, _, y_high, _, _ = self.aimodel_data_factory.create_xy( + mergedohlcv_df, testshift, p.variant_high(), [p.variant_high()], + ) + X_low, _, y_low, _, _ = self.aimodel_data_factory.create_xy( + mergedohlcv_df, testshift, p.variant_low(), [p.variant_low()], + ) + + ytrue_UP = [] + ytrue_DOWN = [] + for i, cur_close in enumerate(y_close[:-1]): + # did the next high value go above the current close+% value? + next_high = y_high[i+1] + thr_UP = self.thr_UP(cur_close) + ytrue_UP.append(next_high > thr_UP) + + # did the next low value go below the current close-% value? + next_low = y_low[i+1] + thr_DOWN = self.thr_DOWN(cur_close) + ytrue_DOWN.append(next_low < thr_DOWN) + + d_UP = SimModelData1Dir(X_high[1:,:], ytrue_UP) # or is it [:-1,:] + d_DOWN = SimModelData1Dir(X_low[1:,:], ytrue_DOWN) # "" + # note: alternatively, each input X could be h+l+c rather than just h or l + d = SimModelData(d_UP, d_DOWN) + + return d + + @enforce_types + def thr_UP(self, cur_close: float) -> float: + return cur_close * (1 + self.class_thr) + + @enforce_types + def thr_DOWN(self, cur_close: float) -> float: + return cur_close * (1 - self.class_thr) + + diff --git a/pdr_backend/sim/sim_model_factory.py b/pdr_backend/sim/sim_model_factory.py new file mode 100644 index 000000000..9501aa5c7 --- /dev/null +++ b/pdr_backend/sim/sim_model_factory.py @@ -0,0 +1,31 @@ + +@enforce_types +class SimModelFactory: + def __init__(self, aimodel_ss: AimodelSS): + self.aimodel_ss = aimodel_ss + + def do_build(self, prev_model: Optional[SimModel], test_i: int) -> bool: + """Update/train model?""" + return prev_model is None or \ + test_i % self.aimodel_ss.train_every_n_epochs == 0 + + def build(self, sim_model_data: SimModelData) -> SimModel: + d = sim_model_data + model_f = AimodelFactory(self.aimodel_ss) + + model_UP = model_f.build( + d.UP.X_train, + d.UP.ytrue_train, + None, + None, + ) + + model_DOWN = model_f.build( + d.DOWN.X_train, + d.DOWN.ytrue_train, + None, + None, + ) + + sim_model = SimModel(model_UP, model_DOWN) + return sim_model diff --git a/pdr_backend/sim/sim_model_prediction.py b/pdr_backend/sim/sim_model_prediction.py new file mode 100644 index 000000000..30b79b522 --- /dev/null +++ b/pdr_backend/sim/sim_model_prediction.py @@ -0,0 +1,89 @@ +from enforce_typing import enforce_types + + +@enforce_types +class SimModelPrediction: + def __init__( + self, + conf_thr: float, + prob_up_UP: float, + prob_up_DOWN: float, + ): + # ppss.trader_ss.sim_confidence_threshold + self.conf_thr = conf_thr + + # core attributes + self.prob_up_UP = prob_up_UP + self.prob_up_DOWN = prob_up_DOWN + + # derived attributes + self.prob_down_DOWN = 1.0 - self.prob_up_DOWN + + if self.models_in_conflict(): + self.conf_up = 0.0 + self.conf_down = 0.0 + self.pred_up = False + self.pred_down = False + self.prob_up_MERGED = 0.5 + + elif self.prob_up_UP >= self.prob_down_DOWN: + self.conf_up = (prob_up_UP - 0.5) * 2.0 # to range [0,1] + self.conf_down = 0.0 + self.pred_up = self.conf_up > self.conf_thr + self.pred_down = False + self.prob_up_MERGED = self.prob_up_UP + + else: # prob_down_DOWN > prob_up_UP + self.conf_up = 0.0 + self.conf_down = (self.prob_down_DOWN - 0.5) * 2.0 + self.pred_up = False + self.pred_down = self.conf_down > self.conf_thr + self.prob_up_MERGED = 1.0 - self.prob_down_DOWN + + @enforce_types + def do_trust_models(self) -> bool: + return _do_trust_models( + self.pred_up, self.pred_down, self.prob_up_UP, self.prob_down_DOWN, + ) + + @enforce_types + def models_in_conflict(self) -> bool: + return _models_in_conflict(self.prob_up_UP, self.prob_down_DOWN) + + +@enforce_types +def _do_trust_models( + pred_up:bool, + pred_down:bool, + prob_up_UP:float, + prob_down_DOWN:float, +) -> bool: + """Do we trust the models enough to take prediction / trading action?""" + # preconditions + if pred_up and pred_down: + raise ValueError("can't have pred_up=True and pred_down=True") + if pred_up and prob_down_DOWN > prob_up_UP: + raise ValueError("can't have pred_up=True with prob_DOWN dominant") + if pred_down and prob_up_UP > prob_down_DOWN: + raise ValueError("can't have pred_down=True with prob_UP dominant") + if not (0.0 <= prob_up_UP <= 1.0): + raise ValueError(prob_up_UP) + if not (0.0 <= prob_down_DOWN <= 1.0): + raise ValueError(prob_down_DOWN) + + # main test + return (pred_up or pred_down) \ + and not _models_in_conflict(prob_up_UP, prob_down_DOWN) + +@enforce_types +def _models_in_conflict(prob_up_UP: float, prob_down_DOWN: float) -> bool: + """Does the UP model conflict with the DOWN model?""" + # preconditions + if not (0.0 <= prob_up_UP <= 1.0): + raise ValueError(prob_up_UP) + if not (0.0 <= prob_down_DOWN <= 1.0): + raise ValueError(prob_down_DOWN) + + # main test + return (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \ + (prob_up_UP < 0.5 and prob_down_DOWN < 0.5) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 4e6a760e9..a907507ad 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -31,7 +31,7 @@ def __init__( self, ): self.st = None - self.aimodel_plotdata = None + self.aimodel_plotdata_ = None self.multi_id = None @staticmethod @@ -103,9 +103,7 @@ def init_state(self, multi_id): os.makedirs(f"sim_state/{multi_id}") - def save_state( - self, sim_state, aimodel_plotdata: AimodelPlotdata, is_final: bool = False - ): + def save_state(self, sim_state, aimodel_plotdata, is_final: bool = False): root_path = f"sim_state/{self.multi_id}" ts = ( datetime.now().strftime("%Y%m%d_%H%M%S.%f")[:-3] @@ -141,7 +139,7 @@ def save_state( @enforce_types def plot_pdr_profit_vs_time(self): - y = list(np.cumsum(self.st.pdr_profits_OCEAN)) + y = list(np.cumsum(self.st.profits.pdr_profits_OCEAN)) ylabel = "predictoor profit (OCEAN)" title = f"Predictoor profit vs time. Current: {y[-1]:.2f} OCEAN" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) @@ -150,7 +148,7 @@ def plot_pdr_profit_vs_time(self): @enforce_types def plot_trader_profit_vs_time(self): - y = list(np.cumsum(self.st.trader_profits_USD)) + y = list(np.cumsum(self.st.profits.trader_profits_USD)) ylabel = "trader profit (USD)" title = f"Trader profit vs time. Current: ${y[-1]:.2f}" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) @@ -159,8 +157,8 @@ def plot_trader_profit_vs_time(self): @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.probs_up_MERGED - y = self.st.pdr_profits_OCEAN + x = self.st.classif_base.UP.probs_up + y = self.st.profits.pdr_profits_OCEAN fig = go.Figure( go.Scatter( x=x, @@ -179,8 +177,8 @@ def plot_pdr_profit_vs_ptrue(self): @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.probs_up_MERGED - y = self.st.trader_profits_USD + x = self.st.classif_base.UP.probs_up + y = self.st.profits.trader_profits_USD fig = go.Figure( go.Scatter( x=x, @@ -200,15 +198,15 @@ def plot_trader_profit_vs_ptrue(self): @enforce_types def plot_model_performance_vs_time(self): # set titles - aim = self.st.aim - s1 = f"accuracy = {aim.acc_ests[-1]*100:.2f}% " - s1 += f"[{aim.acc_ls[-1]*100:.2f}%, {aim.acc_us[-1]*100:.2f}%]" + metrics = self.st.metrics.UP + s1 = f"accuracy = {metrics.acc_ests[-1]*100:.2f}% " + s1 += f"[{metrics.acc_ls[-1]*100:.2f}%, {metrics.acc_us[-1]*100:.2f}%]" - s2 = f"f1={aim.f1s[-1]:.4f}" - s2 += f" [recall={aim.recalls[-1]:.4f}" - s2 += f", precision={aim.precisions[-1]:.4f}]" + s2 = f"f1={metrics.f1s[-1]:.4f}" + s2 += f" [recall={metrics.recalls[-1]:.4f}" + s2 += f", precision={metrics.precisions[-1]:.4f}]" - s3 = f"log loss = {aim.losses[-1]:.4f}" + s3 = f"log loss = {metrics.losses[-1]:.4f}" # make subplots fig = make_subplots( @@ -246,12 +244,12 @@ def plot_model_performance_vs_time(self): @enforce_types def _add_subplot_accuracy_vs_time(self, fig, row): - aim = self.st.aim - acc_ests = [100 * a for a in aim.acc_ests] + metrics = self.st.metrics.UP + acc_ests = [100 * a for a in metrics.acc_ests] df = pd.DataFrame(acc_ests, columns=["accuracy"]) - df["acc_ls"] = [100 * a for a in aim.acc_ls] - df["acc_us"] = [100 * a for a in aim.acc_us] - df["time"] = range(len(aim.acc_ests)) + df["acc_ls"] = [100 * a for a in metrics.acc_ls] + df["acc_us"] = [100 * a for a in metrics.acc_us] + df["time"] = range(len(metrics.acc_ests)) fig.add_traces( [ @@ -296,11 +294,11 @@ def _add_subplot_accuracy_vs_time(self, fig, row): @enforce_types def _add_subplot_f1_precision_recall_vs_time(self, fig, row): - aim = self.st.aim - df = pd.DataFrame(aim.f1s, columns=["f1"]) - df["precisions"] = aim.precisions - df["recalls"] = aim.recalls - df["time"] = range(len(aim.f1s)) + metrics = self.st.metrics.UP + df = pd.DataFrame(metrics.f1s, columns=["f1"]) + df["precisions"] = metrics.precisions + df["recalls"] = metrics.recalls + df["time"] = range(len(metrics.f1s)) fig.add_traces( [ @@ -344,9 +342,9 @@ def _add_subplot_f1_precision_recall_vs_time(self, fig, row): @enforce_types def _add_subplot_log_loss_vs_time(self, fig, row): - aim = self.st.aim - df = pd.DataFrame(aim.losses, columns=["log loss"]) - df["time"] = range(len(aim.losses)) + metrics = self.st.metrics.UP + df = pd.DataFrame(metrics.losses, columns=["log loss"]) + df["time"] = range(len(metrics.losses)) fig.add_trace( go.Scatter(x=df["time"], y=df["log loss"], mode="lines", name="log loss"), @@ -356,111 +354,10 @@ def _add_subplot_log_loss_vs_time(self, fig, row): fig.update_yaxes(title_text="log loss", row=3, col=1) @enforce_types - def plot_prediction_residuals_dist(self): - if _model_is_classif(self.st): - return _empty_fig("(Nothing to show because model is a classifier.)") - - # calc data - d = DistPlotdataFactory.build(self.st.aim.yerrs) - - # initialize subplots - s1, s2, s3 = "Residuals distribution", "", "" - fig = make_subplots( - rows=3, - cols=1, - subplot_titles=(s1, s2, s3), - vertical_spacing=0.02, - shared_xaxes=True, - ) - - # fill in subplots - add_pdf(fig, d, row=1, col=1) - add_cdf(fig, d, row=2, col=1) - add_nq(fig, d, row=3, col=1) - - # global: set minor ticks - minor = {"ticks": "inside", "showgrid": True} - fig.update_yaxes(minor=minor, row=1, col=1) - for row in [2, 3, 4, 5]: - fig.update_yaxes(minor=minor, row=row, col=1) - fig.update_xaxes(minor=minor, row=row, col=1) - - return fig - - @enforce_types - def plot_prediction_residuals_other(self): - if _model_is_classif(self.st): - return _empty_fig() - - # calc data - nlags = 10 # magic number alert # FIX ME: have spinner, like ARIMA feeds - d = AutocorrelationPlotdataFactory.build(self.st.aim.yerrs, nlags=nlags) - - # initialize subplots - s1 = "Residuals vs time" - s2 = "Residuals correlogram" - fig = make_subplots( - rows=2, - cols=1, - subplot_titles=(s1, s2), - vertical_spacing=0.12, - ) - - # fill in subplots - self._add_subplot_residual_vs_time(fig, row=1, col=1) - add_corr_traces( - fig, - d.acf_results, - row=2, - col=1, - ylabel="autocorrelation (ACF)", - ) - - return fig - - @enforce_types - def _add_subplot_residual_vs_time(self, fig, row, col): - y = self.st.aim.yerrs - self._add_subplot_y_vs_time(fig, y, "residual", "markers", row, col) - - @enforce_types - def _add_subplot_y_vs_time(self, fig, y, ylabel, mode, row, col): - assert mode in ["markers", "lines"], mode - line, marker = None, None - if mode == "markers": - marker = {"color": "black", "size": 2} - elif mode == "lines": - line = {"color": "#636EFA"} - - x = list(range(len(y))) + def plot_prediction_residuals_dist(self): + return _empty_fig("(Nothing to show because model is a classifier.)") + - fig.add_traces( - [ - # points: y vs time - go.Scatter( - x=x, - y=y, - mode=mode, - marker=marker, - line=line, - showlegend=False, - ), - # line: horizontal error = 0 - go.Scatter( - x=[min(x), max(x)], - y=[0.0, 0.0], - mode="lines", - line={"color": "grey", "dash": "dot"}, - showlegend=False, - ), - ], - rows=[row] * 2, - cols=[col] * 2, - ) - fig.update_xaxes(title="time", row=row, col=col) - fig.update_yaxes(title=ylabel, row=row, col=col) - - return fig @enforce_types @@ -471,7 +368,7 @@ def file_age_in_seconds(pathname): @enforce_types def _model_is_classif(sim_state) -> bool: - yerrs = sim_state.aim.yerrs + yerrs = sim_state.metrics.yerrs return min(yerrs) == max(yerrs) == 0.0 diff --git a/pdr_backend/sim/sim_predictoor.py b/pdr_backend/sim/sim_predictoor.py new file mode 100644 index 000000000..6e2876928 --- /dev/null +++ b/pdr_backend/sim/sim_predictoor.py @@ -0,0 +1,30 @@ +from enforce_typing import enforce_types + +from pdr_backend.ppss.predictoor_ss import PredictoorSS + +class SimPredictoor: + @enforce_types + def __init__(self, pdr_ss: PredictoorSS): + self.pdr_ss = pdr_ss + + @property + def max_stake_amt(self) -> float: + return self.pdr_ss.stake_amount.amt_eth + + @enforce_types + def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: + """@return (stake_up, stake_down)""" + if not p.do_trust_models(): + stake_up = 0 + stake_down = 0 + elif p.prob_up_UP >= p.prob_down_DOWN: + stake_amt = self.max_stake_amt * p.conf_up + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - p.prob_up_MERGED) + else: # p.prob_down_DOWN > p.prob_up_UP + stake_amt = self.max_stake_amt * p.conf_down + stake_up = stake_amt * prob_up_MERGED + stake_down = stake_amt * (1.0 - p.prob_up_MERGED) + + return (stake_up, stake_down) + diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 400f825ad..8318d65b1 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,13 +1,40 @@ from typing import Dict, List, Optional, Union from enforce_typing import enforce_types +from pdr_backend.sim.constants import Dirn, UP, DOWN @enforce_types -class AimodelMetrics: +class ClassifMetrics(dict): + def __init__(self): + self[UP] = ClassifMetrics1Dir() + self[DOWN] = ClassifMetrics1Dir() + + def update(self, classif_base: ClassifBaseData): + self[UP].update(classif_base.UP.metrics()) + self[DOWN].update(classif_base.DOWN.metrics()) + + @staticmethod + def recent_metrics_names() -> List[str]: + names = [] + names += self[UP].recent_metrics_names() + names += self[DOWN].recent_metrics_names() + return names + + @staticmethod + def recent_metrics(self) -> Dict[str, float]: + metrics = {} + metrics.update(self[UP].recent_metrics()) + metrics.update(self[DOWN].recent_metrics()) + return metrics + +@enforce_types +class ClassifMetrics1Dir: # pylint: disable=too-many-instance-attributes - def __init__(self): + def __init__(self, dirn: Dirn): + self.dirn = dirn + # 'i' is iteration number i self.acc_ests: List[float] = [] # [i] : %-correct self.acc_ls: List[float] = [] # [i] : %-correct-lower @@ -18,10 +45,10 @@ def __init__(self): self.recalls: List[float] = [] # [i] : recall self.losses: List[float] = [] # [i] : log-loss - - self.yerrs: List[float] = [] # [i] : regressor pred'n errs, w/ sign - - def update(self, acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr): + + def update(self, metrics): + acc_est, acc_l, acc_u, f1, precision, recall, loss = metrics + self.acc_ests.append(acc_est) self.acc_ls.append(acc_l) self.acc_us.append(acc_u) @@ -32,8 +59,6 @@ def update(self, acc_est, acc_l, acc_u, f1, precision, recall, loss, yerr): self.losses.append(loss) - self.yerrs.append(yerr) - @staticmethod def recent_metrics_names() -> List[str]: return [ @@ -44,69 +69,160 @@ def recent_metrics_names() -> List[str]: "precision", "recall", "loss", - "yerr", ] def recent_metrics(self) -> Dict[str, Union[int, float, None]]: """Return most recent aimodel metrics""" if not self.acc_ests: - return {key: None for key in AimodelMetrics.recent_metrics_names()} + return {key: None for key in ClassifMetrics.recent_metrics_names()} + dirn = self.dirn return { - "acc_est": self.acc_ests[-1], - "acc_l": self.acc_ls[-1], - "acc_u": self.acc_us[-1], - "f1": self.f1s[-1], - "precision": self.precisions[-1], - "recall": self.recalls[-1], - "loss": self.losses[-1], - "yerr": self.yerrs[-1], + f"acc_est_{dirn}": self.acc_ests[-1], + f"acc_l_{dirn}": self.acc_ls[-1], + f"acc_u_{dirn}": self.acc_us[-1], + f"f1_{dirn}": self.f1s[-1], + f"precision_{dirn}": self.precisions[-1], + f"recall_{dirn}": self.recalls[-1], + f"loss_{dirn}": self.losses[-1], } - -# pylint: disable=too-many-instance-attributes @enforce_types -class SimState: +class ClassifBaseData(dict): + def __init__(self): + self[UP] = ClassifBaseData1Dir() + self[DOWN] = ClassifBaseData1Dir() + + def update(self, true_up_UP, prob_up_UP, sim_model_p: SimModelPrediction): + self[UP].update(true_up_UP, sim_model_p.prob_up_UP) + self[DOWN].update(true_up_DOWN, sim_model_p.prob_up_DOWN) + +class ClassifBaseData1Dir: + @enforce_types def __init__(self): - self.init_loop_attributes() - self.iter_number = 0 - - def init_loop_attributes(self): # 'i' is iteration number i + self.ytrues: List[bool] = [] # [i] : true value + self.probs_up: List[float] = [] # [i] : model's pred. prob. + + @enforce_types + def update(self, true_up: float, prob_up: float): + self.ytrues.append(true_up) + self.probs_up.append(prob_up) + + @property + def ytrues_hat(self) -> List[bool]: + """@return [i] : model pred. value""" + return [p > 0.5 for p in self.probs_up] + + @property + def n_correct(self) -> int: + return sum(t == t_hat for t, t_hat in zip(self.ytrues, self.ytrues_hat)) - # base data for UP classifier - self.ytrues_UP: List[bool] = [] # [i] : true value - self.ytrues_hat_UP: List[bool] = [] # [i] : model pred. value - self.probs_up_UP: List[float] = [] # [i] : model's pred. prob. + @property + def n_trials(self) -> int: + return len(self.ytrues) + + @enforce_types + def accuracy(self) -> Tuple[float, float, float]: + n_correct, n_trials = self.n_correct, self.n_trials + acc_est = n_correct / n_trials + acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) + return (acc_est, acc_l, acc_u) + + @enforce_types + def precision_recall_f1(self) -> Tuple[float, float, float]: + (precision, recall, f1, _) = precision_recall_fscore_support( + self.ytrues, + self.ytrues_hat, + average="binary", + zero_division=0.0, + ) + return (precision, recall, f1) - # base data for DOWN classifier - self.ytrues_DOWN: List[bool] = [] # [i] : true value - self.ytrues_hat_DOWN: List[bool] = [] # [i] : model pred. value - self.probs_up_DOWN: List[float] = [] # [i] : model's pred. prob. + @enforce_types + def log_loss(self) -> float: + if min(ytrues) == max(ytrues): + return 3.0 # magic number + return log_loss(self.ytrues, self.probs_up) - # merged values - self.probs_up_MERGED: List[float] = [] # [i] : merged pred. prob. + @enforce_types + def metrics(self) -> List[float]: + return \ + list(self.accuracy()) + \ + list(precision_recall_f1) + \ + [self.log_loss()] - # aimodel metrics - self.aim = AimodelMetrics() - # profits +@enforce_types +class Profits: + def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit self.trader_profits_USD: List[float] = [] # [i] : trader-profit + + def update_trader_profit(self, trader_profit_USD: float): + self.trader_profits_USD.append(trader_profit_USD) + + def update_pdr_profit( + self, others_stake, others_accuracy, + stake_up, stake_down, true_up_close): + others_stake_correct = others_stake * others_accuracy + tot_stake = others_stake + stake_up + stake_down + + acct_up_profit = acct_down_profit = 0.0 + acct_up_profit -= stake_up + acct_down_profit -= stake_down + if true_up_close: + tot_stake_correct = others_stake_correct + stake_up + percent_to_me = stake_up / tot_stake_correct + acct_up_profit += (revenue + tot_stake) * percent_to_me + else: + tot_stake_correct = others_stake_correct + stake_down + percent_to_me = stake_down / tot_stake_correct + acct_down_profit += (revenue + tot_stake) * percent_to_me + pdr_profit_OCEAN = acct_up_profit + acct_down_profit + + self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) + + @staticmethod + def recent_metrics_names() -> List[str]: + return ["pdr_profit_OCEAN", "trader_profit_USD"] + + @staticmethod + def recent_metrics(self) -> Dict[str, float]: + return { + "pdr_profit_OCEAN" : self.pdr_profits_OCEAN[-1], + "trader_profit_USD" : self.trader_profits_USD[-1], + } + +# pylint: disable=too-many-instance-attributes +@enforce_types +class SimState: + def __init__(self): + self.init_loop_attributes() + + def init_loop_attributes(self): + self.iter_number = 0 + self.sim_model_data: Optional[SimModelData] = None + self.sim_model: Optional[SimModel] = None + self.classif_base = ClassifBaseData() + self.classif_metrics = ClassifMetrics() + self.profits = Profits() @staticmethod def recent_metrics_names() -> List[str]: - return AimodelMetrics.recent_metrics_names() + [ - "pdr_profit_OCEAN", - "trader_profit_USD", - ] + return ClassifMetrics.recent_metrics_names() + \ + [ + "pdr_profit_OCEAN", + "trader_profit_USD", + ] def recent_metrics( - self, extras: Optional[List[str]] = None + self, + extras: Optional[List[str]] = None ) -> List[Union[int, float]]: """Return most recent aimodel metrics + profit metrics""" - rm = self.aim.recent_metrics().copy() - rm.update( + metrics = self.metrics.recent_metrics().copy() + metrics.update( { "pdr_profit_OCEAN": self.pdr_profits_OCEAN[-1], "trader_profit_USD": self.trader_profits_USD[-1], @@ -114,10 +230,6 @@ def recent_metrics( ) if extras and "prob_up" in extras: - rm["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN - - return rm + metrics["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN - @property - def n_correct(self) -> int: - return sum((p > 0.5) == t for p, t in zip(self.probs_up, self.ytrues)) + return metrics diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index 8c772e58e..9a6b29406 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -57,12 +57,9 @@ def close_short_position(self, buy_price: float) -> float: def trade_iter( self, cur_close: float, - pred_up, - pred_down, - conf_up: float, - conf_down: float, high: float, low: float, + p: SimModelPrediction, ) -> float: """ @description @@ -74,17 +71,21 @@ def trade_iter( @arguments cur_close -- current price of the token - pred_up -- prediction that the price will go up - pred_down -- prediction that the price will go down - conf_up -- confidence in the prediction that the price will go up - conf_down -- confidence in the prediction that the price will go down - high -- highest price reached during the period - low -- lowest price reached during the period + high -- highest price reached during the previous period + low -- lowest price reached during the previous period + p -- SimModelPrediction. Includes attributes: + pred_up -- prediction that the price will go up + pred_down -- prediction that the price will go down + conf_up -- confidence in the prediction that the price will go up + conf_down -- confidence in the prediction that the price will go down @return profit -- profit made by the trader in this iteration """ + pred_up, pred_down, conf_up, conf_down = \ + p.pred_up, p.pred_down, p.conf_up, p.conf_down + trade_amt = self.ppss.trader_ss.buy_amt_usd.amt_eth if self.position_open == "": if pred_up: diff --git a/pdr_backend/sim/test/resources.py b/pdr_backend/sim/test/resources.py new file mode 100644 index 000000000..4ed3b30d7 --- /dev/null +++ b/pdr_backend/sim/test/resources.py @@ -0,0 +1,17 @@ +import numpy as np + +from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir + +@enforce_types +def get_sim_model_data() -> SimModelData: + X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) + ytrue_UP = np.array([True, True, False, True]) + + X_DOWN = np.array([[11.,12.,13.,14.],[15.,16.,17.,18.]]) + ytrue_DOWN = np.array([False, True, False, True]) + + data_UP = SimModelData1Dir(X_UP, ytrue_UP) + data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) + data = SimModelData(data_UP, data_DOWN) + + return data diff --git a/pdr_backend/sim/test/test_sim_constants.py b/pdr_backend/sim/test/test_sim_constants.py new file mode 100644 index 000000000..3a878e391 --- /dev/null +++ b/pdr_backend/sim/test/test_sim_constants.py @@ -0,0 +1,19 @@ +from enforce_typing import enforce_types + +from pdr_backend.sim.constants import Dirn, UP, DOWN + +@enforce_types +def test_sim_constants(): + assert UP == Dirn.UP + assert DOWN == Dirn.DOWN + + assert UP in Dirn + assert DOWN in Dirn + + assert UP == 1 + assert DOWN == 2 + assert 0 not in Dirn + assert 3 not in Dirn + assert "up" not in Dirn + + diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index ed8601b00..4f529b4e0 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -8,6 +8,8 @@ from pdr_backend.aimodel.aimodel import Aimodel from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedsets from pdr_backend.ppss.lake_ss import LakeSS, lake_ss_test_dict +from pdr_backend.ppss.aimodel_ss import aimodel_ss_test_dict +from pdr_backend.ppss.aimodel_data_ss import aimodel_data_ss_test_dict from pdr_backend.ppss.ppss import PPSS, fast_test_yaml_str from pdr_backend.ppss.predictoor_ss import PredictoorSS, predictoor_ss_test_dict from pdr_backend.ppss.sim_ss import SimSS, sim_ss_test_dict @@ -33,28 +35,32 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): # lake ss lake_dir = os.path.join(tmpdir, "parquet_data") - d = lake_ss_test_dict(lake_dir, feeds=feedsets.feed_strs) - assert "st_timestr" in d - d["st_timestr"] = "2023-06-18" - d["fin_timestr"] = "2023-06-19" - ppss.lake_ss = LakeSS(d) - - # predictoor ss - d = predictoor_ss_test_dict(feedset_list) - assert "max_n_train" in d["aimodel_data_ss"] - assert "autoregressive_n" in d["aimodel_data_ss"] - assert "approach" in d["aimodel_ss"] - assert "train_every_n_epochs" in d["aimodel_ss"] - d["aimodel_data_ss"]["max_n_train"] = 20 - d["aimodel_data_ss"]["autoregressive_n"] = 1 - d["aimodel_ss"]["approach"] = "ClassifLinearRidge" - d["aimodel_ss"]["train_every_n_epochs"] = 2 - ppss.predictoor_ss = PredictoorSS(d) + lake_d = lake_ss_test_dict( + lake_dir, + feeds=feedsets.feed_strs, + st_timestr = "2023-06-18", + fin_timestr = "2023-06-19", + ) + ppss.lake_ss = LakeSS(lake_d) + + # predictoor ss + pdr_d = predictoor_ss_test_dict( + feedset_list, + aimodel_data_ss_dict=aimodel_data_ss_test_dict( + max_n_train=20, + autoregressive_n=1, + ), + aimodel_ss_dict=aimodel_ss_test_dict( + approach="ClassifLinearRidge", + train_every_n_epochs=2, + ), + ) + ppss.predictoor_ss = PredictoorSS(pdr_d) # sim ss log_dir = os.path.join(tmpdir, "logs") - d = sim_ss_test_dict(log_dir, test_n=5) - ppss.sim_ss = SimSS(d) + sim_d = sim_ss_test_dict(log_dir, test_n=5) + ppss.sim_ss = SimSS(sim_d) # go feedsets = ppss.predictoor_ss.predict_train_feedsets diff --git a/pdr_backend/sim/test/test_sim_model.py b/pdr_backend/sim/test/test_sim_model.py new file mode 100644 index 000000000..02698359e --- /dev/null +++ b/pdr_backend/sim/test/test_sim_model.py @@ -0,0 +1,12 @@ +from enforce_typing import enforce_types + +from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.test.resources import get_sim_model_data + +@enforce_types +def test_sim_model(): + model_UP = Mock(foo) + model_DOWN = Mock(foo) + model = SimModel(ppss, model_UP, model_DOWN) + p = model.predict_next(d) + diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py new file mode 100644 index 000000000..9904b508c --- /dev/null +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -0,0 +1,51 @@ +from enforce_typing import enforce_types +import numpy as np +from numpy.testing import assert_array_equal + +from pdr_backend.sim.constants import Dirn, UP, DOWN +from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir + +@enforce_types +def test_sim_model_data_1dir(): + # build data + X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) # 4 x 2 + ytrue_UP = np.array([True, True, False, True]) # 4 + + # basic tests + assert_array_equal(X_UP, data_UP.X) + assert_array_equal(ytrue_UP, data_UP.ytrue) + + # test properties + assert data_UP.st == 0 + assert data_UP.fin == (4 - 1) == 3 + assert_array_equal(data_UP.X_train, X_UP[0:3,:]) + assert_array_equal(data_UP.X_test, X_UP[3:3+1,:]) + assert_array_equal(data_UP.ytrue_train, ytrue_train[0:3]) + +@enforce_types +def test_sim_model_data_both_dirs(): + # build data + X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) + ytrue_UP = np.array([True, True, False, True]) + + X_DOWN = np.array([[11.,12.,13.,14.],[15.,16.,17.,18.]]) + ytrue_DOWN = np.array([False, True, False, True]) + + data_UP = SimModelData1Dir(X_UP, ytrue_UP) + data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) + data = SimModelData(data_UP, data_DOWN) + + # basic tests + assert sorted(data.keys()) == sorted([UP, DOWN]) + for key in data: + assert isinstance(key, Dirn) + assert isinstance(data[UP], SimModelData1Dir) + assert isinstance(data[DOWN], SimModelData1Dir) + + assert_array_equal(X_UP, data[UP].X) + assert_array_equal(ytrue_UP, data[UP].ytrue) + + assert_array_equal(X_DOWN, data[DOWN].X) + assert_array_equal(ytrue_DOWN, data[DOWN].ytrue) + + diff --git a/pdr_backend/sim/test/test_sim_model_data_factory.py b/pdr_backend/sim/test/test_sim_model_data_factory.py new file mode 100644 index 000000000..e67edf9a3 --- /dev/null +++ b/pdr_backend/sim/test/test_sim_model_data_factory.py @@ -0,0 +1,87 @@ +from enforce_typing import enforce_types + +from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset +from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS +from pdr_backend.ppss.ppss import mock_feed_ppss +from pdr_backend.ppss.predictoor_ss import PredictoorSS +from pdr_backend.sim.sim_model_data_factory import SimModelDataFactory + + +@enforce_types +def test_sim_model_data_factory__basic(tmpdir): + #base data + data_f = _factory(tmpdir) + + # attributes + assert isinstance(data_f.ppss, PPSS) + assert isinstance(data_f.predict_train_feedset, PredictTrainFeedset) + + # properties + assert isinstance(data_f.pdr_ss, PredictoorSS) + assert isinstance(data_f.aimodel_data_ss, AimodelDataSS) + assert 0.0 < data_f.class_thr < 1.0 + assert isinstance(data_f.aimodel_data_factory, AimodelDataFactory) + +@enforce_types +def test_sim_model_data_factory__testshift(tmpdir): + #base data + data_f = _factory(tmpdir) + test_i = 3 + + # do work + test_n = data_f.ppss.sim_ss.test_n + + # test + assert data_f.testshift(test_i) == test_n - test_i - 1 + +@enforce_types +def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): + #base data + data_f = _factory(tmpdir) + cur_close = 8.0 + class_thr = data_f.ppss.predictoor_ss.aimodel_ss.class_thr + + # do work + thr_UP = data_f.thr_UP(cur_close) + thr_DOWN = data_f.thr_DOWN(cur_close) + + # test + assert class_thr > 0.0 + assert thr_DOWN < cur_close < thr_UP + assert thr_UP == cur_close * (1 + class_thr) + assert thr_DOWN == cur_close * (1 - class_thr) + +@enforce_types +def test_sim_model_data_factory_build(tmpdir): + #base data + data_f = _factory(tmpdir) + test_i = 3 + + # do work + data = data_f.build(test_i, mergedohlcv_df) + + # test + # - keep this light for now. Maybe expand when things stabilize + assert isinstance(data, SimModelData) + assert 9.0 < np.min(data[UP].X) < np.max(data[UP].X) < 9.8 # all 'high' + assert 0.0 < np.min(data[DOWN].X) < np.max(data[DOWN].X) < 0.8 # all 'low' + + +@enforce_types +def _factory(tmpdir) -> SimModelDataFactory: + feed, ppss = mock_feed_ppss("5m", "binance", "BTC/USDT") + predict_train_feedset = PredictTrainFeedset(predict=feed, train_on=[feed]) + data_f = SimModelDataFactory(ppss, predict_train_feedset) + return data_f + +@enforce_types +def _merged_ohlcv_df() -> pl.DataFrame: + return pl.DataFrame( + { + # - every column in df is ordered from youngest to oldest + # - high values are 9.x, 9.y, ..; low are 0.x, 0.y, ..; close is b/w + "timestamp": [1, 2, 3, 4, 5, 6, 7], + "binanceus:ETH/USDT:high": [9.1, 9.6, 9.8, 9.4, 9.2, 9.5, 9.7], + "binanceus:ETH/USDT:low" : [0.1, 0.6, 0.8, 0.4, 0.2, 0.5, 0.7], + "binanceus:ETH/USDT:close": [2.1, 3.6, 4.6, 5.6, 6.2, 7.5, 8.7], + } diff --git a/pdr_backend/sim/test/test_sim_model_factory.py b/pdr_backend/sim/test/test_sim_model_factory.py new file mode 100644 index 000000000..519fb41e2 --- /dev/null +++ b/pdr_backend/sim/test/test_sim_model_factory.py @@ -0,0 +1,48 @@ +from enforce_typing import enforce_types + +from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.test.resources import get_sim_model_data + +from pdr_backend.ppss.aimodel_ss import AimodelSS, aimodel_ss_test_dict +from pdr_backend.sim.constants import Dirn, UP, DOWN +from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_prediction import SimModelPrediction + + +@enforce_types +def test_sim_model_factory__attributes(): + f: SimModelFactory = _get_sim_model_factory() + assert isinstance(f.aimodel_ss, AimodelSS) + +@enforce_types +def test_sim_model_factory__do_build(): + f: SimModelFactory = _get_sim_model_factory() + + # case: no previous model + assert f.do_build(None, 0) + + # case: have previous model; then on proper iter? + prev_model = Mock() + assert f.do_build(prev_model, test_i=3) + assert not f.do_build(prev_model, test_i=4) + +@enforce_types +def test_sim_model_factory__build(): + f: SimModelFactory = _get_sim_model_factory() + + sim_model_data = get_sim_model_data() + model = f.do_build(sim_model_data) + assert isinstance(model, SimModel) + + p = model.predict_next(d) + assert isinstance(p, SimModelPrediction) + +@enforce_types +def _get_sim_model_factory() -> SimModelFactory: + aimodel_ss = _get_aimodel_ss() + return SimModelFactory(aimodel_ss) + +@enforce_types +def _get_aimodel_ss() -> AimodelSS: + d = aimodel_ss_test_dict() + return AimodelSS(d) diff --git a/pdr_backend/sim/test/test_sim_model_prediction.py b/pdr_backend/sim/test/test_sim_model_prediction.py new file mode 100644 index 000000000..9ec7ea52c --- /dev/null +++ b/pdr_backend/sim/test/test_sim_model_prediction.py @@ -0,0 +1,118 @@ +rom enforce_typing import enforce_types + +from pdr_backend.sim.sim_model_prediction import ( + SimModelPrediction, + _do_trust_models, + _models_in_conflict, +) + +@enforce_types +def test_sim_model_prediction_case1_models_in_conflict(): + p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.3) + assert p.conf_thr == 0.1 + assert p.prob_up_UP == 0.6 + assert p.prob_up_DOWN = 0.3 + + assert p.prob_down_DOWN = 1.0 - 0.3 == 0.7 + + assert p.models_in_conflict() + assert p.conf_up == 0.0 + assert p.conf_down == 0.0 + assert not p.pred_up + assert not p.pred_down + assert p.prob_up_MERGED == 0.5 + +@enforce_types +def test_sim_model_prediction_case2_up_dominates(): + p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.7) + assert p.conf_thr == 0.1 + assert p.prob_up_UP == 0.6 + assert p.prob_up_DOWN = 0.7 + + assert p.prob_down_DOWN = 1.0 - 0.7 == 0.3 + + assert not p.models_in_conflict() + assert p.prob_up_UP >= p.prob_down_DOWN + assert p.conf_up == (0.6 - 0.5) * 2.0 == 0.1 * 2.0 == 0.2 + assert p.conf_down == 0.0 + assert p.pred_up == p.conf_up > p.conf_thr == 0.2 > 0.1 == True + assert not p.pred_down + assert p.prob_up_MERGED == 0.6 + + # setup like above, but now with higher confidence level, which it can't exceed + p = SimModelPrediction(conf_thr = 0.3, prob_up_UP = 0.6, prob_up_DOWN = 0.7) + assert not p.pred_up + +@enforce_types +def test_sim_model_prediction_case3_down_dominates(): + p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.4, prob_up_DOWN = 0.3) + assert p.conf_thr == 0.1 + assert p.prob_up_UP == 0.4 + assert p.prob_up_DOWN = 0.3 + + assert p.prob_down_DOWN = 1.0 - 0.3 == 0.7 + + assert not p.models_in_conflict() + assert not (p.prob_up_UP >= p.prob_down_DOWN) + assert prob_down_DOWN > prob_up_UP + assert p.conf_up == 0.0 + assert p.conf_down == (0.7 - 0.5) * 2.0 == 0.2 * 2.0 == 0.4 + assert not p.pred_up + assert p.pred_down == (p.conf_down > p.conf_thr) == 0.4 > 0.1 == True + assert p.prob_up_MERGED == 1.0 - 0.7 == 0.3 + + # setup like above, but now with higher confidence level, which it can't exceed + p = SimModelPrediction(conf_thr = 0.5, prob_up_UP = 0.4, prob_up_DOWN = 0.3) + assert not p.pred_down + + +@enforce_types +def test_do_trust_models(): + with pytest.raises(ValueError): + _ = _do_trust_models(True, True, 0.4, 0.6) + + with pytest.raises(ValueError): + _ = _do_trust_models(True, False, 0.4, 0.6) + + with pytest.raises(ValueError): + _ = _do_trust_models(False, True, 0.6, 0.6) + + with pytest.raises(ValueError): + _do_trust_models(True, False, -1.4, 0.6) + with pytest.raises(ValueError): + _do_trust_models(True, False, 1.4, 0.6) + + with pytest.raises(ValueError): + _do_trust_models(True, False, 0.4, -1.6) + with pytest.raises(ValueError): + _do_trust_models(True, False, 0.4, 1.6) + + assert _do_trust_models(True, False, 0.4, 0.6) + assert _do_trust_models(False, True, 0.4, 0.6) + + assert not _do_trust_models(False, False, 0.4, 0.6) + assert not _do_trust_models(False, True, 0.4, 0.4) + assert not _do_trust_models(False, True, 0.6, 0.6) + + + +@enforce_types +def test_models_in_conflict(): + with pytest.raises(ValueError): + _models_in_conflict(-1.6, 0.6) + with pytest.raises(ValueError): + _models_in_conflict(1.6, 0.6) + + with pytest.raises(ValueError): + _models_in_conflict(0.6, -1.6) + with pytest.raises(ValueError): + _models_in_conflict(0.6, 1.6) + + assert _models_in_conflict(0.6, 0.6) + assert _models_in_conflict(0.4, 0.4) + + assert not _models_in_conflict(0.6, 0.4) + assert not _models_in_conflict(0.4, 0.6) + assert not _models_in_conflict(0.5, 0.5) + + diff --git a/pdr_backend/sim/test/test_sim_predictoor.py b/pdr_backend/sim/test/test_sim_predictoor.py new file mode 100644 index 000000000..71b484fef --- /dev/null +++ b/pdr_backend/sim/test/test_sim_predictoor.py @@ -0,0 +1,46 @@ +from enforce_typing import enforce_types + +from pdr_backend.ppss.predictoor_ss import PredictoorSS, predictoor_ss_test_dict +from pdr_backend.sim.sim_predictoor import SimPredictoor + +@enforce_types +def test_sim_pdr__attributes(): + sim_pdr = _sim_pdr() + assert isinstance(sim_pdr.pdr_ss, PredictoorSS) + + +@enforce_types +def test_sim_pdr__properties(): + sim_pdr = _sim_pdr() + max_stake_amt = cls.pdr_ss.stake_amount.amt_eth + assert isinstance(max_stake_amt, float) + assert max_stake_amt > 0.0 + + +@enforce_types +def test_sim_pdr__predict_iter(): + # base data + sim_pdr = _sim_pdr() + + # case 1: don't trust models + stake_up, stake_down = sim_pdr.predict_iter(False, False, 0.4, 0.6) + assert stake_up == stake_down == 0.0 + + # case 2: UP dominates + stake_up, stake_down = sim_pdr.predict_iter(True, False, 0.6, 0.4) + assert 0.0 < stake_down < stake_up < 1.0 + assert (stake_up + stake_down) <= sim_pdr.max_stake_amt + + # case 3: DOWN dominates + stake_up, stake_down = sim_pdr.predict_iter(False, True, 0.4, 0.6) + assert 0.0 < stake_up < stake_down < 1.0 + assert (stake_up + stake_down) <= sim_pdr.max_stake_amt + + +@enforce_types +def _sim_pdr() -> SimPredictoor: + pdr_d = predictoor_ss_test_dict() + pdr_ss = PredictoorSS(pdr_d) + sim_pdr = SimPredictoor(pdr_ss) + return sim_pdr + diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 1bf6f54cc..79a39f19a 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -2,10 +2,8 @@ from unittest.mock import Mock - import pytest - from pdr_backend.ppss.exchange_mgr_ss import ExchangeMgrSS from pdr_backend.sim.sim_trader import SimTrader diff --git a/ppss.yaml b/ppss.yaml index 8fc04d396..eb5fbc432 100644 --- a/ppss.yaml +++ b/ppss.yaml @@ -30,6 +30,7 @@ predictoor_ss: aimodel_data_ss: # used by AimodelDataFactory max_n_train: 1000 # no. epochs to train model on autoregressive_n: 2 # no. epochs that model looks back, to predict next + class_thr: 0.002 # 0.05 = 5%. UP class needs > this, DOWN < this transform: None # None -> raw value of signal; RelDiff -> rel % diff aimodel_ss: # used by AimodelFactory From 0cfa98261ca4675ad60400eb66a4169101accad1 Mon Sep 17 00:00:00 2001 From: trentmc Date: Mon, 1 Jul 2024 11:58:12 +0200 Subject: [PATCH 12/50] wip getting tests to work, and adding tests --- pdr_backend/aimodel/aimodel_data_factory.py | 6 +- pdr_backend/aimodel/aimodel_factory.py | 2 + pdr_backend/ppss/aimodel_data_ss.py | 58 ++++++-- pdr_backend/ppss/aimodel_ss.py | 16 ++- pdr_backend/ppss/sim_ss.py | 27 +++- pdr_backend/ppss/test/test_aimodel_data_ss.py | 40 +++++- pdr_backend/ppss/test/test_aimodel_ss.py | 24 ++++ pdr_backend/ppss/test/test_lake_ss.py | 2 +- pdr_backend/ppss/test/test_sim_ss.py | 18 +++ pdr_backend/sim/sim_engine.py | 6 +- pdr_backend/sim/sim_logger.py | 4 +- pdr_backend/sim/sim_model.py | 1 + pdr_backend/sim/sim_model_data.py | 29 ++-- pdr_backend/sim/sim_model_data_factory.py | 30 ++-- pdr_backend/sim/sim_model_factory.py | 38 +++-- pdr_backend/sim/sim_predictoor.py | 7 +- pdr_backend/sim/sim_state.py | 134 +++++++++--------- pdr_backend/sim/sim_trader.py | 34 +++-- pdr_backend/sim/test/resources.py | 21 ++- pdr_backend/sim/test/test_sim_constants.py | 4 +- pdr_backend/sim/test/test_sim_logger.py | 21 ++- pdr_backend/sim/test/test_sim_model.py | 23 ++- pdr_backend/sim/test/test_sim_model_data.py | 21 +-- .../sim/test/test_sim_model_data_factory.py | 59 +++++--- .../sim/test/test_sim_model_factory.py | 32 ++--- .../sim/test/test_sim_model_prediction.py | 93 ++++++------ pdr_backend/sim/test/test_sim_predictoor.py | 23 +-- pdr_backend/sim/test/test_sim_state.py | 87 +++++++++++- pdr_backend/sim/test/test_sim_trader.py | 9 +- 29 files changed, 599 insertions(+), 270 deletions(-) diff --git a/pdr_backend/aimodel/aimodel_data_factory.py b/pdr_backend/aimodel/aimodel_data_factory.py index 70d259cda..b34bf70c1 100644 --- a/pdr_backend/aimodel/aimodel_data_factory.py +++ b/pdr_backend/aimodel/aimodel_data_factory.py @@ -16,7 +16,6 @@ logger = logging.getLogger("aimodel_data_factory") -@enforce_types class AimodelDataFactory: """ Roles: @@ -51,6 +50,7 @@ class AimodelDataFactory: - "timestamp" values are ut: int is unix time, UTC, in ms (not s) """ + @enforce_types def __init__(self, ss: PredictoorSS): self.ss = ss @@ -71,9 +71,10 @@ def ycont_to_ytrue(ycont: np.ndarray, y_thr: float) -> np.ndarray: return ybool @staticmethod - def testshift(self, test_n: int, test_i: int) -> int: + def testshift(test_n: int, test_i: int) -> int: return test_n - test_i - 1 + @enforce_types def create_xy( self, mergedohlcv_df: pl.DataFrame, @@ -208,6 +209,7 @@ def create_xy( # return return X, ytran, yraw, x_df, xrecent + @enforce_types def get_highlow( self, mergedohlcv_df: pl.DataFrame, feed: ArgFeed, testshift: int ) -> tuple: diff --git a/pdr_backend/aimodel/aimodel_factory.py b/pdr_backend/aimodel/aimodel_factory.py index 7fff9c19c..33c67b23c 100644 --- a/pdr_backend/aimodel/aimodel_factory.py +++ b/pdr_backend/aimodel/aimodel_factory.py @@ -76,6 +76,7 @@ def _build_wrapped_regr( ss = self.ss assert ss.do_regr assert ycont is not None + assert X.shape[0] == ycont.shape[0], (X.shape[0], ycont.shape[0]) do_constant = min(ycont) == max(ycont) or ss.approach == "RegrConstant" # weight newest sample 10x, and 2nd-newest sample 5x @@ -141,6 +142,7 @@ def _build_direct_classif( ) -> Aimodel: ss = self.ss assert not ss.do_regr + assert X.shape[0] == len(ytrue), (X.shape[0], len(ytrue)) n_True, n_False = sum(ytrue), sum(np.invert(ytrue)) smallest_n = min(n_True, n_False) do_constant = (smallest_n == 0) or ss.approach == "ClassifConstant" diff --git a/pdr_backend/ppss/aimodel_data_ss.py b/pdr_backend/ppss/aimodel_data_ss.py index 9b71c58d8..3783182dd 100644 --- a/pdr_backend/ppss/aimodel_data_ss.py +++ b/pdr_backend/ppss/aimodel_data_ss.py @@ -20,15 +20,33 @@ def __init__(self, d: dict): self.d = d # test inputs - if not 0 < self.max_n_train: - raise ValueError(self.max_n_train) - if not 0 < self.autoregressive_n < np.inf: - raise ValueError(self.autoregressive_n) - if not 0 < self.class_thr <= 1.0: - raise ValueError(self.class_thr) - if self.transform not in TRANSFORM_OPTIONS: - raise ValueError(self.transform) - + self.validate_max_n_train(self.max_n_train) + self.validate_autoregressive_n(self.autoregressive_n) + self.validate_class_thr(self.class_thr) + self.validate_transform(self.transform) + + # -------------------------------- + # validators + @staticmethod + def validate_max_n_train(max_n_train: int): + if not 0 < max_n_train: + raise ValueError(max_n_train) + + @staticmethod + def validate_autoregressive_n(autoregressive_n: int): + if not (0 < autoregressive_n < np.inf): + raise ValueError(autoregressive_n) + + @staticmethod + def validate_class_thr(class_thr: float): + if not 0 <= class_thr <= 1.0: + raise ValueError(class_thr) + + @staticmethod + def validate_transform(transform: str): + if transform not in TRANSFORM_OPTIONS: + raise ValueError(transform) + # -------------------------------- # yaml properties @@ -50,13 +68,31 @@ def autoregressive_n(self) -> int: @property def class_thr(self) -> float: """eg 0.05 = 5%. UP class needs > this, DOWN < this""" - return self.d["class_thr1"] + return self.d["class_thr"] @property def transform(self) -> int: """eg 'RelDiff'""" return self.d["transform"] - + + # -------------------------------- + # setters + def set_max_n_train(self, max_n_train:int): + self.validate_max_n_train(max_n_train) + self.d["max_n_train"] = max_n_train + + def set_autoregressive_n(self, autoregressive_n:int): + self.validate_autoregressive_n(autoregressive_n) + self.d["autoregressive_n"] = autoregressive_n + + def set_class_thr(self, class_thr:float): + self.validate_class_thr(class_thr) + self.d["class_thr"] = class_thr + + def set_transform(self, transform:str): + self.validate_transform(transform) + self.d["transform"] = transform + # ========================================================================= # utilities for testing diff --git a/pdr_backend/ppss/aimodel_ss.py b/pdr_backend/ppss/aimodel_ss.py index b03a8f4b7..6affb66f3 100644 --- a/pdr_backend/ppss/aimodel_ss.py +++ b/pdr_backend/ppss/aimodel_ss.py @@ -56,6 +56,13 @@ def __init__(self, d: dict): raise ValueError(self.calibrate_probs) if self.calibrate_regr not in CALIBRATE_REGR_OPTIONS: raise ValueError(self.calibrate_regr) + self.validate_train_every_n_epochs(self.train_every_n_epochs) + + # -------------------------------- + # validators -- add as needed, when setters are added + def validate_train_every_n_epochs(self, n: int): + if n <= 0: + raise ValueError(n) # -------------------------------- # yaml properties @@ -130,6 +137,12 @@ def weight_recent_n(self) -> Tuple[int, int]: if self.weight_recent == "10000x": return 10000, 0 raise ValueError(self.weight_recent) + + # -------------------------------- + # setters (only add as needed) + def set_train_every_n_epochs(self, n: int): + self.validate_train_every_n_epochs(n) + self.d["train_every_n_epochs"] = n # ========================================================================= @@ -143,14 +156,15 @@ def aimodel_ss_test_dict( balance_classes: Optional[str] = None, calibrate_probs: Optional[str] = None, calibrate_regr: Optional[str] = None, + train_every_n_epochs: Optional[int] = None, ) -> dict: """Use this function's return dict 'd' to construct AimodelSS(d)""" d = { "approach": approach or "ClassifLinearRidge", "weight_recent": weight_recent or "10x_5x", "balance_classes": balance_classes or "SMOTE", - "train_every_n_epochs": 1, "calibrate_probs": calibrate_probs or "CalibratedClassifierCV_Sigmoid", "calibrate_regr": calibrate_regr or "None", + "train_every_n_epochs": 1 if train_every_n_epochs is None else train_every_n_epochs } return d diff --git a/pdr_backend/ppss/sim_ss.py b/pdr_backend/ppss/sim_ss.py index 8c8a9f9da..914a6ebdf 100644 --- a/pdr_backend/ppss/sim_ss.py +++ b/pdr_backend/ppss/sim_ss.py @@ -26,15 +26,21 @@ def __init__(self, d: dict): s = f"Couldn't find log dir, so created one at: {self.log_dir}" logger.warning(s) - # check test_n - test_n = d["test_n"] + # validate data + self.validate_test_n(self.test_n) + self.validate_tradetype(self.tradetype) + + # -------------------------------- + # validators + @staticmethod + def validate_test_n(test_n: int): if not isinstance(test_n, int): raise TypeError(test_n) if not 0 < test_n < np.inf: raise ValueError(test_n) - - # check tradetype - tradetype = d["tradetype"] + + @staticmethod + def validate_tradetype(tradetype: str): if not isinstance(tradetype, str): raise TypeError(tradetype) if tradetype not in TRADETYPE_OPTIONS: @@ -67,6 +73,17 @@ def is_final_iter(self, iter_i: int) -> bool: return (iter_i + 1) == self.test_n + # -------------------------------- + # setters + def set_test_n(self, test_n:int): + self.validate_test_n(test_n) + self.d["test_n"] = test_n + + def set_tradetype(self, tradetype:str): + self.validate_tradetype(tradetype) + self.d["tradetype"] = tradetype + + # ========================================================================= # utilities for testing diff --git a/pdr_backend/ppss/test/test_aimodel_data_ss.py b/pdr_backend/ppss/test/test_aimodel_data_ss.py index 6223c44f0..0017faea4 100644 --- a/pdr_backend/ppss/test/test_aimodel_data_ss.py +++ b/pdr_backend/ppss/test/test_aimodel_data_ss.py @@ -35,13 +35,13 @@ def test_aimodel_data_ss__nondefault_values(): assert ss.autoregressive_n == 13 ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.06)) - assert ss.class_thr = 0.06 + assert ss.class_thr == 0.06 ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.0)) - assert ss.class_thr = 0.0 + assert ss.class_thr == 0.0 ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1.0)) - assert ss.class_thr = 1.0 + assert ss.class_thr == 1.0 ss = AimodelDataSS(aimodel_data_ss_test_dict(transform="RelDiff")) assert ss.transform == "RelDiff" @@ -81,3 +81,37 @@ def test_aimodel_data_ss__bad_inputs(): with pytest.raises(TypeError): AimodelDataSS(aimodel_data_ss_test_dict(transform=3.1)) + +@enforce_types +def test_aimodel_data_ss__setters(): + d = aimodel_data_ss_test_dict() + ss = AimodelDataSS(d) + + # max_n_train + ss.set_max_n_train(32) + assert ss.max_n_train == 32 + with pytest.raises(ValueError): + ss.set_max_n_train(0) + with pytest.raises(ValueError): + ss.set_max_n_train(-5) + + # autoregressive_n + ss.set_autoregressive_n(12) + assert ss.autoregressive_n == 12 + with pytest.raises(ValueError): + ss.set_autoregressive_n(0) + with pytest.raises(ValueError): + ss.set_autoregressive_n(-5) + + # class_thr + ss.set_class_thr(0.34) + assert ss.class_thr == 0.34 + with pytest.raises(ValueError): + ss.set_class_thr(-0.1) + + # transform + ss.set_transform("RelDiff") + assert ss.transform == "RelDiff" + with pytest.raises(ValueError): + ss.set_transform("foo") + diff --git a/pdr_backend/ppss/test/test_aimodel_ss.py b/pdr_backend/ppss/test/test_aimodel_ss.py index 0259c8235..075b852ef 100644 --- a/pdr_backend/ppss/test/test_aimodel_ss.py +++ b/pdr_backend/ppss/test/test_aimodel_ss.py @@ -26,6 +26,7 @@ def test_aimodel_ss__default_values(): ss.calibrate_probs == d["calibrate_probs"] == "CalibratedClassifierCV_Sigmoid" ) assert ss.calibrate_regr == d["calibrate_regr"] == "None" + assert ss.train_every_n_epochs == d["train_every_n_epochs"] == 1 # str assert "AimodelSS" in str(ss) @@ -65,6 +66,9 @@ def test_aimodel_ss__nondefault_values(): ss = AimodelSS(aimodel_ss_test_dict(calibrate_regr=calibrate_regr)) assert ss.calibrate_regr == calibrate_regr and calibrate_regr in str(ss) + ss = AimodelSS(aimodel_ss_test_dict(train_every_n_epochs=44)) + assert ss.train_every_n_epochs == 44 + @enforce_types def test_aimodel_ss__bad_inputs(): @@ -83,6 +87,12 @@ def test_aimodel_ss__bad_inputs(): with pytest.raises(ValueError): AimodelSS(aimodel_ss_test_dict(calibrate_regr="foo")) + + with pytest.raises(ValueError): + AimodelSS(aimodel_ss_test_dict(train_every_n_epochs=0)) + + with pytest.raises(ValueError): + AimodelSS(aimodel_ss_test_dict(train_every_n_epochs=-5)) @enforce_types @@ -96,3 +106,17 @@ def test_aimodel_ss__calibrate_probs_skmethod(): ss = AimodelSS(d) assert ss.calibrate_probs_skmethod(100) == "sigmoid" # because N is small assert ss.calibrate_probs_skmethod(1000) == "isotonic" + +@enforce_types +def test_aimodel_ss__setters(): + d = aimodel_ss_test_dict() + ss = AimodelSS(d) + + # train_every_n_epochs + ss.set_train_every_n_epochs(77) + assert ss.train_every_n_epochs == 77 + with pytest.raises(ValueError): + ss.set_train_every_n_epochs(0) + with pytest.raises(ValueError): + ss.set_train_every_n_epochs(-5) + diff --git a/pdr_backend/ppss/test/test_lake_ss.py b/pdr_backend/ppss/test/test_lake_ss.py index 73cc889b8..8856c3f6e 100644 --- a/pdr_backend/ppss/test/test_lake_ss.py +++ b/pdr_backend/ppss/test/test_lake_ss.py @@ -137,5 +137,5 @@ def test_lake_ss_test_dict_3_nondefault_time_settings(tmpdir): timeframe="1h", ) assert d["st_timestr"] == "2023-01-20" - assert d["fin_timesetr"] == "2023-01-21" + assert d["fin_timestr"] == "2023-01-21" assert d["timeframe"] == "1h" diff --git a/pdr_backend/ppss/test/test_sim_ss.py b/pdr_backend/ppss/test/test_sim_ss.py index dd1192c3c..bff6dd7db 100644 --- a/pdr_backend/ppss/test/test_sim_ss.py +++ b/pdr_backend/ppss/test/test_sim_ss.py @@ -103,6 +103,24 @@ def test_sim_ss_is_final_iter(tmpdir): with pytest.raises(ValueError): _ = ss.is_final_iter(11) +@enforce_types +def test_sim_ss_setters(tmpdir): + d = sim_ss_test_dict(_logdir(tmpdir)) + ss = SimSS(d) + + # test_n + ss.set_test_n(32) + assert ss.test_n == 32 + with pytest.raises(ValueError): + ss.set_test_n(0) + with pytest.raises(ValueError): + ss.set_test_n(-5) + + # tradetype + ss.set_tradetype("livereal") + assert ss.tradetype == "livereal" + with pytest.raises(ValueError): + ss.set_tradetype("foo") # ==================================================================== # helper funcs diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 61faad92a..f3d31f4d6 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -1,7 +1,7 @@ import logging import os import uuid -from typing import Optional +from typing import Optional, Tuple import numpy as np import polars as pl @@ -17,7 +17,9 @@ from pdr_backend.cli.arg_timeframe import ArgTimeframe from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset from pdr_backend.lake.ohlcv_data_factory import OhlcvDataFactory +from pdr_backend.ppss.aimodel_ss import AimodelSS from pdr_backend.ppss.ppss import PPSS +from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_logger import SimLogLine from pdr_backend.sim.sim_model import SimModel @@ -75,7 +77,7 @@ def timeframe(self) -> ArgTimeframe: return self.predict_feed.timeframe @property - def pdr_ss(self) -> PredictorSS: + def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss @property diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index ae1845553..e5e10f2eb 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -9,11 +9,9 @@ @enforce_types -# pylint: disable=too-many-instance-attributes class SimLogLine: def __init__(self, ppss, st, test_i, ut): self.st = st - self.test_n = ppss.sim_ss.test_n self.test_i = test_i self.ut = ut @@ -24,7 +22,7 @@ def log(self): s += f" dt={self.ut.to_timestr()[:-7]}" s += " â•‘" - s += f"pdr_profit = {compactSmallNum(self.st.pdr_profits_OCEAN[-1])} OCEAN" + s += f"pdr_profit={compactSmallNum(self.st.pdr_profits_OCEAN[-1])} OCEAN" s += f" (cumul {compactSmallNum(sum(self.st.pdr_profits_OCEAN))} OCEAN)" s += " â•‘" diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index 4273a54cb..ab27489b3 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -1,6 +1,7 @@ from enforce_typing import enforce_types from pdr_backend.aimodel.aimodel import Aimodel +from pdr_backend.ppss.ppss import PPSS from pdr_backend.sim.constants import UP, DOWN from pdr_backend.sim.sim_model_data import SimModelData from pdr_backend.sim.sim_model_prediction import SimModelPrediction diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index 9bd3331f8..99605e865 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -1,22 +1,18 @@ +from typing import List, Union + from enforce_typing import enforce_types import numpy as np from pdr_backend.sim.constants import UP, DOWN -@enforce_types -class SimModelData(dict): - def __init__( - self, - data_UP: SimModelData1Dir, - data_DOWN: SimModelData1Dir, - ): - self[UP] = data_UP - self[DOWN] = data_DOWN - class SimModelData1Dir: @enforce_types - def __init__(self, X: np.ndarray, ytrue: np.ndarray): + def __init__(self, X: np.ndarray, ytrue: np.ndarray): + assert len(X.shape) == 2 + assert len(ytrue.shape) == 1 + assert X.shape[0] == ytrue.shape[0], (X.shape[0], ytrue.shape[0]) + self.X: np.ndarray = X self.ytrue: np.ndarray = ytrue @@ -37,6 +33,15 @@ def X_test(self) -> np.ndarray: return self.X[self.fin : self.fin + 1, :] @property - def ytrue_train(self) -> List[bool]: + def ytrue_train(self) -> np.ndarray: return self.ytrue[self.st:self.fin] +@enforce_types +class SimModelData(dict): + def __init__( + self, + data_UP: SimModelData1Dir, + data_DOWN: SimModelData1Dir, + ): + self[UP] = data_UP + self[DOWN] = data_DOWN diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index 9304336a5..13d4bf9d1 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -1,11 +1,15 @@ from enforce_typing import enforce_types +import numpy as np import polars as pl +from pdr_backend.cli.arg_feed import ArgFeed +from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory from pdr_backend.cli.predict_train_feedset import PredictTrainFeedset from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS from pdr_backend.ppss.ppss import PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS +from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir class SimModelDataFactory: @enforce_types @@ -23,32 +27,29 @@ def aimodel_data_ss(self) -> AimodelDataSS: @property def class_thr(self) -> float: - class_thr = self.aimodel_data_ss.class_thr - - @property - def aimodel_data_factory(self) -> AimodelDataFactory: - return AimodelDataFactory(self.pdr_ss) + return self.aimodel_data_ss.class_thr @enforce_types def testshift(self, test_i: int) -> int: test_n = self.ppss.sim_ss.test_n - data_f = self.aimodel_data_factory - return data_f.testshift(test_n, test_i) + return AimodelDataFactory.testshift(test_n, test_i) @enforce_types def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: - """Construct sim model data""" + """Construct sim model data""" + df = mergedohlcv_df testshift = self.testshift(test_i) # eg [99, 98, .., 2, 1, 0] + data_f = AimodelDataFactory(self.pdr_ss) - p = self.predict_train_feedset.predict_feed + p: ArgFeed = self.predict_train_feedset.predict _, _, y_close, _, _ = data_f.create_xy( - mergedohlcv_df, testshift, p.variant_close(), [p.variant_close()], + df, testshift, p.variant_close(), ArgFeeds([p.variant_close()]), ) - X_high, _, y_high, _, _ = self.aimodel_data_factory.create_xy( - mergedohlcv_df, testshift, p.variant_high(), [p.variant_high()], + X_high, _, y_high, _, _ = data_f.create_xy( + df, testshift, p.variant_high(), ArgFeeds([p.variant_high()]), ) - X_low, _, y_low, _, _ = self.aimodel_data_factory.create_xy( - mergedohlcv_df, testshift, p.variant_low(), [p.variant_low()], + X_low, _, y_low, _, _ = data_f.create_xy( + df, testshift, p.variant_low(), ArgFeeds([p.variant_low()]), ) ytrue_UP = [] @@ -64,6 +65,7 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: thr_DOWN = self.thr_DOWN(cur_close) ytrue_DOWN.append(next_low < thr_DOWN) + ytrue_UP, ytrue_DOWN = np.array(ytrue_UP), np.array(ytrue_DOWN) d_UP = SimModelData1Dir(X_high[1:,:], ytrue_UP) # or is it [:-1,:] d_DOWN = SimModelData1Dir(X_low[1:,:], ytrue_DOWN) # "" # note: alternatively, each input X could be h+l+c rather than just h or l diff --git a/pdr_backend/sim/sim_model_factory.py b/pdr_backend/sim/sim_model_factory.py index 9501aa5c7..20ba084d0 100644 --- a/pdr_backend/sim/sim_model_factory.py +++ b/pdr_backend/sim/sim_model_factory.py @@ -1,31 +1,45 @@ +from typing import Optional + +from enforce_typing import enforce_types + +from pdr_backend.aimodel.aimodel_factory import AimodelFactory +from pdr_backend.ppss.ppss import PPSS +from pdr_backend.sim.constants import UP, DOWN +from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_data import SimModelData -@enforce_types class SimModelFactory: - def __init__(self, aimodel_ss: AimodelSS): - self.aimodel_ss = aimodel_ss + @enforce_types + def __init__(self, ppss: PPSS): + self.ppss = ppss + + @property + def aimodel_ss(self): + return self.ppss.predictoor_ss.aimodel_ss + @enforce_types def do_build(self, prev_model: Optional[SimModel], test_i: int) -> bool: """Update/train model?""" - return prev_model is None or \ - test_i % self.aimodel_ss.train_every_n_epochs == 0 + n = self.aimodel_ss.train_every_n_epochs + return prev_model is None or test_i % n == 0 - def build(self, sim_model_data: SimModelData) -> SimModel: - d = sim_model_data + @enforce_types + def build(self, data: SimModelData) -> SimModel: model_f = AimodelFactory(self.aimodel_ss) model_UP = model_f.build( - d.UP.X_train, - d.UP.ytrue_train, + data[UP].X_train, + data[UP].ytrue_train, None, None, ) model_DOWN = model_f.build( - d.DOWN.X_train, - d.DOWN.ytrue_train, + data[DOWN].X_train, + data[DOWN].ytrue_train, None, None, ) - sim_model = SimModel(model_UP, model_DOWN) + sim_model = SimModel(self.ppss, model_UP, model_DOWN) return sim_model diff --git a/pdr_backend/sim/sim_predictoor.py b/pdr_backend/sim/sim_predictoor.py index 6e2876928..809bfa8a9 100644 --- a/pdr_backend/sim/sim_predictoor.py +++ b/pdr_backend/sim/sim_predictoor.py @@ -1,6 +1,9 @@ +from typing import Tuple + from enforce_typing import enforce_types from pdr_backend.ppss.predictoor_ss import PredictoorSS +from pdr_backend.sim.sim_model_prediction import SimModelPrediction class SimPredictoor: @enforce_types @@ -19,11 +22,11 @@ def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: stake_down = 0 elif p.prob_up_UP >= p.prob_down_DOWN: stake_amt = self.max_stake_amt * p.conf_up - stake_up = stake_amt * prob_up_MERGED + stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) else: # p.prob_down_DOWN > p.prob_up_UP stake_amt = self.max_stake_amt * p.conf_down - stake_up = stake_amt * prob_up_MERGED + stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) return (stake_up, stake_down) diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 8318d65b1..0234acfc9 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,7 +1,74 @@ -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from enforce_typing import enforce_types + from pdr_backend.sim.constants import Dirn, UP, DOWN +from pdr_backend.sim.sim_model_prediction import SimModelPrediction + +@enforce_types +class ClassifBaseData(dict): + def __init__(self): + self[UP] = ClassifBaseData1Dir() + self[DOWN] = ClassifBaseData1Dir() + + def update(self, true_up_UP, prob_up_UP, sim_model_p: SimModelPrediction): + self[UP].update(true_up_UP, sim_model_p.prob_up_UP) + self[DOWN].update(true_up_DOWN, sim_model_p.prob_up_DOWN) + +class ClassifBaseData1Dir: + @enforce_types + def __init__(self): + # 'i' is iteration number i + self.ytrues: List[bool] = [] # [i] : true value + self.probs_up: List[float] = [] # [i] : model's pred. prob. + + @enforce_types + def update(self, true_up: float, prob_up: float): + self.ytrues.append(true_up) + self.probs_up.append(prob_up) + + @property + def ytrues_hat(self) -> List[bool]: + """@return [i] : model pred. value""" + return [p > 0.5 for p in self.probs_up] + + @property + def n_correct(self) -> int: + return sum(t == t_hat for t, t_hat in zip(self.ytrues, self.ytrues_hat)) + + @property + def n_trials(self) -> int: + return len(self.ytrues) + + @enforce_types + def accuracy(self) -> Tuple[float, float, float]: + n_correct, n_trials = self.n_correct, self.n_trials + acc_est = n_correct / n_trials + acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) + return (acc_est, acc_l, acc_u) + + @enforce_types + def precision_recall_f1(self) -> Tuple[float, float, float]: + (precision, recall, f1, _) = precision_recall_fscore_support( + self.ytrues, + self.ytrues_hat, + average="binary", + zero_division=0.0, + ) + return (precision, recall, f1) + + @enforce_types + def log_loss(self) -> float: + if min(ytrues) == max(ytrues): + return 3.0 # magic number + return log_loss(self.ytrues, self.probs_up) + + @enforce_types + def metrics(self) -> List[float]: + return \ + list(self.accuracy()) + \ + list(precision_recall_f1) + \ + [self.log_loss()] @enforce_types @@ -87,71 +154,6 @@ def recent_metrics(self) -> Dict[str, Union[int, float, None]]: f"loss_{dirn}": self.losses[-1], } -@enforce_types -class ClassifBaseData(dict): - def __init__(self): - self[UP] = ClassifBaseData1Dir() - self[DOWN] = ClassifBaseData1Dir() - - def update(self, true_up_UP, prob_up_UP, sim_model_p: SimModelPrediction): - self[UP].update(true_up_UP, sim_model_p.prob_up_UP) - self[DOWN].update(true_up_DOWN, sim_model_p.prob_up_DOWN) - -class ClassifBaseData1Dir: - @enforce_types - def __init__(self): - # 'i' is iteration number i - self.ytrues: List[bool] = [] # [i] : true value - self.probs_up: List[float] = [] # [i] : model's pred. prob. - - @enforce_types - def update(self, true_up: float, prob_up: float): - self.ytrues.append(true_up) - self.probs_up.append(prob_up) - - @property - def ytrues_hat(self) -> List[bool]: - """@return [i] : model pred. value""" - return [p > 0.5 for p in self.probs_up] - - @property - def n_correct(self) -> int: - return sum(t == t_hat for t, t_hat in zip(self.ytrues, self.ytrues_hat)) - - @property - def n_trials(self) -> int: - return len(self.ytrues) - - @enforce_types - def accuracy(self) -> Tuple[float, float, float]: - n_correct, n_trials = self.n_correct, self.n_trials - acc_est = n_correct / n_trials - acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) - return (acc_est, acc_l, acc_u) - - @enforce_types - def precision_recall_f1(self) -> Tuple[float, float, float]: - (precision, recall, f1, _) = precision_recall_fscore_support( - self.ytrues, - self.ytrues_hat, - average="binary", - zero_division=0.0, - ) - return (precision, recall, f1) - - @enforce_types - def log_loss(self) -> float: - if min(ytrues) == max(ytrues): - return 3.0 # magic number - return log_loss(self.ytrues, self.probs_up) - - @enforce_types - def metrics(self) -> List[float]: - return \ - list(self.accuracy()) + \ - list(precision_recall_f1) + \ - [self.log_loss()] - @enforce_types class Profits: diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index 9a6b29406..ec20d0dbe 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -1,7 +1,9 @@ import logging + from enforce_typing import enforce_types -from pdr_backend.exchange.exchange_mgr import ExchangeMgr +from pdr_backend.exchange.exchange_mgr import ExchangeMgr +from pdr_backend.sim.sim_model_prediction import SimModelPrediction logger = logging.getLogger("sim_trader") @@ -52,14 +54,27 @@ def close_short_position(self, buy_price: float) -> float: self.position_open = "" profit = self.position_worth - usdcoin_amt_send return profit - - # pylint: disable = too-many-return-statements + def trade_iter( self, cur_close: float, high: float, low: float, p: SimModelPrediction, + ) -> float: + return self._trade_iter( + cur_close, p.pred_up, p.pred_down, p.conf_up, p.conf_down, high, low, + ) + + def _trade_iter( + self, + cur_close: float, + pred_up, + pred_down, + conf_up: float, + conf_down: float, + high: float, + low: float, ) -> float: """ @description @@ -73,19 +88,14 @@ def trade_iter( cur_close -- current price of the token high -- highest price reached during the previous period low -- lowest price reached during the previous period - p -- SimModelPrediction. Includes attributes: - pred_up -- prediction that the price will go up - pred_down -- prediction that the price will go down - conf_up -- confidence in the prediction that the price will go up - conf_down -- confidence in the prediction that the price will go down - + pred_up -- prediction that the price will go up + pred_down -- prediction that the price will go down + conf_up -- confidence in the prediction that the price will go up + conf_down -- confidence in the prediction that the price will go down @return profit -- profit made by the trader in this iteration """ - pred_up, pred_down, conf_up, conf_down = \ - p.pred_up, p.pred_down, p.conf_up, p.conf_down - trade_amt = self.ppss.trader_ss.buy_amt_usd.amt_eth if self.position_open == "": if pred_up: diff --git a/pdr_backend/sim/test/resources.py b/pdr_backend/sim/test/resources.py index 4ed3b30d7..f6f36b8df 100644 --- a/pdr_backend/sim/test/resources.py +++ b/pdr_backend/sim/test/resources.py @@ -1,17 +1,28 @@ +from enforce_typing import enforce_types import numpy as np from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir @enforce_types -def get_sim_model_data() -> SimModelData: - X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) - ytrue_UP = np.array([True, True, False, True]) - - X_DOWN = np.array([[11.,12.,13.,14.],[15.,16.,17.,18.]]) +def get_Xy_UP() -> tuple: + X_UP = np.array([[1.,2.],[3.,4.],[5.,6.],[7.,8.]]) # 4 x 2 + ytrue_UP = np.array([True, True, False, True]) # 4 + return (X_UP, ytrue_UP) + +@enforce_types +def get_Xy_DOWN() -> tuple: + X_DOWN = np.array([[11.,12.],[13.,14.],[15.,16.],[17.,18.]]) ytrue_DOWN = np.array([False, True, False, True]) + return (X_DOWN, ytrue_DOWN) +@enforce_types +def get_sim_model_data() -> SimModelData: + (X_UP, ytrue_UP) = get_Xy_UP() + (X_DOWN, ytrue_DOWN) = get_Xy_DOWN() + data_UP = SimModelData1Dir(X_UP, ytrue_UP) data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) data = SimModelData(data_UP, data_DOWN) return data + diff --git a/pdr_backend/sim/test/test_sim_constants.py b/pdr_backend/sim/test/test_sim_constants.py index 3a878e391..b24cc7420 100644 --- a/pdr_backend/sim/test/test_sim_constants.py +++ b/pdr_backend/sim/test/test_sim_constants.py @@ -10,8 +10,8 @@ def test_sim_constants(): assert UP in Dirn assert DOWN in Dirn - assert UP == 1 - assert DOWN == 2 + assert 1 in Dirn + assert 2 in Dirn assert 0 not in Dirn assert 3 not in Dirn assert "up" not in Dirn diff --git a/pdr_backend/sim/test/test_sim_logger.py b/pdr_backend/sim/test/test_sim_logger.py index 64cb8daf2..bb6fd4cef 100644 --- a/pdr_backend/sim/test/test_sim_logger.py +++ b/pdr_backend/sim/test/test_sim_logger.py @@ -1,6 +1,8 @@ import os from unittest.mock import Mock +from enforce_typing import enforce_types + from pdr_backend.ppss.ppss import PPSS, fast_test_yaml_str from pdr_backend.ppss.sim_ss import SimSS, sim_ss_test_dict from pdr_backend.sim.sim_logger import SimLogLine @@ -8,7 +10,8 @@ from pdr_backend.util.time_types import UnixTimeMs -def test_compact_num(tmpdir, caplog): +@enforce_types +def test_sim_logger(tmpdir, caplog): s = fast_test_yaml_str(tmpdir) ppss = PPSS(yaml_str=s, network="development") @@ -34,16 +37,12 @@ def test_compact_num(tmpdir, caplog): st.trader_profits_USD = [2.0, 3.0, 4.0, 5.0, 6.0] ut = UnixTimeMs(1701634400000) - log_line = SimLogLine(ppss, st, 1, ut, 0.5, 0.6) - log_line.log_line() - - assert "pdr_profit=0.50 up" in caplog.text - assert "prcsn=0.100" in caplog.text + log_line = SimLogLine(ppss, st, 1, ut) + log_line.log() + assert "pdr_profit=" in caplog.text + assert "tdr_profit=" in caplog.text assert f"Iter #2/{ppss.sim_ss.test_n}" in caplog.text - log_line = SimLogLine(ppss, st, 1, ut, 0.003, 0.4) - log_line.log_line() - - assert "pdr_profit=3.00e-3 up" in caplog.text - assert "prcsn=0.100" in caplog.text + log_line = SimLogLine(ppss, st, 1, ut) + log_line.log() assert f"Iter #2/{ppss.sim_ss.test_n}" in caplog.text diff --git a/pdr_backend/sim/test/test_sim_model.py b/pdr_backend/sim/test/test_sim_model.py index 02698359e..df5cce9e9 100644 --- a/pdr_backend/sim/test/test_sim_model.py +++ b/pdr_backend/sim/test/test_sim_model.py @@ -1,12 +1,29 @@ +from unittest.mock import Mock + from enforce_typing import enforce_types +import numpy as np +from pdr_backend.aimodel.aimodel import Aimodel +from pdr_backend.ppss.ppss import PPSS from pdr_backend.sim.sim_model import SimModel -from pdr_backend.sim.test.resources import get_sim_model_data +from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir +from pdr_backend.sim.sim_model_prediction import SimModelPrediction @enforce_types def test_sim_model(): - model_UP = Mock(foo) - model_DOWN = Mock(foo) + ppss = Mock(spec=PPSS) + ppss.trader_ss = Mock() + ppss.trader_ss.sim_confidence_threshold = 0.1 + + model_UP = Mock(spec=Aimodel) + model_UP.predict_ptrue = Mock(return_value=np.array([0.1, 0.2, 0.3])) + model_DOWN = Mock(spec=Aimodel) + model_DOWN.predict_ptrue = Mock(return_value=np.array([0.7, 0.8, 0.9])) model = SimModel(ppss, model_UP, model_DOWN) + + data_UP = Mock(spec=SimModelData1Dir) + data_DOWN = Mock(spec=SimModelData1Dir) + d = SimModelData(data_UP, data_DOWN) p = model.predict_next(d) + assert isinstance(p, SimModelPrediction) diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py index 9904b508c..94ffecc50 100644 --- a/pdr_backend/sim/test/test_sim_model_data.py +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -4,12 +4,17 @@ from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir +from pdr_backend.sim.test.resources import get_Xy_UP, get_Xy_DOWN @enforce_types def test_sim_model_data_1dir(): # build data - X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) # 4 x 2 - ytrue_UP = np.array([True, True, False, True]) # 4 + (X_UP, ytrue_UP) = get_Xy_UP() + + assert X_UP.shape == (4,2) + assert ytrue_UP.shape == (4,) + ytrue_UP_train = ytrue_UP[:3] + data_UP = SimModelData1Dir(X_UP, ytrue_UP) # basic tests assert_array_equal(X_UP, data_UP.X) @@ -20,23 +25,21 @@ def test_sim_model_data_1dir(): assert data_UP.fin == (4 - 1) == 3 assert_array_equal(data_UP.X_train, X_UP[0:3,:]) assert_array_equal(data_UP.X_test, X_UP[3:3+1,:]) - assert_array_equal(data_UP.ytrue_train, ytrue_train[0:3]) + assert_array_equal(data_UP.ytrue_train, ytrue_UP_train) @enforce_types def test_sim_model_data_both_dirs(): # build data - X_UP = np.array([[1.,2.,3.,4.],[5.,6.,7.,8.]]) - ytrue_UP = np.array([True, True, False, True]) - - X_DOWN = np.array([[11.,12.,13.,14.],[15.,16.,17.,18.]]) - ytrue_DOWN = np.array([False, True, False, True]) + (X_UP, ytrue_UP) = get_Xy_UP() + (X_DOWN, ytrue_DOWN) = get_Xy_DOWN() data_UP = SimModelData1Dir(X_UP, ytrue_UP) data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) data = SimModelData(data_UP, data_DOWN) # basic tests - assert sorted(data.keys()) == sorted([UP, DOWN]) + assert UP in data + assert DOWN in data for key in data: assert isinstance(key, Dirn) assert isinstance(data[UP], SimModelData1Dir) diff --git a/pdr_backend/sim/test/test_sim_model_data_factory.py b/pdr_backend/sim/test/test_sim_model_data_factory.py index e67edf9a3..de0a53b0a 100644 --- a/pdr_backend/sim/test/test_sim_model_data_factory.py +++ b/pdr_backend/sim/test/test_sim_model_data_factory.py @@ -1,9 +1,16 @@ from enforce_typing import enforce_types +import numpy as np +import polars as pl +from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory +from pdr_backend.cli.arg_feed import ArgFeed +from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS -from pdr_backend.ppss.ppss import mock_feed_ppss +from pdr_backend.ppss.ppss import mock_ppss, PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS +from pdr_backend.sim.constants import UP, DOWN +from pdr_backend.sim.sim_model_data import SimModelData from pdr_backend.sim.sim_model_data_factory import SimModelDataFactory @@ -19,8 +26,8 @@ def test_sim_model_data_factory__basic(tmpdir): # properties assert isinstance(data_f.pdr_ss, PredictoorSS) assert isinstance(data_f.aimodel_data_ss, AimodelDataSS) + assert isinstance(data_f.class_thr, float) assert 0.0 < data_f.class_thr < 1.0 - assert isinstance(data_f.aimodel_data_factory, AimodelDataFactory) @enforce_types def test_sim_model_data_factory__testshift(tmpdir): @@ -39,7 +46,7 @@ def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): #base data data_f = _factory(tmpdir) cur_close = 8.0 - class_thr = data_f.ppss.predictoor_ss.aimodel_ss.class_thr + class_thr: float = data_f.ppss.predictoor_ss.aimodel_data_ss.class_thr # do work thr_UP = data_f.thr_UP(cur_close) @@ -52,10 +59,11 @@ def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): assert thr_DOWN == cur_close * (1 - class_thr) @enforce_types -def test_sim_model_data_factory_build(tmpdir): +def test_sim_model_data_factory__build(tmpdir): #base data + mergedohlcv_df = _merged_ohlcv_df() data_f = _factory(tmpdir) - test_i = 3 + test_i = 0 # do work data = data_f.build(test_i, mergedohlcv_df) @@ -63,25 +71,40 @@ def test_sim_model_data_factory_build(tmpdir): # test # - keep this light for now. Maybe expand when things stabilize assert isinstance(data, SimModelData) - assert 9.0 < np.min(data[UP].X) < np.max(data[UP].X) < 9.8 # all 'high' - assert 0.0 < np.min(data[DOWN].X) < np.max(data[DOWN].X) < 0.8 # all 'low' + assert 9.0 < np.min(data[UP].X) < np.max(data[UP].X) < 9.99 # all 'high' + assert 0.0 < np.min(data[DOWN].X) < np.max(data[DOWN].X) < 0.99 # all 'low' @enforce_types def _factory(tmpdir) -> SimModelDataFactory: - feed, ppss = mock_feed_ppss("5m", "binance", "BTC/USDT") - predict_train_feedset = PredictTrainFeedset(predict=feed, train_on=[feed]) + s = "binanceus ETH/USDT c 5m" + + feedset_list = [{"predict": s, "train_on":s}] + ppss = mock_ppss(feedset_list) + + # change settings so that the factory can work with a small # datapoints + ppss.predictoor_ss.aimodel_data_ss.set_max_n_train(4) + ppss.predictoor_ss.aimodel_data_ss.set_autoregressive_n(1) + ppss.sim_ss.set_test_n(2) + + predict_feed = ArgFeed.from_str(s) + train_feeds = ArgFeeds([predict_feed]) + + predict_train_feedset = PredictTrainFeedset( + predict=predict_feed, train_on=train_feeds) + data_f = SimModelDataFactory(ppss, predict_train_feedset) return data_f @enforce_types def _merged_ohlcv_df() -> pl.DataFrame: - return pl.DataFrame( - { - # - every column in df is ordered from youngest to oldest - # - high values are 9.x, 9.y, ..; low are 0.x, 0.y, ..; close is b/w - "timestamp": [1, 2, 3, 4, 5, 6, 7], - "binanceus:ETH/USDT:high": [9.1, 9.6, 9.8, 9.4, 9.2, 9.5, 9.7], - "binanceus:ETH/USDT:low" : [0.1, 0.6, 0.8, 0.4, 0.2, 0.5, 0.7], - "binanceus:ETH/USDT:close": [2.1, 3.6, 4.6, 5.6, 6.2, 7.5, 8.7], - } + d = { + # - every column in df is ordered from youngest to oldest + # - high values are 9.x, 9.y, ..; low are 0.x, 0.y, ..; close is b/w + "timestamp": [1, 2, 3, 4, 5, 6, 7], + "binanceus:ETH/USDT:high": [9.1, 9.6, 9.8, 9.4, 9.2, 9.5, 9.7], + "binanceus:ETH/USDT:low" : [0.1, 0.6, 0.8, 0.4, 0.2, 0.5, 0.7], + "binanceus:ETH/USDT:close": [2.1, 3.6, 4.6, 5.6, 6.2, 7.5, 8.7], + } + return pl.DataFrame(d) + diff --git a/pdr_backend/sim/test/test_sim_model_factory.py b/pdr_backend/sim/test/test_sim_model_factory.py index 519fb41e2..1ced2f1cc 100644 --- a/pdr_backend/sim/test/test_sim_model_factory.py +++ b/pdr_backend/sim/test/test_sim_model_factory.py @@ -1,48 +1,48 @@ -from enforce_typing import enforce_types +from unittest.mock import Mock -from pdr_backend.sim.sim_model import SimModel -from pdr_backend.sim.test.resources import get_sim_model_data +from enforce_typing import enforce_types +from pdr_backend.ppss.ppss import fast_test_yaml_str, PPSS from pdr_backend.ppss.aimodel_ss import AimodelSS, aimodel_ss_test_dict from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_factory import SimModelFactory from pdr_backend.sim.sim_model_prediction import SimModelPrediction +from pdr_backend.sim.test.resources import get_sim_model_data @enforce_types def test_sim_model_factory__attributes(): f: SimModelFactory = _get_sim_model_factory() - assert isinstance(f.aimodel_ss, AimodelSS) + assert isinstance(f.ppss, PPSS) @enforce_types def test_sim_model_factory__do_build(): f: SimModelFactory = _get_sim_model_factory() + f.aimodel_ss.set_train_every_n_epochs(13) # case: no previous model assert f.do_build(None, 0) # case: have previous model; then on proper iter? - prev_model = Mock() - assert f.do_build(prev_model, test_i=3) - assert not f.do_build(prev_model, test_i=4) + prev_model = Mock(spec=SimModel) + assert f.do_build(prev_model, test_i=(13*4)) + assert not f.do_build(prev_model, test_i=(13*4+1)) @enforce_types def test_sim_model_factory__build(): f: SimModelFactory = _get_sim_model_factory() - sim_model_data = get_sim_model_data() - model = f.do_build(sim_model_data) + data = get_sim_model_data() + model = f.build(data) assert isinstance(model, SimModel) - p = model.predict_next(d) + p = model.predict_next(data) assert isinstance(p, SimModelPrediction) @enforce_types def _get_sim_model_factory() -> SimModelFactory: - aimodel_ss = _get_aimodel_ss() - return SimModelFactory(aimodel_ss) + s = fast_test_yaml_str() + ppss = PPSS(yaml_str=s, network="development") + return SimModelFactory(ppss) -@enforce_types -def _get_aimodel_ss() -> AimodelSS: - d = aimodel_ss_test_dict() - return AimodelSS(d) diff --git a/pdr_backend/sim/test/test_sim_model_prediction.py b/pdr_backend/sim/test/test_sim_model_prediction.py index 9ec7ea52c..e62adfff4 100644 --- a/pdr_backend/sim/test/test_sim_model_prediction.py +++ b/pdr_backend/sim/test/test_sim_model_prediction.py @@ -1,4 +1,6 @@ -rom enforce_typing import enforce_types +from enforce_typing import enforce_types +import pytest +from pytest import approx from pdr_backend.sim.sim_model_prediction import ( SimModelPrediction, @@ -11,9 +13,9 @@ def test_sim_model_prediction_case1_models_in_conflict(): p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.3) assert p.conf_thr == 0.1 assert p.prob_up_UP == 0.6 - assert p.prob_up_DOWN = 0.3 + assert p.prob_up_DOWN == 0.3 - assert p.prob_down_DOWN = 1.0 - 0.3 == 0.7 + assert p.prob_down_DOWN == 1.0 - 0.3 == 0.7 assert p.models_in_conflict() assert p.conf_up == 0.0 @@ -27,17 +29,17 @@ def test_sim_model_prediction_case2_up_dominates(): p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.7) assert p.conf_thr == 0.1 assert p.prob_up_UP == 0.6 - assert p.prob_up_DOWN = 0.7 + assert p.prob_up_DOWN == 0.7 - assert p.prob_down_DOWN = 1.0 - 0.7 == 0.3 + assert p.prob_down_DOWN == approx(1.0 - 0.7) == approx(0.3) assert not p.models_in_conflict() assert p.prob_up_UP >= p.prob_down_DOWN - assert p.conf_up == (0.6 - 0.5) * 2.0 == 0.1 * 2.0 == 0.2 + assert p.conf_up == approx((0.6 - 0.5) * 2.0) == approx(0.1 * 2.0) == 0.2 assert p.conf_down == 0.0 - assert p.pred_up == p.conf_up > p.conf_thr == 0.2 > 0.1 == True + assert p.pred_up == (p.conf_up > p.conf_thr) == (0.2 > 0.1) == True assert not p.pred_down - assert p.prob_up_MERGED == 0.6 + assert p.prob_up_MERGED == approx(0.6) # setup like above, but now with higher confidence level, which it can't exceed p = SimModelPrediction(conf_thr = 0.3, prob_up_UP = 0.6, prob_up_DOWN = 0.7) @@ -48,18 +50,18 @@ def test_sim_model_prediction_case3_down_dominates(): p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.4, prob_up_DOWN = 0.3) assert p.conf_thr == 0.1 assert p.prob_up_UP == 0.4 - assert p.prob_up_DOWN = 0.3 + assert p.prob_up_DOWN == 0.3 - assert p.prob_down_DOWN = 1.0 - 0.3 == 0.7 + assert p.prob_down_DOWN == approx(1.0 - 0.3) == approx(0.7) assert not p.models_in_conflict() assert not (p.prob_up_UP >= p.prob_down_DOWN) - assert prob_down_DOWN > prob_up_UP + assert p.prob_down_DOWN > p.prob_up_UP assert p.conf_up == 0.0 - assert p.conf_down == (0.7 - 0.5) * 2.0 == 0.2 * 2.0 == 0.4 + assert p.conf_down == approx((0.7 - 0.5) * 2.0) == approx(0.2 * 2.0) == 0.4 assert not p.pred_up - assert p.pred_down == (p.conf_down > p.conf_thr) == 0.4 > 0.1 == True - assert p.prob_up_MERGED == 1.0 - 0.7 == 0.3 + assert p.pred_down == (p.conf_down > p.conf_thr) == (0.4 > 0.1) == True + assert p.prob_up_MERGED == approx(1.0 - 0.7) == approx(0.3) # setup like above, but now with higher confidence level, which it can't exceed p = SimModelPrediction(conf_thr = 0.5, prob_up_UP = 0.4, prob_up_DOWN = 0.3) @@ -67,47 +69,44 @@ def test_sim_model_prediction_case3_down_dominates(): @enforce_types -def test_do_trust_models(): - with pytest.raises(ValueError): - _ = _do_trust_models(True, True, 0.4, 0.6) +def test_do_trust_models_unhappy_path(): + for tup in [ + (True, True, 0.4, 0.6), + (True, False, 0.4, 0.6), + (False, True, 0.6, 0.4), + (True, False, -1.4, 0.6), + (True, False, 1.4, 0.6), + (True, False, 0.4, -1.6), + (True, False, 0.4, 1.6), + ]: + (pred_up, pred_down, prob_up_UP, prob_down_DOWN) = tup + with pytest.raises(ValueError): + _do_trust_models(pred_up, pred_down, prob_up_UP, prob_down_DOWN) + print(f"Should have failed on {tup}") - with pytest.raises(ValueError): - _ = _do_trust_models(True, False, 0.4, 0.6) - - with pytest.raises(ValueError): - _ = _do_trust_models(False, True, 0.6, 0.6) - - with pytest.raises(ValueError): - _do_trust_models(True, False, -1.4, 0.6) - with pytest.raises(ValueError): - _do_trust_models(True, False, 1.4, 0.6) - - with pytest.raises(ValueError): - _do_trust_models(True, False, 0.4, -1.6) - with pytest.raises(ValueError): - _do_trust_models(True, False, 0.4, 1.6) - - assert _do_trust_models(True, False, 0.4, 0.6) +@enforce_types +def test_do_trust_models_happy_path(): + assert _do_trust_models(True, False, 0.6, 0.4) assert _do_trust_models(False, True, 0.4, 0.6) assert not _do_trust_models(False, False, 0.4, 0.6) assert not _do_trust_models(False, True, 0.4, 0.4) assert not _do_trust_models(False, True, 0.6, 0.6) - - @enforce_types -def test_models_in_conflict(): - with pytest.raises(ValueError): - _models_in_conflict(-1.6, 0.6) - with pytest.raises(ValueError): - _models_in_conflict(1.6, 0.6) - - with pytest.raises(ValueError): - _models_in_conflict(0.6, -1.6) - with pytest.raises(ValueError): - _models_in_conflict(0.6, 1.6) - +def test_models_in_conflict_unhappy_path(): + for (prob_up_UP, prob_down_DOWN) in [ + (-1.6, 0.6), + (1.6, 0.6), + (0.6, -1.6), + (0.6, 1.6), + ]: + with pytest.raises(ValueError): + _models_in_conflict(prob_up_UP, prob_down_DOWN) + +@enforce_types +def test_models_in_conflict_happy_path(): + assert _models_in_conflict(0.6, 0.6) assert _models_in_conflict(0.4, 0.4) diff --git a/pdr_backend/sim/test/test_sim_predictoor.py b/pdr_backend/sim/test/test_sim_predictoor.py index 71b484fef..407cc5590 100644 --- a/pdr_backend/sim/test/test_sim_predictoor.py +++ b/pdr_backend/sim/test/test_sim_predictoor.py @@ -1,38 +1,45 @@ from enforce_typing import enforce_types from pdr_backend.ppss.predictoor_ss import PredictoorSS, predictoor_ss_test_dict +from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_predictoor import SimPredictoor @enforce_types -def test_sim_pdr__attributes(): +def test_sim_predictoor__attributes(): sim_pdr = _sim_pdr() assert isinstance(sim_pdr.pdr_ss, PredictoorSS) @enforce_types -def test_sim_pdr__properties(): +def test_sim_predictoor__properties(): sim_pdr = _sim_pdr() - max_stake_amt = cls.pdr_ss.stake_amount.amt_eth - assert isinstance(max_stake_amt, float) + max_stake_amt = sim_pdr.pdr_ss.stake_amount.amt_eth + assert isinstance(max_stake_amt, float | int) assert max_stake_amt > 0.0 @enforce_types -def test_sim_pdr__predict_iter(): +def test_sim_predictoor__predict_iter(): # base data sim_pdr = _sim_pdr() # case 1: don't trust models - stake_up, stake_down = sim_pdr.predict_iter(False, False, 0.4, 0.6) + p = SimModelPrediction(conf_thr=0.9, prob_up_UP=0.4, prob_up_DOWN=0.4) + assert not p.do_trust_models() + stake_up, stake_down = sim_pdr.predict_iter(p) assert stake_up == stake_down == 0.0 # case 2: UP dominates - stake_up, stake_down = sim_pdr.predict_iter(True, False, 0.6, 0.4) + p = SimModelPrediction(conf_thr=0.1, prob_up_UP=0.6, prob_up_DOWN=0.6) + assert p.do_trust_models() + stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_down < stake_up < 1.0 assert (stake_up + stake_down) <= sim_pdr.max_stake_amt # case 3: DOWN dominates - stake_up, stake_down = sim_pdr.predict_iter(False, True, 0.4, 0.6) + p = SimModelPrediction(conf_thr=0.1, prob_up_UP=0.4, prob_up_DOWN=0.4) + assert p.do_trust_models() + stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_up < stake_down < 1.0 assert (stake_up + stake_down) <= sim_pdr.max_stake_amt diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index ffd0f9d5b..15c8710a8 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -1 +1,86 @@ -# do nothing here, it's all tested in test_sim_engine.py +from enforce_typing import enforce_types + +from pdr_backend.sim.sim_state import ( + ClassifBaseData, + ClassifBaseData1Dir, +) + +@enforce_types +def test_classif_base_data_1dir(): + d = ClassifBaseData1Dir() + assert d.ytrues == [] + assert d.probs_up == [] + assert d.ytrues_hat == [] + assert d.n_correct == 0 + assert d.n_trials == 0 + + # true = up, guess = up (correct guess) + d.update(true_up=True, prob_up=0.6) + assert d.ytrues == [True] + assert d.probs_up == [0.6] + assert d.ytrues_hat == [True] + assert d.n_correct == 1 + assert d.n_trials == 1 + assert len(d.accuracy()) == 3 + assert d.accuracy()[0] == 1.0/1.0 + + # true = down, guess = down (correct guess) + d.update(true_up=False, prob_up=0.3) + assert d.ytrues == [True, False] + assert d.probs_up == [0.6, 0.3] + assert d.ytrues_hat == [True, False] + assert d.n_correct == 2 + assert d.n_trials == 2 + assert d.accuracy()[0] == 2.0/2.0 + + # true = up, guess = down (incorrect guess) + d.update(true_up=True, prob_up=0.4) + assert d.ytrues == [True, False, True] + assert d.probs_up == [0.6, 0.3, 0.4] + assert d.ytrues_hat == [True, False, False] + assert d.n_correct == 2 + assert d.n_trials == 3 + assert d.accuracy()[0] == approx(2.0/3.0) + + # true = down, guess = up (incorrect guess) + d.update(true_up=False, prob_up=0.7) + assert d.ytrues == [True, False, True, False] + assert d.probs_up == [0.6, 0.3, 0.4, 0.7] + assert d.ytrues_hat == [True, False, False, True] + assert d.n_correct == 2 + assert d.n_trials == 4 + assert d.accuracy()[0] == approx(2.0/4.0) + +@enforce_types +def test_classif_base_data(): + d_UP = ClassifBaseData1Dir() + d_DOWN = ClassifBaseData1Dir() + d = ClassifBaseData(d_UP, d_DOWN) + assert id(d[UP]) == id(d_UP) + assert id(d[DOWN]) == id(d_DOWN) + + p = SimModelPrediction(FIXME) + d.update(true_up_UP, prob_up_UP, p) + + raise NotImplementedError("Build the rest of me") + + +@enforce_types +def test_classif_metrics_1dir(): + raise NotImplementedError("Build me") + + +@enforce_types +def test_classif_metrics(): + raise NotImplementedError("Build me") + + +@enforce_types +def test_profits(): + raise NotImplementedError("Build me") + + + +@enforce_types +def test_sim_state(): + raise NotImplementedError("Build me") diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 79a39f19a..358dd3650 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -6,6 +6,7 @@ from pdr_backend.ppss.exchange_mgr_ss import ExchangeMgrSS from pdr_backend.sim.sim_trader import SimTrader +from pdr_backend.sim.sim_model_prediction import SimModelPrediction FEE_PERCENT = 0.01 @@ -66,7 +67,7 @@ def test_close_short_position(sim_trader): def test_trade_iter_open_long(sim_trader): sim_trader._buy = Mock(return_value=10) - sim_trader.trade_iter(100, True, False, 0.5, 0, 110, 90) + sim_trader._trade_iter(100, True, False, 0.5, 0, 110, 90) assert sim_trader.position_open == "long" assert sim_trader.position_worth == 1500 assert sim_trader.position_size == 10 @@ -74,7 +75,7 @@ def test_trade_iter_open_long(sim_trader): def test_trade_iter_open_short(sim_trader): sim_trader._sell = Mock(return_value=1500) - sim_trader.trade_iter(100, False, True, 0, 0.5, 110, 90) + sim_trader._trade_iter(100, False, True, 0, 0.5, 110, 90) assert sim_trader.position_open == "short" assert sim_trader.position_worth == 1500 assert sim_trader.position_size == 15 @@ -86,7 +87,7 @@ def test_trade_iter_close_long_take_profit_percent(sim_trader): sim_trader.position_worth = 1000 sim_trader.tp = 110 sim_trader._sell = Mock(return_value=1100) - profit = sim_trader.trade_iter(100, False, False, 0, 0, 110, 90) + profit = sim_trader._trade_iter(100, False, False, 0, 0, 110, 90) assert profit == 100 # 1100 - 1000 assert sim_trader.position_open == "" @@ -97,7 +98,7 @@ def test_trade_iter_close_short_stop_loss_percent(sim_trader): sim_trader.position_worth = 1000 sim_trader.sl = 110 sim_trader._buy = Mock() - profit = sim_trader.trade_iter(100, False, False, 0, 0, 110, 90) + profit = sim_trader._trade_iter(100, False, False, 0, 0, 110, 90) assert profit == -100 # 1100 - 1000 assert sim_trader.position_open == "" From 928d740b417719578edc4502558b81f505fa20f4 Mon Sep 17 00:00:00 2001 From: trentmc Date: Tue, 2 Jul 2024 08:41:41 +0200 Subject: [PATCH 13/50] wip --- pdr_backend/sim/constants.py | 10 + pdr_backend/sim/sim_engine.py | 6 +- pdr_backend/sim/sim_model.py | 6 +- pdr_backend/sim/sim_model_prediction.py | 64 ++--- pdr_backend/sim/sim_state.py | 220 ++++++++-------- pdr_backend/sim/test/test_sim_constants.py | 7 + .../sim/test/test_sim_model_prediction.py | 63 ++--- pdr_backend/sim/test/test_sim_state.py | 247 +++++++++++++++--- 8 files changed, 412 insertions(+), 211 deletions(-) diff --git a/pdr_backend/sim/constants.py b/pdr_backend/sim/constants.py index 16004ae17..91cbb68fd 100644 --- a/pdr_backend/sim/constants.py +++ b/pdr_backend/sim/constants.py @@ -1,8 +1,18 @@ from enum import Enum +from enforce_typing import enforce_types + class Dirn(Enum): UP = 1 DOWN = 2 UP = Dirn.UP DOWN = Dirn.DOWN + +@enforce_types +def dirn_str(dirn: Dirn): + if dirn == UP: + return "UP" + if dirn == DOWN: + return "DOWN" + raise ValueError(dirn) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index f3d31f4d6..beafaeeeb 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -152,11 +152,11 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # observe true price change true_up_close = next_close > cur_close - true_up_UP = next_high > y_thr_UP # did next high go > prev close+% ? - true_up_DOWN = next_low < y_thr_DOWN # did next low go < prev close-% ? + true_UP = next_high > y_thr_UP # did next high go > prev close+% ? + true_DOWN = next_low < y_thr_DOWN # did next low go < prev close-% ? # update state - classifier data - st.classif_base.update(true_up_UP, true_up_DOWN, sim_model_p) + st.classif_base.update(true_UP, true_DOWN, sim_model_p) st.classif_metrics.update(st.classif_base) # update predictoor profit diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index ab27489b3..e2db03f29 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -16,8 +16,8 @@ def __init__(self, ppss: PPSS, model_UP: Aimodel, model_DOWN: Aimodel): def predict_next(self, d: SimModelData) -> SimModelPrediction: conf_thr = self.ppss.trader_ss.sim_confidence_threshold - prob_up_UP = self[UP].predict_ptrue(d[UP].X_test)[0] - prob_up_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] + prob_UP = self[UP].predict_ptrue(d[UP].X_test)[0] + prob_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] - return SimModelPrediction(conf_thr, prob_up_UP, prob_up_DOWN) + return SimModelPrediction(conf_thr, prob_UP, prob_DOWN) diff --git a/pdr_backend/sim/sim_model_prediction.py b/pdr_backend/sim/sim_model_prediction.py index 30b79b522..428e1f3f2 100644 --- a/pdr_backend/sim/sim_model_prediction.py +++ b/pdr_backend/sim/sim_model_prediction.py @@ -6,19 +6,21 @@ class SimModelPrediction: def __init__( self, conf_thr: float, - prob_up_UP: float, - prob_up_DOWN: float, + + # UP model's probability that next high will go > prev close+% + prob_UP: float, + + # DOWN model's probability that next low will go < prev close-% + prob_DOWN: float, ): # ppss.trader_ss.sim_confidence_threshold self.conf_thr = conf_thr # core attributes - self.prob_up_UP = prob_up_UP - self.prob_up_DOWN = prob_up_DOWN + self.prob_UP = prob_UP + self.prob_DOWN = prob_DOWN - # derived attributes - self.prob_down_DOWN = 1.0 - self.prob_up_DOWN - + # derived attributes if self.models_in_conflict(): self.conf_up = 0.0 self.conf_down = 0.0 @@ -26,64 +28,64 @@ def __init__( self.pred_down = False self.prob_up_MERGED = 0.5 - elif self.prob_up_UP >= self.prob_down_DOWN: - self.conf_up = (prob_up_UP - 0.5) * 2.0 # to range [0,1] + elif self.prob_UP >= self.prob_DOWN: + self.conf_up = (prob_UP - 0.5) * 2.0 # to range [0,1] self.conf_down = 0.0 self.pred_up = self.conf_up > self.conf_thr self.pred_down = False - self.prob_up_MERGED = self.prob_up_UP + self.prob_up_MERGED = self.prob_UP - else: # prob_down_DOWN > prob_up_UP + else: # prob_DOWN > prob_UP self.conf_up = 0.0 - self.conf_down = (self.prob_down_DOWN - 0.5) * 2.0 + self.conf_down = (self.prob_DOWN - 0.5) * 2.0 self.pred_up = False self.pred_down = self.conf_down > self.conf_thr - self.prob_up_MERGED = 1.0 - self.prob_down_DOWN + self.prob_up_MERGED = 1.0 - self.prob_DOWN @enforce_types def do_trust_models(self) -> bool: return _do_trust_models( - self.pred_up, self.pred_down, self.prob_up_UP, self.prob_down_DOWN, + self.pred_up, self.pred_down, self.prob_UP, self.prob_DOWN, ) @enforce_types def models_in_conflict(self) -> bool: - return _models_in_conflict(self.prob_up_UP, self.prob_down_DOWN) + return _models_in_conflict(self.prob_UP, self.prob_DOWN) @enforce_types def _do_trust_models( pred_up:bool, pred_down:bool, - prob_up_UP:float, - prob_down_DOWN:float, + prob_UP:float, + prob_DOWN:float, ) -> bool: """Do we trust the models enough to take prediction / trading action?""" # preconditions + if not (0.0 <= prob_UP <= 1.0): + raise ValueError(prob_UP) + if not (0.0 <= prob_DOWN <= 1.0): + raise ValueError(prob_DOWN) if pred_up and pred_down: raise ValueError("can't have pred_up=True and pred_down=True") - if pred_up and prob_down_DOWN > prob_up_UP: + if pred_up and prob_DOWN > prob_UP: raise ValueError("can't have pred_up=True with prob_DOWN dominant") - if pred_down and prob_up_UP > prob_down_DOWN: + if pred_down and prob_UP > prob_DOWN: raise ValueError("can't have pred_down=True with prob_UP dominant") - if not (0.0 <= prob_up_UP <= 1.0): - raise ValueError(prob_up_UP) - if not (0.0 <= prob_down_DOWN <= 1.0): - raise ValueError(prob_down_DOWN) # main test return (pred_up or pred_down) \ - and not _models_in_conflict(prob_up_UP, prob_down_DOWN) + and not _models_in_conflict(prob_UP, prob_DOWN) @enforce_types -def _models_in_conflict(prob_up_UP: float, prob_down_DOWN: float) -> bool: +def _models_in_conflict(prob_UP: float, prob_DOWN: float) -> bool: """Does the UP model conflict with the DOWN model?""" # preconditions - if not (0.0 <= prob_up_UP <= 1.0): - raise ValueError(prob_up_UP) - if not (0.0 <= prob_down_DOWN <= 1.0): - raise ValueError(prob_down_DOWN) + if not (0.0 <= prob_UP <= 1.0): + raise ValueError(prob_UP) + if not (0.0 <= prob_DOWN <= 1.0): + raise ValueError(prob_DOWN) # main test - return (prob_up_UP > 0.5 and prob_down_DOWN > 0.5) or \ - (prob_up_UP < 0.5 and prob_down_DOWN < 0.5) + return (prob_UP > 0.5 and prob_DOWN > 0.5) or \ + (prob_UP < 0.5 and prob_DOWN < 0.5) diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 0234acfc9..2249d33f2 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,44 +1,38 @@ from typing import Dict, List, Optional, Tuple, Union from enforce_typing import enforce_types +from sklearn.metrics import log_loss, precision_recall_fscore_support +from statsmodels.stats.proportion import proportion_confint -from pdr_backend.sim.constants import Dirn, UP, DOWN +from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN from pdr_backend.sim.sim_model_prediction import SimModelPrediction -@enforce_types -class ClassifBaseData(dict): - def __init__(self): - self[UP] = ClassifBaseData1Dir() - self[DOWN] = ClassifBaseData1Dir() - - def update(self, true_up_UP, prob_up_UP, sim_model_p: SimModelPrediction): - self[UP].update(true_up_UP, sim_model_p.prob_up_UP) - self[DOWN].update(true_up_DOWN, sim_model_p.prob_up_DOWN) +PERFS_NAMES = ["acc_est", "acc_l", "acc_u", "f1", "precision", "recall", "loss"] -class ClassifBaseData1Dir: +class TrueVsPredVals: @enforce_types def __init__(self): # 'i' is iteration number i - self.ytrues: List[bool] = [] # [i] : true value - self.probs_up: List[float] = [] # [i] : model's pred. prob. + self.truevals: List[bool] = [] # [i] : true value + self.predprobs: List[float] = [] # [i] : model's pred. prob. @enforce_types - def update(self, true_up: float, prob_up: float): - self.ytrues.append(true_up) - self.probs_up.append(prob_up) + def update(self, trueval: bool, predprob: float): + self.truevals.append(trueval) + self.predprobs.append(predprob) @property - def ytrues_hat(self) -> List[bool]: + def predvals(self) -> List[bool]: """@return [i] : model pred. value""" - return [p > 0.5 for p in self.probs_up] + return [p > 0.5 for p in self.predprobs] @property def n_correct(self) -> int: - return sum(t == t_hat for t, t_hat in zip(self.ytrues, self.ytrues_hat)) + return sum(tv == pv for tv, pv in zip(self.truevals, self.predvals)) @property def n_trials(self) -> int: - return len(self.ytrues) + return len(self.truevals) @enforce_types def accuracy(self) -> Tuple[float, float, float]: @@ -50,8 +44,8 @@ def accuracy(self) -> Tuple[float, float, float]: @enforce_types def precision_recall_f1(self) -> Tuple[float, float, float]: (precision, recall, f1, _) = precision_recall_fscore_support( - self.ytrues, - self.ytrues_hat, + self.truevals, + self.predvals, average="binary", zero_division=0.0, ) @@ -59,44 +53,40 @@ def precision_recall_f1(self) -> Tuple[float, float, float]: @enforce_types def log_loss(self) -> float: - if min(ytrues) == max(ytrues): + if min(self.truevals) == max(self.truevals): return 3.0 # magic number - return log_loss(self.ytrues, self.probs_up) + return log_loss(self.truevals, self.predprobs) @enforce_types - def metrics(self) -> List[float]: - return \ + def perf_values(self) -> List[float]: + perfs_list = \ list(self.accuracy()) + \ list(precision_recall_f1) + \ [self.log_loss()] - + return perfs_list @enforce_types -class ClassifMetrics(dict): +class ClassifBaseData(dict): def __init__(self): - self[UP] = ClassifMetrics1Dir() - self[DOWN] = ClassifMetrics1Dir() - - def update(self, classif_base: ClassifBaseData): - self[UP].update(classif_base.UP.metrics()) - self[DOWN].update(classif_base.DOWN.metrics()) + self[UP] = TrueVsPredVals() + self[DOWN] = TrueVsPredVals() - @staticmethod - def recent_metrics_names() -> List[str]: - names = [] - names += self[UP].recent_metrics_names() - names += self[DOWN].recent_metrics_names() - return names + def update( + self, + + # to establish UP model's accuracy: did next high go > prev close+% ? + true_UP: bool, + + # to establish DOWN model's accuracy: did next low go < prev close-% ? + true_DOWN: bool, + + sim_model_p: SimModelPrediction, + ): + self[UP].update(true_UP, sim_model_p.prob_UP) + self[DOWN].update(true_DOWN, sim_model_p.prob_DOWN) - @staticmethod - def recent_metrics(self) -> Dict[str, float]: - metrics = {} - metrics.update(self[UP].recent_metrics()) - metrics.update(self[DOWN].recent_metrics()) - return metrics - @enforce_types -class ClassifMetrics1Dir: +class HistPerfs1Dir: # pylint: disable=too-many-instance-attributes def __init__(self, dirn: Dirn): @@ -113,8 +103,9 @@ def __init__(self, dirn: Dirn): self.losses: List[float] = [] # [i] : log-loss - def update(self, metrics): - acc_est, acc_l, acc_u, f1, precision, recall, loss = metrics + def update(self, perfs_list: list): + """perfs_list typically comes from TrueVsPredVals.perf_values()""" + acc_est, acc_l, acc_u, f1, precision, recall, loss = perfs_list self.acc_ests.append(acc_est) self.acc_ls.append(acc_l) @@ -126,47 +117,79 @@ def update(self, metrics): self.losses.append(loss) - @staticmethod - def recent_metrics_names() -> List[str]: + def recent_metrics_names(self) -> List[str]: + s = dirn_str(self.dirn) return [ - "acc_est", - "acc_l", - "acc_u", - "f1", - "precision", - "recall", - "loss", + f"acc_est_{s}", + f"acc_l_{s}", + f"acc_u_{s}", + f"f1_{s}", + f"precision_{s}", + f"recall_{s}", + f"loss_{s}", ] - def recent_metrics(self) -> Dict[str, Union[int, float, None]]: + def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: """Return most recent aimodel metrics""" if not self.acc_ests: - return {key: None for key in ClassifMetrics.recent_metrics_names()} + return {key: None for key in HistPerfs.recent_metrics_names()} - dirn = self.dirn + s = dirn_str(self.dirn) return { - f"acc_est_{dirn}": self.acc_ests[-1], - f"acc_l_{dirn}": self.acc_ls[-1], - f"acc_u_{dirn}": self.acc_us[-1], - f"f1_{dirn}": self.f1s[-1], - f"precision_{dirn}": self.precisions[-1], - f"recall_{dirn}": self.recalls[-1], - f"loss_{dirn}": self.losses[-1], + f"acc_est_{s}": self.acc_ests[-1], + f"acc_l_{s}": self.acc_ls[-1], + f"acc_u_{s}": self.acc_us[-1], + f"f1_{s}": self.f1s[-1], + f"precision_{s}": self.precisions[-1], + f"recall_{s}": self.recalls[-1], + f"loss_{s}": self.losses[-1], } +@enforce_types +class HistPerfs(dict): + def __init__(self): + self[UP] = HistPerfs1Dir(UP) + self[DOWN] = HistPerfs1Dir(DOWN) + + def update_recent(self, classif_base: ClassifBaseData): + self[UP].update(classif_base.UP.recent_metrics_values()) + self[DOWN].update(classif_base.DOWN.recent_metrics_values()) + + def recent_metrics_names(self) -> List[str]: + names = [] + names += self[UP].recent_metrics_names() + names += self[DOWN].recent_metrics_names() + return names + + def recent_metrics_values(self) -> Dict[str, float]: + metrics = {} + metrics.update(self[UP].recent_metrics_values()) + metrics.update(self[DOWN].recent_metrics_values()) + return metrics + + @enforce_types class Profits: def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit self.trader_profits_USD: List[float] = [] # [i] : trader-profit - - def update_trader_profit(self, trader_profit_USD: float): - self.trader_profits_USD.append(trader_profit_USD) - def update_pdr_profit( - self, others_stake, others_accuracy, - stake_up, stake_down, true_up_close): + @staticmethod + def calc_pdr_profit( + others_stake:float, + others_accuracy: float, + stake_up: float, + stake_down: float, + revenue: float, + true_up_close: bool, + ): + assert others_stake >= 0 + assert 0.0 <= others_accuracy <= 1.0 + assert stake_up >= 0.0 + assert stake_down >= 0.0 + assert revenue >= 0.0 + others_stake_correct = others_stake * others_accuracy tot_stake = others_stake + stake_up + stake_down @@ -182,15 +205,18 @@ def update_pdr_profit( percent_to_me = stake_down / tot_stake_correct acct_down_profit += (revenue + tot_stake) * percent_to_me pdr_profit_OCEAN = acct_up_profit + acct_down_profit - + return pdr_profit_OCEAN + + def update_pdr_profit(self, pdr_profit_OCEAN): self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) + + def update_trader_profit(self, trader_profit_USD: float): + self.trader_profits_USD.append(trader_profit_USD) - @staticmethod - def recent_metrics_names() -> List[str]: + def recent_metrics_names(self) -> List[str]: return ["pdr_profit_OCEAN", "trader_profit_USD"] - @staticmethod - def recent_metrics(self) -> Dict[str, float]: + def recent_metrics_values(self) -> Dict[str, float]: return { "pdr_profit_OCEAN" : self.pdr_profits_OCEAN[-1], "trader_profit_USD" : self.trader_profits_USD[-1], @@ -207,31 +233,17 @@ def init_loop_attributes(self): self.sim_model_data: Optional[SimModelData] = None self.sim_model: Optional[SimModel] = None self.classif_base = ClassifBaseData() - self.classif_metrics = ClassifMetrics() + self.hist_perfs = HistPerfs() self.profits = Profits() - @staticmethod - def recent_metrics_names() -> List[str]: - return ClassifMetrics.recent_metrics_names() + \ - [ - "pdr_profit_OCEAN", - "trader_profit_USD", - ] - - def recent_metrics( - self, - extras: Optional[List[str]] = None - ) -> List[Union[int, float]]: - """Return most recent aimodel metrics + profit metrics""" - metrics = self.metrics.recent_metrics().copy() - metrics.update( - { - "pdr_profit_OCEAN": self.pdr_profits_OCEAN[-1], - "trader_profit_USD": self.trader_profits_USD[-1], - } - ) - - if extras and "prob_up" in extras: - metrics["prob_up"] = self.probs_up_UP[-1] # FIXME: account for DOWN + def recent_metrics_names(self) -> List[str]: + return self.hist_perfs.recent_metrics_names() + \ + self.profits.recent_metrics_names() + def recent_metrics_values(self) -> List[Union[int, float]]: + """Return most recent aimodel metrics + profit metrics""" + metrics = {} + metrics.update(self.hist_perfs.recent_metrics_values()) + metrics.update(self.profits.recent_metrics_values()) return metrics + diff --git a/pdr_backend/sim/test/test_sim_constants.py b/pdr_backend/sim/test/test_sim_constants.py index b24cc7420..7897b9798 100644 --- a/pdr_backend/sim/test/test_sim_constants.py +++ b/pdr_backend/sim/test/test_sim_constants.py @@ -15,5 +15,12 @@ def test_sim_constants(): assert 0 not in Dirn assert 3 not in Dirn assert "up" not in Dirn + + assert dirn_str(UP) == "UP" + assert dirn_str(DOWN) == "DOWN" + with pytest.raises(ValueError): + _ = dirn_str(3) + with pytest.raises(TypeError): + _ = dirn_str("not an int") diff --git a/pdr_backend/sim/test/test_sim_model_prediction.py b/pdr_backend/sim/test/test_sim_model_prediction.py index e62adfff4..c3a6634b6 100644 --- a/pdr_backend/sim/test/test_sim_model_prediction.py +++ b/pdr_backend/sim/test/test_sim_model_prediction.py @@ -10,12 +10,10 @@ @enforce_types def test_sim_model_prediction_case1_models_in_conflict(): - p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.3) + p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.6, prob_DOWN = 0.7) assert p.conf_thr == 0.1 - assert p.prob_up_UP == 0.6 - assert p.prob_up_DOWN == 0.3 - - assert p.prob_down_DOWN == 1.0 - 0.3 == 0.7 + assert p.prob_UP == 0.6 + assert p.prob_DOWN == 0.7 assert p.models_in_conflict() assert p.conf_up == 0.0 @@ -26,62 +24,60 @@ def test_sim_model_prediction_case1_models_in_conflict(): @enforce_types def test_sim_model_prediction_case2_up_dominates(): - p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.6, prob_up_DOWN = 0.7) + p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.6, prob_DOWN = 0.3) assert p.conf_thr == 0.1 - assert p.prob_up_UP == 0.6 - assert p.prob_up_DOWN == 0.7 - - assert p.prob_down_DOWN == approx(1.0 - 0.7) == approx(0.3) + assert p.prob_UP == 0.6 + assert p.prob_DOWN == 0.3 assert not p.models_in_conflict() - assert p.prob_up_UP >= p.prob_down_DOWN + assert p.prob_UP >= p.prob_DOWN assert p.conf_up == approx((0.6 - 0.5) * 2.0) == approx(0.1 * 2.0) == 0.2 assert p.conf_down == 0.0 assert p.pred_up == (p.conf_up > p.conf_thr) == (0.2 > 0.1) == True assert not p.pred_down assert p.prob_up_MERGED == approx(0.6) - # setup like above, but now with higher confidence level, which it can't exceed - p = SimModelPrediction(conf_thr = 0.3, prob_up_UP = 0.6, prob_up_DOWN = 0.7) + # setup like above, but now with higher conf thr, which it can't exceed + p = SimModelPrediction(conf_thr = 0.3, prob_UP = 0.6, prob_DOWN = 0.3) assert not p.pred_up @enforce_types def test_sim_model_prediction_case3_down_dominates(): - p = SimModelPrediction(conf_thr = 0.1, prob_up_UP = 0.4, prob_up_DOWN = 0.3) + p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.4, prob_DOWN = 0.7) assert p.conf_thr == 0.1 - assert p.prob_up_UP == 0.4 - assert p.prob_up_DOWN == 0.3 - - assert p.prob_down_DOWN == approx(1.0 - 0.3) == approx(0.7) + assert p.prob_UP == 0.4 + assert p.prob_DOWN == 0.7 assert not p.models_in_conflict() - assert not (p.prob_up_UP >= p.prob_down_DOWN) - assert p.prob_down_DOWN > p.prob_up_UP + assert not (p.prob_UP >= p.prob_DOWN) + assert p.prob_DOWN > p.prob_UP assert p.conf_up == 0.0 assert p.conf_down == approx((0.7 - 0.5) * 2.0) == approx(0.2 * 2.0) == 0.4 assert not p.pred_up assert p.pred_down == (p.conf_down > p.conf_thr) == (0.4 > 0.1) == True assert p.prob_up_MERGED == approx(1.0 - 0.7) == approx(0.3) - # setup like above, but now with higher confidence level, which it can't exceed - p = SimModelPrediction(conf_thr = 0.5, prob_up_UP = 0.4, prob_up_DOWN = 0.3) + # setup like above, but now with higher conf thr, which it can't exceed + p = SimModelPrediction(conf_thr = 0.5, prob_UP = 0.4, prob_DOWN = 0.7) assert not p.pred_down @enforce_types def test_do_trust_models_unhappy_path(): for tup in [ - (True, True, 0.4, 0.6), - (True, False, 0.4, 0.6), - (False, True, 0.6, 0.4), + # out of range (True, False, -1.4, 0.6), (True, False, 1.4, 0.6), (True, False, 0.4, -1.6), (True, False, 0.4, 1.6), + # values conflict + (True, True, 0.4, 0.6), # pred_up and pred_down + (True, False, 0.4, 0.6), # pred_up and prob_DOWN > prob_UP + (False, True, 0.6, 0.4), # pred_down and prob_UP > prob_DOWN ]: - (pred_up, pred_down, prob_up_UP, prob_down_DOWN) = tup + (pred_up, pred_down, prob_UP, prob_DOWN) = tup with pytest.raises(ValueError): - _do_trust_models(pred_up, pred_down, prob_up_UP, prob_down_DOWN) + _do_trust_models(pred_up, pred_down, prob_UP, prob_DOWN) print(f"Should have failed on {tup}") @enforce_types @@ -95,18 +91,17 @@ def test_do_trust_models_happy_path(): @enforce_types def test_models_in_conflict_unhappy_path(): - for (prob_up_UP, prob_down_DOWN) in [ - (-1.6, 0.6), - (1.6, 0.6), - (0.6, -1.6), - (0.6, 1.6), + for (prob_UP, prob_DOWN) in [ + (-1.6, 0.6), + (1.6, 0.6), + (0.6, -1.6), + (0.6, 1.6), ]: with pytest.raises(ValueError): - _models_in_conflict(prob_up_UP, prob_down_DOWN) + _models_in_conflict(prob_UP, prob_DOWN) @enforce_types def test_models_in_conflict_happy_path(): - assert _models_in_conflict(0.6, 0.6) assert _models_in_conflict(0.4, 0.4) diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index 15c8710a8..d05f3a548 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -1,86 +1,261 @@ from enforce_typing import enforce_types +import numpy as np +import pytest +from pytest import approx +from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN +from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_state import ( ClassifBaseData, - ClassifBaseData1Dir, + TrueVsPredVals, + HistPerfs, + HistPerfs1Dir, + Profits, + SimState, + PERFS_NAMES, ) @enforce_types def test_classif_base_data_1dir(): - d = ClassifBaseData1Dir() - assert d.ytrues == [] - assert d.probs_up == [] - assert d.ytrues_hat == [] + d = TrueVsPredVals() + assert d.truevals == [] + assert d.predprobs == [] + assert d.predvals == [] assert d.n_correct == 0 assert d.n_trials == 0 # true = up, guess = up (correct guess) - d.update(true_up=True, prob_up=0.6) - assert d.ytrues == [True] - assert d.probs_up == [0.6] - assert d.ytrues_hat == [True] + d.update(trueval=True, predprob=0.6) + assert d.truevals == [True] + assert d.predprobs == [0.6] + assert d.predvals == [True] assert d.n_correct == 1 assert d.n_trials == 1 assert len(d.accuracy()) == 3 assert d.accuracy()[0] == 1.0/1.0 # true = down, guess = down (correct guess) - d.update(true_up=False, prob_up=0.3) - assert d.ytrues == [True, False] - assert d.probs_up == [0.6, 0.3] - assert d.ytrues_hat == [True, False] + d.update(trueval=False, predprob=0.3) + assert d.truevals == [True, False] + assert d.predprobs == [0.6, 0.3] + assert d.predvals == [True, False] assert d.n_correct == 2 assert d.n_trials == 2 assert d.accuracy()[0] == 2.0/2.0 # true = up, guess = down (incorrect guess) - d.update(true_up=True, prob_up=0.4) - assert d.ytrues == [True, False, True] - assert d.probs_up == [0.6, 0.3, 0.4] - assert d.ytrues_hat == [True, False, False] + d.update(trueval=True, predprob=0.4) + assert d.truevals == [True, False, True] + assert d.predprobs == [0.6, 0.3, 0.4] + assert d.predvals == [True, False, False] assert d.n_correct == 2 assert d.n_trials == 3 assert d.accuracy()[0] == approx(2.0/3.0) # true = down, guess = up (incorrect guess) - d.update(true_up=False, prob_up=0.7) - assert d.ytrues == [True, False, True, False] - assert d.probs_up == [0.6, 0.3, 0.4, 0.7] - assert d.ytrues_hat == [True, False, False, True] + d.update(trueval=False, predprob=0.7) + assert d.truevals == [True, False, True, False] + assert d.predprobs == [0.6, 0.3, 0.4, 0.7] + assert d.predvals == [True, False, False, True] assert d.n_correct == 2 assert d.n_trials == 4 assert d.accuracy()[0] == approx(2.0/4.0) + # test classifier metrics + (acc_est, acc_l, acc_u) = d.accuracy() + assert acc_est == approx(0.5) + assert acc_l == approx(0.010009003864986377) + assert acc_u == approx(0.9899909961350136) + + (precision, recall, f1) = d.precision_recall_f1() + assert precision == approx(0.5) + assert recall == approx(0.5) + assert f1 == approx(0.5) + + loss = d.log_loss() + assert loss == approx(0.7469410259762035) + @enforce_types def test_classif_base_data(): - d_UP = ClassifBaseData1Dir() - d_DOWN = ClassifBaseData1Dir() - d = ClassifBaseData(d_UP, d_DOWN) - assert id(d[UP]) == id(d_UP) - assert id(d[DOWN]) == id(d_DOWN) + d = ClassifBaseData() + assert isinstance(d[UP], TrueVsPredVals) + assert isinstance(d[DOWN], TrueVsPredVals) + + # true was UP, both models were correct, 2x in a row + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.3) + d.update(true_UP=True, true_DOWN=False, sim_model_p=p) + d.update(true_UP=True, true_DOWN=False, sim_model_p=p) + assert d[UP].truevals == [True, True] + assert d[UP].predprobs == [0.6, 0.6] + assert d[DOWN].truevals == [False, False] + assert d[DOWN].predprobs == [0.3, 0.3] + + +@pytest.mark.parametrize("dirn", [UP, DOWN]) +def test_hist_perfs_1dir(dirn): + m = HistPerfs1Dir(dirn) + + assert m.acc_ests == m.acc_ls == m.acc_us == [] + assert m.f1s == m.precisions == m.recalls == [] + assert m.losses == [] + + m.update(list(np.arange(0.1, 7.1, 1.0))) # 0.1, 1.1, ..., 6.1 + m.update(list(np.arange(0.2, 7.2, 1.0))) # 0.2, 1.2, ..., 6.2 - p = SimModelPrediction(FIXME) - d.update(true_up_UP, prob_up_UP, p) + assert m.acc_ests == [0.1, 0.2] + assert m.acc_ls == [1.1, 1.2] + assert m.acc_us == [2.1, 2.2] + assert m.f1s == [3.1, 3.2] + assert m.precisions == [4.1, 4.2] + assert m.recalls == [5.1, 5.2] + assert m.losses == [6.1, 6.2] - raise NotImplementedError("Build the rest of me") + dirn_s = dirn_str(dirn) + assert len(PERFS_NAMES) == 7 + target_metrics_names = [f"{name}_{dirn_s}" for name in PERFS_NAMES] + assert m.recent_metrics_names() == target_metrics_names + assert m.recent_metrics_names()[0] == "acc_est_UP" + assert m.recent_metrics_names()[-1] == "loss_UP" + metrics = m.recent_metrics_values() + assert len(metrics) == 7 + assert metrics == { + f"acc_est_{dirn_s}": 0.1, + f"acc_l_{dirn_s}": 1.1, + f"acc_u_{dirn_s}": 2.1, + f"f1_{dirn_s}": 3.1, + f"precision_{dirn_s}": 4.1, + f"recall_{dirn_s}": 5.1, + f"loss_{dirn_s}": 6.1, + } @enforce_types -def test_classif_metrics_1dir(): - raise NotImplementedError("Build me") +def test_hist_perfs(): + hist_perfs = HistPerfs() + assert isinstance(hist_perfs[UP], HistPerfs1Dir) + assert isinstance(hist_perfs[DOWN], HistPerfs1Dir) + + classif_base_data = ClassifBaseData() + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.3) + classif_base_data.update(true_UP=True, true_DOWN=False, sim_model_p=p) + classif_base_data.update(true_UP=True, true_DOWN=False, sim_model_p=p) + + hist_perfs.update(classif_base_data) + + target_metrics_names = [f"{name}_{dirn_str(dirn)}" + for dirn in [UP, DOWN] + for name in PERFS_NAMES] + assert len(target_metrics_names) == 7 * 2 + recent_names = hist_perfs.recent_metrics_names() + assert recent_names == target_metrics_names + assert recent_names[0] == "acc_est_UP" + assert recent_names[1] == "acc_l_UP" + assert recent_names[-1] == "loss_DOWN" + + metrics = hist_perfs.recent_metrics_values() + assert len(metrics) == 7 * 2 + assert sorted(target_metrics_names) == sorted(metrics.keys()) + + # given the metrics: accuracy, f1/precision/recall, and loss, + # then nothing should be outside [0.0, 3.0] + assert 0.0 <= min(metrics.values()) <= max(metrics.values()) <= 3.0 @enforce_types -def test_classif_metrics(): - raise NotImplementedError("Build me") +def test_profits__attributes(): + p = Profits() + assert p.pdr_profits_OCEAN == [] + assert p.trader_profits_USD == [] + + p.update_pdr_profit(2.1) + p.update_pdr_profit(2.2) + assert p.pdr_profits_OCEAN == [2.1, 2.2] + + p.update_trader_profit(3.1) + p.update_trader_profit(3.2) + assert p.trader_profits_USD == [3.1, 3.2] + assert Profits.recent_metrics_names() == \ + ["pdr_profit_OCEAN", "trader_profit_USD"] @enforce_types -def test_profits(): - raise NotImplementedError("Build me") +def test_profits__calc_pdr_profit(): + # true = up, guess = up (correct guess), others fully wrong + profit = Profits.calc_pdr_profit( + others_stake = 2000.0, + others_accuracy = 0.0, + stake_up = 1000.0, + stake_down = 0.0, + revenue = 2.0, + true_up_close = True, + ) + assert profit == 2002.0 + # true = down, guess = down (correct guess), others fully wrong + profit = Profits.calc_pdr_profit( + others_stake = 2000.0, + others_accuracy = 0.0, + stake_up = 0.0, + stake_down = 1000.0, + revenue = 2.0, + true_up_close = False, + ) + assert profit == 2002.0 + + # true = up, guess = down (incorrect guess), others fully right + profit = Profits.calc_pdr_profit( + others_stake = 2000.0, + others_accuracy = 100.0, + stake_up = 0.0, + stake_down = 1000.0, + revenue = 2.0, + true_up_close = True, + ) + assert profit == -1000.0 + + # true = down, guess = up (incorrect guess), others fully right + profit = Profits.calc_pdr_profit( + others_stake = 2000.0, + others_accuracy = 100.0, + stake_up = 1000.0, + stake_down = 0.0, + revenue = 2.0, + true_up_close = False, + ) + assert profit == -1000.0 + + # true = up, guess = up AND down (half-correct), others fully wrong + profit = Profits.calc_pdr_profit( + others_stake = 1000.0, + others_accuracy = 0.00, + stake_up = 1000.0, + stake_down = 100.0, + revenue = 2.0, + true_up_close = True, + ) + assert profit == 502.0 + @enforce_types def test_sim_state(): - raise NotImplementedError("Build me") + state = SimState() + assert state.iter_number == 0 + assert state.sim_model_data is None + assert state.sim_model is None + assert isinstance(state.classif_base, ClassifBaseData) + assert isinstance(state.hist_perfs, HistPerfs) + assert isinstance(state.profits, Profits) + + state.iter_number = 1 + state.sim_model = "foo" + state.init_loop_attributes() + assert state.iter_number == 0 + assert state.sim_model is None + + names = state.recent_metrics_names() + assert len(names) == 7 * 2 + 2 + + # FIXME: need to add some data to state first + metrics = state.recent_metrics_values() + assert len(metrics) == 7 * 2 + 2 From c9ad2fbfae97a70303157d35751c4605a5bd1d3c Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 11:26:29 +0200 Subject: [PATCH 14/50] many more tests passing, including all of test_sim_state.py --- pdr_backend/aimodel/test/test_true_vs_pred.py | 73 +++ pdr_backend/aimodel/true_vs_pred.py | 74 ++++ pdr_backend/sim/sim_engine.py | 35 +- pdr_backend/sim/sim_model.py | 15 +- pdr_backend/sim/sim_state.py | 206 +++------ pdr_backend/sim/test/test_sim_state.py | 418 ++++++++++-------- 6 files changed, 472 insertions(+), 349 deletions(-) create mode 100644 pdr_backend/aimodel/test/test_true_vs_pred.py create mode 100644 pdr_backend/aimodel/true_vs_pred.py diff --git a/pdr_backend/aimodel/test/test_true_vs_pred.py b/pdr_backend/aimodel/test/test_true_vs_pred.py new file mode 100644 index 000000000..9b4a27406 --- /dev/null +++ b/pdr_backend/aimodel/test/test_true_vs_pred.py @@ -0,0 +1,73 @@ +from enforce_typing import enforce_types +from pytest import approx + +from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred + + +@enforce_types +def test_true_vs_pred(): + d = TrueVsPred() + assert d.truevals == [] + assert d.predprobs == [] + assert d.predvals == [] + assert d.n_correct == 0 + assert d.n_trials == 0 + + # true = up, guess = up (correct guess) + d.update(trueval=True, predprob=0.6) + assert d.truevals == [True] + assert d.predprobs == [0.6] + assert d.predvals == [True] + assert d.n_correct == 1 + assert d.n_trials == 1 + assert len(d.accuracy()) == 3 + assert d.accuracy()[0] == 1.0/1.0 + + # true = down, guess = down (correct guess) + d.update(trueval=False, predprob=0.3) + assert d.truevals == [True, False] + assert d.predprobs == [0.6, 0.3] + assert d.predvals == [True, False] + assert d.n_correct == 2 + assert d.n_trials == 2 + assert d.accuracy()[0] == 2.0/2.0 + + # true = up, guess = down (incorrect guess) + d.update(trueval=True, predprob=0.4) + assert d.truevals == [True, False, True] + assert d.predprobs == [0.6, 0.3, 0.4] + assert d.predvals == [True, False, False] + assert d.n_correct == 2 + assert d.n_trials == 3 + assert d.accuracy()[0] == approx(2.0/3.0) + + # true = down, guess = up (incorrect guess) + d.update(trueval=False, predprob=0.7) + assert d.truevals == [True, False, True, False] + assert d.predprobs == [0.6, 0.3, 0.4, 0.7] + assert d.predvals == [True, False, False, True] + assert d.n_correct == 2 + assert d.n_trials == 4 + assert d.accuracy()[0] == approx(2.0/4.0) + + # test performance values + (acc_est, acc_l, acc_u) = d.accuracy() + assert acc_est == approx(0.5) + assert acc_l == approx(0.010009003864986377) + assert acc_u == approx(0.9899909961350136) + + (precision, recall, f1) = d.precision_recall_f1() + assert precision == approx(0.5) + assert recall == approx(0.5) + assert f1 == approx(0.5) + + loss = d.log_loss() + assert loss == approx(0.7469410259762035) + + assert d.perf_names() == PERF_NAMES + assert len(d.perf_values()) == len(PERF_NAMES) + + target_values = [acc_est, acc_l, acc_u, precision, recall, f1, loss] + values = d.perf_values() + for val, target_val in zip(values, target_values): + assert val == approx(target_val) diff --git a/pdr_backend/aimodel/true_vs_pred.py b/pdr_backend/aimodel/true_vs_pred.py new file mode 100644 index 000000000..68063f76c --- /dev/null +++ b/pdr_backend/aimodel/true_vs_pred.py @@ -0,0 +1,74 @@ +from typing import List, Tuple + +from enforce_typing import enforce_types + +from sklearn.metrics import log_loss, precision_recall_fscore_support +from statsmodels.stats.proportion import proportion_confint + +PERF_NAMES = ["acc_est", "acc_l", "acc_u", "f1", "precision", "recall", "loss"] + +class TrueVsPred: + """ + True vs pred vals for a single aimodel, or for a history of models, + + the performances that derive from true vs pred value info + """ + + @enforce_types + def __init__(self): + # 'i' is iteration number i + self.truevals: List[bool] = [] # [i] : true value + self.predprobs: List[float] = [] # [i] : model's pred. prob. + + @enforce_types + def update(self, trueval: bool, predprob: float): + self.truevals.append(trueval) + self.predprobs.append(predprob) + + @property + def predvals(self) -> List[bool]: + """@return [i] : model pred. value""" + return [p > 0.5 for p in self.predprobs] + + @property + def n_correct(self) -> int: + return sum(tv == pv for tv, pv in zip(self.truevals, self.predvals)) + + @property + def n_trials(self) -> int: + return len(self.truevals) + + @enforce_types + def accuracy(self) -> Tuple[float, float, float]: + n_correct, n_trials = self.n_correct, self.n_trials + acc_est = n_correct / n_trials + acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) + return (acc_est, acc_l, acc_u) + + @enforce_types + def precision_recall_f1(self) -> Tuple[float, float, float]: + (precision, recall, f1, _) = precision_recall_fscore_support( + self.truevals, + self.predvals, + average="binary", + zero_division=0.0, + ) + return (precision, recall, f1) + + @enforce_types + def log_loss(self) -> float: + if min(self.truevals) == max(self.truevals): + return 3.0 # magic number + return log_loss(self.truevals, self.predprobs) + + @enforce_types + def perf_names(self) -> List[str]: + return PERF_NAMES + + @enforce_types + def perf_values(self) -> List[float]: + perfs_list = \ + list(self.accuracy()) + \ + list(self.precision_recall_f1()) + \ + [self.log_loss()] + assert len(perfs_list) == len(PERF_NAMES) + return perfs_list diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index beafaeeeb..bfdb3baa4 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -134,38 +134,39 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # build model st = self.st data_f = SimModelDataFactory(self.ppss) - model_d: SimModelData = data_f.build(iter_i, mergdohlcv_df) + model_data: SimModelData = data_f.build(iter_i, mergdohlcv_df) model_f = SimModelFactory(self.aimodel_ss) if model_f.do_build(st.sim_model, test_i): - st.sim_model = model_f.train(test_i, model_d) + st.sim_model = model_f.train(test_i, model_data) # make prediction - sim_model_p: SimModelPrediction = self.model.predict_next() + predprob = self.model.predict_next() # {UP: predprob_UP, DOWN: ..} + conf_thr = self.ppss.trader_ss.sim_confidence_threshold + sim_model_p = SimModelPrediction(conf_thr, predprob[UP], predprob[DOWN]) + # predictoor takes action (stake) stake_up, stake_down = self.pdr.predict_iter(sim_model_p) # trader takes action (trade) - trader_profit = self.trader.trade_iter( + trader_profit_USD = self.trader.trade_iter( cur_close, cur_high, cur_low, sim_model_p, ) # observe true price change - true_up_close = next_close > cur_close - true_UP = next_high > y_thr_UP # did next high go > prev close+% ? - true_DOWN = next_low < y_thr_DOWN # did next low go < prev close-% ? - - # update state - classifier data - st.classif_base.update(true_UP, true_DOWN, sim_model_p) - st.classif_metrics.update(st.classif_base) - - # update predictoor profit - st.profit.update_pdr_profit( + trueval_up_close = next_close > cur_close + trueval = { + UP: next_high > y_thr_UP, # did next high go > prev close+% ? + DOWN: next_low < y_thr_DOWN, # did next low go < prev close-% ? + } + + # calc predictoor profit + pdr_profit_OCEAN = HistProfits.calc_pdr_profit( others_stake, pdr_ss.others_stake_accuracy, - stake_up, stake_down, true_up_close) + stake_up, stake_down, trueval_up_close) - # update trader profit - self.profits.update_trader_profit(trader_profit) + # update state + st.update(trueval, predprob, pdr_profit_OCEAN, trader_profit_USD) # log ut = self._calc_ut(mergedohlcv_df) diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index e2db03f29..5ef8977e0 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -9,15 +9,12 @@ @enforce_types class SimModel(dict): - def __init__(self, ppss: PPSS, model_UP: Aimodel, model_DOWN: Aimodel): - self.ppss: PPSS = ppss + def __init__(self, model_UP: Aimodel, model_DOWN: Aimodel): self[UP] = model_UP self[DOWN] = model_DOWN - - def predict_next(self, d: SimModelData) -> SimModelPrediction: - conf_thr = self.ppss.trader_ss.sim_confidence_threshold - prob_UP = self[UP].predict_ptrue(d[UP].X_test)[0] - prob_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] - - return SimModelPrediction(conf_thr, prob_UP, prob_DOWN) + + def predict_next(self, d: SimModelData) -> dict: + predprob_UP = self[UP].predict_ptrue(d[UP].X_test)[0] + predprob_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] + return {UP: predprob_UP, DOWN: predprob_DOWN} diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 2249d33f2..2ddddb44a 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,94 +1,18 @@ from typing import Dict, List, Optional, Tuple, Union from enforce_typing import enforce_types -from sklearn.metrics import log_loss, precision_recall_fscore_support -from statsmodels.stats.proportion import proportion_confint +from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN -from pdr_backend.sim.sim_model_prediction import SimModelPrediction -PERFS_NAMES = ["acc_est", "acc_l", "acc_u", "f1", "precision", "recall", "loss"] - -class TrueVsPredVals: - @enforce_types - def __init__(self): - # 'i' is iteration number i - self.truevals: List[bool] = [] # [i] : true value - self.predprobs: List[float] = [] # [i] : model's pred. prob. - - @enforce_types - def update(self, trueval: bool, predprob: float): - self.truevals.append(trueval) - self.predprobs.append(predprob) - - @property - def predvals(self) -> List[bool]: - """@return [i] : model pred. value""" - return [p > 0.5 for p in self.predprobs] - - @property - def n_correct(self) -> int: - return sum(tv == pv for tv, pv in zip(self.truevals, self.predvals)) - - @property - def n_trials(self) -> int: - return len(self.truevals) - - @enforce_types - def accuracy(self) -> Tuple[float, float, float]: - n_correct, n_trials = self.n_correct, self.n_trials - acc_est = n_correct / n_trials - acc_l, acc_u = proportion_confint(count=n_correct, nobs=n_trials) - return (acc_est, acc_l, acc_u) - - @enforce_types - def precision_recall_f1(self) -> Tuple[float, float, float]: - (precision, recall, f1, _) = precision_recall_fscore_support( - self.truevals, - self.predvals, - average="binary", - zero_division=0.0, - ) - return (precision, recall, f1) - - @enforce_types - def log_loss(self) -> float: - if min(self.truevals) == max(self.truevals): - return 3.0 # magic number - return log_loss(self.truevals, self.predprobs) - - @enforce_types - def perf_values(self) -> List[float]: - perfs_list = \ - list(self.accuracy()) + \ - list(precision_recall_f1) + \ - [self.log_loss()] - return perfs_list -@enforce_types -class ClassifBaseData(dict): - def __init__(self): - self[UP] = TrueVsPredVals() - self[DOWN] = TrueVsPredVals() - - def update( - self, - - # to establish UP model's accuracy: did next high go > prev close+% ? - true_UP: bool, - - # to establish DOWN model's accuracy: did next low go < prev close-% ? - true_DOWN: bool, - - sim_model_p: SimModelPrediction, - ): - self[UP].update(true_UP, sim_model_p.prob_UP) - self[DOWN].update(true_DOWN, sim_model_p.prob_DOWN) +#============================================================================= +# HistPerfs @enforce_types -class HistPerfs1Dir: - # pylint: disable=too-many-instance-attributes - +class HistPerfs: + """Historical performances, for 1 model dir'n (eg UP)""" + def __init__(self, dirn: Dirn): self.dirn = dirn @@ -104,7 +28,7 @@ def __init__(self, dirn: Dirn): self.losses: List[float] = [] # [i] : log-loss def update(self, perfs_list: list): - """perfs_list typically comes from TrueVsPredVals.perf_values()""" + """perfs_list typically comes from TrueVsPred.perf_values()""" acc_est, acc_l, acc_u, f1, precision, recall, loss = perfs_list self.acc_ests.append(acc_est) @@ -117,22 +41,15 @@ def update(self, perfs_list: list): self.losses.append(loss) - def recent_metrics_names(self) -> List[str]: - s = dirn_str(self.dirn) - return [ - f"acc_est_{s}", - f"acc_l_{s}", - f"acc_u_{s}", - f"f1_{s}", - f"precision_{s}", - f"recall_{s}", - f"loss_{s}", - ] - + def metrics_names(self) -> List[str]: + """@return e.g. ['acc_est_UP', 'acc_l_UP', ..., 'loss_UP]""" + return [f"{name}_{dirn_str(self.dirn)}" + for name in PERF_NAMES] + def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: """Return most recent aimodel metrics""" if not self.acc_ests: - return {key: None for key in HistPerfs.recent_metrics_names()} + return {name: None for name in self.metrics_names()} s = dirn_str(self.dirn) return { @@ -144,33 +61,14 @@ def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: f"recall_{s}": self.recalls[-1], f"loss_{s}": self.losses[-1], } - - -@enforce_types -class HistPerfs(dict): - def __init__(self): - self[UP] = HistPerfs1Dir(UP) - self[DOWN] = HistPerfs1Dir(DOWN) - - def update_recent(self, classif_base: ClassifBaseData): - self[UP].update(classif_base.UP.recent_metrics_values()) - self[DOWN].update(classif_base.DOWN.recent_metrics_values()) - - def recent_metrics_names(self) -> List[str]: - names = [] - names += self[UP].recent_metrics_names() - names += self[DOWN].recent_metrics_names() - return names - - def recent_metrics_values(self) -> Dict[str, float]: - metrics = {} - metrics.update(self[UP].recent_metrics_values()) - metrics.update(self[DOWN].recent_metrics_values()) - return metrics +#============================================================================= +# HistProfits + +PROFIT_NAMES = ["pdr_profit_OCEAN", "trader_profit_USD"] @enforce_types -class Profits: +class HistProfits: def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit self.trader_profits_USD: List[float] = [] # [i] : trader-profit @@ -190,39 +88,39 @@ def calc_pdr_profit( assert stake_down >= 0.0 assert revenue >= 0.0 + amt_sent = stake_up + stake_down others_stake_correct = others_stake * others_accuracy tot_stake = others_stake + stake_up + stake_down - - acct_up_profit = acct_down_profit = 0.0 - acct_up_profit -= stake_up - acct_down_profit -= stake_down if true_up_close: tot_stake_correct = others_stake_correct + stake_up percent_to_me = stake_up / tot_stake_correct - acct_up_profit += (revenue + tot_stake) * percent_to_me + amt_received = (revenue + tot_stake) * percent_to_me else: tot_stake_correct = others_stake_correct + stake_down percent_to_me = stake_down / tot_stake_correct - acct_down_profit += (revenue + tot_stake) * percent_to_me - pdr_profit_OCEAN = acct_up_profit + acct_down_profit + amt_received = (revenue + tot_stake) * percent_to_me + pdr_profit_OCEAN = amt_received - amt_sent return pdr_profit_OCEAN - def update_pdr_profit(self, pdr_profit_OCEAN): + def update(self, pdr_profit_OCEAN: float, trader_profit_USD: float): self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) - - def update_trader_profit(self, trader_profit_USD: float): self.trader_profits_USD.append(trader_profit_USD) - def recent_metrics_names(self) -> List[str]: - return ["pdr_profit_OCEAN", "trader_profit_USD"] + def metrics_names(self) -> List[str]: + return PROFIT_NAMES def recent_metrics_values(self) -> Dict[str, float]: + if not self.pdr_profits_OCEAN: + return {name: None for name in self.metrics_names()} + return { "pdr_profit_OCEAN" : self.pdr_profits_OCEAN[-1], "trader_profit_USD" : self.trader_profits_USD[-1], } - -# pylint: disable=too-many-instance-attributes + +#============================================================================= +# SimState + @enforce_types class SimState: def __init__(self): @@ -232,18 +130,42 @@ def init_loop_attributes(self): self.iter_number = 0 self.sim_model_data: Optional[SimModelData] = None self.sim_model: Optional[SimModel] = None - self.classif_base = ClassifBaseData() - self.hist_perfs = HistPerfs() - self.profits = Profits() + self.true_vs_pred = {UP: TrueVsPred(), DOWN: TrueVsPred()} + self.hist_perfs = {UP: HistPerfs(UP), DOWN: HistPerfs(DOWN)} + self.hist_profits = HistProfits() - def recent_metrics_names(self) -> List[str]: - return self.hist_perfs.recent_metrics_names() + \ - self.profits.recent_metrics_names() + def update( + self, + trueval: dict, + predprob: dict, + pdr_profit_OCEAN: float, + trader_profit_USD: float, + ): + """ + @arguments + trueval -- dict of {UP: trueval_UP, DOWN: trueval_DOWN} + predprob -- dict of {UP: predprob_UP, DOWN: predprob_DOWN} + pdr_profit_OCEAN -- + trader_profit_USD -- + """ + self.true_vs_pred[UP].update(trueval[UP], predprob[UP]) + self.true_vs_pred[DOWN].update(trueval[DOWN], predprob[DOWN]) + + self.hist_perfs[UP].update(self.true_vs_pred[UP].perf_values()) + self.hist_perfs[DOWN].update(self.true_vs_pred[DOWN].perf_values()) + + self.hist_profits.update(pdr_profit_OCEAN, trader_profit_USD) + + def metrics_names(self) -> List[str]: + return self.hist_perfs[UP].metrics_names() + \ + self.hist_perfs[DOWN].metrics_names() + \ + self.hist_profits.metrics_names() def recent_metrics_values(self) -> List[Union[int, float]]: """Return most recent aimodel metrics + profit metrics""" metrics = {} - metrics.update(self.hist_perfs.recent_metrics_values()) - metrics.update(self.profits.recent_metrics_values()) + metrics.update(self.hist_perfs[UP].recent_metrics_values()) + metrics.update(self.hist_perfs[DOWN].recent_metrics_values()) + metrics.update(self.hist_profits.recent_metrics_values()) return metrics diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index d05f3a548..f5b00781c 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -3,185 +3,112 @@ import pytest from pytest import approx +from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN -from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_state import ( - ClassifBaseData, - TrueVsPredVals, HistPerfs, - HistPerfs1Dir, - Profits, + HistProfits, + PROFIT_NAMES, SimState, - PERFS_NAMES, ) -@enforce_types -def test_classif_base_data_1dir(): - d = TrueVsPredVals() - assert d.truevals == [] - assert d.predprobs == [] - assert d.predvals == [] - assert d.n_correct == 0 - assert d.n_trials == 0 - - # true = up, guess = up (correct guess) - d.update(trueval=True, predprob=0.6) - assert d.truevals == [True] - assert d.predprobs == [0.6] - assert d.predvals == [True] - assert d.n_correct == 1 - assert d.n_trials == 1 - assert len(d.accuracy()) == 3 - assert d.accuracy()[0] == 1.0/1.0 - - # true = down, guess = down (correct guess) - d.update(trueval=False, predprob=0.3) - assert d.truevals == [True, False] - assert d.predprobs == [0.6, 0.3] - assert d.predvals == [True, False] - assert d.n_correct == 2 - assert d.n_trials == 2 - assert d.accuracy()[0] == 2.0/2.0 - - # true = up, guess = down (incorrect guess) - d.update(trueval=True, predprob=0.4) - assert d.truevals == [True, False, True] - assert d.predprobs == [0.6, 0.3, 0.4] - assert d.predvals == [True, False, False] - assert d.n_correct == 2 - assert d.n_trials == 3 - assert d.accuracy()[0] == approx(2.0/3.0) - - # true = down, guess = up (incorrect guess) - d.update(trueval=False, predprob=0.7) - assert d.truevals == [True, False, True, False] - assert d.predprobs == [0.6, 0.3, 0.4, 0.7] - assert d.predvals == [True, False, False, True] - assert d.n_correct == 2 - assert d.n_trials == 4 - assert d.accuracy()[0] == approx(2.0/4.0) - - # test classifier metrics - (acc_est, acc_l, acc_u) = d.accuracy() - assert acc_est == approx(0.5) - assert acc_l == approx(0.010009003864986377) - assert acc_u == approx(0.9899909961350136) - - (precision, recall, f1) = d.precision_recall_f1() - assert precision == approx(0.5) - assert recall == approx(0.5) - assert f1 == approx(0.5) - - loss = d.log_loss() - assert loss == approx(0.7469410259762035) +#============================================================================= +# test HistPerfs -@enforce_types -def test_classif_base_data(): - d = ClassifBaseData() - assert isinstance(d[UP], TrueVsPredVals) - assert isinstance(d[DOWN], TrueVsPredVals) +@pytest.mark.parametrize("dirn", [UP, DOWN]) +def test_hist_perfs__basic_init(dirn): + dirn_s = dirn_str(dirn) + + hist_perfs = HistPerfs(dirn) - # true was UP, both models were correct, 2x in a row - p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.3) - d.update(true_UP=True, true_DOWN=False, sim_model_p=p) - d.update(true_UP=True, true_DOWN=False, sim_model_p=p) - assert d[UP].truevals == [True, True] - assert d[UP].predprobs == [0.6, 0.6] - assert d[DOWN].truevals == [False, False] - assert d[DOWN].predprobs == [0.3, 0.3] + assert hist_perfs.acc_ests == hist_perfs.acc_ls == hist_perfs.acc_us == [] + assert hist_perfs.f1s == hist_perfs.precisions == hist_perfs.recalls == [] + assert hist_perfs.losses == [] + assert len(PERF_NAMES) == 7 + target_names = [f"{name}_{dirn_s}" for name in PERF_NAMES] + assert hist_perfs.metrics_names() == target_names + assert hist_perfs.metrics_names()[0] == f"acc_est_{dirn_s}" + assert hist_perfs.metrics_names()[-1] == f"loss_{dirn_s}" + values = hist_perfs.recent_metrics_values() + assert len(values) == 7 + assert sorted(values.keys()) == sorted(target_names) + for val in values.values(): + assert val is None + @pytest.mark.parametrize("dirn", [UP, DOWN]) -def test_hist_perfs_1dir(dirn): - m = HistPerfs1Dir(dirn) - - assert m.acc_ests == m.acc_ls == m.acc_us == [] - assert m.f1s == m.precisions == m.recalls == [] - assert m.losses == [] +def test_hist_perfs__main(dirn): + dirn_s = dirn_str(dirn) + + hist_perfs = HistPerfs(dirn) - m.update(list(np.arange(0.1, 7.1, 1.0))) # 0.1, 1.1, ..., 6.1 - m.update(list(np.arange(0.2, 7.2, 1.0))) # 0.2, 1.2, ..., 6.2 + perfs_list1 = list(np.arange(0.1, 7.1, 1.0)) # 0.1, 1.1, ..., 6.1 + perfs_list2 = list(np.arange(0.2, 7.2, 1.0)) # 0.2, 1.2, ..., 6.2 + hist_perfs.update(perfs_list1) + hist_perfs.update(perfs_list2) - assert m.acc_ests == [0.1, 0.2] - assert m.acc_ls == [1.1, 1.2] - assert m.acc_us == [2.1, 2.2] - assert m.f1s == [3.1, 3.2] - assert m.precisions == [4.1, 4.2] - assert m.recalls == [5.1, 5.2] - assert m.losses == [6.1, 6.2] + assert hist_perfs.acc_ests == [0.1, 0.2] + assert hist_perfs.acc_ls == [1.1, 1.2] + assert hist_perfs.acc_us == [2.1, 2.2] + assert hist_perfs.f1s == [3.1, 3.2] + assert hist_perfs.precisions == [4.1, 4.2] + assert hist_perfs.recalls == [5.1, 5.2] + assert hist_perfs.losses == [6.1, 6.2] - dirn_s = dirn_str(dirn) - assert len(PERFS_NAMES) == 7 - target_metrics_names = [f"{name}_{dirn_s}" for name in PERFS_NAMES] - assert m.recent_metrics_names() == target_metrics_names - assert m.recent_metrics_names()[0] == "acc_est_UP" - assert m.recent_metrics_names()[-1] == "loss_UP" - - metrics = m.recent_metrics_values() - assert len(metrics) == 7 - assert metrics == { - f"acc_est_{dirn_s}": 0.1, - f"acc_l_{dirn_s}": 1.1, - f"acc_u_{dirn_s}": 2.1, - f"f1_{dirn_s}": 3.1, - f"precision_{dirn_s}": 4.1, - f"recall_{dirn_s}": 5.1, - f"loss_{dirn_s}": 6.1, + values = hist_perfs.recent_metrics_values() + assert len(values) == 7 + assert f"acc_est_{dirn_s}" in values.keys() + assert values == { + f"acc_est_{dirn_s}": 0.2, + f"acc_l_{dirn_s}": 1.2, + f"acc_u_{dirn_s}": 2.2, + f"f1_{dirn_s}": 3.2, + f"precision_{dirn_s}": 4.2, + f"recall_{dirn_s}": 5.2, + f"loss_{dirn_s}": 6.2, } -@enforce_types -def test_hist_perfs(): - hist_perfs = HistPerfs() - assert isinstance(hist_perfs[UP], HistPerfs1Dir) - assert isinstance(hist_perfs[DOWN], HistPerfs1Dir) - - classif_base_data = ClassifBaseData() - p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.3) - classif_base_data.update(true_UP=True, true_DOWN=False, sim_model_p=p) - classif_base_data.update(true_UP=True, true_DOWN=False, sim_model_p=p) - hist_perfs.update(classif_base_data) - target_metrics_names = [f"{name}_{dirn_str(dirn)}" - for dirn in [UP, DOWN] - for name in PERFS_NAMES] - assert len(target_metrics_names) == 7 * 2 - - recent_names = hist_perfs.recent_metrics_names() - assert recent_names == target_metrics_names - assert recent_names[0] == "acc_est_UP" - assert recent_names[1] == "acc_l_UP" - assert recent_names[-1] == "loss_DOWN" - - metrics = hist_perfs.recent_metrics_values() - assert len(metrics) == 7 * 2 - assert sorted(target_metrics_names) == sorted(metrics.keys()) +#============================================================================= +# test HistProfits + +@enforce_types +def test_hist_profits__basic_init(): + hist_profits = HistProfits() + + assert hist_profits.pdr_profits_OCEAN == [] + assert hist_profits.trader_profits_USD == [] - # given the metrics: accuracy, f1/precision/recall, and loss, - # then nothing should be outside [0.0, 3.0] - assert 0.0 <= min(metrics.values()) <= max(metrics.values()) <= 3.0 + target_names = PROFIT_NAMES + names = hist_profits.metrics_names() + assert names == target_names + + values = hist_profits.recent_metrics_values() + assert sorted(values.keys()) == sorted(names) + for val in values.values(): + assert val is None + @enforce_types -def test_profits__attributes(): - p = Profits() - assert p.pdr_profits_OCEAN == [] - assert p.trader_profits_USD == [] - - p.update_pdr_profit(2.1) - p.update_pdr_profit(2.2) - assert p.pdr_profits_OCEAN == [2.1, 2.2] +def test_hist_profits__update(): + hist_profits = HistProfits() - p.update_trader_profit(3.1) - p.update_trader_profit(3.2) - assert p.trader_profits_USD == [3.1, 3.2] + hist_profits.update(2.1, 3.1) + hist_profits.update(2.2, 3.2) + assert hist_profits.pdr_profits_OCEAN == [2.1, 2.2] + assert hist_profits.trader_profits_USD == [3.1, 3.2] - assert Profits.recent_metrics_names() == \ - ["pdr_profit_OCEAN", "trader_profit_USD"] + values = hist_profits.recent_metrics_values() + assert values == {"pdr_profit_OCEAN" : 2.2, + "trader_profit_USD": 3.2} + @enforce_types -def test_profits__calc_pdr_profit(): +def test_hist_profits__calc_pdr_profit(): # true = up, guess = up (correct guess), others fully wrong - profit = Profits.calc_pdr_profit( + profit = HistProfits.calc_pdr_profit( others_stake = 2000.0, others_accuracy = 0.0, stake_up = 1000.0, @@ -192,7 +119,7 @@ def test_profits__calc_pdr_profit(): assert profit == 2002.0 # true = down, guess = down (correct guess), others fully wrong - profit = Profits.calc_pdr_profit( + profit = HistProfits.calc_pdr_profit( others_stake = 2000.0, others_accuracy = 0.0, stake_up = 0.0, @@ -203,9 +130,9 @@ def test_profits__calc_pdr_profit(): assert profit == 2002.0 # true = up, guess = down (incorrect guess), others fully right - profit = Profits.calc_pdr_profit( + profit = HistProfits.calc_pdr_profit( others_stake = 2000.0, - others_accuracy = 100.0, + others_accuracy = 1.0, stake_up = 0.0, stake_down = 1000.0, revenue = 2.0, @@ -214,9 +141,9 @@ def test_profits__calc_pdr_profit(): assert profit == -1000.0 # true = down, guess = up (incorrect guess), others fully right - profit = Profits.calc_pdr_profit( + profit = HistProfits.calc_pdr_profit( others_stake = 2000.0, - others_accuracy = 100.0, + others_accuracy = 1.0, stake_up = 1000.0, stake_down = 0.0, revenue = 2.0, @@ -225,7 +152,18 @@ def test_profits__calc_pdr_profit(): assert profit == -1000.0 # true = up, guess = up AND down (half-correct), others fully wrong - profit = Profits.calc_pdr_profit( + # summary: I should get back all my stake $, plus stake $ of others + # calculations: + # - sent (by me) = stake_up + stake_down = 1000 + 100 = 1100 + # - tot_stake (by all) = others_stake + stake_up + stake_down + # = 1000 + 1000 + 100 = 2100 + # - tot_stake_correct (by all) = others_stake_correct + stake_up + # = 1000*0.0 + 1000 = 1000 + # - percent_to_me = stake_up / tot_stake_correct = 1000/1000 = 1.0 + # - rec'd (to me) = (revenue + tot_stake) * percent_to_me + # = (2 + 3100) * 1.0 = 3102 + # - profit = received - sent = 2102 - 1100 = 1002 + profit = HistProfits.calc_pdr_profit( others_stake = 1000.0, others_accuracy = 0.00, stake_up = 1000.0, @@ -233,29 +171,147 @@ def test_profits__calc_pdr_profit(): revenue = 2.0, true_up_close = True, ) - assert profit == 502.0 + assert profit == 1002.0 + + # true = up, guess = lots up & some down, others 30% accurate + # summary: + # calculations: + # - amt_sent = stake_up + stake_down = 1000 + 100 = 1100 + # - others_stake_correct = 1000 * 0.3 = 300 + # - tot_stake = others_stake + stake_up + stake_down + # = 1000 + 1000 + 100 = 2100 + # - tot_stake_correct = others_stake_correct + stake_up + # = 1000*0.30 + 1000 = 300 + 1000 = 1300 + # - percent_to_me = stake_up / tot_stake_correct + # = 1000/1300 = 0.7692307692307693 + # - amt_received = (revenue + tot_stake) * percent_to_me + # = (2 + 2100) * 0.769230 = 1616.9230769 + # - profit = received - sent = 1616.9230769 - 1100 = 516.923 + profit = HistProfits.calc_pdr_profit( + others_stake = 1000.0, + others_accuracy = 0.30, + stake_up = 1000.0, + stake_down = 100.0, + revenue = 2.0, + true_up_close = True, + ) + assert profit == approx(516.923) + + +@enforce_types +def test_hist_profits__calc_pdr_profit__unhappy_path(): + o_stake = 2000.0 + o_accuracy = 0.51 + stake_up = 1000.0 + stake_down = 100.0 + revenue = 15.0 + true_up_close = True + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + -0.1, o_accuracy, stake_up, stake_down, revenue, true_up_close, + ) + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + o_stake, -0.1, stake_up, stake_down, revenue, true_up_close, + ) + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + o_stake, +1.1, stake_up, stake_down, revenue, true_up_close, + ) + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + o_stake, o_accuracy, -0.1, stake_down, revenue, true_up_close, + ) + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + o_stake, o_accuracy, stake_up, -0.1, revenue, true_up_close, + ) + + with pytest.raises(AssertionError): + HistProfits.calc_pdr_profit( + o_stake, o_accuracy, stake_up, stake_down, -0.1, true_up_close, + ) + + +#============================================================================= +# test SimState +@enforce_types +def test_sim_state__basic_init(): + st = SimState() + assert st.iter_number == 0 + assert st.sim_model_data is None + assert st.sim_model is None + assert isinstance(st.true_vs_pred[UP], TrueVsPred) + assert isinstance(st.true_vs_pred[DOWN], TrueVsPred) + assert isinstance(st.hist_perfs[UP], HistPerfs) + assert isinstance(st.hist_perfs[DOWN], HistPerfs) + assert isinstance(st.hist_profits, HistProfits) + + target_names = [f"{name}_{dirn_str(dirn)}" + for dirn in [UP, DOWN] + for name in PERF_NAMES] + PROFIT_NAMES + assert len(target_names) == 7 * 2 + 2 + names = st.metrics_names() + assert names == target_names + values = st.recent_metrics_values() + assert len(values) == 7 * 2 + 2 + assert sorted(values.keys()) == sorted(names) + for val in values.values(): + assert val is None + + @enforce_types -def test_sim_state(): - state = SimState() - assert state.iter_number == 0 - assert state.sim_model_data is None - assert state.sim_model is None - assert isinstance(state.classif_base, ClassifBaseData) - assert isinstance(state.hist_perfs, HistPerfs) - assert isinstance(state.profits, Profits) - - state.iter_number = 1 - state.sim_model = "foo" - state.init_loop_attributes() - assert state.iter_number == 0 - assert state.sim_model is None - - names = state.recent_metrics_names() - assert len(names) == 7 * 2 + 2 - - # FIXME: need to add some data to state first - metrics = state.recent_metrics_values() - assert len(metrics) == 7 * 2 + 2 +def test_sim_state__init_loop_attributes(): + # init + st = SimState() + + # change after init + st.iter_number = 1 + st.sim_model = "foo" + + # should go back to init state + st.init_loop_attributes() + + # check + assert st.iter_number == 0 + assert st.sim_model is None + + +@enforce_types +def test_sim_state__main(): + st = SimState() + + # update + trueval = {UP:True, DOWN:False} + predprob = {UP:0.6, DOWN:0.3} + + st.update(trueval, predprob, pdr_profit_OCEAN=1.4, trader_profit_USD=1.5) + st.update(trueval, predprob, pdr_profit_OCEAN=2.4, trader_profit_USD=2.5) + + # test true_vs_pred + assert st.true_vs_pred[UP].truevals == [True, True] + assert st.true_vs_pred[UP].predprobs == [0.6, 0.6] + assert st.true_vs_pred[DOWN].truevals == [False, False] + assert st.true_vs_pred[DOWN].predprobs == [0.3, 0.3] + + # test hist_perfs + values = st.recent_metrics_values() + assert len(values) == 7 * 2 + 2 + for name, val in values.items(): + if name == "pdr_profit_OCEAN": + assert val == 2.4 + elif name == "trader_profit_USD": + assert val == 2.5 + elif "loss" in name: + assert 0.0 <= val <= 3.0 + else: # hist_perfs value + assert 0.0 <= val <= 1.0, (name, val) + From c73deb18e1ccc612fad8adb68cef97920f78afc7 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 12:55:02 +0200 Subject: [PATCH 15/50] fixing many bugs --- pdr_backend/sim/constants.py | 5 +-- pdr_backend/sim/sim_engine.py | 31 ++++++++++--------- pdr_backend/sim/sim_model.py | 21 ++++++++++--- pdr_backend/sim/sim_model_data.py | 8 +++-- pdr_backend/sim/sim_model_data_factory.py | 19 ++++++------ pdr_backend/sim/sim_model_factory.py | 12 +++---- pdr_backend/sim/sim_predictoor.py | 4 +-- pdr_backend/sim/test/test_sim_constants.py | 16 +++++++--- pdr_backend/sim/test/test_sim_engine.py | 7 +++-- pdr_backend/sim/test/test_sim_logger.py | 1 + pdr_backend/sim/test/test_sim_model.py | 27 +++++++--------- pdr_backend/sim/test/test_sim_model_data.py | 7 ++++- .../sim/test/test_sim_model_data_factory.py | 13 ++------ .../sim/test/test_sim_model_factory.py | 12 +++---- pdr_backend/sim/test/test_sim_predictoor.py | 6 ++-- 15 files changed, 103 insertions(+), 86 deletions(-) diff --git a/pdr_backend/sim/constants.py b/pdr_backend/sim/constants.py index 91cbb68fd..684c0097d 100644 --- a/pdr_backend/sim/constants.py +++ b/pdr_backend/sim/constants.py @@ -1,10 +1,11 @@ -from enum import Enum +from enum import IntEnum from enforce_typing import enforce_types -class Dirn(Enum): +class Dirn(IntEnum): UP = 1 DOWN = 2 + UP = Dirn.UP DOWN = Dirn.DOWN diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index bfdb3baa4..10ba38f34 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -23,6 +23,9 @@ from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_logger import SimLogLine from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_data_factory import SimModelDataFactory +from pdr_backend.sim.sim_model_factory import SimModelFactory +from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_plotter import SimPlotter from pdr_backend.sim.sim_predictoor import SimPredictoor from pdr_backend.sim.sim_state import SimState @@ -81,12 +84,12 @@ def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss @property - def aimodel_ss_ss(self) -> AimodelSS: + def aimodel_ss(self) -> AimodelSS: return self.pdr_ss.aimodel_ss @property def transform(self) -> str: - return self.aimodel_ss.transform + return self.pdr_ss.aimodel_data_ss.transform @property def others_stake(self) -> float: @@ -122,25 +125,25 @@ def run(self): mergedohlcv_df = f.get_mergedohlcv_df() # main loop! - for test_i in range(self.ppss.sim_ss.test_n): - self.run_one_iter(test_i, mergedohlcv_df) + for iter_i in range(self.ppss.sim_ss.test_n): + self.run_one_iter(iter_i, mergedohlcv_df) # done logger.info("Done all iters.") # pylint: disable=too-many-statements# pylint: disable=too-many-statements @enforce_types - def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): + def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): # build model st = self.st - data_f = SimModelDataFactory(self.ppss) - model_data: SimModelData = data_f.build(iter_i, mergdohlcv_df) - model_f = SimModelFactory(self.aimodel_ss) - if model_f.do_build(st.sim_model, test_i): - st.sim_model = model_f.train(test_i, model_data) + data_factory = SimModelDataFactory(self.ppss) + model_data: SimModelData = data_factory.build(iter_i, mergedohlcv_df) + model_factory = SimModelFactory(self.aimodel_ss) + if model_factory.do_build(st.sim_model, iter_i): + st.sim_model = model_factory.build(model_data) # make prediction - predprob = self.model.predict_next() # {UP: predprob_UP, DOWN: ..} + predprob = self.st.sim_model.predict_next(model_data.X_test) conf_thr = self.ppss.trader_ss.sim_confidence_threshold sim_model_p = SimModelPrediction(conf_thr, predprob[UP], predprob[DOWN]) @@ -170,15 +173,15 @@ def run_one_iter(self, test_i: int, mergedohlcv_df: pl.DataFrame): # log ut = self._calc_ut(mergedohlcv_df) - SimLogLine(ppss, st, test_i, ut).log() + SimLogLine(ppss, st, iter_i, ut).log() # plot - do_save_state, is_final_state = self._do_save_state(test_i) + do_save_state, is_final_state = self._do_save_state(iter_i) if do_save_state: d_UP = self._aimodel_plotdata_1dirn(UP) d_DOWN = self._aimodel_plotdata_1dirn(DOWN) d = {UP: d_UP, DOWN: d_DOWN} - st.iter_number = test_i + st.iter_number = iter_i self.sim_plotter.save_state(st, d, is_final_state) @enforce_types diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index 5ef8977e0..bded6ed5f 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -13,8 +13,21 @@ def __init__(self, model_UP: Aimodel, model_DOWN: Aimodel): self[UP] = model_UP self[DOWN] = model_DOWN - def predict_next(self, d: SimModelData) -> dict: - predprob_UP = self[UP].predict_ptrue(d[UP].X_test)[0] - predprob_DOWN = self[DOWN].predict_ptrue(d[DOWN].X_test)[0] - return {UP: predprob_UP, DOWN: predprob_DOWN} + def predict_next(self, X_test: dict) -> dict: + """ + @arguments + X_test_dict -- dict of {UP: X_test_UP_array, DOWN: X_test_DOWN_arr.} + It expects each array to have exactly 1 row, to predict from + + @return + predprob -- dict of {UP: predprob_UP_float, DOWN: predprob_DOWN_float} + """ + assert X_test[UP].shape[0] == 1 + assert X_test[DOWN].shape[0] == 1 + + predprob_UP = self[UP].predict_ptrue(X_test[UP])[0] + predprob_DOWN = self[DOWN].predict_ptrue(X_test[UP])[0] + predprob = {UP: predprob_UP, DOWN: predprob_DOWN} + + return predprob diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index 99605e865..c1730e438 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Tuple, Union from enforce_typing import enforce_types import numpy as np @@ -36,8 +36,8 @@ def X_test(self) -> np.ndarray: def ytrue_train(self) -> np.ndarray: return self.ytrue[self.st:self.fin] -@enforce_types class SimModelData(dict): + @enforce_types def __init__( self, data_UP: SimModelData1Dir, @@ -45,3 +45,7 @@ def __init__( ): self[UP] = data_UP self[DOWN] = data_DOWN + + @property + def X_test(self) -> Tuple[np.ndarray, np.ndarray]: + return {UP: self[UP].X_test, DOWN: self[DOWN].X_test} diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index 13d4bf9d1..c1b868e4f 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -5,7 +5,6 @@ from pdr_backend.cli.arg_feed import ArgFeed from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory -from pdr_backend.cli.predict_train_feedset import PredictTrainFeedset from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS from pdr_backend.ppss.ppss import PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS @@ -13,21 +12,20 @@ class SimModelDataFactory: @enforce_types - def __init__(self, ppss: PPSS, predict_train_feedset: PredictTrainFeedset): + def __init__(self, ppss: PPSS): self.ppss = ppss - self.predict_train_feedset = predict_train_feedset - + @property def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss - - @property - def aimodel_data_ss(self) -> AimodelDataSS: - return self.pdr_ss.aimodel_data_ss @property def class_thr(self) -> float: - return self.aimodel_data_ss.class_thr + return self.pdr_ss.aimodel_data_ss.class_thr + + @property + def predict_feed(self) -> ArgFeed: + return self.pdr_ss.predict_train_feedsets[0].predict @enforce_types def testshift(self, test_i: int) -> int: @@ -41,7 +39,8 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: testshift = self.testshift(test_i) # eg [99, 98, .., 2, 1, 0] data_f = AimodelDataFactory(self.pdr_ss) - p: ArgFeed = self.predict_train_feedset.predict + p: ArgFeed = self.predict_feed + _, _, y_close, _, _ = data_f.create_xy( df, testshift, p.variant_close(), ArgFeeds([p.variant_close()]), ) diff --git a/pdr_backend/sim/sim_model_factory.py b/pdr_backend/sim/sim_model_factory.py index 20ba084d0..9e10b14fe 100644 --- a/pdr_backend/sim/sim_model_factory.py +++ b/pdr_backend/sim/sim_model_factory.py @@ -3,19 +3,15 @@ from enforce_typing import enforce_types from pdr_backend.aimodel.aimodel_factory import AimodelFactory -from pdr_backend.ppss.ppss import PPSS +from pdr_backend.ppss.aimodel_ss import AimodelSS from pdr_backend.sim.constants import UP, DOWN from pdr_backend.sim.sim_model import SimModel from pdr_backend.sim.sim_model_data import SimModelData class SimModelFactory: @enforce_types - def __init__(self, ppss: PPSS): - self.ppss = ppss - - @property - def aimodel_ss(self): - return self.ppss.predictoor_ss.aimodel_ss + def __init__(self, aimodel_ss: AimodelSS): + self.aimodel_ss = aimodel_ss @enforce_types def do_build(self, prev_model: Optional[SimModel], test_i: int) -> bool: @@ -41,5 +37,5 @@ def build(self, data: SimModelData) -> SimModel: None, ) - sim_model = SimModel(self.ppss, model_UP, model_DOWN) + sim_model = SimModel(model_UP, model_DOWN) return sim_model diff --git a/pdr_backend/sim/sim_predictoor.py b/pdr_backend/sim/sim_predictoor.py index 809bfa8a9..157f7b43b 100644 --- a/pdr_backend/sim/sim_predictoor.py +++ b/pdr_backend/sim/sim_predictoor.py @@ -20,11 +20,11 @@ def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: if not p.do_trust_models(): stake_up = 0 stake_down = 0 - elif p.prob_up_UP >= p.prob_down_DOWN: + elif p.prob_UP >= p.prob_DOWN: stake_amt = self.max_stake_amt * p.conf_up stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) - else: # p.prob_down_DOWN > p.prob_up_UP + else: # p.prob_DOWN > p.prob_UP stake_amt = self.max_stake_amt * p.conf_down stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) diff --git a/pdr_backend/sim/test/test_sim_constants.py b/pdr_backend/sim/test/test_sim_constants.py index 7897b9798..afe186d64 100644 --- a/pdr_backend/sim/test/test_sim_constants.py +++ b/pdr_backend/sim/test/test_sim_constants.py @@ -1,9 +1,10 @@ from enforce_typing import enforce_types +import pytest -from pdr_backend.sim.constants import Dirn, UP, DOWN +from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN @enforce_types -def test_sim_constants(): +def test_sim_constants__basic(): assert UP == Dirn.UP assert DOWN == Dirn.DOWN @@ -16,11 +17,18 @@ def test_sim_constants(): assert 3 not in Dirn assert "up" not in Dirn +@enforce_types +def test_sim_constants__dirn_str(): assert dirn_str(UP) == "UP" assert dirn_str(DOWN) == "DOWN" - with pytest.raises(ValueError): + with pytest.raises(TypeError): _ = dirn_str(3) with pytest.raises(TypeError): _ = dirn_str("not an int") - + +@enforce_types +def test_sim_constants__can_sort(): + # this is possible because Dirn inherits from IntEnum, vs Enum :) + assert sorted([Dirn.UP, Dirn.DOWN]) == [Dirn.UP, Dirn.DOWN] + assert sorted([UP, DOWN]) == [UP, DOWN] diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index 4f529b4e0..4cda0cfef 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -66,10 +66,13 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): feedsets = ppss.predictoor_ss.predict_train_feedsets sim_engine = SimEngine(ppss, feedsets[0]) - assert sim_engine.model is None + assert sim_engine.st.sim_model is None sim_engine.run() - assert isinstance(sim_engine.model, Aimodel) + # basic test that engine ran + assert isinstance(sim_engine.st.sim_model, SimModel) + + # basic tests for plots app = Dash("pdr_backend.sim.sim_dash") app.config["suppress_callback_exceptions"] = True app.run_id = sim_engine.multi_id diff --git a/pdr_backend/sim/test/test_sim_logger.py b/pdr_backend/sim/test/test_sim_logger.py index bb6fd4cef..48af87c96 100644 --- a/pdr_backend/sim/test/test_sim_logger.py +++ b/pdr_backend/sim/test/test_sim_logger.py @@ -21,6 +21,7 @@ def test_sim_logger(tmpdir, caplog): st = Mock(spec=SimState) st.ytrues = [True, False, True, False, True] + st.recent_metrics = Mock() st.recent_metrics.return_value = { "pdr_profit_OCEAN": 1.0, "trader_profit_USD": 2.0, diff --git a/pdr_backend/sim/test/test_sim_model.py b/pdr_backend/sim/test/test_sim_model.py index df5cce9e9..90fd36c11 100644 --- a/pdr_backend/sim/test/test_sim_model.py +++ b/pdr_backend/sim/test/test_sim_model.py @@ -3,27 +3,22 @@ from enforce_typing import enforce_types import numpy as np +from pdr_backend.sim.constants import UP, DOWN from pdr_backend.aimodel.aimodel import Aimodel -from pdr_backend.ppss.ppss import PPSS from pdr_backend.sim.sim_model import SimModel -from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir -from pdr_backend.sim.sim_model_prediction import SimModelPrediction @enforce_types -def test_sim_model(): - ppss = Mock(spec=PPSS) - ppss.trader_ss = Mock() - ppss.trader_ss.sim_confidence_threshold = 0.1 - +def test_sim_model(): model_UP = Mock(spec=Aimodel) - model_UP.predict_ptrue = Mock(return_value=np.array([0.1, 0.2, 0.3])) + model_UP.predict_ptrue = Mock(return_value=np.array([0.2])) + model_DOWN = Mock(spec=Aimodel) - model_DOWN.predict_ptrue = Mock(return_value=np.array([0.7, 0.8, 0.9])) - model = SimModel(ppss, model_UP, model_DOWN) + model_DOWN.predict_ptrue = Mock(return_value=np.array([0.8])) + model = SimModel(model_UP, model_DOWN) - data_UP = Mock(spec=SimModelData1Dir) - data_DOWN = Mock(spec=SimModelData1Dir) - d = SimModelData(data_UP, data_DOWN) - p = model.predict_next(d) - assert isinstance(p, SimModelPrediction) + X_test = {UP: np.array([[1.0]]), DOWN: np.array([[2.0]])} + + predprob = model.predict_next(X_test) + assert predprob == {UP: 0.2, DOWN: 0.8} + diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py index 94ffecc50..4b3113957 100644 --- a/pdr_backend/sim/test/test_sim_model_data.py +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -26,7 +26,8 @@ def test_sim_model_data_1dir(): assert_array_equal(data_UP.X_train, X_UP[0:3,:]) assert_array_equal(data_UP.X_test, X_UP[3:3+1,:]) assert_array_equal(data_UP.ytrue_train, ytrue_UP_train) - + + @enforce_types def test_sim_model_data_both_dirs(): # build data @@ -51,4 +52,8 @@ def test_sim_model_data_both_dirs(): assert_array_equal(X_DOWN, data[DOWN].X) assert_array_equal(ytrue_DOWN, data[DOWN].ytrue) + assert sorted(data.X_test.keys()) == [UP, DOWN] + assert_array_equal(data.X_test[UP], data[UP].X_test) + assert_array_equal(data.X_test[DOWN], data[DOWN].X_test) + diff --git a/pdr_backend/sim/test/test_sim_model_data_factory.py b/pdr_backend/sim/test/test_sim_model_data_factory.py index de0a53b0a..87ee5485e 100644 --- a/pdr_backend/sim/test/test_sim_model_data_factory.py +++ b/pdr_backend/sim/test/test_sim_model_data_factory.py @@ -21,11 +21,9 @@ def test_sim_model_data_factory__basic(tmpdir): # attributes assert isinstance(data_f.ppss, PPSS) - assert isinstance(data_f.predict_train_feedset, PredictTrainFeedset) # properties assert isinstance(data_f.pdr_ss, PredictoorSS) - assert isinstance(data_f.aimodel_data_ss, AimodelDataSS) assert isinstance(data_f.class_thr, float) assert 0.0 < data_f.class_thr < 1.0 @@ -78,7 +76,6 @@ def test_sim_model_data_factory__build(tmpdir): @enforce_types def _factory(tmpdir) -> SimModelDataFactory: s = "binanceus ETH/USDT c 5m" - feedset_list = [{"predict": s, "train_on":s}] ppss = mock_ppss(feedset_list) @@ -86,14 +83,8 @@ def _factory(tmpdir) -> SimModelDataFactory: ppss.predictoor_ss.aimodel_data_ss.set_max_n_train(4) ppss.predictoor_ss.aimodel_data_ss.set_autoregressive_n(1) ppss.sim_ss.set_test_n(2) - - predict_feed = ArgFeed.from_str(s) - train_feeds = ArgFeeds([predict_feed]) - - predict_train_feedset = PredictTrainFeedset( - predict=predict_feed, train_on=train_feeds) - - data_f = SimModelDataFactory(ppss, predict_train_feedset) + + data_f = SimModelDataFactory(ppss) return data_f @enforce_types diff --git a/pdr_backend/sim/test/test_sim_model_factory.py b/pdr_backend/sim/test/test_sim_model_factory.py index 1ced2f1cc..13a948a71 100644 --- a/pdr_backend/sim/test/test_sim_model_factory.py +++ b/pdr_backend/sim/test/test_sim_model_factory.py @@ -2,7 +2,6 @@ from enforce_typing import enforce_types -from pdr_backend.ppss.ppss import fast_test_yaml_str, PPSS from pdr_backend.ppss.aimodel_ss import AimodelSS, aimodel_ss_test_dict from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_model import SimModel @@ -14,7 +13,7 @@ @enforce_types def test_sim_model_factory__attributes(): f: SimModelFactory = _get_sim_model_factory() - assert isinstance(f.ppss, PPSS) + assert isinstance(f.aimodel_ss, AimodelSS) @enforce_types def test_sim_model_factory__do_build(): @@ -37,12 +36,11 @@ def test_sim_model_factory__build(): model = f.build(data) assert isinstance(model, SimModel) - p = model.predict_next(data) - assert isinstance(p, SimModelPrediction) + p = model.predict_next(data.X_test) + assert p is not None # don't test further; leave that to test_sim_model.py @enforce_types def _get_sim_model_factory() -> SimModelFactory: - s = fast_test_yaml_str() - ppss = PPSS(yaml_str=s, network="development") - return SimModelFactory(ppss) + aimodel_ss = AimodelSS(aimodel_ss_test_dict()) + return SimModelFactory(aimodel_ss) diff --git a/pdr_backend/sim/test/test_sim_predictoor.py b/pdr_backend/sim/test/test_sim_predictoor.py index 407cc5590..3c286c061 100644 --- a/pdr_backend/sim/test/test_sim_predictoor.py +++ b/pdr_backend/sim/test/test_sim_predictoor.py @@ -24,20 +24,20 @@ def test_sim_predictoor__predict_iter(): sim_pdr = _sim_pdr() # case 1: don't trust models - p = SimModelPrediction(conf_thr=0.9, prob_up_UP=0.4, prob_up_DOWN=0.4) + p = SimModelPrediction(conf_thr=0.9, prob_UP=0.4, prob_DOWN=0.4) assert not p.do_trust_models() stake_up, stake_down = sim_pdr.predict_iter(p) assert stake_up == stake_down == 0.0 # case 2: UP dominates - p = SimModelPrediction(conf_thr=0.1, prob_up_UP=0.6, prob_up_DOWN=0.6) + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.4) assert p.do_trust_models() stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_down < stake_up < 1.0 assert (stake_up + stake_down) <= sim_pdr.max_stake_amt # case 3: DOWN dominates - p = SimModelPrediction(conf_thr=0.1, prob_up_UP=0.4, prob_up_DOWN=0.4) + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.4, prob_DOWN=0.6) assert p.do_trust_models() stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_up < stake_down < 1.0 From 96a16d034276ffa4ae3a7a3935dd75d4b001861d Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 14:24:54 +0200 Subject: [PATCH 16/50] many bug fixes --- pdr_backend/cli/arg_feed.py | 8 +- pdr_backend/cli/cli_module.py | 2 +- pdr_backend/sim/sim_engine.py | 128 ++++++++++++++---------- pdr_backend/sim/sim_logger.py | 10 +- pdr_backend/sim/sim_model.py | 11 +- pdr_backend/sim/sim_model_prediction.py | 8 +- pdr_backend/sim/sim_trader.py | 4 +- pdr_backend/sim/test/test_sim_engine.py | 16 ++- pdr_backend/sim/test/test_sim_trader.py | 14 ++- 9 files changed, 120 insertions(+), 81 deletions(-) diff --git a/pdr_backend/cli/arg_feed.py b/pdr_backend/cli/arg_feed.py index ed7daac70..6bea2d3e4 100644 --- a/pdr_backend/cli/arg_feed.py +++ b/pdr_backend/cli/arg_feed.py @@ -90,15 +90,15 @@ def from_str(feed_str: str, do_verify: bool = True) -> "ArgFeed": return feed def variant_close(self) -> "ArgFeed": - return self._variant_signal("close") + return self.variant_signal("close") def variant_high(self) -> "ArgFeed": - return self._variant_signal("high") + return self.variant_signal("high") def variant_low(self) -> "ArgFeed": - return self._variant_signal("low") + return self.variant_signal("low") - def _variant_signal(self, signal_str: "str") -> "ArgFeed": + def variant_signal(self, signal_str: str) -> "ArgFeed": return ArgFeed( self.exchange, signal_str, diff --git a/pdr_backend/cli/cli_module.py b/pdr_backend/cli/cli_module.py index acf064f2e..cd53d9802 100644 --- a/pdr_backend/cli/cli_module.py +++ b/pdr_backend/cli/cli_module.py @@ -89,7 +89,7 @@ def do_sim(args, nested_args=None): feedset = ppss.predictoor_ss.predict_train_feedsets[0] if len(ppss.predictoor_ss.predict_train_feedsets) > 0: logger.warning("Multiple predict feeds provided, using the first one") - sim_engine = SimEngine(ppss, feedset) + sim_engine = SimEngine(ppss) sim_engine.run() diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 10ba38f34..74798f709 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -14,6 +14,7 @@ from pdr_backend.aimodel.aimodel_factory import AimodelFactory from pdr_backend.aimodel.aimodel_plotdata import AimodelPlotdata from pdr_backend.cli.arg_feed import ArgFeed +from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.cli.arg_timeframe import ArgTimeframe from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset from pdr_backend.lake.ohlcv_data_factory import OhlcvDataFactory @@ -28,7 +29,7 @@ from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_plotter import SimPlotter from pdr_backend.sim.sim_predictoor import SimPredictoor -from pdr_backend.sim.sim_state import SimState +from pdr_backend.sim.sim_state import HistProfits, SimState from pdr_backend.sim.sim_trader import SimTrader from pdr_backend.util.strutil import shift_one_earlier from pdr_backend.util.time_types import UnixTimeMs @@ -42,23 +43,20 @@ class SimEngine: def __init__( self, ppss: PPSS, - predict_train_feedset: PredictTrainFeedset, multi_id: Optional[str] = None, ): - predict_feed = predict_train_feedset.predict - assert predict_feed.signal == "close", \ - "only operates on close predictions" - - self.predict_train_feedset = predict_train_feedset self.ppss = ppss + + assert self.predict_feed.signal == "close", \ + "only operates on close predictions" # can be disabled by calling disable_realtime_state() self.do_state_updates = True self.st = SimState() - self.pdr = SimPredictoor(ppss.predictoor_ss) - self.trader = SimTrader(ppss, predict_feed) + self.sim_predictoor = SimPredictoor(ppss.predictoor_ss) + self.sim_trader = SimTrader(ppss) self.sim_plotter = SimPlotter() @@ -69,36 +67,32 @@ def __init__( else: self.multi_id = str(uuid.uuid4()) - assert self.transform == "None" + assert self.pdr_ss.aimodel_data_ss.transform == "None" + @property + def pdr_ss(self) -> PredictoorSS: + return self.ppss.predictoor_ss + @property def predict_feed(self) -> ArgFeed: - return self.predict_train_feedset.predict_feed + return self.pdr_ss.predict_train_feedsets[0].predict @property def timeframe(self) -> ArgTimeframe: return self.predict_feed.timeframe - - @property - def pdr_ss(self) -> PredictoorSS: - return self.ppss.predictoor_ss @property - def aimodel_ss(self) -> AimodelSS: - return self.pdr_ss.aimodel_ss - - @property - def transform(self) -> str: - return self.pdr_ss.aimodel_data_ss.transform + def others_stake(self) -> float: + return self.pdr_ss.others_stake.amt_eth @property - def others_stake(self) -> float: - return self.others_stale.amt_eth + def others_accuracy(self) -> float: + return self.pdr_ss.others_accuracy @property def revenue(self) -> float: return self.pdr_ss.revenue.amt_eth - + @enforce_types def _init_loop_attributes(self): filebase = f"out_{UnixTimeMs.now()}.txt" @@ -134,29 +128,45 @@ def run(self): # pylint: disable=too-many-statements# pylint: disable=too-many-statements @enforce_types def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): - # build model + # base data st = self.st - data_factory = SimModelDataFactory(self.ppss) - model_data: SimModelData = data_factory.build(iter_i, mergedohlcv_df) - model_factory = SimModelFactory(self.aimodel_ss) + df = mergedohlcv_df + sim_model_data_f = SimModelDataFactory(self.ppss) + testshift = sim_model_data_f.testshift(iter_i) + + # observe current price value, and related thresholds for classifier + cur_close = self._curval(df, testshift, "close") + cur_high = self._curval(df, testshift, "high") + cur_low = self._curval(df, testshift, "low") + y_thr_UP = sim_model_data_f.thr_UP(cur_close) + y_thr_DOWN = sim_model_data_f.thr_DOWN(cur_close) + + # build model + model_factory = SimModelFactory(self.pdr_ss.aimodel_ss) + st.sim_model_data: SimModelData = sim_model_data_f.build(iter_i, df) if model_factory.do_build(st.sim_model, iter_i): - st.sim_model = model_factory.build(model_data) + st.sim_model = model_factory.build(st.sim_model_data) # make prediction - predprob = self.st.sim_model.predict_next(model_data.X_test) + predprob = self.st.sim_model.predict_next(st.sim_model_data.X_test) conf_thr = self.ppss.trader_ss.sim_confidence_threshold sim_model_p = SimModelPrediction(conf_thr, predprob[UP], predprob[DOWN]) # predictoor takes action (stake) - stake_up, stake_down = self.pdr.predict_iter(sim_model_p) + stake_up, stake_down = self.sim_predictoor.predict_iter(sim_model_p) # trader takes action (trade) - trader_profit_USD = self.trader.trade_iter( + trader_profit_USD = self.sim_trader.trade_iter( cur_close, cur_high, cur_low, sim_model_p, ) - # observe true price change + # observe next price values + next_close = self._nextval(df, testshift, "close") + next_high = self._nextval(df, testshift, "high") + next_low = self._nextval(df, testshift, "low") + + # observe price change prev -> next, and related changes for classifier trueval_up_close = next_close > cur_close trueval = { UP: next_high > y_thr_UP, # did next high go > prev close+% ? @@ -165,31 +175,38 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): # calc predictoor profit pdr_profit_OCEAN = HistProfits.calc_pdr_profit( - others_stake, pdr_ss.others_stake_accuracy, - stake_up, stake_down, trueval_up_close) + self.others_stake, self.others_accuracy, + stake_up, stake_down, self.revenue, trueval_up_close) # update state st.update(trueval, predprob, pdr_profit_OCEAN, trader_profit_USD) # log - ut = self._calc_ut(mergedohlcv_df) - SimLogLine(ppss, st, iter_i, ut).log() + ut = self._calc_ut(df, testshift) + SimLogLine(self.ppss, self.st, iter_i, ut).log() # plot do_save_state, is_final_state = self._do_save_state(iter_i) if do_save_state: - d_UP = self._aimodel_plotdata_1dirn(UP) - d_DOWN = self._aimodel_plotdata_1dirn(DOWN) - d = {UP: d_UP, DOWN: d_DOWN} + d = self._aimodel_plotdata() st.iter_number = iter_i self.sim_plotter.save_state(st, d, is_final_state) + @enforce_types + def _aimodel_plotdata(self) -> dict: + d_UP = self._aimodel_plotdata_1dir(UP) + d_DOWN = self._aimodel_plotdata_1dir(DOWN) + return {UP: d_UP, DOWN: d_DOWN} + @enforce_types def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: st = self.st - model, model_data = st.sim_model[dirn], st.sim_model_data[dirn] + model = st.sim_model[dirn] + model_data = st.sim_model_data[dirn] + colnames = model_data.colnames colnames = [shift_one_earlier(c) for c in colnames] + most_recent_x = model_data.X[-1, :] slicing_x = most_recent_x d = AimodelPlotdata( @@ -204,20 +221,27 @@ def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: return d @enforce_types - def _recent_close(self, mergedohlcv_df, testshift: int) -> Tuple[float, float]: - """@return -- (cur_close, next_close)""" - p = self.predict_feed_trainset.predict_feed - _, _, yraw_close, _, _ = data_f.create_xy( - mergedohlcv_df, - testshift, - p.variant_close(), - [p.variant_close()], + def _curval(self, df, testshift: int, signal_str: str) -> float: + # float() so not np.float64, bc applying ">" gives np.bool -> problems + return float(self._yraw(df, testshift, signal_str)[-2]) + + @enforce_types + def _nextval(self, df, testshift: int, signal_str: str) -> float: + # float() so not np.float64, bc applying ">" gives np.bool -> problems + return float(self._yraw(df, testshift, signal_str)[-1]) + + @enforce_types + def _yraw(self, mergedohlcv_df, testshift: int, signal_str: str): + assert signal_str in ["close", "high", "low"] + feed = self.predict_feed.variant_signal(signal_str) + aimodel_data_f = AimodelDataFactory(self.pdr_ss) + _, _, yraw, _, _ = aimodel_data_f.create_xy( + mergedohlcv_df, testshift, feed, ArgFeeds([feed]), ) - cur_close, next_close = yraw_close[-2], yraw_close[-1] - return (cur_close, next_close) + return yraw @enforce_types - def _calc_ut(self, mergedohlcv_df) -> UnixTimeMs: + def _calc_ut(self, mergedohlcv_df, testshift: int) -> UnixTimeMs: recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) ut = UnixTimeMs(recent_ut - testshift * self.timeframe.ms) return ut diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index e5e10f2eb..23905404f 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -22,11 +22,13 @@ def log(self): s += f" dt={self.ut.to_timestr()[:-7]}" s += " â•‘" - s += f"pdr_profit={compactSmallNum(self.st.pdr_profits_OCEAN[-1])} OCEAN" - s += f" (cumul {compactSmallNum(sum(self.st.pdr_profits_OCEAN))} OCEAN)" + pdr_profits = self.st.hist_profits.pdr_profits_OCEAN + s += f"pdr_profit={compactSmallNum(pdr_profits[-1])} OCEAN" + s += f" (cumul {compactSmallNum(sum(pdr_profits))} OCEAN)" s += " â•‘" - s += f" tdr_profit=${self.st.trader_profits_USD[-1]:6.2f}" - s += f" (cumul ${sum(self.st.trader_profits_USD):6.2f})" + trader_profits = self.st.hist_profits.trader_profits_USD + s += f" tdr_profit=${trader_profits[-1]:6.2f}" + s += f" (cumul ${sum(trader_profits):6.2f})" logger.info(s) diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index bded6ed5f..57eb1910a 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -25,9 +25,12 @@ def predict_next(self, X_test: dict) -> dict: assert X_test[UP].shape[0] == 1 assert X_test[DOWN].shape[0] == 1 - predprob_UP = self[UP].predict_ptrue(X_test[UP])[0] - predprob_DOWN = self[DOWN].predict_ptrue(X_test[UP])[0] - predprob = {UP: predprob_UP, DOWN: predprob_DOWN} + prob_UP = self[UP].predict_ptrue(X_test[UP])[0] + prob_DOWN = self[DOWN].predict_ptrue(X_test[UP])[0] + + # ensure not np.float64. Why: applying ">" gives np.bool --> problems + prob_UP, prob_DOWN = float(prob_UP), float(prob_DOWN) + + return {UP: prob_UP, DOWN: prob_DOWN} - return predprob diff --git a/pdr_backend/sim/sim_model_prediction.py b/pdr_backend/sim/sim_model_prediction.py index 428e1f3f2..9dc9406e0 100644 --- a/pdr_backend/sim/sim_model_prediction.py +++ b/pdr_backend/sim/sim_model_prediction.py @@ -13,6 +13,9 @@ def __init__( # DOWN model's probability that next low will go < prev close-% prob_DOWN: float, ): + # ensure not np.float64. Why: applying ">" gives np.bool --> problems + prob_UP, prob_DOWN = float(prob_UP), float(prob_DOWN) + # ppss.trader_ss.sim_confidence_threshold self.conf_thr = conf_thr @@ -42,13 +45,12 @@ def __init__( self.pred_down = self.conf_down > self.conf_thr self.prob_up_MERGED = 1.0 - self.prob_DOWN - @enforce_types def do_trust_models(self) -> bool: - return _do_trust_models( + do_trust = _do_trust_models( self.pred_up, self.pred_down, self.prob_UP, self.prob_DOWN, ) + return do_trust - @enforce_types def models_in_conflict(self) -> bool: return _models_in_conflict(self.prob_UP, self.prob_DOWN) diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index ec20d0dbe..c2ef6cce1 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -10,7 +10,7 @@ # pylint: disable=too-many-instance-attributes class SimTrader: - def __init__(self, ppss, predict_feed): + def __init__(self, ppss): self.ppss = ppss self.position_open = "" # long, short, "" @@ -27,7 +27,7 @@ def __init__(self, ppss, predict_feed): "mock" if mock else ppss.predictoor_ss.exchange_str, ) - self.predict_feed = predict_feed + self.predict_feed = ppss.predictoor_ss.predict_train_feedsets[0].predict assert isinstance(self.tokcoin, str) assert isinstance(self.usdcoin, str) diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index 4cda0cfef..670a4b8e2 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -17,10 +17,19 @@ from pdr_backend.sim.dash_plots.view_elements import get_layout from pdr_backend.sim.sim_engine import SimEngine +@enforce_types +# pylint: disable=unused-argument +def test_sim_engine_no_plot(tmpdir): + _test_sim_engine(tmpdir, None, None) + +@enforce_types +# pylint: disable=unused-argument +def test_sim_engine_with_plot(tmpdir, check_chromedriver, dash_duo): + _test_sim_engine(tmpdir, check_chromedriver, dash_duo) @enforce_types # pylint: disable=unused-argument -def test_sim_engine(tmpdir, check_chromedriver, dash_duo): +def _test_sim_engine(tmpdir, check_chromedriver, dash_duo): s = fast_test_yaml_str(tmpdir) ppss = PPSS(yaml_str=s, network="development") @@ -63,8 +72,7 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): ppss.sim_ss = SimSS(sim_d) # go - feedsets = ppss.predictoor_ss.predict_train_feedsets - sim_engine = SimEngine(ppss, feedsets[0]) + sim_engine = SimEngine(ppss) assert sim_engine.st.sim_model is None sim_engine.run() @@ -73,6 +81,8 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): assert isinstance(sim_engine.st.sim_model, SimModel) # basic tests for plots + if check_chromedriver is None: + return app = Dash("pdr_backend.sim.sim_dash") app.config["suppress_callback_exceptions"] = True app.run_id = sim_engine.multi_id diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 358dd3650..2b8c0fcc1 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -21,20 +21,18 @@ def mock_ppss(): ppss.sim_ss.tradetype = "histmock" ppss.exchange_mgr_ss = Mock(spec=ExchangeMgrSS) ppss.predictoor_ss.exchange_str = "mock" - return ppss - + ppss.predictoor_ss.predict_train_feedsets = [Mock()] -@pytest.fixture -def mock_predict_feed(): predict_feed = Mock() predict_feed.pair.base_str = "ETH" predict_feed.pair.quote_str = "USDT" - return predict_feed - + ppss.predictoor_ss.predict_train_feedsets[0].predict = predict_feed + + return ppss @pytest.fixture -def sim_trader(mock_ppss, mock_predict_feed): - return SimTrader(mock_ppss, mock_predict_feed) +def sim_trader(mock_ppss): + return SimTrader(mock_ppss) def test_initial_state(sim_trader): From 81078f6e1b324e04c1792bddb4c0338f1b4b49b7 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 14:47:31 +0200 Subject: [PATCH 17/50] test_sim_engine.py working --- pdr_backend/aimodel/aimodel_plotdata.py | 7 ++++--- pdr_backend/aimodel/aimodel_plotter.py | 2 ++ pdr_backend/sim/sim_engine.py | 2 +- pdr_backend/sim/sim_model_data.py | 8 +++++++- pdr_backend/sim/sim_model_data_factory.py | 14 ++++++++++---- pdr_backend/sim/test/test_sim_engine.py | 12 ++---------- pdr_backend/sim/test/test_sim_model_data.py | 9 ++++++--- 7 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pdr_backend/aimodel/aimodel_plotdata.py b/pdr_backend/aimodel/aimodel_plotdata.py index 6f85f3aa5..d97b9bdbd 100644 --- a/pdr_backend/aimodel/aimodel_plotdata.py +++ b/pdr_backend/aimodel/aimodel_plotdata.py @@ -16,8 +16,8 @@ def __init__( model: Aimodel, X_train: np.ndarray, ytrue_train: np.ndarray, - ycont_train: np.ndarray, - y_thr: float, + ycont_train: Optional[np.ndarray], + y_thr: Optional[float], colnames: List[str], slicing_x: np.ndarray, sweep_vars: Optional[List[int]] = None, @@ -41,7 +41,8 @@ def __init__( assert len(colnames) == n, (len(colnames), n) assert slicing_x.shape[0] == n, (slicing_x.shape[0], n) assert ytrue_train.shape[0] == N, (ytrue_train.shape[0], N) - assert ycont_train.shape[0] == N, (ycont_train.shape[0], N) + if ycont_train is not None: + assert ycont_train.shape[0] == N, (ycont_train.shape[0], N) assert sweep_vars is None or len(sweep_vars) in [1, 2] # set values diff --git a/pdr_backend/aimodel/aimodel_plotter.py b/pdr_backend/aimodel/aimodel_plotter.py index e5ae34205..6028ec666 100644 --- a/pdr_backend/aimodel/aimodel_plotter.py +++ b/pdr_backend/aimodel/aimodel_plotter.py @@ -134,6 +134,8 @@ def _plot_lineplot_1var(aimodel_plotdata: AimodelPlotdata): # line plot: regressor response, training data if d.model.do_regr: assert mesh_ycont_hat is not None + assert y_thr is not None + assert ycont is not None fig.add_trace( go.Scatter( x=mesh_chosen_x, diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 74798f709..dff197847 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -215,7 +215,7 @@ def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: model_data.ytrue_train, None, None, - colnames, + model_data.colnames, slicing_x, ) return d diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index c1730e438..61e9e295f 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -8,13 +8,19 @@ class SimModelData1Dir: @enforce_types - def __init__(self, X: np.ndarray, ytrue: np.ndarray): + def __init__( + self, + X: np.ndarray, + ytrue: np.ndarray, + colnames:List[str], + ): assert len(X.shape) == 2 assert len(ytrue.shape) == 1 assert X.shape[0] == ytrue.shape[0], (X.shape[0], ytrue.shape[0]) self.X: np.ndarray = X self.ytrue: np.ndarray = ytrue + self.colnames: List[str] = colnames @property def st(self) -> int: diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index c1b868e4f..e6eda81ad 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -44,10 +44,10 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: _, _, y_close, _, _ = data_f.create_xy( df, testshift, p.variant_close(), ArgFeeds([p.variant_close()]), ) - X_high, _, y_high, _, _ = data_f.create_xy( + X_high, _, y_high, x_high_df, _ = data_f.create_xy( df, testshift, p.variant_high(), ArgFeeds([p.variant_high()]), ) - X_low, _, y_low, _, _ = data_f.create_xy( + X_low, _, y_low, x_low_df, _ = data_f.create_xy( df, testshift, p.variant_low(), ArgFeeds([p.variant_low()]), ) @@ -65,8 +65,14 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: ytrue_DOWN.append(next_low < thr_DOWN) ytrue_UP, ytrue_DOWN = np.array(ytrue_UP), np.array(ytrue_DOWN) - d_UP = SimModelData1Dir(X_high[1:,:], ytrue_UP) # or is it [:-1,:] - d_DOWN = SimModelData1Dir(X_low[1:,:], ytrue_DOWN) # "" + + colnames_UP = list(x_high_df.columns) + colnames_DOWN = list(x_low_df.columns) + + # Q: I used X[1:,:], but should it be X[:-1,:] ? + d_UP = SimModelData1Dir(X_high[1:,:], ytrue_UP, colnames_UP) + d_DOWN = SimModelData1Dir(X_low[1:,:], ytrue_DOWN, colnames_DOWN) + # note: alternatively, each input X could be h+l+c rather than just h or l d = SimModelData(d_UP, d_DOWN) diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index 670a4b8e2..4e5b41429 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -16,20 +16,12 @@ from pdr_backend.sim.dash_plots.callbacks import get_callbacks from pdr_backend.sim.dash_plots.view_elements import get_layout from pdr_backend.sim.sim_engine import SimEngine +from pdr_backend.sim.sim_model import SimModel -@enforce_types -# pylint: disable=unused-argument -def test_sim_engine_no_plot(tmpdir): - _test_sim_engine(tmpdir, None, None) - -@enforce_types -# pylint: disable=unused-argument -def test_sim_engine_with_plot(tmpdir, check_chromedriver, dash_duo): - _test_sim_engine(tmpdir, check_chromedriver, dash_duo) @enforce_types # pylint: disable=unused-argument -def _test_sim_engine(tmpdir, check_chromedriver, dash_duo): +def test_sim_engine(tmpdir, check_chromedriver, dash_duo): s = fast_test_yaml_str(tmpdir) ppss = PPSS(yaml_str=s, network="development") diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py index 4b3113957..72877b6eb 100644 --- a/pdr_backend/sim/test/test_sim_model_data.py +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -14,11 +14,12 @@ def test_sim_model_data_1dir(): assert X_UP.shape == (4,2) assert ytrue_UP.shape == (4,) ytrue_UP_train = ytrue_UP[:3] - data_UP = SimModelData1Dir(X_UP, ytrue_UP) + data_UP = SimModelData1Dir(X_UP, ytrue_UP, ["x0", "x1"]) # basic tests assert_array_equal(X_UP, data_UP.X) assert_array_equal(ytrue_UP, data_UP.ytrue) + assert data_UP.colnames == ["x0", "x1"] # test properties assert data_UP.st == 0 @@ -33,9 +34,11 @@ def test_sim_model_data_both_dirs(): # build data (X_UP, ytrue_UP) = get_Xy_UP() (X_DOWN, ytrue_DOWN) = get_Xy_DOWN() + colnames_UP = ["x0_high", "x1_high"] + colnames_DOWN = ["x0_low", "x1_low"] - data_UP = SimModelData1Dir(X_UP, ytrue_UP) - data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) + data_UP = SimModelData1Dir(X_UP, ytrue_UP, colnames_UP) + data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN, colnames_DOWN) data = SimModelData(data_UP, data_DOWN) # basic tests From 0f7e1a91be4d86edd7fead444b9908aefb7bf1a1 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 18:00:22 +0200 Subject: [PATCH 18/50] more bug fixees --- logging.yaml | 2 +- pdr_backend/sim/multisim_engine.py | 29 +--- pdr_backend/sim/sim_state.py | 101 ++++++++++--- pdr_backend/sim/sim_trader.py | 38 ++--- pdr_backend/sim/test/resources.py | 7 +- pdr_backend/sim/test/test_multisim_engine.py | 2 +- pdr_backend/sim/test/test_sim_logger.py | 5 +- pdr_backend/sim/test/test_sim_state.py | 140 ++++++++++++++----- pdr_backend/sim/test/test_sim_trader.py | 84 ++++++----- 9 files changed, 275 insertions(+), 133 deletions(-) diff --git a/logging.yaml b/logging.yaml index d54dc5148..5abdf2e8e 100644 --- a/logging.yaml +++ b/logging.yaml @@ -23,7 +23,7 @@ loggers: level: INFO aimodel_data_factory: - level: DEBUG + level: INFO sim_engine: level: INFO diff --git a/pdr_backend/sim/multisim_engine.py b/pdr_backend/sim/multisim_engine.py index 9ea32598e..45e6d8df9 100644 --- a/pdr_backend/sim/multisim_engine.py +++ b/pdr_backend/sim/multisim_engine.py @@ -66,30 +66,15 @@ async def run_one(self, run_i: int): point_i = self.ss.point_i(run_i) logger.info("Multisim run_i=%s: start. Vals=%s", run_i, point_i) ppss = self.ppss_from_point(point_i) - feedset = ppss.predictoor_ss.predict_train_feedsets[0] multi_id = str(uuid.uuid4()) - sim_engine = SimEngine(ppss, feedset, multi_id) + sim_engine = SimEngine(ppss, multi_id) sim_engine.disable_realtime_state() sim_engine.run() st = sim_engine.st - recent_metrics = st.recent_metrics() - - # below, the "[1:]" is to avoid the first sample, which may be off - run_metrics = { - "acc_est": recent_metrics["acc_est"], - "acc_l": recent_metrics["acc_l"], - "acc_u": recent_metrics["acc_u"], - "f1": np.mean(st.aim.f1s), - "precision": np.mean(st.aim.precisions[1:]), - "recall": np.mean(st.aim.recalls[1:]), - "loss": np.mean(st.aim.losses[1:]), - "yerr": np.mean(st.aim.yerrs[1:]), - "pdr_profit_OCEAN": np.sum(st.pdr_profits_OCEAN), - "trader_profit_USD": np.sum(st.trader_profits_USD), - } - run_metrics_list = list(run_metrics.values()) + metrics_values = st.final_metrics_values() + metrics_list = [metrics_values[name] for name in st.metrics_names()] async with lock: - self.update_csv(run_i, run_metrics_list, point_i) + self.update_csv(run_i, metrics_list, point_i) logger.info("Multisim run_i=%s: done", run_i) logger.info("Multisim engine: done. Output file: %s", self.csv_file) @@ -113,7 +98,7 @@ def csv_header(self) -> List[str]: # put metrics first, because point_meta names/values can be superlong header = [] header += ["run_number"] - header += SimState.recent_metrics_names() + header += SimState.metrics_names() header += list(self.ss.point_meta.keys()) return header @@ -122,7 +107,7 @@ def spaces(self) -> List[int]: buf = 3 spaces = [] spaces += [len("run_number") + buf] - spaces += [max(len(name), 6) + buf for name in SimState.recent_metrics_names()] + spaces += [max(len(name), 6) + buf for name in SimState.metrics_names()] for var, cand_vals in self.ss.point_meta.items(): var_len = len(var) @@ -155,7 +140,7 @@ def update_csv( @arguments run_i - it's run #i - run_metrics -- output of SimState.recent_metrics() for run #i + run_metrics -- output of SimState.recent_metrics_values() for run #i point_i -- value of each sweep param, for run #i """ assert os.path.exists(self.csv_file), self.csv_file diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 2ddddb44a..201b832f8 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional, Tuple, Union from enforce_typing import enforce_types +import numpy as np from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN @@ -9,10 +10,10 @@ #============================================================================= # HistPerfs -@enforce_types class HistPerfs: """Historical performances, for 1 model dir'n (eg UP)""" + @enforce_types def __init__(self, dirn: Dirn): self.dirn = dirn @@ -27,6 +28,7 @@ def __init__(self, dirn: Dirn): self.losses: List[float] = [] # [i] : log-loss + @enforce_types def update(self, perfs_list: list): """perfs_list typically comes from TrueVsPred.perf_values()""" acc_est, acc_l, acc_u, f1, precision, recall, loss = perfs_list @@ -40,16 +42,22 @@ def update(self, perfs_list: list): self.recalls.append(recall) self.losses.append(loss) + + @enforce_types + def metrics_names_instance(self) -> List[str]: + """@return e.g. ['acc_est_UP', 'acc_l_UP', ..., 'loss_UP]""" + return HistPerfs.metrics_names_static(self.dirn) - def metrics_names(self) -> List[str]: + @staticmethod + def metrics_names_static(dirn) -> List[str]: """@return e.g. ['acc_est_UP', 'acc_l_UP', ..., 'loss_UP]""" - return [f"{name}_{dirn_str(self.dirn)}" + return [f"{name}_{dirn_str(dirn)}" for name in PERF_NAMES] - def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: - """Return most recent aimodel metrics""" - if not self.acc_ests: - return {name: None for name in self.metrics_names()} + @enforce_types + def recent_metrics_values(self) -> Dict[str, float]: + """Return most recent metrics""" + assert self.have_data(), "only works for >0 entries" s = dirn_str(self.dirn) return { @@ -62,12 +70,32 @@ def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: f"loss_{s}": self.losses[-1], } + @enforce_types + def final_metrics_values(self) -> Dict[str, float]: + """Return *final* metrics, rather than most recent. """ + assert self.have_data(), "only works for >0 entries" + + s = dirn_str(self.dirn) + return { + f"acc_est_{s}": self.acc_ests[-1], + f"acc_l_{s}": self.acc_ls[-1], + f"acc_u_{s}": self.acc_us[-1], + f"f1_{s}": np.mean(self.f1s), + f"precision_{s}": np.mean(self.precisions), + f"recall_{s}": np.mean(self.recalls), + f"loss_{s}": np.mean(self.losses), + } + + @enforce_types + def have_data(self) -> bool: + return bool(self.acc_ests) + + #============================================================================= # HistProfits PROFIT_NAMES = ["pdr_profit_OCEAN", "trader_profit_USD"] -@enforce_types class HistProfits: def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit @@ -102,30 +130,50 @@ def calc_pdr_profit( pdr_profit_OCEAN = amt_received - amt_sent return pdr_profit_OCEAN + @enforce_types def update(self, pdr_profit_OCEAN: float, trader_profit_USD: float): self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) self.trader_profits_USD.append(trader_profit_USD) - def metrics_names(self) -> List[str]: + @staticmethod + def metrics_names() -> List[str]: return PROFIT_NAMES + @enforce_types def recent_metrics_values(self) -> Dict[str, float]: - if not self.pdr_profits_OCEAN: - return {name: None for name in self.metrics_names()} + """Return most recent metrics""" + assert self.have_data(), "only works for >0 entries" return { "pdr_profit_OCEAN" : self.pdr_profits_OCEAN[-1], "trader_profit_USD" : self.trader_profits_USD[-1], } + @enforce_types + def final_metrics_values(self) -> Dict[str, float]: + """Return *final* metrics, rather than most recent. """ + assert self.have_data(), "only works for >0 entries" + + return { + "pdr_profit_OCEAN" : np.sum(self.pdr_profits_OCEAN), + "trader_profit_USD" : np.sum(self.trader_profits_USD), + } + + @enforce_types + def have_data(self) -> bool: + return bool(self.pdr_profits_OCEAN) + + + #============================================================================= # SimState -@enforce_types class SimState: + @enforce_types def __init__(self): self.init_loop_attributes() + @enforce_types def init_loop_attributes(self): self.iter_number = 0 self.sim_model_data: Optional[SimModelData] = None @@ -134,6 +182,7 @@ def init_loop_attributes(self): self.hist_perfs = {UP: HistPerfs(UP), DOWN: HistPerfs(DOWN)} self.hist_profits = HistProfits() + @enforce_types def update( self, trueval: dict, @@ -156,16 +205,32 @@ def update( self.hist_profits.update(pdr_profit_OCEAN, trader_profit_USD) - def metrics_names(self) -> List[str]: - return self.hist_perfs[UP].metrics_names() + \ - self.hist_perfs[DOWN].metrics_names() + \ - self.hist_profits.metrics_names() + @staticmethod + def metrics_names() -> List[str]: + return HistPerfs.metrics_names_static(UP) + \ + HistPerfs.metrics_names_static(DOWN) + \ + HistProfits.metrics_names() - def recent_metrics_values(self) -> List[Union[int, float]]: + @enforce_types + def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: """Return most recent aimodel metrics + profit metrics""" metrics = {} metrics.update(self.hist_perfs[UP].recent_metrics_values()) metrics.update(self.hist_perfs[DOWN].recent_metrics_values()) metrics.update(self.hist_profits.recent_metrics_values()) return metrics - + + @enforce_types + def final_metrics_values(self) -> Dict[str, Union[int, float, None]]: + """Return *final* metrics, rather than most recent. """ + metrics = {} + metrics.update(self.hist_perfs[UP].final_metrics_values()) + metrics.update(self.hist_perfs[DOWN].final_metrics_values()) + metrics.update(self.hist_profits.final_metrics_values()) + return metrics + + @enforce_types + def have_data(self) -> bool: + return self.hist_perfs[UP].have_data() \ + and self.hist_perfs[DOWN].have_data() \ + and self.hist_profits().have_data() diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index c2ef6cce1..1802c9f69 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -10,16 +10,17 @@ # pylint: disable=too-many-instance-attributes class SimTrader: + @enforce_types def __init__(self, ppss): self.ppss = ppss - self.position_open = "" # long, short, "" - self.position_size = 0 # amount of tokens in position - self.position_worth = 0 # amount of USD in position - self.tp = 0.0 # take profit - self.sl = 0.0 # stop loss - self.tp_percent = self.ppss.trader_ss.take_profit_percent - self.sl_percent = self.ppss.trader_ss.stop_loss_percent + self.position_open: str = "" # long, short, "" + self.position_size: float = 0. # amount of tokens in position + self.position_worth: float = 0. # amount of USD in position + self.tp: float = 0. # take profit + self.sl: float = 0. # stop loss + self.tp_percent: float = self.ppss.trader_ss.take_profit_percent + self.sl_percent: float = self.ppss.trader_ss.stop_loss_percent mock = self.ppss.sim_ss.tradetype in ["histmock"] exchange_mgr = ExchangeMgr(self.ppss.exchange_mgr_ss) @@ -41,20 +42,23 @@ def usdcoin(self) -> str: """Return e.g. 'USDT'""" return self.predict_feed.pair.quote_str + @enforce_types def close_long_position(self, sell_price: float) -> float: tokcoin_amt_send = self.position_size usd_received = self._sell(sell_price, tokcoin_amt_send) self.position_open = "" profit = usd_received - self.position_worth - return profit + return float(profit) + @enforce_types def close_short_position(self, buy_price: float) -> float: usdcoin_amt_send = self.position_size * buy_price self._buy(buy_price, usdcoin_amt_send) self.position_open = "" profit = self.position_worth - usdcoin_amt_send - return profit + return float(profit) + @enforce_types def trade_iter( self, cur_close: float, @@ -62,15 +66,17 @@ def trade_iter( low: float, p: SimModelPrediction, ) -> float: - return self._trade_iter( + profit_USD = self._trade_iter( cur_close, p.pred_up, p.pred_down, p.conf_up, p.conf_down, high, low, ) + return float(profit_USD) + @enforce_types def _trade_iter( self, cur_close: float, - pred_up, - pred_down, + pred_up: bool, + pred_down: bool, conf_up: float, conf_down: float, high: float, @@ -117,7 +123,7 @@ def _trade_iter( self.position_size = tokcoin_amt_send self.tp = cur_close - (cur_close * self.tp_percent) self.sl = cur_close + (cur_close * self.sl_percent) - return 0 + return 0. # Check for take profit or stop loss if self.position_open == "long": @@ -140,7 +146,7 @@ def _trade_iter( if not pred_down: return self.close_short_position(cur_close) - return 0 + return 0. @enforce_types def _buy(self, price: float, usdcoin_amt_send: float) -> float: @@ -172,7 +178,7 @@ def _buy(self, price: float, usdcoin_amt_send: float) -> float: self.usdcoin, ) - return tokcoin_amt_recd + return float(tokcoin_amt_recd) @enforce_types def _sell(self, price: float, tokcoin_amt_send: float) -> float: @@ -206,4 +212,4 @@ def _sell(self, price: float, tokcoin_amt_send: float) -> float: self.usdcoin, ) - return usdcoin_amt_recd + return float(usdcoin_amt_recd) diff --git a/pdr_backend/sim/test/resources.py b/pdr_backend/sim/test/resources.py index f6f36b8df..d73169241 100644 --- a/pdr_backend/sim/test/resources.py +++ b/pdr_backend/sim/test/resources.py @@ -20,8 +20,11 @@ def get_sim_model_data() -> SimModelData: (X_UP, ytrue_UP) = get_Xy_UP() (X_DOWN, ytrue_DOWN) = get_Xy_DOWN() - data_UP = SimModelData1Dir(X_UP, ytrue_UP) - data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN) + colnames_UP = ["x0_high", "x1_high"] + colnames_DOWN = ["x0_low", "x1_low"] + + data_UP = SimModelData1Dir(X_UP, ytrue_UP, colnames_UP) + data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN, colnames_DOWN) data = SimModelData(data_UP, data_DOWN) return data diff --git a/pdr_backend/sim/test/test_multisim_engine.py b/pdr_backend/sim/test/test_multisim_engine.py index 71b4b308d..52b666d64 100644 --- a/pdr_backend/sim/test/test_multisim_engine.py +++ b/pdr_backend/sim/test/test_multisim_engine.py @@ -24,7 +24,7 @@ def test_multisim1(tmpdir): multisim_engine.run() # csv ok? - target_columns = ["run_number"] + SimState.recent_metrics_names() + [param] + target_columns = ["run_number"] + SimState.metrics_names() + [param] assert multisim_engine.csv_header() == target_columns assert os.path.exists(multisim_engine.csv_file) df = multisim_engine.load_csv() diff --git a/pdr_backend/sim/test/test_sim_logger.py b/pdr_backend/sim/test/test_sim_logger.py index 48af87c96..e0a2b7210 100644 --- a/pdr_backend/sim/test/test_sim_logger.py +++ b/pdr_backend/sim/test/test_sim_logger.py @@ -34,8 +34,9 @@ def test_sim_logger(tmpdir, caplog): "recall": 0.2, "loss": 0.3, } - st.pdr_profits_OCEAN = [1.0, 2.0, 3.0, 4.0, 5.0] - st.trader_profits_USD = [2.0, 3.0, 4.0, 5.0, 6.0] + st.hist_profits = Mock() + st.hist_profits.pdr_profits_OCEAN = [1.0, 2.0, 3.0, 4.0, 5.0] + st.hist_profits.trader_profits_USD = [2.0, 3.0, 4.0, 5.0, 6.0] ut = UnixTimeMs(1701634400000) log_line = SimLogLine(ppss, st, 1, ut) diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index f5b00781c..a45d3eee6 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -15,39 +15,47 @@ #============================================================================= # test HistPerfs +@enforce_types @pytest.mark.parametrize("dirn", [UP, DOWN]) def test_hist_perfs__basic_init(dirn): + # set data dirn_s = dirn_str(dirn) - hist_perfs = HistPerfs(dirn) + # test empty raw state assert hist_perfs.acc_ests == hist_perfs.acc_ls == hist_perfs.acc_us == [] assert hist_perfs.f1s == hist_perfs.precisions == hist_perfs.recalls == [] assert hist_perfs.losses == [] + # test names assert len(PERF_NAMES) == 7 - target_names = [f"{name}_{dirn_s}" for name in PERF_NAMES] - assert hist_perfs.metrics_names() == target_names - assert hist_perfs.metrics_names()[0] == f"acc_est_{dirn_s}" - assert hist_perfs.metrics_names()[-1] == f"loss_{dirn_s}" - - values = hist_perfs.recent_metrics_values() - assert len(values) == 7 - assert sorted(values.keys()) == sorted(target_names) - for val in values.values(): - assert val is None + target_names = _target_perfs_names(dirn_s) + assert hist_perfs.metrics_names_instance() == target_names + assert hist_perfs.metrics_names_instance()[0] == f"acc_est_{dirn_s}" + assert hist_perfs.metrics_names_instance()[-1] == f"loss_{dirn_s}" + assert HistPerfs.metrics_names_static(dirn) == target_names + + # test can't call for metrics + assert not hist_perfs.have_data() + with pytest.raises(AssertionError): + _ = hist_perfs.recent_metrics_values() + with pytest.raises(AssertionError): + _ = hist_perfs.final_metrics_values() +@enforce_types @pytest.mark.parametrize("dirn", [UP, DOWN]) def test_hist_perfs__main(dirn): + # set data dirn_s = dirn_str(dirn) - + target_names = [f"{name}_{dirn_s}" for name in PERF_NAMES] hist_perfs = HistPerfs(dirn) perfs_list1 = list(np.arange(0.1, 7.1, 1.0)) # 0.1, 1.1, ..., 6.1 perfs_list2 = list(np.arange(0.2, 7.2, 1.0)) # 0.2, 1.2, ..., 6.2 hist_perfs.update(perfs_list1) hist_perfs.update(perfs_list2) - + + # test raw state assert hist_perfs.acc_ests == [0.1, 0.2] assert hist_perfs.acc_ls == [1.1, 1.2] assert hist_perfs.acc_us == [2.1, 2.2] @@ -56,8 +64,13 @@ def test_hist_perfs__main(dirn): assert hist_perfs.recalls == [5.1, 5.2] assert hist_perfs.losses == [6.1, 6.2] + # test can call for metrics + assert hist_perfs.have_data() + + # test *recent* metrics values = hist_perfs.recent_metrics_values() assert len(values) == 7 + assert sorted(values.keys()) == sorted(target_names) assert f"acc_est_{dirn_s}" in values.keys() assert values == { f"acc_est_{dirn_s}": 0.2, @@ -68,42 +81,78 @@ def test_hist_perfs__main(dirn): f"recall_{dirn_s}": 5.2, f"loss_{dirn_s}": 6.2, } + + # test *final* metrics + values = hist_perfs.final_metrics_values() + assert len(values) == 7 + assert sorted(values.keys()) == sorted(target_names) + assert f"acc_est_{dirn_s}" in values.keys() + assert values == { + f"acc_est_{dirn_s}": 0.2, + f"acc_l_{dirn_s}": 1.2, + f"acc_u_{dirn_s}": 2.2, + f"f1_{dirn_s}": np.mean([3.1, 3.2]), + f"precision_{dirn_s}": np.mean([4.1, 4.2]), + f"recall_{dirn_s}": np.mean([5.1, 5.2]), + f"loss_{dirn_s}": np.mean([6.1, 6.2]), + } +@enforce_types +def _target_perfs_names(dirn_s: str): + return [f"{name}_{dirn_s}" for name in PERF_NAMES] #============================================================================= # test HistProfits @enforce_types def test_hist_profits__basic_init(): + # set data hist_profits = HistProfits() + # test empty raw data assert hist_profits.pdr_profits_OCEAN == [] assert hist_profits.trader_profits_USD == [] + # test names target_names = PROFIT_NAMES - names = hist_profits.metrics_names() + names = HistProfits.metrics_names() assert names == target_names - - values = hist_profits.recent_metrics_values() - assert sorted(values.keys()) == sorted(names) - for val in values.values(): - assert val is None - + # test can't call for metrics + assert not hist_profits.have_data() + with pytest.raises(AssertionError): + _ = hist_profits.recent_metrics_values() + with pytest.raises(AssertionError): + _ = hist_profits.final_metrics_values() + + @enforce_types def test_hist_profits__update(): + # set data hist_profits = HistProfits() - hist_profits.update(2.1, 3.1) hist_profits.update(2.2, 3.2) + + # test raw values assert hist_profits.pdr_profits_OCEAN == [2.1, 2.2] assert hist_profits.trader_profits_USD == [3.1, 3.2] + # test can call for metrics + assert hist_profits.have_data() + + # test *recent* metrics + target_names = PROFIT_NAMES values = hist_profits.recent_metrics_values() + assert sorted(values.keys()) == sorted(target_names) assert values == {"pdr_profit_OCEAN" : 2.2, "trader_profit_USD": 3.2} + # test *final* metrics + values = hist_profits.final_metrics_values() + assert sorted(values.keys()) == sorted(target_names) + assert values == {"pdr_profit_OCEAN" : np.sum([2.1, 2.2]), + "trader_profit_USD": np.sum([3.1, 3.2])} @enforce_types def test_hist_profits__calc_pdr_profit(): @@ -243,7 +292,10 @@ def test_hist_profits__calc_pdr_profit__unhappy_path(): @enforce_types def test_sim_state__basic_init(): + # set data st = SimState() + + # test empty raw state assert st.iter_number == 0 assert st.sim_model_data is None assert st.sim_model is None @@ -253,21 +305,19 @@ def test_sim_state__basic_init(): assert isinstance(st.hist_perfs[DOWN], HistPerfs) assert isinstance(st.hist_profits, HistProfits) - target_names = [f"{name}_{dirn_str(dirn)}" - for dirn in [UP, DOWN] - for name in PERF_NAMES] + PROFIT_NAMES + # test names + target_names = _target_state_names() assert len(target_names) == 7 * 2 + 2 - - names = st.metrics_names() + names = SimState.metrics_names() assert names == target_names - values = st.recent_metrics_values() - assert len(values) == 7 * 2 + 2 - assert sorted(values.keys()) == sorted(names) - for val in values.values(): - assert val is None + # test can't call for metrics + assert not st.have_data() + with pytest.raises(AssertionError): + _ = st.recent_metrics_values() + with pytest.raises(AssertionError): + _ = st.final_metrics_values() - @enforce_types def test_sim_state__init_loop_attributes(): # init @@ -288,6 +338,7 @@ def test_sim_state__init_loop_attributes(): @enforce_types def test_sim_state__main(): st = SimState() + target_names = _target_state_names() # update trueval = {UP:True, DOWN:False} @@ -295,16 +346,17 @@ def test_sim_state__main(): st.update(trueval, predprob, pdr_profit_OCEAN=1.4, trader_profit_USD=1.5) st.update(trueval, predprob, pdr_profit_OCEAN=2.4, trader_profit_USD=2.5) - - # test true_vs_pred + + # test raw state -- true_vs_pred assert st.true_vs_pred[UP].truevals == [True, True] assert st.true_vs_pred[UP].predprobs == [0.6, 0.6] assert st.true_vs_pred[DOWN].truevals == [False, False] assert st.true_vs_pred[DOWN].predprobs == [0.3, 0.3] - # test hist_perfs + # test *recent* metrics values = st.recent_metrics_values() assert len(values) == 7 * 2 + 2 + assert sorted(values.keys()) == sorted(target_names) for name, val in values.items(): if name == "pdr_profit_OCEAN": assert val == 2.4 @@ -315,3 +367,21 @@ def test_sim_state__main(): else: # hist_perfs value assert 0.0 <= val <= 1.0, (name, val) + # test *final* metrics + values = st.final_metrics_values() + assert sorted(values.keys()) == sorted(target_names) + for name, val in values.items(): + if name == "pdr_profit_OCEAN": + assert val == np.sum([1.4, 2.4]) + elif name == "trader_profit_USD": + assert val == np.sum([1.5, 2.5]) + elif "loss" in name: + assert 0.0 <= val <= 3.0 + else: # hist_perfs value + assert 0.0 <= val <= 1.0, (name, val) + +@enforce_types +def _target_state_names(): + return [f"{name}_{dirn_str(dirn)}" + for dirn in [UP, DOWN] + for name in PERF_NAMES] + PROFIT_NAMES diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 2b8c0fcc1..27e1dae53 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -2,6 +2,7 @@ from unittest.mock import Mock +from enforce_typing import enforce_types import pytest from pdr_backend.ppss.exchange_mgr_ss import ExchangeMgrSS @@ -11,6 +12,7 @@ FEE_PERCENT = 0.01 +@enforce_types @pytest.fixture def mock_ppss(): ppss = Mock() @@ -30,86 +32,96 @@ def mock_ppss(): return ppss +@enforce_types @pytest.fixture def sim_trader(mock_ppss): return SimTrader(mock_ppss) +@enforce_types def test_initial_state(sim_trader): assert sim_trader.position_open == "" - assert sim_trader.position_size == 0 - assert sim_trader.position_worth == 0 - assert sim_trader.tp == 0.0 - assert sim_trader.sl == 0.0 + assert sim_trader.position_size == 0. + assert sim_trader.position_worth == 0. + assert sim_trader.tp == 0. + assert sim_trader.sl == 0. +@enforce_types def test_close_long_position(sim_trader): sim_trader.position_open = "long" - sim_trader.position_size = 10 - sim_trader.position_worth = 1000 - sim_trader._sell = Mock(return_value=1100) - profit = sim_trader.close_long_position(110) - assert profit == 100 + sim_trader.position_size = 10. + sim_trader.position_worth = 1000. + sim_trader._sell = Mock(return_value=1100.) + profit = sim_trader.close_long_position(110.) + assert profit == 100. assert sim_trader.position_open == "" +@enforce_types def test_close_short_position(sim_trader): sim_trader.position_open = "short" - sim_trader.position_size = 10 - sim_trader.position_worth = 1000 + sim_trader.position_size = 10. + sim_trader.position_worth = 1000. sim_trader._buy = Mock() - profit = sim_trader.close_short_position(90) - assert profit == 100 + profit = sim_trader.close_short_position(90.) + assert profit == 100. assert sim_trader.position_open == "" +@enforce_types def test_trade_iter_open_long(sim_trader): - sim_trader._buy = Mock(return_value=10) - sim_trader._trade_iter(100, True, False, 0.5, 0, 110, 90) + sim_trader._buy = Mock(return_value=10.) + sim_trader._trade_iter(100., True, False, 0.5, 0., 110., 90.) assert sim_trader.position_open == "long" - assert sim_trader.position_worth == 1500 - assert sim_trader.position_size == 10 + assert sim_trader.position_worth == 1500. + assert sim_trader.position_size == 10. +@enforce_types def test_trade_iter_open_short(sim_trader): - sim_trader._sell = Mock(return_value=1500) - sim_trader._trade_iter(100, False, True, 0, 0.5, 110, 90) + sim_trader._sell = Mock(return_value=1500.) + sim_trader._trade_iter(100., False, True, 0., 0.5, 110., 90.) assert sim_trader.position_open == "short" - assert sim_trader.position_worth == 1500 - assert sim_trader.position_size == 15 + assert sim_trader.position_worth == 1500. + assert sim_trader.position_size == 15. +@enforce_types def test_trade_iter_close_long_take_profit_percent(sim_trader): sim_trader.position_open = "long" - sim_trader.position_size = 10 - sim_trader.position_worth = 1000 - sim_trader.tp = 110 - sim_trader._sell = Mock(return_value=1100) - profit = sim_trader._trade_iter(100, False, False, 0, 0, 110, 90) - assert profit == 100 # 1100 - 1000 + sim_trader.position_size = 10. + sim_trader.position_worth = 1000. + sim_trader.tp = 110. + sim_trader._sell = Mock(return_value=1100.) + profit = sim_trader._trade_iter(100., False, False, 0., 0., 110., 90.) + assert profit == 100. # 1100 - 1000 assert sim_trader.position_open == "" +@enforce_types def test_trade_iter_close_short_stop_loss_percent(sim_trader): sim_trader.position_open = "short" - sim_trader.position_size = 10 - sim_trader.position_worth = 1000 - sim_trader.sl = 110 + sim_trader.position_size = 10. + sim_trader.position_worth = 1000. + sim_trader.sl = 110. sim_trader._buy = Mock() - profit = sim_trader._trade_iter(100, False, False, 0, 0, 110, 90) - assert profit == -100 # 1100 - 1000 + profit = sim_trader._trade_iter(100., False, False, 0., 0., 110., 90.) + assert profit == -100. # 1100 - 1000 assert sim_trader.position_open == "" +@enforce_types def test_buy(sim_trader): sim_trader.exchange.create_market_buy_order = Mock() - tokcoin_amt_recd = sim_trader._buy(100.0, 1000.0) - assert tokcoin_amt_recd == (1000 / 100) * (1 - FEE_PERCENT) + tokcoin_amt_recd = sim_trader._buy(100., 1000.) + assert tokcoin_amt_recd == (1000. / 100.) * (1. - FEE_PERCENT) sim_trader.exchange.create_market_buy_order.assert_called_once() +@enforce_types def test_sell(sim_trader): sim_trader.exchange.create_market_sell_order = Mock() - usdcoin_amt_recd = sim_trader._sell(100.0, 10.0) - assert usdcoin_amt_recd == (100 * 10) * (1 - FEE_PERCENT) + usdcoin_amt_recd = sim_trader._sell(100., 10.) + assert usdcoin_amt_recd == (100. * 10.) * (1. - FEE_PERCENT) sim_trader.exchange.create_market_sell_order.assert_called_once() From caf5cbf830821e1fe1aafb6d84b94c645adf4e2e Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 18:03:22 +0200 Subject: [PATCH 19/50] deprecate residuals plots --- pdr_backend/sim/dash_plots/view_elements.py | 16 ---------------- pdr_backend/sim/sim_plotter.py | 5 ----- pdr_backend/sim/test/test_dash_plots.py | 2 -- pdr_backend/sim/test/test_sim_engine.py | 1 - 4 files changed, 24 deletions(-) diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index 8be74bb09..feca5ed49 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -10,8 +10,6 @@ "model_performance_vs_time", "aimodel_varimps", "aimodel_response", - "prediction_residuals_dist", - "prediction_residuals_other", ] empty_selected_vars = dcc.Checklist([], [], id="selected_vars") @@ -127,20 +125,6 @@ def get_tabs(figures): ], "className": "model_response_tab", }, - { - "name": "Model residuals", - "components": [ - side_by_side_graphs( - figures, - name1="prediction_residuals_dist", - name2="prediction_residuals_other", - height="100%", - width1="60%", - width2="40%", - ), - ], - "className": "model_residuals_tab", - }, ] diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index a907507ad..83aaad5e4 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -353,11 +353,6 @@ def _add_subplot_log_loss_vs_time(self, fig, row): ) fig.update_yaxes(title_text="log loss", row=3, col=1) - @enforce_types - def plot_prediction_residuals_dist(self): - return _empty_fig("(Nothing to show because model is a classifier.)") - - @enforce_types diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py index d670728a4..e8ae8e6f0 100644 --- a/pdr_backend/sim/test/test_dash_plots.py +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -55,8 +55,6 @@ def test_get_figures_by_state(): mock_sim_plotter.plot_pdr_profit_vs_ptrue.return_value = Figure() mock_sim_plotter.plot_trader_profit_vs_ptrue.return_value = Figure() mock_sim_plotter.plot_model_performance_vs_time.return_value = Figure() - mock_sim_plotter.plot_prediction_residuals_dist.return_value = Figure() - mock_sim_plotter.plot_prediction_residuals_other.return_value = Figure() plotdata = Mock() plotdata.colnames = ["var1", "var2"] diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index 4e5b41429..d6cf02234 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -101,7 +101,6 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): "trader_profit_tab": ["trader_profit_vs_time", "trader_profit_vs_ptrue"], "model_performance_tab": ["model_performance_vs_time"], "model_response_tab": ["aimodel_response", "aimodel_varimps"], - "model_residuals_tab": ["prediction_residuals_other"], } for tab_name, figures in tabs.items(): From c442a59f8be01f1e42e47cb760e03ca56fe8d23c Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 18:04:15 +0200 Subject: [PATCH 20/50] black --- pdr_backend/aimodel/test/test_true_vs_pred.py | 20 +- pdr_backend/aimodel/true_vs_pred.py | 14 +- pdr_backend/cli/arg_feed.py | 11 +- pdr_backend/ppss/aimodel_data_ss.py | 30 +-- pdr_backend/ppss/aimodel_ss.py | 8 +- pdr_backend/ppss/sim_ss.py | 13 +- pdr_backend/ppss/test/test_aimodel_data_ss.py | 24 +-- pdr_backend/ppss/test/test_aimodel_ss.py | 6 +- pdr_backend/ppss/test/test_sim_ss.py | 6 +- pdr_backend/sim/constants.py | 4 +- pdr_backend/sim/sim_engine.py | 63 +++--- pdr_backend/sim/sim_model.py | 15 +- pdr_backend/sim/sim_model_data.py | 20 +- pdr_backend/sim/sim_model_data_factory.py | 48 +++-- pdr_backend/sim/sim_model_factory.py | 13 +- pdr_backend/sim/sim_model_prediction.py | 34 ++-- pdr_backend/sim/sim_plotter.py | 1 - pdr_backend/sim/sim_predictoor.py | 6 +- pdr_backend/sim/sim_state.py | 75 +++---- pdr_backend/sim/sim_trader.py | 26 ++- pdr_backend/sim/test/resources.py | 16 +- pdr_backend/sim/test/test_sim_constants.py | 6 +- pdr_backend/sim/test/test_sim_engine.py | 6 +- pdr_backend/sim/test/test_sim_model.py | 7 +- pdr_backend/sim/test/test_sim_model_data.py | 19 +- .../sim/test/test_sim_model_data_factory.py | 39 ++-- .../sim/test/test_sim_model_factory.py | 18 +- .../sim/test/test_sim_model_prediction.py | 56 ++--- pdr_backend/sim/test/test_sim_predictoor.py | 12 +- pdr_backend/sim/test/test_sim_state.py | 191 +++++++++++------- pdr_backend/sim/test/test_sim_trader.py | 75 +++---- 31 files changed, 484 insertions(+), 398 deletions(-) diff --git a/pdr_backend/aimodel/test/test_true_vs_pred.py b/pdr_backend/aimodel/test/test_true_vs_pred.py index 9b4a27406..8391cf70e 100644 --- a/pdr_backend/aimodel/test/test_true_vs_pred.py +++ b/pdr_backend/aimodel/test/test_true_vs_pred.py @@ -21,8 +21,8 @@ def test_true_vs_pred(): assert d.n_correct == 1 assert d.n_trials == 1 assert len(d.accuracy()) == 3 - assert d.accuracy()[0] == 1.0/1.0 - + assert d.accuracy()[0] == 1.0 / 1.0 + # true = down, guess = down (correct guess) d.update(trueval=False, predprob=0.3) assert d.truevals == [True, False] @@ -30,8 +30,8 @@ def test_true_vs_pred(): assert d.predvals == [True, False] assert d.n_correct == 2 assert d.n_trials == 2 - assert d.accuracy()[0] == 2.0/2.0 - + assert d.accuracy()[0] == 2.0 / 2.0 + # true = up, guess = down (incorrect guess) d.update(trueval=True, predprob=0.4) assert d.truevals == [True, False, True] @@ -39,8 +39,8 @@ def test_true_vs_pred(): assert d.predvals == [True, False, False] assert d.n_correct == 2 assert d.n_trials == 3 - assert d.accuracy()[0] == approx(2.0/3.0) - + assert d.accuracy()[0] == approx(2.0 / 3.0) + # true = down, guess = up (incorrect guess) d.update(trueval=False, predprob=0.7) assert d.truevals == [True, False, True, False] @@ -48,25 +48,25 @@ def test_true_vs_pred(): assert d.predvals == [True, False, False, True] assert d.n_correct == 2 assert d.n_trials == 4 - assert d.accuracy()[0] == approx(2.0/4.0) + assert d.accuracy()[0] == approx(2.0 / 4.0) # test performance values (acc_est, acc_l, acc_u) = d.accuracy() assert acc_est == approx(0.5) assert acc_l == approx(0.010009003864986377) assert acc_u == approx(0.9899909961350136) - + (precision, recall, f1) = d.precision_recall_f1() assert precision == approx(0.5) assert recall == approx(0.5) assert f1 == approx(0.5) - + loss = d.log_loss() assert loss == approx(0.7469410259762035) assert d.perf_names() == PERF_NAMES assert len(d.perf_values()) == len(PERF_NAMES) - + target_values = [acc_est, acc_l, acc_u, precision, recall, f1, loss] values = d.perf_values() for val, target_val in zip(values, target_values): diff --git a/pdr_backend/aimodel/true_vs_pred.py b/pdr_backend/aimodel/true_vs_pred.py index 68063f76c..9b32596f5 100644 --- a/pdr_backend/aimodel/true_vs_pred.py +++ b/pdr_backend/aimodel/true_vs_pred.py @@ -6,13 +6,14 @@ from statsmodels.stats.proportion import proportion_confint PERF_NAMES = ["acc_est", "acc_l", "acc_u", "f1", "precision", "recall", "loss"] - + + class TrueVsPred: """ True vs pred vals for a single aimodel, or for a history of models, + the performances that derive from true vs pred value info """ - + @enforce_types def __init__(self): # 'i' is iteration number i @@ -57,7 +58,7 @@ def precision_recall_f1(self) -> Tuple[float, float, float]: @enforce_types def log_loss(self) -> float: if min(self.truevals) == max(self.truevals): - return 3.0 # magic number + return 3.0 # magic number return log_loss(self.truevals, self.predprobs) @enforce_types @@ -66,9 +67,8 @@ def perf_names(self) -> List[str]: @enforce_types def perf_values(self) -> List[float]: - perfs_list = \ - list(self.accuracy()) + \ - list(self.precision_recall_f1()) + \ - [self.log_loss()] + perfs_list = ( + list(self.accuracy()) + list(self.precision_recall_f1()) + [self.log_loss()] + ) assert len(perfs_list) == len(PERF_NAMES) return perfs_list diff --git a/pdr_backend/cli/arg_feed.py b/pdr_backend/cli/arg_feed.py index 6bea2d3e4..d7efe16d9 100644 --- a/pdr_backend/cli/arg_feed.py +++ b/pdr_backend/cli/arg_feed.py @@ -91,20 +91,21 @@ def from_str(feed_str: str, do_verify: bool = True) -> "ArgFeed": def variant_close(self) -> "ArgFeed": return self.variant_signal("close") - + def variant_high(self) -> "ArgFeed": return self.variant_signal("high") - + def variant_low(self) -> "ArgFeed": return self.variant_signal("low") - + def variant_signal(self, signal_str: str) -> "ArgFeed": return ArgFeed( - self.exchange, + self.exchange, signal_str, self.pair, self.timeframe, - ) + ) + @enforce_types def _unpack_feeds_str(feeds_str: str) -> List[ArgFeed]: diff --git a/pdr_backend/ppss/aimodel_data_ss.py b/pdr_backend/ppss/aimodel_data_ss.py index 3783182dd..0de10837a 100644 --- a/pdr_backend/ppss/aimodel_data_ss.py +++ b/pdr_backend/ppss/aimodel_data_ss.py @@ -24,29 +24,29 @@ def __init__(self, d: dict): self.validate_autoregressive_n(self.autoregressive_n) self.validate_class_thr(self.class_thr) self.validate_transform(self.transform) - + # -------------------------------- # validators @staticmethod def validate_max_n_train(max_n_train: int): if not 0 < max_n_train: raise ValueError(max_n_train) - + @staticmethod def validate_autoregressive_n(autoregressive_n: int): if not (0 < autoregressive_n < np.inf): raise ValueError(autoregressive_n) - + @staticmethod def validate_class_thr(class_thr: float): if not 0 <= class_thr <= 1.0: raise ValueError(class_thr) - + @staticmethod def validate_transform(transform: str): if transform not in TRANSFORM_OPTIONS: raise ValueError(transform) - + # -------------------------------- # yaml properties @@ -64,7 +64,7 @@ def autoregressive_n(self) -> int: For diff=2, add to model_inputs (z[t-1]-z[t-2]) - (z[t-2]-z[t-3]), .. """ return self.d["autoregressive_n"] - + @property def class_thr(self) -> float: """eg 0.05 = 5%. UP class needs > this, DOWN < this""" @@ -74,25 +74,25 @@ def class_thr(self) -> float: def transform(self) -> int: """eg 'RelDiff'""" return self.d["transform"] - + # -------------------------------- # setters - def set_max_n_train(self, max_n_train:int): + def set_max_n_train(self, max_n_train: int): self.validate_max_n_train(max_n_train) self.d["max_n_train"] = max_n_train - - def set_autoregressive_n(self, autoregressive_n:int): + + def set_autoregressive_n(self, autoregressive_n: int): self.validate_autoregressive_n(autoregressive_n) self.d["autoregressive_n"] = autoregressive_n - - def set_class_thr(self, class_thr:float): + + def set_class_thr(self, class_thr: float): self.validate_class_thr(class_thr) self.d["class_thr"] = class_thr - - def set_transform(self, transform:str): + + def set_transform(self, transform: str): self.validate_transform(transform) self.d["transform"] = transform - + # ========================================================================= # utilities for testing diff --git a/pdr_backend/ppss/aimodel_ss.py b/pdr_backend/ppss/aimodel_ss.py index 6affb66f3..0f96be136 100644 --- a/pdr_backend/ppss/aimodel_ss.py +++ b/pdr_backend/ppss/aimodel_ss.py @@ -57,7 +57,7 @@ def __init__(self, d: dict): if self.calibrate_regr not in CALIBRATE_REGR_OPTIONS: raise ValueError(self.calibrate_regr) self.validate_train_every_n_epochs(self.train_every_n_epochs) - + # -------------------------------- # validators -- add as needed, when setters are added def validate_train_every_n_epochs(self, n: int): @@ -137,7 +137,7 @@ def weight_recent_n(self) -> Tuple[int, int]: if self.weight_recent == "10000x": return 10000, 0 raise ValueError(self.weight_recent) - + # -------------------------------- # setters (only add as needed) def set_train_every_n_epochs(self, n: int): @@ -165,6 +165,8 @@ def aimodel_ss_test_dict( "balance_classes": balance_classes or "SMOTE", "calibrate_probs": calibrate_probs or "CalibratedClassifierCV_Sigmoid", "calibrate_regr": calibrate_regr or "None", - "train_every_n_epochs": 1 if train_every_n_epochs is None else train_every_n_epochs + "train_every_n_epochs": ( + 1 if train_every_n_epochs is None else train_every_n_epochs + ), } return d diff --git a/pdr_backend/ppss/sim_ss.py b/pdr_backend/ppss/sim_ss.py index 914a6ebdf..a63475f9b 100644 --- a/pdr_backend/ppss/sim_ss.py +++ b/pdr_backend/ppss/sim_ss.py @@ -29,7 +29,7 @@ def __init__(self, d: dict): # validate data self.validate_test_n(self.test_n) self.validate_tradetype(self.tradetype) - + # -------------------------------- # validators @staticmethod @@ -38,7 +38,7 @@ def validate_test_n(test_n: int): raise TypeError(test_n) if not 0 < test_n < np.inf: raise ValueError(test_n) - + @staticmethod def validate_tradetype(tradetype: str): if not isinstance(tradetype, str): @@ -72,17 +72,16 @@ def is_final_iter(self, iter_i: int) -> bool: raise ValueError(iter_i) return (iter_i + 1) == self.test_n - # -------------------------------- # setters - def set_test_n(self, test_n:int): + def set_test_n(self, test_n: int): self.validate_test_n(test_n) self.d["test_n"] = test_n - - def set_tradetype(self, tradetype:str): + + def set_tradetype(self, tradetype: str): self.validate_tradetype(tradetype) self.d["tradetype"] = tradetype - + # ========================================================================= # utilities for testing diff --git a/pdr_backend/ppss/test/test_aimodel_data_ss.py b/pdr_backend/ppss/test/test_aimodel_data_ss.py index 0017faea4..79aed56a6 100644 --- a/pdr_backend/ppss/test/test_aimodel_data_ss.py +++ b/pdr_backend/ppss/test/test_aimodel_data_ss.py @@ -33,13 +33,13 @@ def test_aimodel_data_ss__nondefault_values(): ss = AimodelDataSS(aimodel_data_ss_test_dict(autoregressive_n=13)) assert ss.autoregressive_n == 13 - + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.06)) assert ss.class_thr == 0.06 - + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0.0)) assert ss.class_thr == 0.0 - + ss = AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1.0)) assert ss.class_thr == 1.0 @@ -57,14 +57,14 @@ def test_aimodel_data_ss__bad_inputs(): with pytest.raises(ValueError): AimodelDataSS(aimodel_data_ss_test_dict(class_thr=-0.1)) - + with pytest.raises(ValueError): AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1.1)) - - with pytest.raises(TypeError): # floats only, for simplicity + + with pytest.raises(TypeError): # floats only, for simplicity AimodelDataSS(aimodel_data_ss_test_dict(class_thr=0)) - - with pytest.raises(TypeError): # floats only, for simplicity + + with pytest.raises(TypeError): # floats only, for simplicity AimodelDataSS(aimodel_data_ss_test_dict(class_thr=1)) with pytest.raises(ValueError): @@ -82,6 +82,7 @@ def test_aimodel_data_ss__bad_inputs(): with pytest.raises(TypeError): AimodelDataSS(aimodel_data_ss_test_dict(transform=3.1)) + @enforce_types def test_aimodel_data_ss__setters(): d = aimodel_data_ss_test_dict() @@ -94,7 +95,7 @@ def test_aimodel_data_ss__setters(): ss.set_max_n_train(0) with pytest.raises(ValueError): ss.set_max_n_train(-5) - + # autoregressive_n ss.set_autoregressive_n(12) assert ss.autoregressive_n == 12 @@ -102,16 +103,15 @@ def test_aimodel_data_ss__setters(): ss.set_autoregressive_n(0) with pytest.raises(ValueError): ss.set_autoregressive_n(-5) - + # class_thr ss.set_class_thr(0.34) assert ss.class_thr == 0.34 with pytest.raises(ValueError): ss.set_class_thr(-0.1) - + # transform ss.set_transform("RelDiff") assert ss.transform == "RelDiff" with pytest.raises(ValueError): ss.set_transform("foo") - diff --git a/pdr_backend/ppss/test/test_aimodel_ss.py b/pdr_backend/ppss/test/test_aimodel_ss.py index 075b852ef..62c9b4bc8 100644 --- a/pdr_backend/ppss/test/test_aimodel_ss.py +++ b/pdr_backend/ppss/test/test_aimodel_ss.py @@ -87,10 +87,10 @@ def test_aimodel_ss__bad_inputs(): with pytest.raises(ValueError): AimodelSS(aimodel_ss_test_dict(calibrate_regr="foo")) - + with pytest.raises(ValueError): AimodelSS(aimodel_ss_test_dict(train_every_n_epochs=0)) - + with pytest.raises(ValueError): AimodelSS(aimodel_ss_test_dict(train_every_n_epochs=-5)) @@ -107,6 +107,7 @@ def test_aimodel_ss__calibrate_probs_skmethod(): assert ss.calibrate_probs_skmethod(100) == "sigmoid" # because N is small assert ss.calibrate_probs_skmethod(1000) == "isotonic" + @enforce_types def test_aimodel_ss__setters(): d = aimodel_ss_test_dict() @@ -119,4 +120,3 @@ def test_aimodel_ss__setters(): ss.set_train_every_n_epochs(0) with pytest.raises(ValueError): ss.set_train_every_n_epochs(-5) - diff --git a/pdr_backend/ppss/test/test_sim_ss.py b/pdr_backend/ppss/test/test_sim_ss.py index bff6dd7db..a26c97f0a 100644 --- a/pdr_backend/ppss/test/test_sim_ss.py +++ b/pdr_backend/ppss/test/test_sim_ss.py @@ -103,11 +103,12 @@ def test_sim_ss_is_final_iter(tmpdir): with pytest.raises(ValueError): _ = ss.is_final_iter(11) + @enforce_types def test_sim_ss_setters(tmpdir): d = sim_ss_test_dict(_logdir(tmpdir)) ss = SimSS(d) - + # test_n ss.set_test_n(32) assert ss.test_n == 32 @@ -115,13 +116,14 @@ def test_sim_ss_setters(tmpdir): ss.set_test_n(0) with pytest.raises(ValueError): ss.set_test_n(-5) - + # tradetype ss.set_tradetype("livereal") assert ss.tradetype == "livereal" with pytest.raises(ValueError): ss.set_tradetype("foo") + # ==================================================================== # helper funcs @enforce_types diff --git a/pdr_backend/sim/constants.py b/pdr_backend/sim/constants.py index 684c0097d..fe90dd92c 100644 --- a/pdr_backend/sim/constants.py +++ b/pdr_backend/sim/constants.py @@ -2,14 +2,16 @@ from enforce_typing import enforce_types + class Dirn(IntEnum): UP = 1 DOWN = 2 - + UP = Dirn.UP DOWN = Dirn.DOWN + @enforce_types def dirn_str(dirn: Dirn): if dirn == UP: diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index dff197847..2f67df107 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -46,9 +46,8 @@ def __init__( multi_id: Optional[str] = None, ): self.ppss = ppss - - assert self.predict_feed.signal == "close", \ - "only operates on close predictions" + + assert self.predict_feed.signal == "close", "only operates on close predictions" # can be disabled by calling disable_realtime_state() self.do_state_updates = True @@ -66,13 +65,13 @@ def __init__( self.multi_id = multi_id else: self.multi_id = str(uuid.uuid4()) - + assert self.pdr_ss.aimodel_data_ss.transform == "None" @property def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss - + @property def predict_feed(self) -> ArgFeed: return self.pdr_ss.predict_train_feedsets[0].predict @@ -80,19 +79,19 @@ def predict_feed(self) -> ArgFeed: @property def timeframe(self) -> ArgTimeframe: return self.predict_feed.timeframe - + @property def others_stake(self) -> float: return self.pdr_ss.others_stake.amt_eth - + @property def others_accuracy(self) -> float: return self.pdr_ss.others_accuracy - + @property def revenue(self) -> float: return self.pdr_ss.revenue.amt_eth - + @enforce_types def _init_loop_attributes(self): filebase = f"out_{UnixTimeMs.now()}.txt" @@ -103,7 +102,7 @@ def _init_loop_attributes(self): logger.addHandler(fh) self.st.init_loop_attributes() - + logger.info("Initialize plot data.") self.sim_plotter.init_state(self.multi_id) @@ -117,7 +116,7 @@ def run(self): # ohclv data f = OhlcvDataFactory(self.ppss.lake_ss) mergedohlcv_df = f.get_mergedohlcv_df() - + # main loop! for iter_i in range(self.ppss.sim_ss.test_n): self.run_one_iter(iter_i, mergedohlcv_df) @@ -140,7 +139,7 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): cur_low = self._curval(df, testshift, "low") y_thr_UP = sim_model_data_f.thr_UP(cur_close) y_thr_DOWN = sim_model_data_f.thr_DOWN(cur_close) - + # build model model_factory = SimModelFactory(self.pdr_ss.aimodel_ss) st.sim_model_data: SimModelData = sim_model_data_f.build(iter_i, df) @@ -152,13 +151,16 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): conf_thr = self.ppss.trader_ss.sim_confidence_threshold sim_model_p = SimModelPrediction(conf_thr, predprob[UP], predprob[DOWN]) - + # predictoor takes action (stake) stake_up, stake_down = self.sim_predictoor.predict_iter(sim_model_p) # trader takes action (trade) trader_profit_USD = self.sim_trader.trade_iter( - cur_close, cur_high, cur_low, sim_model_p, + cur_close, + cur_high, + cur_low, + sim_model_p, ) # observe next price values @@ -169,17 +171,22 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): # observe price change prev -> next, and related changes for classifier trueval_up_close = next_close > cur_close trueval = { - UP: next_high > y_thr_UP, # did next high go > prev close+% ? - DOWN: next_low < y_thr_DOWN, # did next low go < prev close-% ? + UP: next_high > y_thr_UP, # did next high go > prev close+% ? + DOWN: next_low < y_thr_DOWN, # did next low go < prev close-% ? } # calc predictoor profit pdr_profit_OCEAN = HistProfits.calc_pdr_profit( - self.others_stake, self.others_accuracy, - stake_up, stake_down, self.revenue, trueval_up_close) - + self.others_stake, + self.others_accuracy, + stake_up, + stake_down, + self.revenue, + trueval_up_close, + ) + # update state - st.update(trueval, predprob, pdr_profit_OCEAN, trader_profit_USD) + st.update(trueval, predprob, pdr_profit_OCEAN, trader_profit_USD) # log ut = self._calc_ut(df, testshift) @@ -203,10 +210,10 @@ def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: st = self.st model = st.sim_model[dirn] model_data = st.sim_model_data[dirn] - + colnames = model_data.colnames colnames = [shift_one_earlier(c) for c in colnames] - + most_recent_x = model_data.X[-1, :] slicing_x = most_recent_x d = AimodelPlotdata( @@ -224,22 +231,25 @@ def _aimodel_plotdata_1dir(self, dirn: Dirn) -> AimodelPlotdata: def _curval(self, df, testshift: int, signal_str: str) -> float: # float() so not np.float64, bc applying ">" gives np.bool -> problems return float(self._yraw(df, testshift, signal_str)[-2]) - + @enforce_types def _nextval(self, df, testshift: int, signal_str: str) -> float: # float() so not np.float64, bc applying ">" gives np.bool -> problems return float(self._yraw(df, testshift, signal_str)[-1]) - + @enforce_types def _yraw(self, mergedohlcv_df, testshift: int, signal_str: str): assert signal_str in ["close", "high", "low"] feed = self.predict_feed.variant_signal(signal_str) aimodel_data_f = AimodelDataFactory(self.pdr_ss) _, _, yraw, _, _ = aimodel_data_f.create_xy( - mergedohlcv_df, testshift, feed, ArgFeeds([feed]), + mergedohlcv_df, + testshift, + feed, + ArgFeeds([feed]), ) return yraw - + @enforce_types def _calc_ut(self, mergedohlcv_df, testshift: int) -> UnixTimeMs: recent_ut = UnixTimeMs(int(mergedohlcv_df["timestamp"].to_list()[-1])) @@ -269,4 +279,3 @@ def _do_save_state(self, i: int) -> Tuple[bool, bool]: return False, False return True, False - diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index 57eb1910a..91cef8771 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -6,31 +6,30 @@ from pdr_backend.sim.sim_model_data import SimModelData from pdr_backend.sim.sim_model_prediction import SimModelPrediction + @enforce_types class SimModel(dict): - + def __init__(self, model_UP: Aimodel, model_DOWN: Aimodel): self[UP] = model_UP self[DOWN] = model_DOWN - + def predict_next(self, X_test: dict) -> dict: """ @arguments X_test_dict -- dict of {UP: X_test_UP_array, DOWN: X_test_DOWN_arr.} It expects each array to have exactly 1 row, to predict from - + @return predprob -- dict of {UP: predprob_UP_float, DOWN: predprob_DOWN_float} """ assert X_test[UP].shape[0] == 1 assert X_test[DOWN].shape[0] == 1 - + prob_UP = self[UP].predict_ptrue(X_test[UP])[0] prob_DOWN = self[DOWN].predict_ptrue(X_test[UP])[0] - + # ensure not np.float64. Why: applying ">" gives np.bool --> problems prob_UP, prob_DOWN = float(prob_UP), float(prob_DOWN) - + return {UP: prob_UP, DOWN: prob_DOWN} - - diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index 61e9e295f..4369e638e 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -5,43 +5,45 @@ from pdr_backend.sim.constants import UP, DOWN + class SimModelData1Dir: - + @enforce_types def __init__( self, X: np.ndarray, ytrue: np.ndarray, - colnames:List[str], + colnames: List[str], ): assert len(X.shape) == 2 assert len(ytrue.shape) == 1 assert X.shape[0] == ytrue.shape[0], (X.shape[0], ytrue.shape[0]) - + self.X: np.ndarray = X self.ytrue: np.ndarray = ytrue self.colnames: List[str] = colnames - + @property def st(self) -> int: return 0 - + @property def fin(self) -> int: return self.X.shape[0] - 1 @property def X_train(self) -> np.ndarray: - return self.X[self.st:self.fin, :] + return self.X[self.st : self.fin, :] @property def X_test(self) -> np.ndarray: return self.X[self.fin : self.fin + 1, :] - + @property def ytrue_train(self) -> np.ndarray: - return self.ytrue[self.st:self.fin] - + return self.ytrue[self.st : self.fin] + + class SimModelData(dict): @enforce_types def __init__( diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index e6eda81ad..17b944f6f 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -10,19 +10,20 @@ from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir + class SimModelDataFactory: @enforce_types def __init__(self, ppss: PPSS): self.ppss = ppss - + @property def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss - + @property def class_thr(self) -> float: return self.pdr_ss.aimodel_data_ss.class_thr - + @property def predict_feed(self) -> ArgFeed: return self.pdr_ss.predict_train_feedsets[0].predict @@ -36,33 +37,42 @@ def testshift(self, test_i: int) -> int: def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: """Construct sim model data""" df = mergedohlcv_df - testshift = self.testshift(test_i) # eg [99, 98, .., 2, 1, 0] + testshift = self.testshift(test_i) # eg [99, 98, .., 2, 1, 0] data_f = AimodelDataFactory(self.pdr_ss) p: ArgFeed = self.predict_feed - + _, _, y_close, _, _ = data_f.create_xy( - df, testshift, p.variant_close(), ArgFeeds([p.variant_close()]), + df, + testshift, + p.variant_close(), + ArgFeeds([p.variant_close()]), ) X_high, _, y_high, x_high_df, _ = data_f.create_xy( - df, testshift, p.variant_high(), ArgFeeds([p.variant_high()]), + df, + testshift, + p.variant_high(), + ArgFeeds([p.variant_high()]), ) X_low, _, y_low, x_low_df, _ = data_f.create_xy( - df, testshift, p.variant_low(), ArgFeeds([p.variant_low()]), + df, + testshift, + p.variant_low(), + ArgFeeds([p.variant_low()]), ) ytrue_UP = [] ytrue_DOWN = [] for i, cur_close in enumerate(y_close[:-1]): # did the next high value go above the current close+% value? - next_high = y_high[i+1] + next_high = y_high[i + 1] thr_UP = self.thr_UP(cur_close) ytrue_UP.append(next_high > thr_UP) - + # did the next low value go below the current close-% value? - next_low = y_low[i+1] + next_low = y_low[i + 1] thr_DOWN = self.thr_DOWN(cur_close) - ytrue_DOWN.append(next_low < thr_DOWN) + ytrue_DOWN.append(next_low < thr_DOWN) ytrue_UP, ytrue_DOWN = np.array(ytrue_UP), np.array(ytrue_DOWN) @@ -70,20 +80,18 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: colnames_DOWN = list(x_low_df.columns) # Q: I used X[1:,:], but should it be X[:-1,:] ? - d_UP = SimModelData1Dir(X_high[1:,:], ytrue_UP, colnames_UP) - d_DOWN = SimModelData1Dir(X_low[1:,:], ytrue_DOWN, colnames_DOWN) - + d_UP = SimModelData1Dir(X_high[1:, :], ytrue_UP, colnames_UP) + d_DOWN = SimModelData1Dir(X_low[1:, :], ytrue_DOWN, colnames_DOWN) + # note: alternatively, each input X could be h+l+c rather than just h or l d = SimModelData(d_UP, d_DOWN) - + return d @enforce_types def thr_UP(self, cur_close: float) -> float: return cur_close * (1 + self.class_thr) - + @enforce_types def thr_DOWN(self, cur_close: float) -> float: - return cur_close * (1 - self.class_thr) - - + return cur_close * (1 - self.class_thr) diff --git a/pdr_backend/sim/sim_model_factory.py b/pdr_backend/sim/sim_model_factory.py index 9e10b14fe..0367f79e0 100644 --- a/pdr_backend/sim/sim_model_factory.py +++ b/pdr_backend/sim/sim_model_factory.py @@ -8,34 +8,35 @@ from pdr_backend.sim.sim_model import SimModel from pdr_backend.sim.sim_model_data import SimModelData -class SimModelFactory: + +class SimModelFactory: @enforce_types def __init__(self, aimodel_ss: AimodelSS): self.aimodel_ss = aimodel_ss - + @enforce_types def do_build(self, prev_model: Optional[SimModel], test_i: int) -> bool: """Update/train model?""" n = self.aimodel_ss.train_every_n_epochs return prev_model is None or test_i % n == 0 - + @enforce_types def build(self, data: SimModelData) -> SimModel: model_f = AimodelFactory(self.aimodel_ss) - + model_UP = model_f.build( data[UP].X_train, data[UP].ytrue_train, None, None, ) - + model_DOWN = model_f.build( data[DOWN].X_train, data[DOWN].ytrue_train, None, None, ) - + sim_model = SimModel(model_UP, model_DOWN) return sim_model diff --git a/pdr_backend/sim/sim_model_prediction.py b/pdr_backend/sim/sim_model_prediction.py index 9dc9406e0..2c24af3d3 100644 --- a/pdr_backend/sim/sim_model_prediction.py +++ b/pdr_backend/sim/sim_model_prediction.py @@ -6,16 +6,14 @@ class SimModelPrediction: def __init__( self, conf_thr: float, - # UP model's probability that next high will go > prev close+% prob_UP: float, - # DOWN model's probability that next low will go < prev close-% prob_DOWN: float, ): # ensure not np.float64. Why: applying ">" gives np.bool --> problems prob_UP, prob_DOWN = float(prob_UP), float(prob_DOWN) - + # ppss.trader_ss.sim_confidence_threshold self.conf_thr = conf_thr @@ -23,31 +21,34 @@ def __init__( self.prob_UP = prob_UP self.prob_DOWN = prob_DOWN - # derived attributes + # derived attributes if self.models_in_conflict(): self.conf_up = 0.0 self.conf_down = 0.0 self.pred_up = False self.pred_down = False self.prob_up_MERGED = 0.5 - + elif self.prob_UP >= self.prob_DOWN: self.conf_up = (prob_UP - 0.5) * 2.0 # to range [0,1] self.conf_down = 0.0 self.pred_up = self.conf_up > self.conf_thr self.pred_down = False self.prob_up_MERGED = self.prob_UP - - else: # prob_DOWN > prob_UP + + else: # prob_DOWN > prob_UP self.conf_up = 0.0 self.conf_down = (self.prob_DOWN - 0.5) * 2.0 self.pred_up = False self.pred_down = self.conf_down > self.conf_thr self.prob_up_MERGED = 1.0 - self.prob_DOWN - + def do_trust_models(self) -> bool: do_trust = _do_trust_models( - self.pred_up, self.pred_down, self.prob_UP, self.prob_DOWN, + self.pred_up, + self.pred_down, + self.prob_UP, + self.prob_DOWN, ) return do_trust @@ -57,10 +58,10 @@ def models_in_conflict(self) -> bool: @enforce_types def _do_trust_models( - pred_up:bool, - pred_down:bool, - prob_UP:float, - prob_DOWN:float, + pred_up: bool, + pred_down: bool, + prob_UP: float, + prob_DOWN: float, ) -> bool: """Do we trust the models enough to take prediction / trading action?""" # preconditions @@ -76,8 +77,8 @@ def _do_trust_models( raise ValueError("can't have pred_down=True with prob_UP dominant") # main test - return (pred_up or pred_down) \ - and not _models_in_conflict(prob_UP, prob_DOWN) + return (pred_up or pred_down) and not _models_in_conflict(prob_UP, prob_DOWN) + @enforce_types def _models_in_conflict(prob_UP: float, prob_DOWN: float) -> bool: @@ -89,5 +90,4 @@ def _models_in_conflict(prob_UP: float, prob_DOWN: float) -> bool: raise ValueError(prob_DOWN) # main test - return (prob_UP > 0.5 and prob_DOWN > 0.5) or \ - (prob_UP < 0.5 and prob_DOWN < 0.5) + return (prob_UP > 0.5 and prob_DOWN > 0.5) or (prob_UP < 0.5 and prob_DOWN < 0.5) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 83aaad5e4..bd86e29f3 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -354,7 +354,6 @@ def _add_subplot_log_loss_vs_time(self, fig, row): fig.update_yaxes(title_text="log loss", row=3, col=1) - @enforce_types def file_age_in_seconds(pathname): stat_result = os.stat(pathname) diff --git a/pdr_backend/sim/sim_predictoor.py b/pdr_backend/sim/sim_predictoor.py index 157f7b43b..1f93cc1be 100644 --- a/pdr_backend/sim/sim_predictoor.py +++ b/pdr_backend/sim/sim_predictoor.py @@ -5,6 +5,7 @@ from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.sim_model_prediction import SimModelPrediction + class SimPredictoor: @enforce_types def __init__(self, pdr_ss: PredictoorSS): @@ -13,7 +14,7 @@ def __init__(self, pdr_ss: PredictoorSS): @property def max_stake_amt(self) -> float: return self.pdr_ss.stake_amount.amt_eth - + @enforce_types def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: """@return (stake_up, stake_down)""" @@ -24,10 +25,9 @@ def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: stake_amt = self.max_stake_amt * p.conf_up stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) - else: # p.prob_DOWN > p.prob_UP + else: # p.prob_DOWN > p.prob_UP stake_amt = self.max_stake_amt * p.conf_down stake_up = stake_amt * p.prob_up_MERGED stake_down = stake_amt * (1.0 - p.prob_up_MERGED) return (stake_up, stake_down) - diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 201b832f8..95a4bf69a 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -7,16 +7,17 @@ from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN -#============================================================================= +# ============================================================================= # HistPerfs + class HistPerfs: """Historical performances, for 1 model dir'n (eg UP)""" - + @enforce_types def __init__(self, dirn: Dirn): self.dirn = dirn - + # 'i' is iteration number i self.acc_ests: List[float] = [] # [i] : %-correct self.acc_ls: List[float] = [] # [i] : %-correct-lower @@ -27,12 +28,12 @@ def __init__(self, dirn: Dirn): self.recalls: List[float] = [] # [i] : recall self.losses: List[float] = [] # [i] : log-loss - + @enforce_types def update(self, perfs_list: list): """perfs_list typically comes from TrueVsPred.perf_values()""" acc_est, acc_l, acc_u, f1, precision, recall, loss = perfs_list - + self.acc_ests.append(acc_est) self.acc_ls.append(acc_l) self.acc_us.append(acc_u) @@ -42,7 +43,7 @@ def update(self, perfs_list: list): self.recalls.append(recall) self.losses.append(loss) - + @enforce_types def metrics_names_instance(self) -> List[str]: """@return e.g. ['acc_est_UP', 'acc_l_UP', ..., 'loss_UP]""" @@ -51,9 +52,8 @@ def metrics_names_instance(self) -> List[str]: @staticmethod def metrics_names_static(dirn) -> List[str]: """@return e.g. ['acc_est_UP', 'acc_l_UP', ..., 'loss_UP]""" - return [f"{name}_{dirn_str(dirn)}" - for name in PERF_NAMES] - + return [f"{name}_{dirn_str(dirn)}" for name in PERF_NAMES] + @enforce_types def recent_metrics_values(self) -> Dict[str, float]: """Return most recent metrics""" @@ -69,12 +69,12 @@ def recent_metrics_values(self) -> Dict[str, float]: f"recall_{s}": self.recalls[-1], f"loss_{s}": self.losses[-1], } - + @enforce_types def final_metrics_values(self) -> Dict[str, float]: - """Return *final* metrics, rather than most recent. """ + """Return *final* metrics, rather than most recent.""" assert self.have_data(), "only works for >0 entries" - + s = dirn_str(self.dirn) return { f"acc_est_{s}": self.acc_ests[-1], @@ -90,12 +90,13 @@ def final_metrics_values(self) -> Dict[str, float]: def have_data(self) -> bool: return bool(self.acc_ests) - -#============================================================================= + +# ============================================================================= # HistProfits PROFIT_NAMES = ["pdr_profit_OCEAN", "trader_profit_USD"] + class HistProfits: def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit @@ -103,7 +104,7 @@ def __init__(self): @staticmethod def calc_pdr_profit( - others_stake:float, + others_stake: float, others_accuracy: float, stake_up: float, stake_down: float, @@ -115,7 +116,7 @@ def calc_pdr_profit( assert stake_up >= 0.0 assert stake_down >= 0.0 assert revenue >= 0.0 - + amt_sent = stake_up + stake_down others_stake_correct = others_stake * others_accuracy tot_stake = others_stake + stake_up + stake_down @@ -134,29 +135,29 @@ def calc_pdr_profit( def update(self, pdr_profit_OCEAN: float, trader_profit_USD: float): self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) self.trader_profits_USD.append(trader_profit_USD) - + @staticmethod def metrics_names() -> List[str]: return PROFIT_NAMES - + @enforce_types def recent_metrics_values(self) -> Dict[str, float]: """Return most recent metrics""" assert self.have_data(), "only works for >0 entries" - + return { - "pdr_profit_OCEAN" : self.pdr_profits_OCEAN[-1], - "trader_profit_USD" : self.trader_profits_USD[-1], + "pdr_profit_OCEAN": self.pdr_profits_OCEAN[-1], + "trader_profit_USD": self.trader_profits_USD[-1], } @enforce_types def final_metrics_values(self) -> Dict[str, float]: - """Return *final* metrics, rather than most recent. """ + """Return *final* metrics, rather than most recent.""" assert self.have_data(), "only works for >0 entries" - + return { - "pdr_profit_OCEAN" : np.sum(self.pdr_profits_OCEAN), - "trader_profit_USD" : np.sum(self.trader_profits_USD), + "pdr_profit_OCEAN": np.sum(self.pdr_profits_OCEAN), + "trader_profit_USD": np.sum(self.trader_profits_USD), } @enforce_types @@ -164,10 +165,10 @@ def have_data(self) -> bool: return bool(self.pdr_profits_OCEAN) - -#============================================================================= +# ============================================================================= # SimState + class SimState: @enforce_types def __init__(self): @@ -202,14 +203,16 @@ def update( self.hist_perfs[UP].update(self.true_vs_pred[UP].perf_values()) self.hist_perfs[DOWN].update(self.true_vs_pred[DOWN].perf_values()) - + self.hist_profits.update(pdr_profit_OCEAN, trader_profit_USD) - + @staticmethod def metrics_names() -> List[str]: - return HistPerfs.metrics_names_static(UP) + \ - HistPerfs.metrics_names_static(DOWN) + \ - HistProfits.metrics_names() + return ( + HistPerfs.metrics_names_static(UP) + + HistPerfs.metrics_names_static(DOWN) + + HistProfits.metrics_names() + ) @enforce_types def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: @@ -222,7 +225,7 @@ def recent_metrics_values(self) -> Dict[str, Union[int, float, None]]: @enforce_types def final_metrics_values(self) -> Dict[str, Union[int, float, None]]: - """Return *final* metrics, rather than most recent. """ + """Return *final* metrics, rather than most recent.""" metrics = {} metrics.update(self.hist_perfs[UP].final_metrics_values()) metrics.update(self.hist_perfs[DOWN].final_metrics_values()) @@ -231,6 +234,8 @@ def final_metrics_values(self) -> Dict[str, Union[int, float, None]]: @enforce_types def have_data(self) -> bool: - return self.hist_perfs[UP].have_data() \ - and self.hist_perfs[DOWN].have_data() \ + return ( + self.hist_perfs[UP].have_data() + and self.hist_perfs[DOWN].have_data() and self.hist_profits().have_data() + ) diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index 1802c9f69..2b4dc0065 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -15,10 +15,10 @@ def __init__(self, ppss): self.ppss = ppss self.position_open: str = "" # long, short, "" - self.position_size: float = 0. # amount of tokens in position - self.position_worth: float = 0. # amount of USD in position - self.tp: float = 0. # take profit - self.sl: float = 0. # stop loss + self.position_size: float = 0.0 # amount of tokens in position + self.position_worth: float = 0.0 # amount of USD in position + self.tp: float = 0.0 # take profit + self.sl: float = 0.0 # stop loss self.tp_percent: float = self.ppss.trader_ss.take_profit_percent self.sl_percent: float = self.ppss.trader_ss.stop_loss_percent @@ -57,7 +57,7 @@ def close_short_position(self, buy_price: float) -> float: self.position_open = "" profit = self.position_worth - usdcoin_amt_send return float(profit) - + @enforce_types def trade_iter( self, @@ -67,10 +67,16 @@ def trade_iter( p: SimModelPrediction, ) -> float: profit_USD = self._trade_iter( - cur_close, p.pred_up, p.pred_down, p.conf_up, p.conf_down, high, low, - ) + cur_close, + p.pred_up, + p.pred_down, + p.conf_up, + p.conf_down, + high, + low, + ) return float(profit_USD) - + @enforce_types def _trade_iter( self, @@ -123,7 +129,7 @@ def _trade_iter( self.position_size = tokcoin_amt_send self.tp = cur_close - (cur_close * self.tp_percent) self.sl = cur_close + (cur_close * self.sl_percent) - return 0. + return 0.0 # Check for take profit or stop loss if self.position_open == "long": @@ -146,7 +152,7 @@ def _trade_iter( if not pred_down: return self.close_short_position(cur_close) - return 0. + return 0.0 @enforce_types def _buy(self, price: float, usdcoin_amt_send: float) -> float: diff --git a/pdr_backend/sim/test/resources.py b/pdr_backend/sim/test/resources.py index d73169241..5899a3894 100644 --- a/pdr_backend/sim/test/resources.py +++ b/pdr_backend/sim/test/resources.py @@ -3,29 +3,31 @@ from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir + @enforce_types def get_Xy_UP() -> tuple: - X_UP = np.array([[1.,2.],[3.,4.],[5.,6.],[7.,8.]]) # 4 x 2 - ytrue_UP = np.array([True, True, False, True]) # 4 + X_UP = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]) # 4 x 2 + ytrue_UP = np.array([True, True, False, True]) # 4 return (X_UP, ytrue_UP) + @enforce_types def get_Xy_DOWN() -> tuple: - X_DOWN = np.array([[11.,12.],[13.,14.],[15.,16.],[17.,18.]]) + X_DOWN = np.array([[11.0, 12.0], [13.0, 14.0], [15.0, 16.0], [17.0, 18.0]]) ytrue_DOWN = np.array([False, True, False, True]) return (X_DOWN, ytrue_DOWN) + @enforce_types -def get_sim_model_data() -> SimModelData: +def get_sim_model_data() -> SimModelData: (X_UP, ytrue_UP) = get_Xy_UP() (X_DOWN, ytrue_DOWN) = get_Xy_DOWN() - + colnames_UP = ["x0_high", "x1_high"] colnames_DOWN = ["x0_low", "x1_low"] - + data_UP = SimModelData1Dir(X_UP, ytrue_UP, colnames_UP) data_DOWN = SimModelData1Dir(X_DOWN, ytrue_DOWN, colnames_DOWN) data = SimModelData(data_UP, data_DOWN) return data - diff --git a/pdr_backend/sim/test/test_sim_constants.py b/pdr_backend/sim/test/test_sim_constants.py index afe186d64..3ef439d9a 100644 --- a/pdr_backend/sim/test/test_sim_constants.py +++ b/pdr_backend/sim/test/test_sim_constants.py @@ -3,12 +3,13 @@ from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN + @enforce_types def test_sim_constants__basic(): assert UP == Dirn.UP assert DOWN == Dirn.DOWN - assert UP in Dirn + assert UP in Dirn assert DOWN in Dirn assert 1 in Dirn @@ -17,6 +18,7 @@ def test_sim_constants__basic(): assert 3 not in Dirn assert "up" not in Dirn + @enforce_types def test_sim_constants__dirn_str(): assert dirn_str(UP) == "UP" @@ -26,9 +28,9 @@ def test_sim_constants__dirn_str(): with pytest.raises(TypeError): _ = dirn_str("not an int") + @enforce_types def test_sim_constants__can_sort(): # this is possible because Dirn inherits from IntEnum, vs Enum :) assert sorted([Dirn.UP, Dirn.DOWN]) == [Dirn.UP, Dirn.DOWN] assert sorted([UP, DOWN]) == [UP, DOWN] - diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index d6cf02234..146c73376 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -39,12 +39,12 @@ def test_sim_engine(tmpdir, check_chromedriver, dash_duo): lake_d = lake_ss_test_dict( lake_dir, feeds=feedsets.feed_strs, - st_timestr = "2023-06-18", - fin_timestr = "2023-06-19", + st_timestr="2023-06-18", + fin_timestr="2023-06-19", ) ppss.lake_ss = LakeSS(lake_d) - # predictoor ss + # predictoor ss pdr_d = predictoor_ss_test_dict( feedset_list, aimodel_data_ss_dict=aimodel_data_ss_test_dict( diff --git a/pdr_backend/sim/test/test_sim_model.py b/pdr_backend/sim/test/test_sim_model.py index 90fd36c11..1760c26e7 100644 --- a/pdr_backend/sim/test/test_sim_model.py +++ b/pdr_backend/sim/test/test_sim_model.py @@ -7,8 +7,9 @@ from pdr_backend.aimodel.aimodel import Aimodel from pdr_backend.sim.sim_model import SimModel + @enforce_types -def test_sim_model(): +def test_sim_model(): model_UP = Mock(spec=Aimodel) model_UP.predict_ptrue = Mock(return_value=np.array([0.2])) @@ -17,8 +18,6 @@ def test_sim_model(): model = SimModel(model_UP, model_DOWN) X_test = {UP: np.array([[1.0]]), DOWN: np.array([[2.0]])} - + predprob = model.predict_next(X_test) assert predprob == {UP: 0.2, DOWN: 0.8} - - diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py index 72877b6eb..96fea13db 100644 --- a/pdr_backend/sim/test/test_sim_model_data.py +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -6,16 +6,17 @@ from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir from pdr_backend.sim.test.resources import get_Xy_UP, get_Xy_DOWN + @enforce_types def test_sim_model_data_1dir(): # build data (X_UP, ytrue_UP) = get_Xy_UP() - - assert X_UP.shape == (4,2) + + assert X_UP.shape == (4, 2) assert ytrue_UP.shape == (4,) ytrue_UP_train = ytrue_UP[:3] data_UP = SimModelData1Dir(X_UP, ytrue_UP, ["x0", "x1"]) - + # basic tests assert_array_equal(X_UP, data_UP.X) assert_array_equal(ytrue_UP, data_UP.ytrue) @@ -24,11 +25,11 @@ def test_sim_model_data_1dir(): # test properties assert data_UP.st == 0 assert data_UP.fin == (4 - 1) == 3 - assert_array_equal(data_UP.X_train, X_UP[0:3,:]) - assert_array_equal(data_UP.X_test, X_UP[3:3+1,:]) + assert_array_equal(data_UP.X_train, X_UP[0:3, :]) + assert_array_equal(data_UP.X_test, X_UP[3 : 3 + 1, :]) assert_array_equal(data_UP.ytrue_train, ytrue_UP_train) - + @enforce_types def test_sim_model_data_both_dirs(): # build data @@ -48,15 +49,13 @@ def test_sim_model_data_both_dirs(): assert isinstance(key, Dirn) assert isinstance(data[UP], SimModelData1Dir) assert isinstance(data[DOWN], SimModelData1Dir) - + assert_array_equal(X_UP, data[UP].X) assert_array_equal(ytrue_UP, data[UP].ytrue) - + assert_array_equal(X_DOWN, data[DOWN].X) assert_array_equal(ytrue_DOWN, data[DOWN].ytrue) assert sorted(data.X_test.keys()) == [UP, DOWN] assert_array_equal(data.X_test[UP], data[UP].X_test) assert_array_equal(data.X_test[DOWN], data[DOWN].X_test) - - diff --git a/pdr_backend/sim/test/test_sim_model_data_factory.py b/pdr_backend/sim/test/test_sim_model_data_factory.py index 87ee5485e..3ad0d6211 100644 --- a/pdr_backend/sim/test/test_sim_model_data_factory.py +++ b/pdr_backend/sim/test/test_sim_model_data_factory.py @@ -16,36 +16,38 @@ @enforce_types def test_sim_model_data_factory__basic(tmpdir): - #base data + # base data data_f = _factory(tmpdir) # attributes assert isinstance(data_f.ppss, PPSS) - + # properties assert isinstance(data_f.pdr_ss, PredictoorSS) assert isinstance(data_f.class_thr, float) assert 0.0 < data_f.class_thr < 1.0 + @enforce_types def test_sim_model_data_factory__testshift(tmpdir): - #base data + # base data data_f = _factory(tmpdir) test_i = 3 - + # do work test_n = data_f.ppss.sim_ss.test_n # test assert data_f.testshift(test_i) == test_n - test_i - 1 - + + @enforce_types def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): - #base data + # base data data_f = _factory(tmpdir) cur_close = 8.0 class_thr: float = data_f.ppss.predictoor_ss.aimodel_data_ss.class_thr - + # do work thr_UP = data_f.thr_UP(cur_close) thr_DOWN = data_f.thr_DOWN(cur_close) @@ -56,46 +58,47 @@ def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): assert thr_UP == cur_close * (1 + class_thr) assert thr_DOWN == cur_close * (1 - class_thr) + @enforce_types def test_sim_model_data_factory__build(tmpdir): - #base data + # base data mergedohlcv_df = _merged_ohlcv_df() data_f = _factory(tmpdir) test_i = 0 - + # do work data = data_f.build(test_i, mergedohlcv_df) # test # - keep this light for now. Maybe expand when things stabilize assert isinstance(data, SimModelData) - assert 9.0 < np.min(data[UP].X) < np.max(data[UP].X) < 9.99 # all 'high' - assert 0.0 < np.min(data[DOWN].X) < np.max(data[DOWN].X) < 0.99 # all 'low' + assert 9.0 < np.min(data[UP].X) < np.max(data[UP].X) < 9.99 # all 'high' + assert 0.0 < np.min(data[DOWN].X) < np.max(data[DOWN].X) < 0.99 # all 'low' + - @enforce_types def _factory(tmpdir) -> SimModelDataFactory: s = "binanceus ETH/USDT c 5m" - feedset_list = [{"predict": s, "train_on":s}] + feedset_list = [{"predict": s, "train_on": s}] ppss = mock_ppss(feedset_list) # change settings so that the factory can work with a small # datapoints ppss.predictoor_ss.aimodel_data_ss.set_max_n_train(4) ppss.predictoor_ss.aimodel_data_ss.set_autoregressive_n(1) ppss.sim_ss.set_test_n(2) - + data_f = SimModelDataFactory(ppss) return data_f + @enforce_types def _merged_ohlcv_df() -> pl.DataFrame: - d = { + d = { # - every column in df is ordered from youngest to oldest # - high values are 9.x, 9.y, ..; low are 0.x, 0.y, ..; close is b/w "timestamp": [1, 2, 3, 4, 5, 6, 7], - "binanceus:ETH/USDT:high": [9.1, 9.6, 9.8, 9.4, 9.2, 9.5, 9.7], - "binanceus:ETH/USDT:low" : [0.1, 0.6, 0.8, 0.4, 0.2, 0.5, 0.7], + "binanceus:ETH/USDT:high": [9.1, 9.6, 9.8, 9.4, 9.2, 9.5, 9.7], + "binanceus:ETH/USDT:low": [0.1, 0.6, 0.8, 0.4, 0.2, 0.5, 0.7], "binanceus:ETH/USDT:close": [2.1, 3.6, 4.6, 5.6, 6.2, 7.5, 8.7], } return pl.DataFrame(d) - diff --git a/pdr_backend/sim/test/test_sim_model_factory.py b/pdr_backend/sim/test/test_sim_model_factory.py index 13a948a71..323ea6e6e 100644 --- a/pdr_backend/sim/test/test_sim_model_factory.py +++ b/pdr_backend/sim/test/test_sim_model_factory.py @@ -14,7 +14,8 @@ def test_sim_model_factory__attributes(): f: SimModelFactory = _get_sim_model_factory() assert isinstance(f.aimodel_ss, AimodelSS) - + + @enforce_types def test_sim_model_factory__do_build(): f: SimModelFactory = _get_sim_model_factory() @@ -25,22 +26,23 @@ def test_sim_model_factory__do_build(): # case: have previous model; then on proper iter? prev_model = Mock(spec=SimModel) - assert f.do_build(prev_model, test_i=(13*4)) - assert not f.do_build(prev_model, test_i=(13*4+1)) + assert f.do_build(prev_model, test_i=(13 * 4)) + assert not f.do_build(prev_model, test_i=(13 * 4 + 1)) + @enforce_types def test_sim_model_factory__build(): f: SimModelFactory = _get_sim_model_factory() - + data = get_sim_model_data() model = f.build(data) assert isinstance(model, SimModel) - + p = model.predict_next(data.X_test) - assert p is not None # don't test further; leave that to test_sim_model.py - + assert p is not None # don't test further; leave that to test_sim_model.py + + @enforce_types def _get_sim_model_factory() -> SimModelFactory: aimodel_ss = AimodelSS(aimodel_ss_test_dict()) return SimModelFactory(aimodel_ss) - diff --git a/pdr_backend/sim/test/test_sim_model_prediction.py b/pdr_backend/sim/test/test_sim_model_prediction.py index c3a6634b6..428769b59 100644 --- a/pdr_backend/sim/test/test_sim_model_prediction.py +++ b/pdr_backend/sim/test/test_sim_model_prediction.py @@ -8,13 +8,14 @@ _models_in_conflict, ) + @enforce_types def test_sim_model_prediction_case1_models_in_conflict(): - p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.6, prob_DOWN = 0.7) + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.7) assert p.conf_thr == 0.1 assert p.prob_UP == 0.6 assert p.prob_DOWN == 0.7 - + assert p.models_in_conflict() assert p.conf_up == 0.0 assert p.conf_down == 0.0 @@ -22,13 +23,14 @@ def test_sim_model_prediction_case1_models_in_conflict(): assert not p.pred_down assert p.prob_up_MERGED == 0.5 + @enforce_types def test_sim_model_prediction_case2_up_dominates(): - p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.6, prob_DOWN = 0.3) + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.6, prob_DOWN=0.3) assert p.conf_thr == 0.1 assert p.prob_UP == 0.6 assert p.prob_DOWN == 0.3 - + assert not p.models_in_conflict() assert p.prob_UP >= p.prob_DOWN assert p.conf_up == approx((0.6 - 0.5) * 2.0) == approx(0.1 * 2.0) == 0.2 @@ -38,16 +40,17 @@ def test_sim_model_prediction_case2_up_dominates(): assert p.prob_up_MERGED == approx(0.6) # setup like above, but now with higher conf thr, which it can't exceed - p = SimModelPrediction(conf_thr = 0.3, prob_UP = 0.6, prob_DOWN = 0.3) + p = SimModelPrediction(conf_thr=0.3, prob_UP=0.6, prob_DOWN=0.3) assert not p.pred_up - + + @enforce_types def test_sim_model_prediction_case3_down_dominates(): - p = SimModelPrediction(conf_thr = 0.1, prob_UP = 0.4, prob_DOWN = 0.7) + p = SimModelPrediction(conf_thr=0.1, prob_UP=0.4, prob_DOWN=0.7) assert p.conf_thr == 0.1 assert p.prob_UP == 0.4 assert p.prob_DOWN == 0.7 - + assert not p.models_in_conflict() assert not (p.prob_UP >= p.prob_DOWN) assert p.prob_DOWN > p.prob_UP @@ -56,42 +59,44 @@ def test_sim_model_prediction_case3_down_dominates(): assert not p.pred_up assert p.pred_down == (p.conf_down > p.conf_thr) == (0.4 > 0.1) == True assert p.prob_up_MERGED == approx(1.0 - 0.7) == approx(0.3) - + # setup like above, but now with higher conf thr, which it can't exceed - p = SimModelPrediction(conf_thr = 0.5, prob_UP = 0.4, prob_DOWN = 0.7) + p = SimModelPrediction(conf_thr=0.5, prob_UP=0.4, prob_DOWN=0.7) assert not p.pred_down @enforce_types def test_do_trust_models_unhappy_path(): for tup in [ - # out of range - (True, False, -1.4, 0.6), - (True, False, 1.4, 0.6), - (True, False, 0.4, -1.6), - (True, False, 0.4, 1.6), - # values conflict - (True, True, 0.4, 0.6), # pred_up and pred_down - (True, False, 0.4, 0.6), # pred_up and prob_DOWN > prob_UP - (False, True, 0.6, 0.4), # pred_down and prob_UP > prob_DOWN + # out of range + (True, False, -1.4, 0.6), + (True, False, 1.4, 0.6), + (True, False, 0.4, -1.6), + (True, False, 0.4, 1.6), + # values conflict + (True, True, 0.4, 0.6), # pred_up and pred_down + (True, False, 0.4, 0.6), # pred_up and prob_DOWN > prob_UP + (False, True, 0.6, 0.4), # pred_down and prob_UP > prob_DOWN ]: (pred_up, pred_down, prob_UP, prob_DOWN) = tup with pytest.raises(ValueError): _do_trust_models(pred_up, pred_down, prob_UP, prob_DOWN) print(f"Should have failed on {tup}") - + + @enforce_types def test_do_trust_models_happy_path(): assert _do_trust_models(True, False, 0.6, 0.4) assert _do_trust_models(False, True, 0.4, 0.6) - + assert not _do_trust_models(False, False, 0.4, 0.6) assert not _do_trust_models(False, True, 0.4, 0.4) assert not _do_trust_models(False, True, 0.6, 0.6) - + + @enforce_types def test_models_in_conflict_unhappy_path(): - for (prob_UP, prob_DOWN) in [ + for prob_UP, prob_DOWN in [ (-1.6, 0.6), (1.6, 0.6), (0.6, -1.6), @@ -100,13 +105,12 @@ def test_models_in_conflict_unhappy_path(): with pytest.raises(ValueError): _models_in_conflict(prob_UP, prob_DOWN) + @enforce_types def test_models_in_conflict_happy_path(): assert _models_in_conflict(0.6, 0.6) assert _models_in_conflict(0.4, 0.4) - + assert not _models_in_conflict(0.6, 0.4) assert not _models_in_conflict(0.4, 0.6) assert not _models_in_conflict(0.5, 0.5) - - diff --git a/pdr_backend/sim/test/test_sim_predictoor.py b/pdr_backend/sim/test/test_sim_predictoor.py index 3c286c061..5d0f8634a 100644 --- a/pdr_backend/sim/test/test_sim_predictoor.py +++ b/pdr_backend/sim/test/test_sim_predictoor.py @@ -4,12 +4,13 @@ from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.sim_predictoor import SimPredictoor + @enforce_types def test_sim_predictoor__attributes(): sim_pdr = _sim_pdr() assert isinstance(sim_pdr.pdr_ss, PredictoorSS) - + @enforce_types def test_sim_predictoor__properties(): sim_pdr = _sim_pdr() @@ -17,7 +18,7 @@ def test_sim_predictoor__properties(): assert isinstance(max_stake_amt, float | int) assert max_stake_amt > 0.0 - + @enforce_types def test_sim_predictoor__predict_iter(): # base data @@ -35,19 +36,18 @@ def test_sim_predictoor__predict_iter(): stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_down < stake_up < 1.0 assert (stake_up + stake_down) <= sim_pdr.max_stake_amt - + # case 3: DOWN dominates p = SimModelPrediction(conf_thr=0.1, prob_UP=0.4, prob_DOWN=0.6) assert p.do_trust_models() stake_up, stake_down = sim_pdr.predict_iter(p) assert 0.0 < stake_up < stake_down < 1.0 assert (stake_up + stake_down) <= sim_pdr.max_stake_amt - - + + @enforce_types def _sim_pdr() -> SimPredictoor: pdr_d = predictoor_ss_test_dict() pdr_ss = PredictoorSS(pdr_d) sim_pdr = SimPredictoor(pdr_ss) return sim_pdr - diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index a45d3eee6..f1d4f6ee1 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -12,9 +12,10 @@ SimState, ) -#============================================================================= +# ============================================================================= # test HistPerfs + @enforce_types @pytest.mark.parametrize("dirn", [UP, DOWN]) def test_hist_perfs__basic_init(dirn): @@ -41,7 +42,8 @@ def test_hist_perfs__basic_init(dirn): _ = hist_perfs.recent_metrics_values() with pytest.raises(AssertionError): _ = hist_perfs.final_metrics_values() - + + @enforce_types @pytest.mark.parametrize("dirn", [UP, DOWN]) def test_hist_perfs__main(dirn): @@ -50,8 +52,8 @@ def test_hist_perfs__main(dirn): target_names = [f"{name}_{dirn_s}" for name in PERF_NAMES] hist_perfs = HistPerfs(dirn) - perfs_list1 = list(np.arange(0.1, 7.1, 1.0)) # 0.1, 1.1, ..., 6.1 - perfs_list2 = list(np.arange(0.2, 7.2, 1.0)) # 0.2, 1.2, ..., 6.2 + perfs_list1 = list(np.arange(0.1, 7.1, 1.0)) # 0.1, 1.1, ..., 6.1 + perfs_list2 = list(np.arange(0.2, 7.2, 1.0)) # 0.2, 1.2, ..., 6.2 hist_perfs.update(perfs_list1) hist_perfs.update(perfs_list2) @@ -81,7 +83,7 @@ def test_hist_perfs__main(dirn): f"recall_{dirn_s}": 5.2, f"loss_{dirn_s}": 6.2, } - + # test *final* metrics values = hist_perfs.final_metrics_values() assert len(values) == 7 @@ -102,14 +104,16 @@ def test_hist_perfs__main(dirn): def _target_perfs_names(dirn_s: str): return [f"{name}_{dirn_s}" for name in PERF_NAMES] -#============================================================================= + +# ============================================================================= # test HistProfits - + + @enforce_types def test_hist_profits__basic_init(): # set data hist_profits = HistProfits() - + # test empty raw data assert hist_profits.pdr_profits_OCEAN == [] assert hist_profits.trader_profits_USD == [] @@ -133,7 +137,7 @@ def test_hist_profits__update(): hist_profits = HistProfits() hist_profits.update(2.1, 3.1) hist_profits.update(2.2, 3.2) - + # test raw values assert hist_profits.pdr_profits_OCEAN == [2.1, 2.2] assert hist_profits.trader_profits_USD == [3.1, 3.2] @@ -145,61 +149,63 @@ def test_hist_profits__update(): target_names = PROFIT_NAMES values = hist_profits.recent_metrics_values() assert sorted(values.keys()) == sorted(target_names) - assert values == {"pdr_profit_OCEAN" : 2.2, - "trader_profit_USD": 3.2} - + assert values == {"pdr_profit_OCEAN": 2.2, "trader_profit_USD": 3.2} + # test *final* metrics values = hist_profits.final_metrics_values() assert sorted(values.keys()) == sorted(target_names) - assert values == {"pdr_profit_OCEAN" : np.sum([2.1, 2.2]), - "trader_profit_USD": np.sum([3.1, 3.2])} + assert values == { + "pdr_profit_OCEAN": np.sum([2.1, 2.2]), + "trader_profit_USD": np.sum([3.1, 3.2]), + } + @enforce_types def test_hist_profits__calc_pdr_profit(): # true = up, guess = up (correct guess), others fully wrong profit = HistProfits.calc_pdr_profit( - others_stake = 2000.0, - others_accuracy = 0.0, - stake_up = 1000.0, - stake_down = 0.0, - revenue = 2.0, - true_up_close = True, + others_stake=2000.0, + others_accuracy=0.0, + stake_up=1000.0, + stake_down=0.0, + revenue=2.0, + true_up_close=True, ) assert profit == 2002.0 - + # true = down, guess = down (correct guess), others fully wrong profit = HistProfits.calc_pdr_profit( - others_stake = 2000.0, - others_accuracy = 0.0, - stake_up = 0.0, - stake_down = 1000.0, - revenue = 2.0, - true_up_close = False, + others_stake=2000.0, + others_accuracy=0.0, + stake_up=0.0, + stake_down=1000.0, + revenue=2.0, + true_up_close=False, ) assert profit == 2002.0 # true = up, guess = down (incorrect guess), others fully right profit = HistProfits.calc_pdr_profit( - others_stake = 2000.0, - others_accuracy = 1.0, - stake_up = 0.0, - stake_down = 1000.0, - revenue = 2.0, - true_up_close = True, + others_stake=2000.0, + others_accuracy=1.0, + stake_up=0.0, + stake_down=1000.0, + revenue=2.0, + true_up_close=True, ) assert profit == -1000.0 # true = down, guess = up (incorrect guess), others fully right profit = HistProfits.calc_pdr_profit( - others_stake = 2000.0, - others_accuracy = 1.0, - stake_up = 1000.0, - stake_down = 0.0, - revenue = 2.0, - true_up_close = False, + others_stake=2000.0, + others_accuracy=1.0, + stake_up=1000.0, + stake_down=0.0, + revenue=2.0, + true_up_close=False, ) assert profit == -1000.0 - + # true = up, guess = up AND down (half-correct), others fully wrong # summary: I should get back all my stake $, plus stake $ of others # calculations: @@ -213,15 +219,15 @@ def test_hist_profits__calc_pdr_profit(): # = (2 + 3100) * 1.0 = 3102 # - profit = received - sent = 2102 - 1100 = 1002 profit = HistProfits.calc_pdr_profit( - others_stake = 1000.0, - others_accuracy = 0.00, - stake_up = 1000.0, - stake_down = 100.0, - revenue = 2.0, - true_up_close = True, + others_stake=1000.0, + others_accuracy=0.00, + stake_up=1000.0, + stake_down=100.0, + revenue=2.0, + true_up_close=True, ) assert profit == 1002.0 - + # true = up, guess = lots up & some down, others 30% accurate # summary: # calculations: @@ -237,16 +243,16 @@ def test_hist_profits__calc_pdr_profit(): # = (2 + 2100) * 0.769230 = 1616.9230769 # - profit = received - sent = 1616.9230769 - 1100 = 516.923 profit = HistProfits.calc_pdr_profit( - others_stake = 1000.0, - others_accuracy = 0.30, - stake_up = 1000.0, - stake_down = 100.0, - revenue = 2.0, - true_up_close = True, + others_stake=1000.0, + others_accuracy=0.30, + stake_up=1000.0, + stake_down=100.0, + revenue=2.0, + true_up_close=True, ) assert profit == approx(516.923) - + @enforce_types def test_hist_profits__calc_pdr_profit__unhappy_path(): o_stake = 2000.0 @@ -258,43 +264,74 @@ def test_hist_profits__calc_pdr_profit__unhappy_path(): with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - -0.1, o_accuracy, stake_up, stake_down, revenue, true_up_close, + -0.1, + o_accuracy, + stake_up, + stake_down, + revenue, + true_up_close, ) with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - o_stake, -0.1, stake_up, stake_down, revenue, true_up_close, + o_stake, + -0.1, + stake_up, + stake_down, + revenue, + true_up_close, ) - + with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - o_stake, +1.1, stake_up, stake_down, revenue, true_up_close, + o_stake, + +1.1, + stake_up, + stake_down, + revenue, + true_up_close, ) with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - o_stake, o_accuracy, -0.1, stake_down, revenue, true_up_close, + o_stake, + o_accuracy, + -0.1, + stake_down, + revenue, + true_up_close, ) with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - o_stake, o_accuracy, stake_up, -0.1, revenue, true_up_close, + o_stake, + o_accuracy, + stake_up, + -0.1, + revenue, + true_up_close, ) with pytest.raises(AssertionError): HistProfits.calc_pdr_profit( - o_stake, o_accuracy, stake_up, stake_down, -0.1, true_up_close, + o_stake, + o_accuracy, + stake_up, + stake_down, + -0.1, + true_up_close, ) - -#============================================================================= + +# ============================================================================= # test SimState + @enforce_types def test_sim_state__basic_init(): # set data st = SimState() - + # test empty raw state assert st.iter_number == 0 assert st.sim_model_data is None @@ -318,11 +355,12 @@ def test_sim_state__basic_init(): with pytest.raises(AssertionError): _ = st.final_metrics_values() + @enforce_types def test_sim_state__init_loop_attributes(): # init st = SimState() - + # change after init st.iter_number = 1 st.sim_model = "foo" @@ -334,19 +372,19 @@ def test_sim_state__init_loop_attributes(): assert st.iter_number == 0 assert st.sim_model is None - + @enforce_types def test_sim_state__main(): st = SimState() target_names = _target_state_names() # update - trueval = {UP:True, DOWN:False} - predprob = {UP:0.6, DOWN:0.3} - + trueval = {UP: True, DOWN: False} + predprob = {UP: 0.6, DOWN: 0.3} + st.update(trueval, predprob, pdr_profit_OCEAN=1.4, trader_profit_USD=1.5) st.update(trueval, predprob, pdr_profit_OCEAN=2.4, trader_profit_USD=2.5) - + # test raw state -- true_vs_pred assert st.true_vs_pred[UP].truevals == [True, True] assert st.true_vs_pred[UP].predprobs == [0.6, 0.6] @@ -364,9 +402,9 @@ def test_sim_state__main(): assert val == 2.5 elif "loss" in name: assert 0.0 <= val <= 3.0 - else: # hist_perfs value + else: # hist_perfs value assert 0.0 <= val <= 1.0, (name, val) - + # test *final* metrics values = st.final_metrics_values() assert sorted(values.keys()) == sorted(target_names) @@ -377,11 +415,12 @@ def test_sim_state__main(): assert val == np.sum([1.5, 2.5]) elif "loss" in name: assert 0.0 <= val <= 3.0 - else: # hist_perfs value + else: # hist_perfs value assert 0.0 <= val <= 1.0, (name, val) + @enforce_types def _target_state_names(): - return [f"{name}_{dirn_str(dirn)}" - for dirn in [UP, DOWN] - for name in PERF_NAMES] + PROFIT_NAMES + return [ + f"{name}_{dirn_str(dirn)}" for dirn in [UP, DOWN] for name in PERF_NAMES + ] + PROFIT_NAMES diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 27e1dae53..7a9f857d7 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -29,9 +29,10 @@ def mock_ppss(): predict_feed.pair.base_str = "ETH" predict_feed.pair.quote_str = "USDT" ppss.predictoor_ss.predict_train_feedsets[0].predict = predict_feed - + return ppss + @enforce_types @pytest.fixture def sim_trader(mock_ppss): @@ -41,87 +42,87 @@ def sim_trader(mock_ppss): @enforce_types def test_initial_state(sim_trader): assert sim_trader.position_open == "" - assert sim_trader.position_size == 0. - assert sim_trader.position_worth == 0. - assert sim_trader.tp == 0. - assert sim_trader.sl == 0. + assert sim_trader.position_size == 0.0 + assert sim_trader.position_worth == 0.0 + assert sim_trader.tp == 0.0 + assert sim_trader.sl == 0.0 @enforce_types def test_close_long_position(sim_trader): sim_trader.position_open = "long" - sim_trader.position_size = 10. - sim_trader.position_worth = 1000. - sim_trader._sell = Mock(return_value=1100.) - profit = sim_trader.close_long_position(110.) - assert profit == 100. + sim_trader.position_size = 10.0 + sim_trader.position_worth = 1000.0 + sim_trader._sell = Mock(return_value=1100.0) + profit = sim_trader.close_long_position(110.0) + assert profit == 100.0 assert sim_trader.position_open == "" @enforce_types def test_close_short_position(sim_trader): sim_trader.position_open = "short" - sim_trader.position_size = 10. - sim_trader.position_worth = 1000. + sim_trader.position_size = 10.0 + sim_trader.position_worth = 1000.0 sim_trader._buy = Mock() - profit = sim_trader.close_short_position(90.) - assert profit == 100. + profit = sim_trader.close_short_position(90.0) + assert profit == 100.0 assert sim_trader.position_open == "" @enforce_types def test_trade_iter_open_long(sim_trader): - sim_trader._buy = Mock(return_value=10.) - sim_trader._trade_iter(100., True, False, 0.5, 0., 110., 90.) + sim_trader._buy = Mock(return_value=10.0) + sim_trader._trade_iter(100.0, True, False, 0.5, 0.0, 110.0, 90.0) assert sim_trader.position_open == "long" - assert sim_trader.position_worth == 1500. - assert sim_trader.position_size == 10. + assert sim_trader.position_worth == 1500.0 + assert sim_trader.position_size == 10.0 @enforce_types def test_trade_iter_open_short(sim_trader): - sim_trader._sell = Mock(return_value=1500.) - sim_trader._trade_iter(100., False, True, 0., 0.5, 110., 90.) + sim_trader._sell = Mock(return_value=1500.0) + sim_trader._trade_iter(100.0, False, True, 0.0, 0.5, 110.0, 90.0) assert sim_trader.position_open == "short" - assert sim_trader.position_worth == 1500. - assert sim_trader.position_size == 15. + assert sim_trader.position_worth == 1500.0 + assert sim_trader.position_size == 15.0 @enforce_types def test_trade_iter_close_long_take_profit_percent(sim_trader): sim_trader.position_open = "long" - sim_trader.position_size = 10. - sim_trader.position_worth = 1000. - sim_trader.tp = 110. - sim_trader._sell = Mock(return_value=1100.) - profit = sim_trader._trade_iter(100., False, False, 0., 0., 110., 90.) - assert profit == 100. # 1100 - 1000 + sim_trader.position_size = 10.0 + sim_trader.position_worth = 1000.0 + sim_trader.tp = 110.0 + sim_trader._sell = Mock(return_value=1100.0) + profit = sim_trader._trade_iter(100.0, False, False, 0.0, 0.0, 110.0, 90.0) + assert profit == 100.0 # 1100 - 1000 assert sim_trader.position_open == "" @enforce_types def test_trade_iter_close_short_stop_loss_percent(sim_trader): sim_trader.position_open = "short" - sim_trader.position_size = 10. - sim_trader.position_worth = 1000. - sim_trader.sl = 110. + sim_trader.position_size = 10.0 + sim_trader.position_worth = 1000.0 + sim_trader.sl = 110.0 sim_trader._buy = Mock() - profit = sim_trader._trade_iter(100., False, False, 0., 0., 110., 90.) - assert profit == -100. # 1100 - 1000 + profit = sim_trader._trade_iter(100.0, False, False, 0.0, 0.0, 110.0, 90.0) + assert profit == -100.0 # 1100 - 1000 assert sim_trader.position_open == "" @enforce_types def test_buy(sim_trader): sim_trader.exchange.create_market_buy_order = Mock() - tokcoin_amt_recd = sim_trader._buy(100., 1000.) - assert tokcoin_amt_recd == (1000. / 100.) * (1. - FEE_PERCENT) + tokcoin_amt_recd = sim_trader._buy(100.0, 1000.0) + assert tokcoin_amt_recd == (1000.0 / 100.0) * (1.0 - FEE_PERCENT) sim_trader.exchange.create_market_buy_order.assert_called_once() @enforce_types def test_sell(sim_trader): sim_trader.exchange.create_market_sell_order = Mock() - usdcoin_amt_recd = sim_trader._sell(100., 10.) - assert usdcoin_amt_recd == (100. * 10.) * (1. - FEE_PERCENT) + usdcoin_amt_recd = sim_trader._sell(100.0, 10.0) + assert usdcoin_amt_recd == (100.0 * 10.0) * (1.0 - FEE_PERCENT) sim_trader.exchange.create_market_sell_order.assert_called_once() From 4e167b3f055f6d9c1acb548b2a5f40eb426cfd4c Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 18:23:59 +0200 Subject: [PATCH 21/50] black & pylint happy --- pdr_backend/aimodel/test/test_true_vs_pred.py | 1 + pdr_backend/cli/cli_module.py | 1 - pdr_backend/ppss/aimodel_data_ss.py | 2 +- pdr_backend/sim/multisim_engine.py | 1 - pdr_backend/sim/sim_engine.py | 9 +------- pdr_backend/sim/sim_logger.py | 1 - pdr_backend/sim/sim_model.py | 3 --- pdr_backend/sim/sim_model_data.py | 2 +- pdr_backend/sim/sim_model_data_factory.py | 1 - pdr_backend/sim/sim_model_prediction.py | 9 ++++---- pdr_backend/sim/sim_plotter.py | 14 ++++------- pdr_backend/sim/sim_state.py | 7 ++++-- pdr_backend/sim/sim_trader.py | 1 + pdr_backend/sim/test/test_sim_engine.py | 1 - pdr_backend/sim/test/test_sim_model_data.py | 1 - .../sim/test/test_sim_model_data_factory.py | 23 ++++++++----------- .../sim/test/test_sim_model_factory.py | 6 ++--- .../sim/test/test_sim_model_prediction.py | 5 ++-- pdr_backend/sim/test/test_sim_state.py | 6 ++--- pdr_backend/sim/test/test_sim_trader.py | 1 - 20 files changed, 35 insertions(+), 60 deletions(-) diff --git a/pdr_backend/aimodel/test/test_true_vs_pred.py b/pdr_backend/aimodel/test/test_true_vs_pred.py index 8391cf70e..c4af647d6 100644 --- a/pdr_backend/aimodel/test/test_true_vs_pred.py +++ b/pdr_backend/aimodel/test/test_true_vs_pred.py @@ -4,6 +4,7 @@ from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred +# pylint: disable=too-many-statements @enforce_types def test_true_vs_pred(): d = TrueVsPred() diff --git a/pdr_backend/cli/cli_module.py b/pdr_backend/cli/cli_module.py index cd53d9802..efc9dead1 100644 --- a/pdr_backend/cli/cli_module.py +++ b/pdr_backend/cli/cli_module.py @@ -86,7 +86,6 @@ def do_sim(args, nested_args=None): network="development", nested_override_args=nested_args, ) - feedset = ppss.predictoor_ss.predict_train_feedsets[0] if len(ppss.predictoor_ss.predict_train_feedsets) > 0: logger.warning("Multiple predict feeds provided, using the first one") sim_engine = SimEngine(ppss) diff --git a/pdr_backend/ppss/aimodel_data_ss.py b/pdr_backend/ppss/aimodel_data_ss.py index 0de10837a..975220b9b 100644 --- a/pdr_backend/ppss/aimodel_data_ss.py +++ b/pdr_backend/ppss/aimodel_data_ss.py @@ -34,7 +34,7 @@ def validate_max_n_train(max_n_train: int): @staticmethod def validate_autoregressive_n(autoregressive_n: int): - if not (0 < autoregressive_n < np.inf): + if not 0 < autoregressive_n < np.inf: raise ValueError(autoregressive_n) @staticmethod diff --git a/pdr_backend/sim/multisim_engine.py b/pdr_backend/sim/multisim_engine.py index 45e6d8df9..65f8cd1bd 100644 --- a/pdr_backend/sim/multisim_engine.py +++ b/pdr_backend/sim/multisim_engine.py @@ -6,7 +6,6 @@ import uuid from typing import List, Union -import numpy as np import pandas as pd from enforce_typing import enforce_types diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 2f67df107..b181dc32f 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -3,27 +3,20 @@ import uuid from typing import Optional, Tuple -import numpy as np import polars as pl from enforce_typing import enforce_types -from sklearn.metrics import log_loss, precision_recall_fscore_support -from statsmodels.stats.proportion import proportion_confint -from pdr_backend.aimodel.aimodel import Aimodel from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory -from pdr_backend.aimodel.aimodel_factory import AimodelFactory from pdr_backend.aimodel.aimodel_plotdata import AimodelPlotdata from pdr_backend.cli.arg_feed import ArgFeed from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.cli.arg_timeframe import ArgTimeframe -from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset from pdr_backend.lake.ohlcv_data_factory import OhlcvDataFactory -from pdr_backend.ppss.aimodel_ss import AimodelSS from pdr_backend.ppss.ppss import PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_logger import SimLogLine -from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_data import SimModelData from pdr_backend.sim.sim_model_data_factory import SimModelDataFactory from pdr_backend.sim.sim_model_factory import SimModelFactory from pdr_backend.sim.sim_model_prediction import SimModelPrediction diff --git a/pdr_backend/sim/sim_logger.py b/pdr_backend/sim/sim_logger.py index 23905404f..8d5ef619e 100644 --- a/pdr_backend/sim/sim_logger.py +++ b/pdr_backend/sim/sim_logger.py @@ -1,6 +1,5 @@ import logging -import numpy as np from enforce_typing import enforce_types from pdr_backend.util.strutil import compactSmallNum diff --git a/pdr_backend/sim/sim_model.py b/pdr_backend/sim/sim_model.py index 91cef8771..2a2d69f25 100644 --- a/pdr_backend/sim/sim_model.py +++ b/pdr_backend/sim/sim_model.py @@ -1,10 +1,7 @@ from enforce_typing import enforce_types from pdr_backend.aimodel.aimodel import Aimodel -from pdr_backend.ppss.ppss import PPSS from pdr_backend.sim.constants import UP, DOWN -from pdr_backend.sim.sim_model_data import SimModelData -from pdr_backend.sim.sim_model_prediction import SimModelPrediction @enforce_types diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index 4369e638e..4f7da925a 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union +from typing import List, Tuple from enforce_typing import enforce_types import numpy as np diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index 17b944f6f..43973cab3 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -5,7 +5,6 @@ from pdr_backend.cli.arg_feed import ArgFeed from pdr_backend.cli.arg_feeds import ArgFeeds from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory -from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS from pdr_backend.ppss.ppss import PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.sim_model_data import SimModelData, SimModelData1Dir diff --git a/pdr_backend/sim/sim_model_prediction.py b/pdr_backend/sim/sim_model_prediction.py index 2c24af3d3..36952e7a5 100644 --- a/pdr_backend/sim/sim_model_prediction.py +++ b/pdr_backend/sim/sim_model_prediction.py @@ -1,6 +1,7 @@ from enforce_typing import enforce_types +# pylint: disable=too-many-instance-attributes @enforce_types class SimModelPrediction: def __init__( @@ -65,9 +66,9 @@ def _do_trust_models( ) -> bool: """Do we trust the models enough to take prediction / trading action?""" # preconditions - if not (0.0 <= prob_UP <= 1.0): + if not 0.0 <= prob_UP <= 1.0: raise ValueError(prob_UP) - if not (0.0 <= prob_DOWN <= 1.0): + if not 0.0 <= prob_DOWN <= 1.0: raise ValueError(prob_DOWN) if pred_up and pred_down: raise ValueError("can't have pred_up=True and pred_down=True") @@ -84,9 +85,9 @@ def _do_trust_models( def _models_in_conflict(prob_UP: float, prob_DOWN: float) -> bool: """Does the UP model conflict with the DOWN model?""" # preconditions - if not (0.0 <= prob_UP <= 1.0): + if not 0.0 <= prob_UP <= 1.0: raise ValueError(prob_UP) - if not (0.0 <= prob_DOWN <= 1.0): + if not 0.0 <= prob_DOWN <= 1.0: raise ValueError(prob_DOWN) # main test diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index bd86e29f3..d3a6641bc 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -11,20 +11,11 @@ import plotly.graph_objects as go from plotly.subplots import make_subplots -from pdr_backend.aimodel.aimodel_plotdata import AimodelPlotdata - -from pdr_backend.statutil.autocorrelation_plotdata import ( - AutocorrelationPlotdataFactory, -) -from pdr_backend.statutil.autocorrelation_plotter import add_corr_traces -from pdr_backend.statutil.dist_plotdata import DistPlotdataFactory -from pdr_backend.statutil.dist_plotter import add_pdf, add_cdf, add_nq - HEIGHT = 7.5 WIDTH = int(HEIGHT * 3.2) -# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-instance-attributes,attribute-defined-outside-init class SimPlotter: @enforce_types def __init__( @@ -48,6 +39,7 @@ def get_all_run_names(): path = Path("sim_state").iterdir() return [str(p).replace("sim_state/", "") for p in path] + @enforce_types def load_state(self, multi_id): root_path = f"sim_state/{multi_id}" @@ -93,6 +85,7 @@ def load_state(self, multi_id): return self.st, "final" + @enforce_types def init_state(self, multi_id): files = glob.glob("sim_state/{multi_id}/*") @@ -103,6 +96,7 @@ def init_state(self, multi_id): os.makedirs(f"sim_state/{multi_id}") + @enforce_types def save_state(self, sim_state, aimodel_plotdata, is_final: bool = False): root_path = f"sim_state/{self.multi_id}" ts = ( diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 95a4bf69a..02289b329 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -1,16 +1,19 @@ -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Union from enforce_typing import enforce_types import numpy as np from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN +from pdr_backend.sim.sim_model import SimModel +from pdr_backend.sim.sim_model_data import SimModelData # ============================================================================= # HistPerfs +# pylint: disable=too-many-instance-attributes class HistPerfs: """Historical performances, for 1 model dir'n (eg UP)""" @@ -237,5 +240,5 @@ def have_data(self) -> bool: return ( self.hist_perfs[UP].have_data() and self.hist_perfs[DOWN].have_data() - and self.hist_profits().have_data() + and self.hist_profits.have_data() ) diff --git a/pdr_backend/sim/sim_trader.py b/pdr_backend/sim/sim_trader.py index 2b4dc0065..52f49e582 100644 --- a/pdr_backend/sim/sim_trader.py +++ b/pdr_backend/sim/sim_trader.py @@ -77,6 +77,7 @@ def trade_iter( ) return float(profit_USD) + # pylint: disable=too-many-return-statements @enforce_types def _trade_iter( self, diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index 146c73376..94adf62d6 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -5,7 +5,6 @@ from enforce_typing import enforce_types from selenium.common.exceptions import NoSuchElementException # type: ignore[import-untyped] -from pdr_backend.aimodel.aimodel import Aimodel from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedsets from pdr_backend.ppss.lake_ss import LakeSS, lake_ss_test_dict from pdr_backend.ppss.aimodel_ss import aimodel_ss_test_dict diff --git a/pdr_backend/sim/test/test_sim_model_data.py b/pdr_backend/sim/test/test_sim_model_data.py index 96fea13db..54486e138 100644 --- a/pdr_backend/sim/test/test_sim_model_data.py +++ b/pdr_backend/sim/test/test_sim_model_data.py @@ -1,5 +1,4 @@ from enforce_typing import enforce_types -import numpy as np from numpy.testing import assert_array_equal from pdr_backend.sim.constants import Dirn, UP, DOWN diff --git a/pdr_backend/sim/test/test_sim_model_data_factory.py b/pdr_backend/sim/test/test_sim_model_data_factory.py index 3ad0d6211..66c858743 100644 --- a/pdr_backend/sim/test/test_sim_model_data_factory.py +++ b/pdr_backend/sim/test/test_sim_model_data_factory.py @@ -2,11 +2,6 @@ import numpy as np import polars as pl -from pdr_backend.aimodel.aimodel_data_factory import AimodelDataFactory -from pdr_backend.cli.arg_feed import ArgFeed -from pdr_backend.cli.arg_feeds import ArgFeeds -from pdr_backend.cli.predict_train_feedsets import PredictTrainFeedset -from pdr_backend.ppss.aimodel_data_ss import AimodelDataSS from pdr_backend.ppss.ppss import mock_ppss, PPSS from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.constants import UP, DOWN @@ -15,9 +10,9 @@ @enforce_types -def test_sim_model_data_factory__basic(tmpdir): +def test_sim_model_data_factory__basic(): # base data - data_f = _factory(tmpdir) + data_f = _factory() # attributes assert isinstance(data_f.ppss, PPSS) @@ -29,9 +24,9 @@ def test_sim_model_data_factory__basic(tmpdir): @enforce_types -def test_sim_model_data_factory__testshift(tmpdir): +def test_sim_model_data_factory__testshift(): # base data - data_f = _factory(tmpdir) + data_f = _factory() test_i = 3 # do work @@ -42,9 +37,9 @@ def test_sim_model_data_factory__testshift(tmpdir): @enforce_types -def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): +def test_sim_model_data_factory__thr_UP__thr_DOWN(): # base data - data_f = _factory(tmpdir) + data_f = _factory() cur_close = 8.0 class_thr: float = data_f.ppss.predictoor_ss.aimodel_data_ss.class_thr @@ -60,10 +55,10 @@ def test_sim_model_data_factory__thr_UP__thr_DOWN(tmpdir): @enforce_types -def test_sim_model_data_factory__build(tmpdir): +def test_sim_model_data_factory__build(): # base data mergedohlcv_df = _merged_ohlcv_df() - data_f = _factory(tmpdir) + data_f = _factory() test_i = 0 # do work @@ -77,7 +72,7 @@ def test_sim_model_data_factory__build(tmpdir): @enforce_types -def _factory(tmpdir) -> SimModelDataFactory: +def _factory() -> SimModelDataFactory: s = "binanceus ETH/USDT c 5m" feedset_list = [{"predict": s, "train_on": s}] ppss = mock_ppss(feedset_list) diff --git a/pdr_backend/sim/test/test_sim_model_factory.py b/pdr_backend/sim/test/test_sim_model_factory.py index 323ea6e6e..5ec497a93 100644 --- a/pdr_backend/sim/test/test_sim_model_factory.py +++ b/pdr_backend/sim/test/test_sim_model_factory.py @@ -3,10 +3,8 @@ from enforce_typing import enforce_types from pdr_backend.ppss.aimodel_ss import AimodelSS, aimodel_ss_test_dict -from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_model import SimModel from pdr_backend.sim.sim_model_factory import SimModelFactory -from pdr_backend.sim.sim_model_prediction import SimModelPrediction from pdr_backend.sim.test.resources import get_sim_model_data @@ -26,8 +24,8 @@ def test_sim_model_factory__do_build(): # case: have previous model; then on proper iter? prev_model = Mock(spec=SimModel) - assert f.do_build(prev_model, test_i=(13 * 4)) - assert not f.do_build(prev_model, test_i=(13 * 4 + 1)) + assert f.do_build(prev_model, test_i=13 * 4) + assert not f.do_build(prev_model, test_i=13 * 4 + 1) @enforce_types diff --git a/pdr_backend/sim/test/test_sim_model_prediction.py b/pdr_backend/sim/test/test_sim_model_prediction.py index 428769b59..d32fafe56 100644 --- a/pdr_backend/sim/test/test_sim_model_prediction.py +++ b/pdr_backend/sim/test/test_sim_model_prediction.py @@ -35,7 +35,7 @@ def test_sim_model_prediction_case2_up_dominates(): assert p.prob_UP >= p.prob_DOWN assert p.conf_up == approx((0.6 - 0.5) * 2.0) == approx(0.1 * 2.0) == 0.2 assert p.conf_down == 0.0 - assert p.pred_up == (p.conf_up > p.conf_thr) == (0.2 > 0.1) == True + assert p.pred_up == (p.conf_up > p.conf_thr) == True # 0.2 > 0.1 assert not p.pred_down assert p.prob_up_MERGED == approx(0.6) @@ -52,12 +52,11 @@ def test_sim_model_prediction_case3_down_dominates(): assert p.prob_DOWN == 0.7 assert not p.models_in_conflict() - assert not (p.prob_UP >= p.prob_DOWN) assert p.prob_DOWN > p.prob_UP assert p.conf_up == 0.0 assert p.conf_down == approx((0.7 - 0.5) * 2.0) == approx(0.2 * 2.0) == 0.4 assert not p.pred_up - assert p.pred_down == (p.conf_down > p.conf_thr) == (0.4 > 0.1) == True + assert p.pred_down == (p.conf_down > p.conf_thr) == True # 0.4 > 0.1 assert p.prob_up_MERGED == approx(1.0 - 0.7) == approx(0.3) # setup like above, but now with higher conf thr, which it can't exceed diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index f1d4f6ee1..ba9e92c3b 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -4,7 +4,7 @@ from pytest import approx from pdr_backend.aimodel.true_vs_pred import PERF_NAMES, TrueVsPred -from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN +from pdr_backend.sim.constants import dirn_str, UP, DOWN from pdr_backend.sim.sim_state import ( HistPerfs, HistProfits, @@ -73,7 +73,7 @@ def test_hist_perfs__main(dirn): values = hist_perfs.recent_metrics_values() assert len(values) == 7 assert sorted(values.keys()) == sorted(target_names) - assert f"acc_est_{dirn_s}" in values.keys() + assert f"acc_est_{dirn_s}" in values assert values == { f"acc_est_{dirn_s}": 0.2, f"acc_l_{dirn_s}": 1.2, @@ -88,7 +88,7 @@ def test_hist_perfs__main(dirn): values = hist_perfs.final_metrics_values() assert len(values) == 7 assert sorted(values.keys()) == sorted(target_names) - assert f"acc_est_{dirn_s}" in values.keys() + assert f"acc_est_{dirn_s}" in values assert values == { f"acc_est_{dirn_s}": 0.2, f"acc_l_{dirn_s}": 1.2, diff --git a/pdr_backend/sim/test/test_sim_trader.py b/pdr_backend/sim/test/test_sim_trader.py index 7a9f857d7..25fb30498 100644 --- a/pdr_backend/sim/test/test_sim_trader.py +++ b/pdr_backend/sim/test/test_sim_trader.py @@ -7,7 +7,6 @@ from pdr_backend.ppss.exchange_mgr_ss import ExchangeMgrSS from pdr_backend.sim.sim_trader import SimTrader -from pdr_backend.sim.sim_model_prediction import SimModelPrediction FEE_PERCENT = 0.01 From b8fed96e33cf8aaa13c2bcd7bf93b4cab72578a9 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 18:32:21 +0200 Subject: [PATCH 22/50] all linters happy --- pdr_backend/ppss/aimodel_data_ss.py | 2 +- pdr_backend/sim/sim_engine.py | 4 ++-- pdr_backend/sim/sim_model_data.py | 6 +++--- pdr_backend/sim/sim_model_data_factory.py | 10 +++++----- pdr_backend/sim/sim_predictoor.py | 4 ++-- pdr_backend/sim/sim_state.py | 8 ++++---- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pdr_backend/ppss/aimodel_data_ss.py b/pdr_backend/ppss/aimodel_data_ss.py index 975220b9b..3c4d4b526 100644 --- a/pdr_backend/ppss/aimodel_data_ss.py +++ b/pdr_backend/ppss/aimodel_data_ss.py @@ -71,7 +71,7 @@ def class_thr(self) -> float: return self.d["class_thr"] @property - def transform(self) -> int: + def transform(self) -> str: """eg 'RelDiff'""" return self.d["transform"] diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index b181dc32f..785c8a78f 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -16,7 +16,6 @@ from pdr_backend.ppss.predictoor_ss import PredictoorSS from pdr_backend.sim.constants import Dirn, UP, DOWN from pdr_backend.sim.sim_logger import SimLogLine -from pdr_backend.sim.sim_model_data import SimModelData from pdr_backend.sim.sim_model_data_factory import SimModelDataFactory from pdr_backend.sim.sim_model_factory import SimModelFactory from pdr_backend.sim.sim_model_prediction import SimModelPrediction @@ -71,6 +70,7 @@ def predict_feed(self) -> ArgFeed: @property def timeframe(self) -> ArgTimeframe: + assert self.predict_feed.timeframe is not None return self.predict_feed.timeframe @property @@ -135,7 +135,7 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): # build model model_factory = SimModelFactory(self.pdr_ss.aimodel_ss) - st.sim_model_data: SimModelData = sim_model_data_f.build(iter_i, df) + st.sim_model_data = sim_model_data_f.build(iter_i, df) if model_factory.do_build(st.sim_model, iter_i): st.sim_model = model_factory.build(st.sim_model_data) diff --git a/pdr_backend/sim/sim_model_data.py b/pdr_backend/sim/sim_model_data.py index 4f7da925a..c129ffd49 100644 --- a/pdr_backend/sim/sim_model_data.py +++ b/pdr_backend/sim/sim_model_data.py @@ -1,9 +1,9 @@ -from typing import List, Tuple +from typing import Dict, List from enforce_typing import enforce_types import numpy as np -from pdr_backend.sim.constants import UP, DOWN +from pdr_backend.sim.constants import Dirn, UP, DOWN class SimModelData1Dir: @@ -55,5 +55,5 @@ def __init__( self[DOWN] = data_DOWN @property - def X_test(self) -> Tuple[np.ndarray, np.ndarray]: + def X_test(self) -> Dict[Dirn, np.ndarray]: return {UP: self[UP].X_test, DOWN: self[DOWN].X_test} diff --git a/pdr_backend/sim/sim_model_data_factory.py b/pdr_backend/sim/sim_model_data_factory.py index 43973cab3..2aa32507e 100644 --- a/pdr_backend/sim/sim_model_data_factory.py +++ b/pdr_backend/sim/sim_model_data_factory.py @@ -60,20 +60,20 @@ def build(self, test_i: int, mergedohlcv_df: pl.DataFrame) -> SimModelData: ArgFeeds([p.variant_low()]), ) - ytrue_UP = [] - ytrue_DOWN = [] + ytrue_UP_list = [] + ytrue_DOWN_list = [] for i, cur_close in enumerate(y_close[:-1]): # did the next high value go above the current close+% value? next_high = y_high[i + 1] thr_UP = self.thr_UP(cur_close) - ytrue_UP.append(next_high > thr_UP) + ytrue_UP_list.append(next_high > thr_UP) # did the next low value go below the current close-% value? next_low = y_low[i + 1] thr_DOWN = self.thr_DOWN(cur_close) - ytrue_DOWN.append(next_low < thr_DOWN) + ytrue_DOWN_list.append(next_low < thr_DOWN) - ytrue_UP, ytrue_DOWN = np.array(ytrue_UP), np.array(ytrue_DOWN) + ytrue_UP, ytrue_DOWN = np.array(ytrue_UP_list), np.array(ytrue_DOWN_list) colnames_UP = list(x_high_df.columns) colnames_DOWN = list(x_low_df.columns) diff --git a/pdr_backend/sim/sim_predictoor.py b/pdr_backend/sim/sim_predictoor.py index 1f93cc1be..2b35c2014 100644 --- a/pdr_backend/sim/sim_predictoor.py +++ b/pdr_backend/sim/sim_predictoor.py @@ -19,8 +19,8 @@ def max_stake_amt(self) -> float: def predict_iter(self, p: SimModelPrediction) -> Tuple[float, float]: """@return (stake_up, stake_down)""" if not p.do_trust_models(): - stake_up = 0 - stake_down = 0 + stake_up = 0.0 + stake_down = 0.0 elif p.prob_UP >= p.prob_DOWN: stake_amt = self.max_stake_amt * p.conf_up stake_up = stake_amt * p.prob_up_MERGED diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 02289b329..28fcef94b 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -83,10 +83,10 @@ def final_metrics_values(self) -> Dict[str, float]: f"acc_est_{s}": self.acc_ests[-1], f"acc_l_{s}": self.acc_ls[-1], f"acc_u_{s}": self.acc_us[-1], - f"f1_{s}": np.mean(self.f1s), - f"precision_{s}": np.mean(self.precisions), - f"recall_{s}": np.mean(self.recalls), - f"loss_{s}": np.mean(self.losses), + f"f1_{s}": float(np.mean(self.f1s)), + f"precision_{s}": float(np.mean(self.precisions)), + f"recall_{s}": float(np.mean(self.recalls)), + f"loss_{s}": float(np.mean(self.losses)), } @enforce_types From 58bdebb87c924577e2fcead4c3bfbe63d32769c0 Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 19:59:41 +0200 Subject: [PATCH 23/50] tweak --- pdr_backend/sim/sim_plotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index d3a6641bc..9d4120fa6 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -22,7 +22,7 @@ def __init__( self, ): self.st = None - self.aimodel_plotdata_ = None + self.aimodel_plotdata = None self.multi_id = None @staticmethod From 220e06ec84fa4020f136f2bcec7923a6bba1be0b Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 20:10:15 +0200 Subject: [PATCH 24/50] plots are running end-to-end --- pdr_backend/sim/dash_plots/callbacks.py | 3 +- pdr_backend/sim/dash_plots/util.py | 11 ++- pdr_backend/sim/sim_plotter.py | 99 +++++++++++++++++-------- 3 files changed, 76 insertions(+), 37 deletions(-) diff --git a/pdr_backend/sim/dash_plots/callbacks.py b/pdr_backend/sim/dash_plots/callbacks.py index 9ed7106ae..c12eeb18d 100644 --- a/pdr_backend/sim/dash_plots/callbacks.py +++ b/pdr_backend/sim/dash_plots/callbacks.py @@ -2,6 +2,7 @@ from dash import Input, Output, State +from pdr_backend.sim.constants import UP from pdr_backend.sim.dash_plots.util import get_figures_by_state from pdr_backend.sim.dash_plots.view_elements import ( get_tabs, @@ -74,7 +75,7 @@ def update_graph_live(n, selected_vars, selected_vars_old, selected_tab): header = get_header_elements(run_id, st, ts) elements = [] - state_options = sim_plotter.aimodel_plotdata.colnames + state_options = sim_plotter.aimodel_plotdata[UP].colnames elements.append(selected_var_checklist(state_options, selected_vars_old)) figures = get_figures_by_state(sim_plotter, selected_vars) diff --git a/pdr_backend/sim/dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py index e74e976cd..533e9b57b 100644 --- a/pdr_backend/sim/dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -1,8 +1,11 @@ +from enforce_typing import enforce_types + from pdr_backend.aimodel import aimodel_plotter +from pdr_backend.sim.constants import UP from pdr_backend.sim.dash_plots.view_elements import figure_names from pdr_backend.sim.sim_plotter import SimPlotter - +@enforce_types def get_figures_by_state(sim_plotter: SimPlotter, selected_vars): figures = {} @@ -13,11 +16,11 @@ def get_figures_by_state(sim_plotter: SimPlotter, selected_vars): if key in ["aimodel_response", "aimodel_varimps"]: sweep_vars = [] for var in selected_vars: - sweep_vars.append(sim_plotter.aimodel_plotdata.colnames.index(var)) - sim_plotter.aimodel_plotdata.sweep_vars = sweep_vars + sweep_vars.append(sim_plotter.aimodel_plotdata[UP].colnames.index(var)) + sim_plotter.aimodel_plotdata[UP].sweep_vars = sweep_vars func_name = getattr(aimodel_plotter, f"plot_{key}") - fig = func_name(sim_plotter.aimodel_plotdata) + fig = func_name(sim_plotter.aimodel_plotdata[UP]) figures[key] = fig diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 9d4120fa6..2831633ab 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -11,6 +11,8 @@ import plotly.graph_objects as go from plotly.subplots import make_subplots +from pdr_backend.sim.constants import UP + HEIGHT = 7.5 WIDTH = int(HEIGHT * 3.2) @@ -133,7 +135,7 @@ def save_state(self, sim_state, aimodel_plotdata, is_final: bool = False): @enforce_types def plot_pdr_profit_vs_time(self): - y = list(np.cumsum(self.st.profits.pdr_profits_OCEAN)) + y = list(np.cumsum(self.st.hist_profits.pdr_profits_OCEAN)) ylabel = "predictoor profit (OCEAN)" title = f"Predictoor profit vs time. Current: {y[-1]:.2f} OCEAN" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) @@ -142,17 +144,56 @@ def plot_pdr_profit_vs_time(self): @enforce_types def plot_trader_profit_vs_time(self): - y = list(np.cumsum(self.st.profits.trader_profits_USD)) + y = list(np.cumsum(self.st.hist_profits.trader_profits_USD)) ylabel = "trader profit (USD)" title = f"Trader profit vs time. Current: ${y[-1]:.2f}" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) self._add_subplot_y_vs_time(fig, y, ylabel, "lines", row=1, col=1) return fig + @enforce_types + def _add_subplot_y_vs_time(self, fig, y, ylabel, mode, row, col): + assert mode in ["markers", "lines"], mode + line, marker = None, None + if mode == "markers": + marker = {"color": "black", "size": 2} + elif mode == "lines": + line = {"color": "#636EFA"} + + x = list(range(len(y))) + + fig.add_traces( + [ + # points: y vs time + go.Scatter( + x=x, + y=y, + mode=mode, + marker=marker, + line=line, + showlegend=False, + ), + # line: horizontal error = 0 + go.Scatter( + x=[min(x), max(x)], + y=[0.0, 0.0], + mode="lines", + line={"color": "grey", "dash": "dot"}, + showlegend=False, + ), + ], + rows=[row] * 2, + cols=[col] * 2, + ) + fig.update_xaxes(title="time", row=row, col=col) + fig.update_yaxes(title=ylabel, row=row, col=col) + + return fig + @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.classif_base.UP.probs_up - y = self.st.profits.pdr_profits_OCEAN + x = self.st.true_vs_pred[UP].predprobs + y = self.st.hist_profits.pdr_profits_OCEAN fig = go.Figure( go.Scatter( x=x, @@ -171,8 +212,8 @@ def plot_pdr_profit_vs_ptrue(self): @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.classif_base.UP.probs_up - y = self.st.profits.trader_profits_USD + x = self.st.true_vs_pred[UP].predprobs + y = self.st.hist_profits.trader_profits_USD fig = go.Figure( go.Scatter( x=x, @@ -192,15 +233,15 @@ def plot_trader_profit_vs_ptrue(self): @enforce_types def plot_model_performance_vs_time(self): # set titles - metrics = self.st.metrics.UP - s1 = f"accuracy = {metrics.acc_ests[-1]*100:.2f}% " - s1 += f"[{metrics.acc_ls[-1]*100:.2f}%, {metrics.acc_us[-1]*100:.2f}%]" + hist_perfs = self.st.hist_perfs[UP] + s1 = f"accuracy = {hist_perfs.acc_ests[-1]*100:.2f}% " + s1 += f"[{hist_perfs.acc_ls[-1]*100:.2f}%, {hist_perfs.acc_us[-1]*100:.2f}%]" - s2 = f"f1={metrics.f1s[-1]:.4f}" - s2 += f" [recall={metrics.recalls[-1]:.4f}" - s2 += f", precision={metrics.precisions[-1]:.4f}]" + s2 = f"f1={hist_perfs.f1s[-1]:.4f}" + s2 += f" [recall={hist_perfs.recalls[-1]:.4f}" + s2 += f", precision={hist_perfs.precisions[-1]:.4f}]" - s3 = f"log loss = {metrics.losses[-1]:.4f}" + s3 = f"log loss = {hist_perfs.losses[-1]:.4f}" # make subplots fig = make_subplots( @@ -238,12 +279,12 @@ def plot_model_performance_vs_time(self): @enforce_types def _add_subplot_accuracy_vs_time(self, fig, row): - metrics = self.st.metrics.UP - acc_ests = [100 * a for a in metrics.acc_ests] + hist_perfs = self.st.hist_perfs[UP] + acc_ests = [100 * a for a in hist_perfs.acc_ests] df = pd.DataFrame(acc_ests, columns=["accuracy"]) - df["acc_ls"] = [100 * a for a in metrics.acc_ls] - df["acc_us"] = [100 * a for a in metrics.acc_us] - df["time"] = range(len(metrics.acc_ests)) + df["acc_ls"] = [100 * a for a in hist_perfs.acc_ls] + df["acc_us"] = [100 * a for a in hist_perfs.acc_us] + df["time"] = range(len(hist_perfs.acc_ests)) fig.add_traces( [ @@ -288,11 +329,11 @@ def _add_subplot_accuracy_vs_time(self, fig, row): @enforce_types def _add_subplot_f1_precision_recall_vs_time(self, fig, row): - metrics = self.st.metrics.UP - df = pd.DataFrame(metrics.f1s, columns=["f1"]) - df["precisions"] = metrics.precisions - df["recalls"] = metrics.recalls - df["time"] = range(len(metrics.f1s)) + hist_perfs = self.st.hist_perfs[UP] + df = pd.DataFrame(hist_perfs.f1s, columns=["f1"]) + df["precisions"] = hist_perfs.precisions + df["recalls"] = hist_perfs.recalls + df["time"] = range(len(hist_perfs.f1s)) fig.add_traces( [ @@ -336,9 +377,9 @@ def _add_subplot_f1_precision_recall_vs_time(self, fig, row): @enforce_types def _add_subplot_log_loss_vs_time(self, fig, row): - metrics = self.st.metrics.UP - df = pd.DataFrame(metrics.losses, columns=["log loss"]) - df["time"] = range(len(metrics.losses)) + hist_perfs = self.st.hist_perfs[UP] + df = pd.DataFrame(hist_perfs.losses, columns=["log loss"]) + df["time"] = range(len(hist_perfs.losses)) fig.add_trace( go.Scatter(x=df["time"], y=df["log loss"], mode="lines", name="log loss"), @@ -354,12 +395,6 @@ def file_age_in_seconds(pathname): return time.time() - stat_result.st_mtime -@enforce_types -def _model_is_classif(sim_state) -> bool: - yerrs = sim_state.metrics.yerrs - return min(yerrs) == max(yerrs) == 0.0 - - @enforce_types def _empty_fig(title=""): fig = go.Figure() From 253fa8b83c9184249deddc10dfcace54b0b855ee Mon Sep 17 00:00:00 2001 From: trentmc Date: Wed, 3 Jul 2024 21:07:50 +0200 Subject: [PATCH 25/50] show both UP and DOWN model performances --- pdr_backend/sim/sim_plotter.py | 71 ++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 2831633ab..ce860dbd9 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -11,7 +11,7 @@ import plotly.graph_objects as go from plotly.subplots import make_subplots -from pdr_backend.sim.constants import UP +from pdr_backend.sim.constants import Dirn, dirn_str, UP, DOWN HEIGHT = 7.5 WIDTH = int(HEIGHT * 3.2) @@ -233,28 +233,27 @@ def plot_trader_profit_vs_ptrue(self): @enforce_types def plot_model_performance_vs_time(self): # set titles - hist_perfs = self.st.hist_perfs[UP] - s1 = f"accuracy = {hist_perfs.acc_ests[-1]*100:.2f}% " - s1 += f"[{hist_perfs.acc_ls[-1]*100:.2f}%, {hist_perfs.acc_us[-1]*100:.2f}%]" - - s2 = f"f1={hist_perfs.f1s[-1]:.4f}" - s2 += f" [recall={hist_perfs.recalls[-1]:.4f}" - s2 += f", precision={hist_perfs.precisions[-1]:.4f}]" - - s3 = f"log loss = {hist_perfs.losses[-1]:.4f}" + titles = [ + self._acc_title(UP), self._acc_title(DOWN), + self._f1_title(UP), self._f1_title(DOWN), + self._loss_title(UP), self._loss_title(DOWN), + ] # make subplots fig = make_subplots( rows=3, - cols=1, - subplot_titles=(s1, s2, s3), + cols=2, + subplot_titles=titles, vertical_spacing=0.08, ) # fill in subplots - self._add_subplot_accuracy_vs_time(fig, row=1) - self._add_subplot_f1_precision_recall_vs_time(fig, row=2) - self._add_subplot_log_loss_vs_time(fig, row=3) + self._add_subplot_accuracy_vs_time(fig, UP, row=1, col=1) + self._add_subplot_accuracy_vs_time(fig, DOWN, row=1, col=2) + self._add_subplot_f1_precision_recall_vs_time(fig, UP, row=2, col=1) + self._add_subplot_f1_precision_recall_vs_time(fig, DOWN, row=2, col=2) + self._add_subplot_log_loss_vs_time(fig, UP, row=3, col=1) + self._add_subplot_log_loss_vs_time(fig, DOWN, row=3, col=2) # global: set minor ticks minor = {"ticks": "inside", "showgrid": True} @@ -266,20 +265,42 @@ def plot_model_performance_vs_time(self): fig.update_layout( { "xaxis": {"matches": "x", "showticklabels": True}, - "xaxis2": {"matches": "x", "showticklabels": True}, "xaxis3": {"matches": "x", "showticklabels": True}, + "xaxis5": {"matches": "x", "showticklabels": True}, + "xaxis2": {"matches": "x2", "showticklabels": True}, + "xaxis4": {"matches": "x2", "showticklabels": True}, + "xaxis6": {"matches": "x2", "showticklabels": True}, } ) fig.update_xaxes(title="time", row=3, col=1) + fig.update_xaxes(title="time", row=3, col=2) # global: don't show legend fig.update_layout(showlegend=False) return fig + def _acc_title(self, dirn: Dirn): + hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) + s = f"{dirn_s} accuracy = {hist_perfs.acc_ests[-1]*100:.2f}% " + s += f"[{hist_perfs.acc_ls[-1]*100:.2f}%, {hist_perfs.acc_us[-1]*100:.2f}%]" + return s + + def _f1_title(self, dirn: Dirn): + hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) + s = f"{dirn_s} f1={hist_perfs.f1s[-1]:.4f}" + s += f" [recall={hist_perfs.recalls[-1]:.4f}" + s += f", precision={hist_perfs.precisions[-1]:.4f}]" + return s + + def _loss_title(self, dirn: Dirn): + hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) + s = f"{dirn_s} log loss = {hist_perfs.losses[-1]:.4f}" + return s + @enforce_types - def _add_subplot_accuracy_vs_time(self, fig, row): - hist_perfs = self.st.hist_perfs[UP] + def _add_subplot_accuracy_vs_time(self, fig, dirn:Dirn, row:int, col:int): + hist_perfs = self.st.hist_perfs[dirn] acc_ests = [100 * a for a in hist_perfs.acc_ests] df = pd.DataFrame(acc_ests, columns=["accuracy"]) df["acc_ls"] = [100 * a for a in hist_perfs.acc_ls] @@ -323,13 +344,13 @@ def _add_subplot_accuracy_vs_time(self, fig, row): ), ], rows=[row] * 4, - cols=[1] * 4, + cols=[col] * 4, ) fig.update_yaxes(title_text="accuracy (%)", row=1, col=1) @enforce_types - def _add_subplot_f1_precision_recall_vs_time(self, fig, row): - hist_perfs = self.st.hist_perfs[UP] + def _add_subplot_f1_precision_recall_vs_time(self, fig, dirn, row, col): + hist_perfs = self.st.hist_perfs[dirn] df = pd.DataFrame(hist_perfs.f1s, columns=["f1"]) df["precisions"] = hist_perfs.precisions df["recalls"] = hist_perfs.recalls @@ -371,20 +392,20 @@ def _add_subplot_f1_precision_recall_vs_time(self, fig, row): ), ], rows=[row] * 4, - cols=[1] * 4, + cols=[col] * 4, ) fig.update_yaxes(title_text="f1, etc", row=2, col=1) @enforce_types - def _add_subplot_log_loss_vs_time(self, fig, row): - hist_perfs = self.st.hist_perfs[UP] + def _add_subplot_log_loss_vs_time(self, fig, dirn:Dirn, row:int, col:int): + hist_perfs = self.st.hist_perfs[dirn] df = pd.DataFrame(hist_perfs.losses, columns=["log loss"]) df["time"] = range(len(hist_perfs.losses)) fig.add_trace( go.Scatter(x=df["time"], y=df["log loss"], mode="lines", name="log loss"), row=row, - col=1, + col=col, ) fig.update_yaxes(title_text="log loss", row=3, col=1) From 91b4f51a491385d050344049d3c7e9a90c6ff4c3 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 11:04:32 +0200 Subject: [PATCH 26/50] add UP and down for pdr profit dist'n vs prob. Other plot tweaks --- pdr_backend/sim/dash_plots/view_elements.py | 4 +- pdr_backend/sim/sim_plotter.py | 97 +++++++++++++++------ 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index feca5ed49..dec49e3ad 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -92,7 +92,7 @@ def get_tabs(figures): "name": "Predictoor Profit", "components": [ single_graph(figures, "pdr_profit_vs_time", width="100%"), - single_graph(figures, "pdr_profit_vs_ptrue", width="50%"), + single_graph(figures, "pdr_profit_vs_ptrue", width="100%"), ], "className": "predictor_profit_tab", }, @@ -100,7 +100,7 @@ def get_tabs(figures): "name": "Trader Profit", "components": [ single_graph(figures, "trader_profit_vs_time", width="100%"), - single_graph(figures, "trader_profit_vs_ptrue", width="50%"), + single_graph(figures, "trader_profit_vs_ptrue", width="100%"), ], "className": "trader_profit_tab", }, diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index ce860dbd9..c56ea3f7d 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -28,6 +28,7 @@ def __init__( self.multi_id = None @staticmethod + @enforce_types def get_latest_run_id(): if not os.path.exists("sim_state"): raise Exception( @@ -37,6 +38,7 @@ def get_latest_run_id(): return str(path).replace("sim_state/", "") @staticmethod + @enforce_types def get_all_run_names(): path = Path("sim_state").iterdir() return [str(p).replace("sim_state/", "") for p in path] @@ -135,11 +137,13 @@ def save_state(self, sim_state, aimodel_plotdata, is_final: bool = False): @enforce_types def plot_pdr_profit_vs_time(self): - y = list(np.cumsum(self.st.hist_profits.pdr_profits_OCEAN)) + profits = self.st.hist_profits.pdr_profits_OCEAN + cum_profits = list(np.cumsum(profits)) ylabel = "predictoor profit (OCEAN)" - title = f"Predictoor profit vs time. Current: {y[-1]:.2f} OCEAN" + title = f"Pdr profit vs time. Current: {cum_profits[-1]:.2f} OCEAN" + title += f". Avg profit per iter: {np.average(profits):.3f} OCEAN" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) - self._add_subplot_y_vs_time(fig, y, ylabel, "lines", row=1, col=1) + self._add_subplot_y_vs_time(fig, cum_profits, ylabel, "lines", row=1, col=1) return fig @enforce_types @@ -192,24 +196,56 @@ def _add_subplot_y_vs_time(self, fig, y, ylabel, mode, row, col): @enforce_types def plot_pdr_profit_vs_ptrue(self): - x = self.st.true_vs_pred[UP].predprobs + # set titles + titles = [self._pdr_profit_title(UP), self._pdr_profit_title(DOWN)] + + # make subplots + fig = make_subplots(rows=1, cols=2, subplot_titles=titles) + + # fill in subplots + self._add_subplot_profit_vs_ptrue(fig, UP, row=1, col=1) + self._add_subplot_profit_vs_ptrue(fig, DOWN, row=1, col=2) + + return fig + + @enforce_types + def _add_subplot_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col:int): + dirn_s = dirn_str(dirn) + x = self.st.true_vs_pred[dirn].predprobs y = self.st.hist_profits.pdr_profits_OCEAN - fig = go.Figure( - go.Scatter( - x=x, - y=y, - mode="markers", - marker={"color": "#636EFA", "size": 2}, - ) + fig.add_traces( + [ + # line: profit vs ptrue scatterplot + go.Scatter( + x=x, + y=y, + mode="markers", + marker={"color": "#636EFA", "size": 2}, + ), + # line: 0.0 horizontal + go.Scatter( + x=[min(x), max(x)], + y=[0.0, 0.0], + mode="lines", + name="", + line_dash="dot", + ), + ], + rows=[row]*2, + cols=[col]*2, ) - fig.add_hline(y=0, line_dash="dot", line_color="grey") - title = f"Predictoor profit dist. avg={np.average(y):.2f} OCEAN" - fig.update_layout(title=title) - fig.update_xaxes(title="prob(up)") - fig.update_yaxes(title="pdr profit (OCEAN)") + fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) + fig.update_yaxes(title="pdr profit (OCEAN)", row=row, col=col) + + # global: don't show legend + fig.update_layout(showlegend=False) return fig + @enforce_types + def _pdr_profit_title(self, dirn:Dirn) -> str: + return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" + @enforce_types def plot_trader_profit_vs_ptrue(self): x = self.st.true_vs_pred[UP].predprobs @@ -280,22 +316,25 @@ def plot_model_performance_vs_time(self): return fig - def _acc_title(self, dirn: Dirn): + @enforce_types + def _acc_title(self, dirn: Dirn) -> str: hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) - s = f"{dirn_s} accuracy = {hist_perfs.acc_ests[-1]*100:.2f}% " - s += f"[{hist_perfs.acc_ls[-1]*100:.2f}%, {hist_perfs.acc_us[-1]*100:.2f}%]" + s = f"{dirn_s} accuracy={hist_perfs.acc_ests[-1]*100:.1f}% " + s += f"[{hist_perfs.acc_ls[-1]*100:.1f}%, {hist_perfs.acc_us[-1]*100:.1f}%]" return s - def _f1_title(self, dirn: Dirn): + @enforce_types + def _f1_title(self, dirn: Dirn) -> str: hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) - s = f"{dirn_s} f1={hist_perfs.f1s[-1]:.4f}" - s += f" [recall={hist_perfs.recalls[-1]:.4f}" - s += f", precision={hist_perfs.precisions[-1]:.4f}]" + s = f"{dirn_s} f1={hist_perfs.f1s[-1]:.2f}" + s += f" [recall={hist_perfs.recalls[-1]:.2f}" + s += f", prec'n={hist_perfs.precisions[-1]:.2f}]" return s - def _loss_title(self, dirn: Dirn): + @enforce_types + def _loss_title(self, dirn: Dirn) -> str: hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) - s = f"{dirn_s} log loss = {hist_perfs.losses[-1]:.4f}" + s = f"{dirn_s} log loss = {hist_perfs.losses[-1]:.2f}" return s @enforce_types @@ -403,7 +442,13 @@ def _add_subplot_log_loss_vs_time(self, fig, dirn:Dirn, row:int, col:int): df["time"] = range(len(hist_perfs.losses)) fig.add_trace( - go.Scatter(x=df["time"], y=df["log loss"], mode="lines", name="log loss"), + go.Scatter( + x=df["time"], + y=df["log loss"], + mode="lines", + name="", + marker_color="#636EFA", + ), row=row, col=col, ) From b76264170a74eefb6d2bc54edea739f0ee90bd32 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 11:36:40 +0200 Subject: [PATCH 27/50] better format plots --- pdr_backend/sim/sim_plotter.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index c56ea3f7d..127339c9d 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -206,13 +206,27 @@ def plot_pdr_profit_vs_ptrue(self): self._add_subplot_profit_vs_ptrue(fig, UP, row=1, col=1) self._add_subplot_profit_vs_ptrue(fig, DOWN, row=1, col=2) + # global: set major x-axis ticks + + # global: set ticks + minor = {"ticks": "inside", "showgrid": True} + rng = [0.5, 1.0] + for col in [1, 2]: + fig.update_xaxes(minor=minor, range=rng, dtick=0.1, row=1, col=col) + fig.update_yaxes(minor=minor, row=1, col=col) + + # global: don't show legend + fig.update_layout(showlegend=False) + return fig @enforce_types def _add_subplot_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col:int): dirn_s = dirn_str(dirn) - x = self.st.true_vs_pred[dirn].predprobs - y = self.st.hist_profits.pdr_profits_OCEAN + x = np.array(self.st.true_vs_pred[dirn].predprobs) + y = np.array(self.st.hist_profits.pdr_profits_OCEAN) + I = (x >= 0.5).nonzero()[0] + x, y = x[I], y[I] fig.add_traces( [ # line: profit vs ptrue scatterplot @@ -236,11 +250,6 @@ def _add_subplot_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col:int): ) fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) fig.update_yaxes(title="pdr profit (OCEAN)", row=row, col=col) - - # global: don't show legend - fig.update_layout(showlegend=False) - - return fig @enforce_types def _pdr_profit_title(self, dirn:Dirn) -> str: From 0c1a800b2b4144697c5e468475ed602dff8cdb76 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 11:47:05 +0200 Subject: [PATCH 28/50] add UP and DOWN for trader profit dist'n --- pdr_backend/sim/sim_plotter.py | 98 ++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 127339c9d..4ec28bd5d 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -148,11 +148,13 @@ def plot_pdr_profit_vs_time(self): @enforce_types def plot_trader_profit_vs_time(self): - y = list(np.cumsum(self.st.hist_profits.trader_profits_USD)) + profits = self.st.hist_profits.trader_profits_USD + cum_profits = list(np.cumsum(profits)) ylabel = "trader profit (USD)" - title = f"Trader profit vs time. Current: ${y[-1]:.2f}" + title = f"Trader profit vs time. Current: ${cum_profits[-1]:.2f}" + title += f". Avg profit per iter: {np.average(profits):.3f} OCEAN" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) - self._add_subplot_y_vs_time(fig, y, ylabel, "lines", row=1, col=1) + self._add_subplot_y_vs_time(fig, cum_profits, ylabel, "lines", row=1, col=1) return fig @enforce_types @@ -197,16 +199,14 @@ def _add_subplot_y_vs_time(self, fig, y, ylabel, mode, row, col): @enforce_types def plot_pdr_profit_vs_ptrue(self): # set titles - titles = [self._pdr_profit_title(UP), self._pdr_profit_title(DOWN)] - + titles = [self._pdr_profit_dist_title(dirn) for dirn in [UP,DOWN]] + # make subplots fig = make_subplots(rows=1, cols=2, subplot_titles=titles) # fill in subplots - self._add_subplot_profit_vs_ptrue(fig, UP, row=1, col=1) - self._add_subplot_profit_vs_ptrue(fig, DOWN, row=1, col=2) - - # global: set major x-axis ticks + self._add_subplot_pdr_profit_vs_ptrue(fig, UP, row=1, col=1) + self._add_subplot_pdr_profit_vs_ptrue(fig, DOWN, row=1, col=2) # global: set ticks minor = {"ticks": "inside", "showgrid": True} @@ -221,7 +221,11 @@ def plot_pdr_profit_vs_ptrue(self): return fig @enforce_types - def _add_subplot_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col:int): + def _pdr_profit_dist_title(self, dirn:Dirn) -> str: + return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" + + @enforce_types + def _add_subplot_pdr_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col): dirn_s = dirn_str(dirn) x = np.array(self.st.true_vs_pred[dirn].predprobs) y = np.array(self.st.hist_profits.pdr_profits_OCEAN) @@ -251,30 +255,66 @@ def _add_subplot_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col:int): fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) fig.update_yaxes(title="pdr profit (OCEAN)", row=row, col=col) - @enforce_types - def _pdr_profit_title(self, dirn:Dirn) -> str: - return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" - @enforce_types def plot_trader_profit_vs_ptrue(self): - x = self.st.true_vs_pred[UP].predprobs - y = self.st.hist_profits.trader_profits_USD - fig = go.Figure( - go.Scatter( - x=x, - y=y, - mode="markers", - marker={"color": "#636EFA", "size": 2}, - ) - ) - fig.add_hline(y=0, line_dash="dot", line_color="grey") - title = f"trader profit dist. avg={np.average(y):.2f} USD" - fig.update_layout(title=title) - fig.update_xaxes(title="prob(up)") - fig.update_yaxes(title="trader profit (USD)") + # set titles + titles = [self._tdr_profit_dist_title(dirn) for dirn in [UP,DOWN]] + + # make subplots + fig = make_subplots(rows=1, cols=2, subplot_titles=titles) + + # fill in subplots + self._add_subplot_tdr_profit_vs_ptrue(fig, UP, row=1, col=1) + self._add_subplot_tdr_profit_vs_ptrue(fig, DOWN, row=1, col=2) + + # global: set ticks + minor = {"ticks": "inside", "showgrid": True} + rng = [0.5, 1.0] + for col in [1, 2]: + fig.update_xaxes(minor=minor, range=rng, dtick=0.1, row=1, col=col) + fig.update_yaxes(minor=minor, row=1, col=col) + + # global: don't show legend + fig.update_layout(showlegend=False) return fig + @enforce_types + def _tdr_profit_dist_title(self, dirn:Dirn) -> str: + return f"Trader profit dist'n vs prob({dirn_str(dirn)})" + + + @enforce_types + def _add_subplot_tdr_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col): + dirn_s = dirn_str(dirn) + x = np.array(self.st.true_vs_pred[dirn].predprobs) + y = np.array(self.st.hist_profits.trader_profits_USD) + I = (x >= 0.5).nonzero()[0] + x, y = x[I], y[I] + fig.add_traces( + [ + # line: profit vs ptrue scatterplot + go.Scatter( + x=x, + y=y, + mode="markers", + marker={"color": "#636EFA", "size": 2}, + ), + # line: 0.0 horizontal + go.Scatter( + x=[min(x), max(x)], + y=[0.0, 0.0], + mode="lines", + name="", + line_dash="dot", + ), + ], + rows=[row]*2, + cols=[col]*2, + ) + fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) + fig.update_yaxes(title="tdr profit (OCEAN)", row=row, col=col) + @enforce_types def plot_model_performance_vs_time(self): # set titles From bad97b3b34d046f18819ea2bcfbeaf00829720eb Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 12:01:23 +0200 Subject: [PATCH 29/50] Simpler code for pdr & trader profit dist'n --- pdr_backend/sim/sim_plotter.py | 97 ++++++++++------------------------ 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 4ec28bd5d..d79eae5c1 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -141,7 +141,7 @@ def plot_pdr_profit_vs_time(self): cum_profits = list(np.cumsum(profits)) ylabel = "predictoor profit (OCEAN)" title = f"Pdr profit vs time. Current: {cum_profits[-1]:.2f} OCEAN" - title += f". Avg profit per iter: {np.average(profits):.3f} OCEAN" + title += f". Avg profit per iter: {np.average(profits):.4f} OCEAN" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) self._add_subplot_y_vs_time(fig, cum_profits, ylabel, "lines", row=1, col=1) return fig @@ -152,7 +152,7 @@ def plot_trader_profit_vs_time(self): cum_profits = list(np.cumsum(profits)) ylabel = "trader profit (USD)" title = f"Trader profit vs time. Current: ${cum_profits[-1]:.2f}" - title += f". Avg profit per iter: {np.average(profits):.3f} OCEAN" + title += f". Avg profit per iter: ${np.average(profits):.4f}" fig = make_subplots(rows=1, cols=1, subplot_titles=(title,)) self._add_subplot_y_vs_time(fig, cum_profits, ylabel, "lines", row=1, col=1) return fig @@ -198,74 +198,22 @@ def _add_subplot_y_vs_time(self, fig, y, ylabel, mode, row, col): @enforce_types def plot_pdr_profit_vs_ptrue(self): - # set titles - titles = [self._pdr_profit_dist_title(dirn) for dirn in [UP,DOWN]] - - # make subplots - fig = make_subplots(rows=1, cols=2, subplot_titles=titles) - - # fill in subplots - self._add_subplot_pdr_profit_vs_ptrue(fig, UP, row=1, col=1) - self._add_subplot_pdr_profit_vs_ptrue(fig, DOWN, row=1, col=2) - - # global: set ticks - minor = {"ticks": "inside", "showgrid": True} - rng = [0.5, 1.0] - for col in [1, 2]: - fig.update_xaxes(minor=minor, range=rng, dtick=0.1, row=1, col=col) - fig.update_yaxes(minor=minor, row=1, col=col) - - # global: don't show legend - fig.update_layout(showlegend=False) - - return fig - - @enforce_types - def _pdr_profit_dist_title(self, dirn:Dirn) -> str: - return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" - - @enforce_types - def _add_subplot_pdr_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col): - dirn_s = dirn_str(dirn) - x = np.array(self.st.true_vs_pred[dirn].predprobs) - y = np.array(self.st.hist_profits.pdr_profits_OCEAN) - I = (x >= 0.5).nonzero()[0] - x, y = x[I], y[I] - fig.add_traces( - [ - # line: profit vs ptrue scatterplot - go.Scatter( - x=x, - y=y, - mode="markers", - marker={"color": "#636EFA", "size": 2}, - ), - # line: 0.0 horizontal - go.Scatter( - x=[min(x), max(x)], - y=[0.0, 0.0], - mode="lines", - name="", - line_dash="dot", - ), - ], - rows=[row]*2, - cols=[col]*2, - ) - fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) - fig.update_yaxes(title="pdr profit (OCEAN)", row=row, col=col) + return self._plot_profit_vs_ptrue(is_pdr=True) @enforce_types def plot_trader_profit_vs_ptrue(self): - # set titles - titles = [self._tdr_profit_dist_title(dirn) for dirn in [UP,DOWN]] + return self._plot_profit_vs_ptrue(is_pdr=False) + @enforce_types + def _plot_profit_vs_ptrue(self, is_pdr: bool): + titles = [self._profit_dist_title(is_pdr, dirn) for dirn in [UP,DOWN]] + # make subplots fig = make_subplots(rows=1, cols=2, subplot_titles=titles) # fill in subplots - self._add_subplot_tdr_profit_vs_ptrue(fig, UP, row=1, col=1) - self._add_subplot_tdr_profit_vs_ptrue(fig, DOWN, row=1, col=2) + self._add_subplot_profit_dist(fig, is_pdr, UP, row=1, col=1) + self._add_subplot_profit_dist(fig, is_pdr, DOWN, row=1, col=2) # global: set ticks minor = {"ticks": "inside", "showgrid": True} @@ -278,17 +226,24 @@ def plot_trader_profit_vs_ptrue(self): fig.update_layout(showlegend=False) return fig - + @enforce_types - def _tdr_profit_dist_title(self, dirn:Dirn) -> str: + def _profit_dist_title(self, is_pdr: bool, dirn:Dirn) -> str: + if is_pdr: + return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" + return f"Trader profit dist'n vs prob({dirn_str(dirn)})" - @enforce_types - def _add_subplot_tdr_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col): + def _add_subplot_profit_dist( + self, fig, is_pdr: bool, dirn:Dirn, row:int, col: int, + ): dirn_s = dirn_str(dirn) x = np.array(self.st.true_vs_pred[dirn].predprobs) - y = np.array(self.st.hist_profits.trader_profits_USD) + if is_pdr: + y = np.array(self.st.hist_profits.pdr_profits_OCEAN) + else: + y = np.array(self.st.hist_profits.trader_profits_USD) I = (x >= 0.5).nonzero()[0] x, y = x[I], y[I] fig.add_traces( @@ -312,8 +267,14 @@ def _add_subplot_tdr_profit_vs_ptrue(self, fig, dirn:Dirn, row:int, col): rows=[row]*2, cols=[col]*2, ) + fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) - fig.update_yaxes(title="tdr profit (OCEAN)", row=row, col=col) + + if is_pdr: + ytitle = "pdr profit (OCEAN)" + else: + ytitle = "trader profit (USD)" + fig.update_yaxes(title=ytitle, row=row, col=col) @enforce_types def plot_model_performance_vs_time(self): From d65ec0e3f1af56286312c38902e6288e684369db Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 12:45:13 +0200 Subject: [PATCH 30/50] add UP and DOWN plots for model responses --- pdr_backend/sim/dash_plots/util.py | 50 +++++++++++++++------ pdr_backend/sim/dash_plots/view_elements.py | 30 ++++++++++--- pdr_backend/sim/test/test_dash_plots.py | 18 +++++--- 3 files changed, 71 insertions(+), 27 deletions(-) diff --git a/pdr_backend/sim/dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py index 533e9b57b..e2a504ff8 100644 --- a/pdr_backend/sim/dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -2,26 +2,48 @@ from pdr_backend.aimodel import aimodel_plotter from pdr_backend.sim.constants import UP -from pdr_backend.sim.dash_plots.view_elements import figure_names +from pdr_backend.sim.dash_plots.view_elements import ( + FIGURE_NAMES, + OTHER_FIGURES, + MODEL_RESPONSE_FIGURES, +) from pdr_backend.sim.sim_plotter import SimPlotter +from pdr_backend.sim.constants import Dirn, UP, DOWN + @enforce_types -def get_figures_by_state(sim_plotter: SimPlotter, selected_vars): +def get_figures_by_state(sim_plotter: SimPlotter, selected_vars: list): figures = {} - for key in figure_names: - if not key.startswith("aimodel"): - fig = getattr(sim_plotter, f"plot_{key}")() - else: - if key in ["aimodel_response", "aimodel_varimps"]: - sweep_vars = [] - for var in selected_vars: - sweep_vars.append(sim_plotter.aimodel_plotdata[UP].colnames.index(var)) - sim_plotter.aimodel_plotdata[UP].sweep_vars = sweep_vars + for figure_name in FIGURE_NAMES: + if figure_name in OTHER_FIGURES: + fig = getattr(sim_plotter, f"plot_{figure_name}")() - func_name = getattr(aimodel_plotter, f"plot_{key}") - fig = func_name(sim_plotter.aimodel_plotdata[UP]) + elif figure_name in MODEL_RESPONSE_FIGURES: + dirn = _figure_name_to_dirn(figure_name) + aimodel_plotdata = sim_plotter.aimodel_plotdata[dirn] + sweep_vars = [aimodel_plotdata.colnames.index(varname) + for varname in selected_vars] + aimodel_plotdata.sweep_vars = sweep_vars + func_name = _figure_name_to_func_name(figure_name) + func = getattr(aimodel_plotter, func_name) + fig = func(aimodel_plotdata) + + else: + raise ValueError(figure_name) - figures[key] = fig + figures[figure_name] = fig return figures + +@enforce_types +def _figure_name_to_dirn(figure_name: str) -> Dirn: + if "UP" in figure_name: + return UP + if "DOWN" in figure_name: + return DOWN + raise ValueError(figure_name) + +@enforce_types +def _figure_name_to_func_name(figure_name: str) -> Dirn: + return f"plot_{figure_name}".replace("_UP","").replace("_DOWN","") diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index dec49e3ad..1233e3343 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -2,20 +2,28 @@ from enforce_typing import enforce_types from plotly.graph_objs import Figure -figure_names = [ +OTHER_FIGURES = [ "pdr_profit_vs_time", "pdr_profit_vs_ptrue", "trader_profit_vs_time", "trader_profit_vs_ptrue", "model_performance_vs_time", - "aimodel_varimps", - "aimodel_response", ] +MODEL_RESPONSE_FIGURES = [ + "aimodel_varimps_UP", + "aimodel_response_UP", + "aimodel_varimps_DOWN", + "aimodel_response_DOWN", +] + +FIGURE_NAMES = OTHER_FIGURES + MODEL_RESPONSE_FIGURES + + empty_selected_vars = dcc.Checklist([], [], id="selected_vars") empty_graphs_template = html.Div( - [dcc.Graph(figure=Figure(), id=key) for key in figure_names] + [dcc.Graph(figure=Figure(), id=name) for name in FIGURE_NAMES] + [empty_selected_vars], style={"display": "none"}, ) @@ -116,12 +124,20 @@ def get_tabs(figures): "components": [ side_by_side_graphs( figures, - name1="aimodel_varimps", - name2="aimodel_response", + name1="aimodel_varimps_UP", + name2="aimodel_response_UP", + height="100%", + width1="30%", + width2="70%", + ), + side_by_side_graphs( + figures, + name1="aimodel_varimps_DOWN", + name2="aimodel_response_DOWN", height="100%", width1="30%", width2="70%", - ) + ), ], "className": "model_response_tab", }, diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py index e8ae8e6f0..396b86e29 100644 --- a/pdr_backend/sim/test/test_dash_plots.py +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -1,23 +1,26 @@ from unittest.mock import Mock, patch +from enforce_typing import enforce_types from plotly.graph_objs import Figure from pdr_backend.sim.dash_plots.util import get_figures_by_state from pdr_backend.sim.dash_plots.view_elements import ( - get_tabs, - figure_names, + FIGURE_NAMES, get_header_elements, + get_tabs, get_waiting_template, selected_var_checklist, ) from pdr_backend.sim.sim_plotter import SimPlotter +@enforce_types def test_get_waiting_template(): result = get_waiting_template("custom message") assert "custom message" in result.children[0].children +@enforce_types def test_get_header_elements(): st = Mock() st.iter_number = 5 @@ -33,14 +36,16 @@ def test_get_header_elements(): assert result[1].className == "finalState" +@enforce_types def test_get_tabs(): - figures = {key: Figure() for key in figure_names} + figures = {name: Figure() for name in FIGURE_NAMES} result = get_tabs(figures) for tab in result: assert "name" in tab assert "components" in tab +@enforce_types def test_selected_var_checklist(): result = selected_var_checklist(["var1", "var2"], ["var1"]) assert result.value == ["var1"] @@ -48,6 +53,7 @@ def test_selected_var_checklist(): assert result.options[1]["label"] == "var2" +@enforce_types def test_get_figures_by_state(): mock_sim_plotter = Mock(spec=SimPlotter) mock_sim_plotter.plot_pdr_profit_vs_time.return_value = Figure() @@ -69,6 +75,6 @@ def test_get_figures_by_state(): result = get_figures_by_state(mock_sim_plotter, ["var1", "var2"]) - for key in figure_names: - assert key in result - assert isinstance(result[key], Figure) + for name in FIGURE_NAMES: + assert name in result + assert isinstance(result[name], Figure) From 23eef8813bf658317a925a0b9d8f4ab410f11f91 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 12:46:50 +0200 Subject: [PATCH 31/50] tweak --- pdr_backend/sim/dash_plots/view_elements.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index 1233e3343..2026d90b5 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -19,12 +19,11 @@ FIGURE_NAMES = OTHER_FIGURES + MODEL_RESPONSE_FIGURES +EMPTY_SELECTED_VARS = dcc.Checklist([], [], id="selected_vars") -empty_selected_vars = dcc.Checklist([], [], id="selected_vars") - -empty_graphs_template = html.Div( +EMPTY_GRAPHS_TEMPLATE = html.Div( [dcc.Graph(figure=Figure(), id=name) for name in FIGURE_NAMES] - + [empty_selected_vars], + + [EMPTY_SELECTED_VARS], style={"display": "none"}, ) @@ -33,7 +32,7 @@ def get_waiting_template(err): return html.Div( [html.H2(f"Error/waiting: {err}", id="sim_state_text")] - + [empty_graphs_template], + + [EMPTY_GRAPHS_TEMPLATE], id="live-graphs", ) @@ -187,7 +186,7 @@ def get_main_container(): return html.Div( [ html.Div( - empty_graphs_template, + EMPTY_GRAPHS_TEMPLATE, id="header", style={ "display": "flex", From 96757652ee2070fe845d525907b7ab24e7c85c1c Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 13:10:49 +0200 Subject: [PATCH 32/50] selecting vars works for UP and DOWN model response plots --- pdr_backend/sim/dash_plots/callbacks.py | 61 ++++++++++++++++----- pdr_backend/sim/dash_plots/util.py | 51 ++++++++--------- pdr_backend/sim/dash_plots/view_elements.py | 21 +++++-- pdr_backend/sim/test/test_dash_plots.py | 6 +- 4 files changed, 88 insertions(+), 51 deletions(-) diff --git a/pdr_backend/sim/dash_plots/callbacks.py b/pdr_backend/sim/dash_plots/callbacks.py index c12eeb18d..c308af773 100644 --- a/pdr_backend/sim/dash_plots/callbacks.py +++ b/pdr_backend/sim/dash_plots/callbacks.py @@ -8,7 +8,8 @@ get_tabs, get_header_elements, get_waiting_template, - selected_var_checklist, + selected_var_UP_checklist, + selected_var_DOWN_checklist, get_tabs_component, ) from pdr_backend.sim.sim_plotter import SimPlotter @@ -39,32 +40,58 @@ def callback_func_start_stop_interval(value, disabled_state): return value == "finalState" @app.callback( - Output("selected_vars", "value"), - Input("aimodel_varimps", "clickData"), - State("selected_vars", "value"), + Output("selected_vars_UP", "value"), + Input("aimodel_varimps_UP", "clickData"), + State("selected_vars_UP", "value"), ) - def update_selected_vars(clickData, selected_vars): + def update_selected_vars_UP(clickData, selected_vars_UP): if clickData is None: - return selected_vars + return selected_vars_UP label = clickData["points"][0]["y"] - if label in selected_vars: - selected_vars.remove(label) + if label in selected_vars_UP: + selected_vars_UP.remove(label) else: - selected_vars.append(label) + selected_vars_UP.append(label) - return selected_vars + return selected_vars_UP + + @app.callback( + Output("selected_vars_DOWN", "value"), + Input("aimodel_varimps_DOWN", "clickData"), + State("selected_vars_DOWN", "value"), + ) + def update_selected_vars_DOWN(clickData, selected_vars_DOWN): + if clickData is None: + return selected_vars_DOWN + + label = clickData["points"][0]["y"] + if label in selected_vars_DOWN: + selected_vars_DOWN.remove(label) + else: + selected_vars_DOWN.append(label) + + return selected_vars_DOWN @app.callback( Output("tabs-container", "children"), Output("header", "children"), Input("interval-component", "n_intervals"), - Input("selected_vars", "value"), - State("selected_vars", "value"), + Input("selected_vars_UP", "value"), + Input("selected_vars_DOWN", "value"), + State("selected_vars_UP", "value"), + State("selected_vars_DOWN", "value"), State("selected-tab", "data"), ) # pylint: disable=unused-argument - def update_graph_live(n, selected_vars, selected_vars_old, selected_tab): + def update_graph_live( + n, + selected_vars_UP, + selected_vars_DOWN, + selected_vars_UP_old, + selected_vars_DOWN_old, + selected_tab, + ): try: run_id = app.run_id if app.run_id else SimPlotter.get_latest_run_id() sim_plotter = SimPlotter() @@ -76,9 +103,13 @@ def update_graph_live(n, selected_vars, selected_vars_old, selected_tab): elements = [] state_options = sim_plotter.aimodel_plotdata[UP].colnames - elements.append(selected_var_checklist(state_options, selected_vars_old)) - figures = get_figures_by_state(sim_plotter, selected_vars) + elements.append(selected_var_UP_checklist(state_options, selected_vars_UP_old)) + elements.append(selected_var_DOWN_checklist(state_options, selected_vars_DOWN_old)) + + figures = get_figures_by_state( + sim_plotter, selected_vars_UP, selected_vars_DOWN, + ) tabs = get_tabs(figures) selected_tab_value = selected_tab if selected_tab else tabs[0]["name"] elements = elements + [get_tabs_component(tabs, selected_tab_value)] diff --git a/pdr_backend/sim/dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py index e2a504ff8..30df11af8 100644 --- a/pdr_backend/sim/dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -1,3 +1,5 @@ +from typing import List + from enforce_typing import enforce_types from pdr_backend.aimodel import aimodel_plotter @@ -12,38 +14,31 @@ @enforce_types -def get_figures_by_state(sim_plotter: SimPlotter, selected_vars: list): - figures = {} - - for figure_name in FIGURE_NAMES: - if figure_name in OTHER_FIGURES: - fig = getattr(sim_plotter, f"plot_{figure_name}")() - - elif figure_name in MODEL_RESPONSE_FIGURES: - dirn = _figure_name_to_dirn(figure_name) +def get_figures_by_state( + sim_plotter: SimPlotter, + selected_vars_UP: List[str], # UP model selected varnames + selected_vars_DOWN: List[str], # DOWN "" +): + figs = {} + + for fig_name in FIGURE_NAMES: + if fig_name in OTHER_FIGURES: + fig = getattr(sim_plotter, f"plot_{fig_name}")() + + elif fig_name in MODEL_RESPONSE_FIGURES: + dirn = UP if "UP" in fig_name else DOWN aimodel_plotdata = sim_plotter.aimodel_plotdata[dirn] - sweep_vars = [aimodel_plotdata.colnames.index(varname) - for varname in selected_vars] - aimodel_plotdata.sweep_vars = sweep_vars - func_name = _figure_name_to_func_name(figure_name) + selected_vars = selected_vars_UP if dirn==UP else selected_vars_DOWN + selected_Is = [aimodel_plotdata.colnames.index(var) + for var in selected_vars] + aimodel_plotdata.sweep_vars = selected_Is + func_name = f"plot_{fig_name}".replace("_UP","").replace("_DOWN","") func = getattr(aimodel_plotter, func_name) fig = func(aimodel_plotdata) else: - raise ValueError(figure_name) - - figures[figure_name] = fig + raise ValueError(fig_name) - return figures + figs[fig_name] = fig -@enforce_types -def _figure_name_to_dirn(figure_name: str) -> Dirn: - if "UP" in figure_name: - return UP - if "DOWN" in figure_name: - return DOWN - raise ValueError(figure_name) - -@enforce_types -def _figure_name_to_func_name(figure_name: str) -> Dirn: - return f"plot_{figure_name}".replace("_UP","").replace("_DOWN","") + return figs diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index 2026d90b5..020282f9d 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -19,11 +19,13 @@ FIGURE_NAMES = OTHER_FIGURES + MODEL_RESPONSE_FIGURES -EMPTY_SELECTED_VARS = dcc.Checklist([], [], id="selected_vars") +EMPTY_SELECTED_VARS_UP = dcc.Checklist([], [], id="selected_vars_UP") +EMPTY_SELECTED_VARS_DOWN = dcc.Checklist([], [], id="selected_vars_DOWN") EMPTY_GRAPHS_TEMPLATE = html.Div( [dcc.Graph(figure=Figure(), id=name) for name in FIGURE_NAMES] - + [EMPTY_SELECTED_VARS], + + [EMPTY_SELECTED_VARS_UP] + + [EMPTY_SELECTED_VARS_DOWN], style={"display": "none"}, ) @@ -144,11 +146,20 @@ def get_tabs(figures): @enforce_types -def selected_var_checklist(state_options, selected_vars_old): +def selected_var_UP_checklist(state_options, selected_vars_UP_old): return dcc.Checklist( options=[{"label": var, "value": var} for var in state_options], - value=selected_vars_old, - id="selected_vars", + value=selected_vars_UP_old, + id="selected_vars_UP", + style={"display": "none"}, + ) + +@enforce_types +def selected_var_DOWN_checklist(state_options, selected_vars_DOWN_old): + return dcc.Checklist( + options=[{"label": var, "value": var} for var in state_options], + value=selected_vars_DOWN_old, + id="selected_vars_DOWN", style={"display": "none"}, ) diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py index 396b86e29..ae545c98c 100644 --- a/pdr_backend/sim/test/test_dash_plots.py +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -9,7 +9,7 @@ get_header_elements, get_tabs, get_waiting_template, - selected_var_checklist, + selected_var_UP_checklist, ) from pdr_backend.sim.sim_plotter import SimPlotter @@ -46,8 +46,8 @@ def test_get_tabs(): @enforce_types -def test_selected_var_checklist(): - result = selected_var_checklist(["var1", "var2"], ["var1"]) +def test_selected_var_UP_checklist(): + result = selected_var_UP_checklist(["var1", "var2"], ["var1"]) assert result.value == ["var1"] assert result.options[0]["label"] == "var1" assert result.options[1]["label"] == "var2" From 944762792c56d6d6b56507c9c19b151380842c6c Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 13:11:41 +0200 Subject: [PATCH 33/50] black --- pdr_backend/sim/dash_plots/callbacks.py | 8 +++- pdr_backend/sim/dash_plots/util.py | 13 +++--- pdr_backend/sim/dash_plots/view_elements.py | 1 + pdr_backend/sim/sim_plotter.py | 46 ++++++++++++--------- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/pdr_backend/sim/dash_plots/callbacks.py b/pdr_backend/sim/dash_plots/callbacks.py index c308af773..f1416f4cd 100644 --- a/pdr_backend/sim/dash_plots/callbacks.py +++ b/pdr_backend/sim/dash_plots/callbacks.py @@ -105,10 +105,14 @@ def update_graph_live( state_options = sim_plotter.aimodel_plotdata[UP].colnames elements.append(selected_var_UP_checklist(state_options, selected_vars_UP_old)) - elements.append(selected_var_DOWN_checklist(state_options, selected_vars_DOWN_old)) + elements.append( + selected_var_DOWN_checklist(state_options, selected_vars_DOWN_old) + ) figures = get_figures_by_state( - sim_plotter, selected_vars_UP, selected_vars_DOWN, + sim_plotter, + selected_vars_UP, + selected_vars_DOWN, ) tabs = get_tabs(figures) selected_tab_value = selected_tab if selected_tab else tabs[0]["name"] diff --git a/pdr_backend/sim/dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py index 30df11af8..ba8839ae6 100644 --- a/pdr_backend/sim/dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -16,8 +16,8 @@ @enforce_types def get_figures_by_state( sim_plotter: SimPlotter, - selected_vars_UP: List[str], # UP model selected varnames - selected_vars_DOWN: List[str], # DOWN "" + selected_vars_UP: List[str], # UP model selected varnames + selected_vars_DOWN: List[str], # DOWN "" ): figs = {} @@ -28,11 +28,12 @@ def get_figures_by_state( elif fig_name in MODEL_RESPONSE_FIGURES: dirn = UP if "UP" in fig_name else DOWN aimodel_plotdata = sim_plotter.aimodel_plotdata[dirn] - selected_vars = selected_vars_UP if dirn==UP else selected_vars_DOWN - selected_Is = [aimodel_plotdata.colnames.index(var) - for var in selected_vars] + selected_vars = selected_vars_UP if dirn == UP else selected_vars_DOWN + selected_Is = [ + aimodel_plotdata.colnames.index(var) for var in selected_vars + ] aimodel_plotdata.sweep_vars = selected_Is - func_name = f"plot_{fig_name}".replace("_UP","").replace("_DOWN","") + func_name = f"plot_{fig_name}".replace("_UP", "").replace("_DOWN", "") func = getattr(aimodel_plotter, func_name) fig = func(aimodel_plotdata) diff --git a/pdr_backend/sim/dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py index 020282f9d..0ca404706 100644 --- a/pdr_backend/sim/dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -154,6 +154,7 @@ def selected_var_UP_checklist(state_options, selected_vars_UP_old): style={"display": "none"}, ) + @enforce_types def selected_var_DOWN_checklist(state_options, selected_vars_DOWN_old): return dcc.Checklist( diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index d79eae5c1..b22d4d5eb 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -206,37 +206,42 @@ def plot_trader_profit_vs_ptrue(self): @enforce_types def _plot_profit_vs_ptrue(self, is_pdr: bool): - titles = [self._profit_dist_title(is_pdr, dirn) for dirn in [UP,DOWN]] - + titles = [self._profit_dist_title(is_pdr, dirn) for dirn in [UP, DOWN]] + # make subplots fig = make_subplots(rows=1, cols=2, subplot_titles=titles) - + # fill in subplots self._add_subplot_profit_dist(fig, is_pdr, UP, row=1, col=1) self._add_subplot_profit_dist(fig, is_pdr, DOWN, row=1, col=2) - + # global: set ticks minor = {"ticks": "inside", "showgrid": True} rng = [0.5, 1.0] for col in [1, 2]: fig.update_xaxes(minor=minor, range=rng, dtick=0.1, row=1, col=col) fig.update_yaxes(minor=minor, row=1, col=col) - + # global: don't show legend fig.update_layout(showlegend=False) return fig - + @enforce_types - def _profit_dist_title(self, is_pdr: bool, dirn:Dirn) -> str: + def _profit_dist_title(self, is_pdr: bool, dirn: Dirn) -> str: if is_pdr: return f"Pdr profit dist'n vs prob({dirn_str(dirn)})" - + return f"Trader profit dist'n vs prob({dirn_str(dirn)})" @enforce_types def _add_subplot_profit_dist( - self, fig, is_pdr: bool, dirn:Dirn, row:int, col: int, + self, + fig, + is_pdr: bool, + dirn: Dirn, + row: int, + col: int, ): dirn_s = dirn_str(dirn) x = np.array(self.st.true_vs_pred[dirn].predprobs) @@ -264,12 +269,12 @@ def _add_subplot_profit_dist( line_dash="dot", ), ], - rows=[row]*2, - cols=[col]*2, + rows=[row] * 2, + cols=[col] * 2, ) - + fig.update_xaxes(title=f"prob({dirn_s})", row=row, col=col) - + if is_pdr: ytitle = "pdr profit (OCEAN)" else: @@ -280,9 +285,12 @@ def _add_subplot_profit_dist( def plot_model_performance_vs_time(self): # set titles titles = [ - self._acc_title(UP), self._acc_title(DOWN), - self._f1_title(UP), self._f1_title(DOWN), - self._loss_title(UP), self._loss_title(DOWN), + self._acc_title(UP), + self._acc_title(DOWN), + self._f1_title(UP), + self._f1_title(DOWN), + self._loss_title(UP), + self._loss_title(DOWN), ] # make subplots @@ -346,9 +354,9 @@ def _loss_title(self, dirn: Dirn) -> str: hist_perfs, dirn_s = self.st.hist_perfs[dirn], dirn_str(dirn) s = f"{dirn_s} log loss = {hist_perfs.losses[-1]:.2f}" return s - + @enforce_types - def _add_subplot_accuracy_vs_time(self, fig, dirn:Dirn, row:int, col:int): + def _add_subplot_accuracy_vs_time(self, fig, dirn: Dirn, row: int, col: int): hist_perfs = self.st.hist_perfs[dirn] acc_ests = [100 * a for a in hist_perfs.acc_ests] df = pd.DataFrame(acc_ests, columns=["accuracy"]) @@ -446,7 +454,7 @@ def _add_subplot_f1_precision_recall_vs_time(self, fig, dirn, row, col): fig.update_yaxes(title_text="f1, etc", row=2, col=1) @enforce_types - def _add_subplot_log_loss_vs_time(self, fig, dirn:Dirn, row:int, col:int): + def _add_subplot_log_loss_vs_time(self, fig, dirn: Dirn, row: int, col: int): hist_perfs = self.st.hist_perfs[dirn] df = pd.DataFrame(hist_perfs.losses, columns=["log loss"]) df["time"] = range(len(hist_perfs.losses)) From 42fbe76ed80224da4c8e058ec0324b33deb1df83 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 13:30:30 +0200 Subject: [PATCH 34/50] sim unit tests passing --- pdr_backend/sim/dash_plots/util.py | 3 +- pdr_backend/sim/test/test_dash_plots.py | 37 ++++++++++++++++++------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/pdr_backend/sim/dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py index ba8839ae6..67dc9406a 100644 --- a/pdr_backend/sim/dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -3,14 +3,13 @@ from enforce_typing import enforce_types from pdr_backend.aimodel import aimodel_plotter -from pdr_backend.sim.constants import UP +from pdr_backend.sim.constants import UP, DOWN from pdr_backend.sim.dash_plots.view_elements import ( FIGURE_NAMES, OTHER_FIGURES, MODEL_RESPONSE_FIGURES, ) from pdr_backend.sim.sim_plotter import SimPlotter -from pdr_backend.sim.constants import Dirn, UP, DOWN @enforce_types diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py index ae545c98c..a11fa099c 100644 --- a/pdr_backend/sim/test/test_dash_plots.py +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -3,6 +3,7 @@ from enforce_typing import enforce_types from plotly.graph_objs import Figure +from pdr_backend.sim.constants import UP, DOWN from pdr_backend.sim.dash_plots.util import get_figures_by_state from pdr_backend.sim.dash_plots.view_elements import ( FIGURE_NAMES, @@ -10,6 +11,7 @@ get_tabs, get_waiting_template, selected_var_UP_checklist, + selected_var_DOWN_checklist, ) from pdr_backend.sim.sim_plotter import SimPlotter @@ -47,10 +49,18 @@ def test_get_tabs(): @enforce_types def test_selected_var_UP_checklist(): - result = selected_var_UP_checklist(["var1", "var2"], ["var1"]) - assert result.value == ["var1"] - assert result.options[0]["label"] == "var1" - assert result.options[1]["label"] == "var2" + result = selected_var_UP_checklist(["var_up1", "var_up2"], ["var_up1"]) + assert result.value == ["var_up1"] + assert result.options[0]["label"] == "var_up1" + assert result.options[1]["label"] == "var_up2" + + +@enforce_types +def test_selected_var_DOWN_checklist(): + result = selected_var_DOWN_checklist(["var_down1", "var_down2"], ["var_down1"]) + assert result.value == ["var_down1"] + assert result.options[0]["label"] == "var_down1" + assert result.options[1]["label"] == "var_down2" @enforce_types @@ -62,19 +72,26 @@ def test_get_figures_by_state(): mock_sim_plotter.plot_trader_profit_vs_ptrue.return_value = Figure() mock_sim_plotter.plot_model_performance_vs_time.return_value = Figure() - plotdata = Mock() - plotdata.colnames = ["var1", "var2"] + plotdata = {UP: Mock(), DOWN: Mock()} + plotdata[UP].colnames = ["var_up1", "var_up2"] + plotdata[DOWN].colnames = ["var_down1", "var_down2"] mock_sim_plotter.aimodel_plotdata = plotdata with patch( "pdr_backend.sim.dash_plots.util.aimodel_plotter" ) as mock_aimodel_plotter: + # *not* with UP or DOWN here, because the plot_*_() calls input Dirn mock_aimodel_plotter.plot_aimodel_response.return_value = Figure() mock_aimodel_plotter.plot_aimodel_varimps.return_value = Figure() - result = get_figures_by_state(mock_sim_plotter, ["var1", "var2"]) + figs = get_figures_by_state( + mock_sim_plotter, + ["var_up1", "var_up2"], + ["var_down1", "var_down2"], + ) - for name in FIGURE_NAMES: - assert name in result - assert isinstance(result[name], Figure) + assert sorted(figs.keys()) == sorted(FIGURE_NAMES) + for fig_name in FIGURE_NAMES: + assert fig_name in figs + assert isinstance(figs[fig_name], Figure) From 3547ef701b245af7ebe6cad865a62fc28d47fbd1 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 13:31:47 +0200 Subject: [PATCH 35/50] all linters pass --- pdr_backend/sim/test/test_dash_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py index a11fa099c..b037bae2a 100644 --- a/pdr_backend/sim/test/test_dash_plots.py +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -54,7 +54,7 @@ def test_selected_var_UP_checklist(): assert result.options[0]["label"] == "var_up1" assert result.options[1]["label"] == "var_up2" - + @enforce_types def test_selected_var_DOWN_checklist(): result = selected_var_DOWN_checklist(["var_down1", "var_down2"], ["var_down1"]) From 5e382d88c93a7e25becbfcd4f830460bf26d9a04 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 15:25:11 +0200 Subject: [PATCH 36/50] bug fix --- pdr_backend/sim/sim_plotter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index b22d4d5eb..74c1bdb6c 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -250,7 +250,8 @@ def _add_subplot_profit_dist( else: y = np.array(self.st.hist_profits.trader_profits_USD) I = (x >= 0.5).nonzero()[0] - x, y = x[I], y[I] + if len(I) > 0: + x, y = x[I], y[I] fig.add_traces( [ # line: profit vs ptrue scatterplot From 862c36609c41d373590495d51748bdb2533da97e Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 15:59:48 +0200 Subject: [PATCH 37/50] move calc_pdr_profit() --- pdr_backend/ppss/predictoor_ss.py | 2 +- pdr_backend/sim/sim_engine.py | 31 +++- pdr_backend/sim/sim_state.py | 29 --- .../test/test_sim_engine_calc_pdr_profit.py | 166 ++++++++++++++++++ ..._sim_engine.py => test_sim_engine_main.py} | 2 +- pdr_backend/sim/test/test_sim_state.py | 162 ----------------- 6 files changed, 198 insertions(+), 194 deletions(-) create mode 100644 pdr_backend/sim/test/test_sim_engine_calc_pdr_profit.py rename pdr_backend/sim/test/{test_sim_engine.py => test_sim_engine_main.py} (98%) diff --git a/pdr_backend/ppss/predictoor_ss.py b/pdr_backend/ppss/predictoor_ss.py index 2c990199f..370828985 100644 --- a/pdr_backend/ppss/predictoor_ss.py +++ b/pdr_backend/ppss/predictoor_ss.py @@ -199,7 +199,7 @@ def predictoor_ss_test_dict( "approach": 1, "stake_amount": 1, "sim_only": { - "others_stake": 2313, + "others_stake": 2313., "others_accuracy": 0.50001, "revenue": 0.93007, }, diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index 785c8a78f..a33b27ad2 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -169,7 +169,7 @@ def run_one_iter(self, iter_i: int, mergedohlcv_df: pl.DataFrame): } # calc predictoor profit - pdr_profit_OCEAN = HistProfits.calc_pdr_profit( + pdr_profit_OCEAN = calc_pdr_profit( self.others_stake, self.others_accuracy, stake_up, @@ -272,3 +272,32 @@ def _do_save_state(self, i: int) -> Tuple[bool, bool]: return False, False return True, False + +@enforce_types +def calc_pdr_profit( + others_stake: float, + others_accuracy: float, + stake_up: float, + stake_down: float, + revenue: float, + true_up_close: bool, +): + assert others_stake >= 0 + assert 0.0 <= others_accuracy <= 1.0 + assert stake_up >= 0.0 + assert stake_down >= 0.0 + assert revenue >= 0.0 + + amt_sent = stake_up + stake_down + others_stake_correct = others_stake * others_accuracy + tot_stake = others_stake + stake_up + stake_down + if true_up_close: + tot_stake_correct = others_stake_correct + stake_up + percent_to_me = stake_up / tot_stake_correct + amt_received = (revenue + tot_stake) * percent_to_me + else: + tot_stake_correct = others_stake_correct + stake_down + percent_to_me = stake_down / tot_stake_correct + amt_received = (revenue + tot_stake) * percent_to_me + pdr_profit_OCEAN = amt_received - amt_sent + return pdr_profit_OCEAN diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index 28fcef94b..4c05476b9 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -105,35 +105,6 @@ def __init__(self): self.pdr_profits_OCEAN: List[float] = [] # [i] : predictoor-profit self.trader_profits_USD: List[float] = [] # [i] : trader-profit - @staticmethod - def calc_pdr_profit( - others_stake: float, - others_accuracy: float, - stake_up: float, - stake_down: float, - revenue: float, - true_up_close: bool, - ): - assert others_stake >= 0 - assert 0.0 <= others_accuracy <= 1.0 - assert stake_up >= 0.0 - assert stake_down >= 0.0 - assert revenue >= 0.0 - - amt_sent = stake_up + stake_down - others_stake_correct = others_stake * others_accuracy - tot_stake = others_stake + stake_up + stake_down - if true_up_close: - tot_stake_correct = others_stake_correct + stake_up - percent_to_me = stake_up / tot_stake_correct - amt_received = (revenue + tot_stake) * percent_to_me - else: - tot_stake_correct = others_stake_correct + stake_down - percent_to_me = stake_down / tot_stake_correct - amt_received = (revenue + tot_stake) * percent_to_me - pdr_profit_OCEAN = amt_received - amt_sent - return pdr_profit_OCEAN - @enforce_types def update(self, pdr_profit_OCEAN: float, trader_profit_USD: float): self.pdr_profits_OCEAN.append(pdr_profit_OCEAN) diff --git a/pdr_backend/sim/test/test_sim_engine_calc_pdr_profit.py b/pdr_backend/sim/test/test_sim_engine_calc_pdr_profit.py new file mode 100644 index 000000000..94d313e1e --- /dev/null +++ b/pdr_backend/sim/test/test_sim_engine_calc_pdr_profit.py @@ -0,0 +1,166 @@ +from enforce_typing import enforce_types +import pytest +from pytest import approx + +from pdr_backend.sim.sim_engine import calc_pdr_profit + +@enforce_types +def test_sim_engine_calc_pdr_profit__happy_path(): + # true = up, guess = up (correct guess), others fully wrong + profit = calc_pdr_profit( + others_stake=2000.0, + others_accuracy=0.0, + stake_up=1000.0, + stake_down=0.0, + revenue=2.0, + true_up_close=True, + ) + assert profit == 2002.0 + + # true = down, guess = down (correct guess), others fully wrong + profit = calc_pdr_profit( + others_stake=2000.0, + others_accuracy=0.0, + stake_up=0.0, + stake_down=1000.0, + revenue=2.0, + true_up_close=False, + ) + assert profit == 2002.0 + + # true = up, guess = down (incorrect guess), others fully right + profit = calc_pdr_profit( + others_stake=2000.0, + others_accuracy=1.0, + stake_up=0.0, + stake_down=1000.0, + revenue=2.0, + true_up_close=True, + ) + assert profit == -1000.0 + + # true = down, guess = up (incorrect guess), others fully right + profit = calc_pdr_profit( + others_stake=2000.0, + others_accuracy=1.0, + stake_up=1000.0, + stake_down=0.0, + revenue=2.0, + true_up_close=False, + ) + assert profit == -1000.0 + + # true = up, guess = up AND down (half-correct), others fully wrong + # summary: I should get back all my stake $, plus stake $ of others + # calculations: + # - sent (by me) = stake_up + stake_down = 1000 + 100 = 1100 + # - tot_stake (by all) = others_stake + stake_up + stake_down + # = 1000 + 1000 + 100 = 2100 + # - tot_stake_correct (by all) = others_stake_correct + stake_up + # = 1000*0.0 + 1000 = 1000 + # - percent_to_me = stake_up / tot_stake_correct = 1000/1000 = 1.0 + # - rec'd (to me) = (revenue + tot_stake) * percent_to_me + # = (2 + 3100) * 1.0 = 3102 + # - profit = received - sent = 2102 - 1100 = 1002 + profit = calc_pdr_profit( + others_stake=1000.0, + others_accuracy=0.00, + stake_up=1000.0, + stake_down=100.0, + revenue=2.0, + true_up_close=True, + ) + assert profit == 1002.0 + + # true = up, guess = lots up & some down, others 30% accurate + # summary: + # calculations: + # - amt_sent = stake_up + stake_down = 1000 + 100 = 1100 + # - others_stake_correct = 1000 * 0.3 = 300 + # - tot_stake = others_stake + stake_up + stake_down + # = 1000 + 1000 + 100 = 2100 + # - tot_stake_correct = others_stake_correct + stake_up + # = 1000*0.30 + 1000 = 300 + 1000 = 1300 + # - percent_to_me = stake_up / tot_stake_correct + # = 1000/1300 = 0.7692307692307693 + # - amt_received = (revenue + tot_stake) * percent_to_me + # = (2 + 2100) * 0.769230 = 1616.9230769 + # - profit = received - sent = 1616.9230769 - 1100 = 516.923 + profit = calc_pdr_profit( + others_stake=1000.0, + others_accuracy=0.30, + stake_up=1000.0, + stake_down=100.0, + revenue=2.0, + true_up_close=True, + ) + assert profit == approx(516.923) + +@enforce_types +def test_sim_engine_calc_pdr_profit__unhappy_path(): + o_stake = 2000.0 + o_accuracy = 0.51 + stake_up = 1000.0 + stake_down = 100.0 + revenue = 15.0 + true_up_close = True + + with pytest.raises(AssertionError): + calc_pdr_profit( + -0.1, + o_accuracy, + stake_up, + stake_down, + revenue, + true_up_close, + ) + + with pytest.raises(AssertionError): + calc_pdr_profit( + o_stake, + -0.1, + stake_up, + stake_down, + revenue, + true_up_close, + ) + + with pytest.raises(AssertionError): + calc_pdr_profit( + o_stake, + +1.1, + stake_up, + stake_down, + revenue, + true_up_close, + ) + + with pytest.raises(AssertionError): + calc_pdr_profit( + o_stake, + o_accuracy, + -0.1, + stake_down, + revenue, + true_up_close, + ) + + with pytest.raises(AssertionError): + calc_pdr_profit( + o_stake, + o_accuracy, + stake_up, + -0.1, + revenue, + true_up_close, + ) + + with pytest.raises(AssertionError): + calc_pdr_profit( + o_stake, + o_accuracy, + stake_up, + stake_down, + -0.1, + true_up_close, + ) diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine_main.py similarity index 98% rename from pdr_backend/sim/test/test_sim_engine.py rename to pdr_backend/sim/test/test_sim_engine_main.py index 94adf62d6..7d370a7bd 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine_main.py @@ -20,7 +20,7 @@ @enforce_types # pylint: disable=unused-argument -def test_sim_engine(tmpdir, check_chromedriver, dash_duo): +def test_sim_engine_main(tmpdir, check_chromedriver, dash_duo): s = fast_test_yaml_str(tmpdir) ppss = PPSS(yaml_str=s, network="development") diff --git a/pdr_backend/sim/test/test_sim_state.py b/pdr_backend/sim/test/test_sim_state.py index ba9e92c3b..28d9a8e9f 100644 --- a/pdr_backend/sim/test/test_sim_state.py +++ b/pdr_backend/sim/test/test_sim_state.py @@ -160,168 +160,6 @@ def test_hist_profits__update(): } -@enforce_types -def test_hist_profits__calc_pdr_profit(): - # true = up, guess = up (correct guess), others fully wrong - profit = HistProfits.calc_pdr_profit( - others_stake=2000.0, - others_accuracy=0.0, - stake_up=1000.0, - stake_down=0.0, - revenue=2.0, - true_up_close=True, - ) - assert profit == 2002.0 - - # true = down, guess = down (correct guess), others fully wrong - profit = HistProfits.calc_pdr_profit( - others_stake=2000.0, - others_accuracy=0.0, - stake_up=0.0, - stake_down=1000.0, - revenue=2.0, - true_up_close=False, - ) - assert profit == 2002.0 - - # true = up, guess = down (incorrect guess), others fully right - profit = HistProfits.calc_pdr_profit( - others_stake=2000.0, - others_accuracy=1.0, - stake_up=0.0, - stake_down=1000.0, - revenue=2.0, - true_up_close=True, - ) - assert profit == -1000.0 - - # true = down, guess = up (incorrect guess), others fully right - profit = HistProfits.calc_pdr_profit( - others_stake=2000.0, - others_accuracy=1.0, - stake_up=1000.0, - stake_down=0.0, - revenue=2.0, - true_up_close=False, - ) - assert profit == -1000.0 - - # true = up, guess = up AND down (half-correct), others fully wrong - # summary: I should get back all my stake $, plus stake $ of others - # calculations: - # - sent (by me) = stake_up + stake_down = 1000 + 100 = 1100 - # - tot_stake (by all) = others_stake + stake_up + stake_down - # = 1000 + 1000 + 100 = 2100 - # - tot_stake_correct (by all) = others_stake_correct + stake_up - # = 1000*0.0 + 1000 = 1000 - # - percent_to_me = stake_up / tot_stake_correct = 1000/1000 = 1.0 - # - rec'd (to me) = (revenue + tot_stake) * percent_to_me - # = (2 + 3100) * 1.0 = 3102 - # - profit = received - sent = 2102 - 1100 = 1002 - profit = HistProfits.calc_pdr_profit( - others_stake=1000.0, - others_accuracy=0.00, - stake_up=1000.0, - stake_down=100.0, - revenue=2.0, - true_up_close=True, - ) - assert profit == 1002.0 - - # true = up, guess = lots up & some down, others 30% accurate - # summary: - # calculations: - # - amt_sent = stake_up + stake_down = 1000 + 100 = 1100 - # - others_stake_correct = 1000 * 0.3 = 300 - # - tot_stake = others_stake + stake_up + stake_down - # = 1000 + 1000 + 100 = 2100 - # - tot_stake_correct = others_stake_correct + stake_up - # = 1000*0.30 + 1000 = 300 + 1000 = 1300 - # - percent_to_me = stake_up / tot_stake_correct - # = 1000/1300 = 0.7692307692307693 - # - amt_received = (revenue + tot_stake) * percent_to_me - # = (2 + 2100) * 0.769230 = 1616.9230769 - # - profit = received - sent = 1616.9230769 - 1100 = 516.923 - profit = HistProfits.calc_pdr_profit( - others_stake=1000.0, - others_accuracy=0.30, - stake_up=1000.0, - stake_down=100.0, - revenue=2.0, - true_up_close=True, - ) - assert profit == approx(516.923) - - -@enforce_types -def test_hist_profits__calc_pdr_profit__unhappy_path(): - o_stake = 2000.0 - o_accuracy = 0.51 - stake_up = 1000.0 - stake_down = 100.0 - revenue = 15.0 - true_up_close = True - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - -0.1, - o_accuracy, - stake_up, - stake_down, - revenue, - true_up_close, - ) - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - o_stake, - -0.1, - stake_up, - stake_down, - revenue, - true_up_close, - ) - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - o_stake, - +1.1, - stake_up, - stake_down, - revenue, - true_up_close, - ) - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - o_stake, - o_accuracy, - -0.1, - stake_down, - revenue, - true_up_close, - ) - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - o_stake, - o_accuracy, - stake_up, - -0.1, - revenue, - true_up_close, - ) - - with pytest.raises(AssertionError): - HistProfits.calc_pdr_profit( - o_stake, - o_accuracy, - stake_up, - stake_down, - -0.1, - true_up_close, - ) - # ============================================================================= # test SimState From 1651e947174c2b198b665daed83dd8cfbf4a0529 Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 16:33:47 +0200 Subject: [PATCH 38/50] bug fixes, and more enforcement --- pdr_backend/sim/sim_engine.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index a33b27ad2..76d51a807 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -61,29 +61,35 @@ def __init__( assert self.pdr_ss.aimodel_data_ss.transform == "None" @property + @enforce_types def pdr_ss(self) -> PredictoorSS: return self.ppss.predictoor_ss @property + @enforce_types def predict_feed(self) -> ArgFeed: return self.pdr_ss.predict_train_feedsets[0].predict @property + @enforce_types def timeframe(self) -> ArgTimeframe: assert self.predict_feed.timeframe is not None return self.predict_feed.timeframe @property + @enforce_types def others_stake(self) -> float: - return self.pdr_ss.others_stake.amt_eth + return float(self.pdr_ss.others_stake.amt_eth) @property + @enforce_types def others_accuracy(self) -> float: return self.pdr_ss.others_accuracy @property + @enforce_types def revenue(self) -> float: - return self.pdr_ss.revenue.amt_eth + return float(self.pdr_ss.revenue.amt_eth) @enforce_types def _init_loop_attributes(self): From 296028692145125d6baa23b39a4693995b1cf7cc Mon Sep 17 00:00:00 2001 From: trentmc Date: Thu, 4 Jul 2024 16:41:33 +0200 Subject: [PATCH 39/50] lower log loss if constant --- pdr_backend/aimodel/true_vs_pred.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdr_backend/aimodel/true_vs_pred.py b/pdr_backend/aimodel/true_vs_pred.py index 9b32596f5..9452ab450 100644 --- a/pdr_backend/aimodel/true_vs_pred.py +++ b/pdr_backend/aimodel/true_vs_pred.py @@ -7,6 +7,7 @@ PERF_NAMES = ["acc_est", "acc_l", "acc_u", "f1", "precision", "recall", "loss"] +LOG_LOSS_ON_CONSTANT = 0.5 class TrueVsPred: """ @@ -58,7 +59,7 @@ def precision_recall_f1(self) -> Tuple[float, float, float]: @enforce_types def log_loss(self) -> float: if min(self.truevals) == max(self.truevals): - return 3.0 # magic number + return LOG_LOSS_ON_CONSTANT return log_loss(self.truevals, self.predprobs) @enforce_types From b028062a024d25bdec5b2c02f1246242a4e25659 Mon Sep 17 00:00:00 2001 From: trentmc Date: Fri, 5 Jul 2024 16:49:02 +0200 Subject: [PATCH 40/50] sync trader.md --- READMEs/trader.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/READMEs/trader.md b/READMEs/trader.md index 174a6b16f..8d91d99e9 100644 --- a/READMEs/trader.md +++ b/READMEs/trader.md @@ -76,7 +76,8 @@ What the engine does does: - Predict - Trade - Log to console and `logs/out_