framework update for inference on kraken data

Author: Axle-Bucamp

app/api/main.py

@@ -156,12 +156,8 @@ def predict_stock(ticker: str, seq: SequenceRequest):
     Raises:
         HTTPException: If the ticker is not found in any group or if the CSV file cannot be read.
     """
-    group_name = get_group_for_ticker(ticker)
-    if not group_name:
-        raise HTTPException(status_code=400, detail="Ticker not found in groups")
-
     try:
-        df = pd.read_csv(f'data/{group_name}_{seq}.csv', parse_dates=['Date'])
+        df = pd.read_csv(f'data/{ticker}_{seq}.csv', parse_dates=['Date'])
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error reading CSV file: {str(e)}")
 

app/sheduler/datamodel/ticker_cluster.py

@@ -15,10 +15,17 @@
     MEMECOIN = ("memecoin", ['BONK-USD', 'SHIB-USD', 'PEPECOIN-USD', 'DOGE-USD'])
 """
 
+class TKGroup(Enum):
+    CRYPTO_STANDARD = ("crypto-standard", ['SAND-USD', 'IMX-USD', "GALA-USD", "AXS-USD", "MANA-USD", "AAVE-USD", "ETH-USD", "CRO-USD", "BTC-USD", "XRP-USD", "ADA-USD", "SOL-USD",  "PEPE-USD", "POPCAT-USD", "DOGE-USD", "TRUMP-USD", "SUI-USD"])
+
+class TKGroupName(str, Enum):
+    cryptoStandard = "crypto-standard"
+
+"""
 class TKGroup(Enum):
     CRYPTO_STANDARD = ("crypto-standard", ["ADA-USD", "SOL-USD", "XRP-USD", "ETH-USD", "BTC-USD"])
     FREEDOM = ("thisiselonmusk", ["XRP-USD", "SHIB-USD", "SOL-USD", "BTC-USD"])
-    WEB3 = ("web3", ["LINK-USD", "SOL"])
+    WEB3 = ("web3", ["LINK-USD", "SOL"]) # "LINK-USD", "SOL"
     GAME = ('gaming', ['SAND', 'IMX', "GALA", "AXS", "MANA"])
     DEFI = ("defi", ['AVAX-USD', 'LINK-USD', 'UNI-USD', 'STX-USD', 'FTM-USD', "INJ-USD"])
     MEMECOIN = ("memecoin", [ "SHIB-USD"])
@@ -37,3 +44,4 @@ class TKGroupName(str, Enum):
     #finance = "finance"
     #energie = "energie"
 
+"""

app/sheduler/finetune.py

@@ -4,13 +4,14 @@
 from torch import save, load
 from torch.utils.data import DataLoader
 from torch import cuda, device
-from hygdra_forecasting.dataloader.dataloader import StockDataset
+from hygdra_forecasting.dataloader.dataloader import StockDataset as standard
+from hygdra_forecasting.dataloader.dataloader_kraken import StockDataset
 from hygdra_forecasting.utils.learning_rate_sheduler import CosineWarmup
 from hygdra_forecasting.model.train import train_model, setup_seed
 import torch.nn as nn
 
 class StockFineTuner:
-    def __init__(self, interval: str = 'days', base_weight: str = 'weight/days/best_model.pth', epoch=5, learnig_rate=0.01):
+    def __init__(self, interval: str = 'days', base_weight: str = 'weight/days/best_model.pth', epoch=100, learnig_rate=0.01):
         self.interval = interval
         self.base_weight = base_weight
         self.device = device('cuda:0') if cuda.is_available() else device('cpu')
@@ -36,8 +37,9 @@ def finetune_one(self, tickers: List[str], path: str):
             epochs=self.epoch,
             learning_rate=self.learning_rate,
             save_epoch=False,
-            lrfn=self.tuning_scheduler,
-            criterion=nn.L1Loss(),
+            # lrfn=self.tuning_scheduler,
+            lrfn=CosineWarmup(self.learning_rate, self.epoch).lrfn,
+            # criterion=nn.L1Loss(), # l1 seems to make it harder
             checkpoint_file=load(self.base_weight)
         )
 
@@ -52,7 +54,7 @@ def finetune_many(self):
 if __name__ == "__main__":
     # {"days" : '1440', "minutes" : '1', "hours" : '60', "thrity" : "30"}
     interval = "minutes"
-    tuner = StockFineTuner(interval=interval, base_weight=f'../../weight/best_model.pth')
+    tuner = StockFineTuner(interval=interval, base_weight=f'weight/minutes/best_model.pth')
     tuner.finetune_many()
     # training is still realy weirdly reset while not in training phase
     # manage non crypto course via kraken

app/sheduler/finetune_temp.py

@@ -1,10 +1,12 @@
 from hygdra_forecasting.model.build import ConvCausalLTSM
 from datamodel.ticker_cluster import TKGroup
 from torch import load, tensor, no_grad, cuda, device
-from hygdra_forecasting.utils.preprocessing import ohlv_to_dataframe_inference, dataframe_to_dataset_inference
-from pandas import DataFrame, DateOffset
+from hygdra_forecasting.utils.ohlv import get_kraken_data_to_json
+from hygdra_forecasting.utils.dataset import dict_to_dataset_inference
+import numpy as np
 import redis
 from os import getenv
+import json
 
 class StockPredictor:
     def __init__(self, interval: str = 'days'):
@@ -27,34 +29,58 @@ def predict(self):
             self.model.load_state_dict(load(f'weight/{self.interval}/best_model.pth')["model_state_dict"]) # {groups_name}.pth
             self.model.eval()
 
-            df, dict_unorm = ohlv_to_dataframe_inference(tickers, interval=self.interval_transform[self.interval])
-            sequences_dict = dataframe_to_dataset_inference(df, tickers)
-            
-            df_result = DataFrame()
-            df_result.index = df[tickers[0] + "_close"].iloc[-100:].index
-            df_result["Date"] = df_result.index
-            
+            df, dict_unorm, index_timestamp = get_kraken_data_to_json(tickers, interval=self.interval_transform[self.interval])
+            sequences_dict = dict_to_dataset_inference(df)
+
+            # 1) join all as before
+            # 2) optimize json dict of dataframe
             for ticker in tickers:
-                sequence = tensor(sequences_dict[ticker]).float()
-                
-                with no_grad():
-                    predictions = self.model(sequence)
-                
-                predictions = predictions.squeeze().numpy()
-                df_result[ticker + "_pred"] = predictions.reshape(-1)[-100:] * dict_unorm[ticker][1] + dict_unorm[ticker][0]
-                df_result[ticker + "_close"] = df[ticker + '_close'].iloc[-100:] * dict_unorm[ticker][1] + dict_unorm[ticker][0]
-                
-            df_result["pred_date"] = df_result.index + DateOffset(days=14)
-            df_result.to_csv(f'data/{groups_name}_{self.interval}.csv')
-            
-            json_data = df_result.to_json(orient="records", date_format="iso")
-            redis_key = f"{groups_name}_{self.interval}"
-            
-            try:
-                self.redis_client.set(redis_key, json_data)
-                print(f"Saved predictions for group '{groups_name}' to Redis with key '{redis_key}'")
-            except Exception as e:
-                print(f"Error saving predictions for group '{groups_name}' to Redis: {e}")
+                try :
+                    ticker = ticker.split("-")[0] + "USD"
+                    sequence = tensor(sequences_dict[ticker]).float()
+                    
+                    with no_grad():
+                        predictions = self.model(sequence)
+                    
+                    date_array = np.array(index_timestamp[ticker], dtype='datetime64[s]')
+                    df[ticker]["Date"] = date_array 
+                    predictions = predictions.squeeze().numpy().reshape(-1)
+                    
+                    df[ticker]["forecasting"] = predictions * dict_unorm[ticker]["close"]["std"] + dict_unorm[ticker]["close"]["mean"]
+                    df[ticker]["close"] = df[ticker]["close"] * dict_unorm[ticker]["close"]["std"] + dict_unorm[ticker]["close"]["std"]
+
+                    # prediction interval # (double check)
+                    offset = None
+                    if self.interval == "minutes":
+                        offset = np.timedelta64(14, "m")
+                    elif self.interval == "thirty":
+                        offset= np.timedelta64(14 * 30, "m")
+                    elif self.interval == "hours":
+                        offset = np.timedelta64(14, "h")
+                    else :
+                        offset = np.timedelta64(14, "D")
+
+                    df[ticker]["pred_date"] = df[ticker]["Date"] + offset
+                    float_keys = ['close', 'forecasting', 'low', 'high', 'open', 'volume', 'upper', 'lower', 'width', 'rsi', 'roc', 'diff', 'percent_change_close']
+                    for key in float_keys:
+                        df[ticker][key] = df[ticker][key].astype(float).tolist()
+
+                    date_keys = ['Date', 'pred_date']
+                    for key in date_keys:
+                        df[ticker][key] = df[ticker][key].astype(str).tolist()
+
+                    redis_key = f"{ticker}_{self.interval}"
+
+                except Exception as e:
+                    print(f"error predicting stock : {e} on stock {ticker}")
+
+                # register data
+                try:
+                    df_json = json.dumps(df[ticker])
+                    self.redis_client.set(redis_key, df_json)
+                    print(f"Saved predictions for group '{groups_name}' to Redis with key '{redis_key}'")
+                except Exception as e:
+                    print(f"Error saving predictions for group '{groups_name}' to Redis: {e}")
 
 if __name__ == "__main__":
     predictor = StockPredictor(interval='days')