diff --git a/scripts/multivariate_forecast/Covid-19_script/OLinear.sh b/scripts/multivariate_forecast/Covid-19_script/OLinear.sh new file mode 100644 index 00000000..f49a60c8 --- /dev/null +++ b/scripts/multivariate_forecast/Covid-19_script/OLinear.sh @@ -0,0 +1,7 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 24}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 4, "d_ff": 1024, "d_model": 1024, "dropout": 0.0, "e_layers": 5, "embed_size": 16, "horizon": 24, "seq_len": 36, "enc_in": 55, "lr": 0.0001, "Q_chan_indep": 0, "q_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_24_ratio0.70.npy"}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 36}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 4, "d_ff": 512, "d_model": 512, "dropout": 0.0, "e_layers": 3, "embed_size": 16, "horizon": 36, "seq_len": 36, "enc_in": 55, "lr": 0.0001, "Q_chan_indep": 0, "q_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy"}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 48}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 4, "d_ff": 512, "d_model": 512, "dropout": 0.0, "e_layers": 5, "embed_size": 16, "horizon": 48, "seq_len": 36, "enc_in": 55, "lr": 0.0001, "Q_chan_indep": 0, "q_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_48_ratio0.70.npy"}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 60}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 4, "d_ff": 512, "d_model": 512, "dropout": 0.0, "e_layers": 3, "embed_size": 16, "horizon": 60, "seq_len": 36, "enc_in": 55, "lr": 0.0001, "Q_chan_indep": 0, "q_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy"}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear" diff --git a/scripts/multivariate_forecast/Weather_script/OLinear.sh b/scripts/multivariate_forecast/Weather_script/OLinear.sh new file mode 100644 index 00000000..f2c4753f --- /dev/null +++ b/scripts/multivariate_forecast/Weather_script/OLinear.sh @@ -0,0 +1,7 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 96}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 96, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_96_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 192}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 192, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_192_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 336}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 336, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 720}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 720, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_720_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear" diff --git a/ts_benchmark/baselines/olinear/__init__.py b/ts_benchmark/baselines/olinear/__init__.py new file mode 100644 index 00000000..aadf05db --- /dev/null +++ b/ts_benchmark/baselines/olinear/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["Olinear",] + +from ts_benchmark.baselines.olinear.olinear import OLinear diff --git a/ts_benchmark/baselines/olinear/layers/RevIN.py b/ts_benchmark/baselines/olinear/layers/RevIN.py new file mode 100644 index 00000000..d60ed0c9 --- /dev/null +++ b/ts_benchmark/baselines/olinear/layers/RevIN.py @@ -0,0 +1,95 @@ +# code from https://github.com/ts-kim/RevIN, with minor modifications + +import torch +import torch.nn as nn + + + +class RevIN(nn.Module): + def __init__(self, num_features: int, eps=1e-5, affine=True, subtract_last=False): + """ + :param num_features: the number of features or channels + :param eps: a value added for numerical stability + :param affine: if True, RevIN has learnable affine parameters + """ + super(RevIN, self).__init__() + self.num_features = num_features + self.eps = eps + self.affine = affine + self.subtract_last = subtract_last + self.mask = None + if self.affine: + self._init_params() + + def forward(self, x, mode: str, mask=None): + # x [b,l,n] + if mode == 'norm': + self._get_statistics(x, mask) + x = self._normalize(x, mask) + elif mode == 'denorm': + x = self._denormalize(x) + else: + raise NotImplementedError + return x + + def _init_params(self): + # initialize RevIN params: (C,) + self.affine_weight = nn.Parameter(torch.ones(self.num_features)) + self.affine_bias = nn.Parameter(torch.zeros(self.num_features)) + + def _get_statistics(self, x, mask=None): + self.mask = mask + dim2reduce = tuple(range(1, x.ndim - 1)) + if self.subtract_last: + self.last = x[:, -1, :].unsqueeze(1) + else: + if mask is None: + self.mean = torch.mean(x, dim=dim2reduce, keepdim=True).detach() + else: + assert isinstance(mask, torch.Tensor) + # print(type(mask)) + x = x.masked_fill(mask, 0) # in case other values are filled + self.mean = (torch.sum(x, dim=1) / torch.sum(~mask, dim=1)).unsqueeze(1).detach() + # self.mean could be nan or inf + self.mean = torch.nan_to_num(self.mean, nan=0.0, posinf=0.0, neginf=0.0) + + if mask is None: + self.stdev = torch.sqrt(torch.var(x, dim=dim2reduce, keepdim=True, unbiased=False) + self.eps).detach() + else: + self.stdev = (torch.sqrt(torch.sum((x - self.mean) ** 2, dim=1) / torch.sum(~mask, dim=1) + self.eps) + .unsqueeze(1).detach()) + self.stdev = torch.nan_to_num(self.stdev, nan=0.0, posinf=None, neginf=None) + + def _normalize(self, x, mask=None): + self.mask = mask + if self.subtract_last: + x = x - self.last + else: + x = x - self.mean + + x = x / self.stdev + + # x should be zero, if the values are masked + if mask is not None: + # forward fill + # x, mask2 = forward_fill(x, mask) + # x = x.masked_fill(mask2, 0) + + # mean imputation + x = x.masked_fill(mask, 0) + + if self.affine: + x = x * self.affine_weight + x = x + self.affine_bias + return x + + def _denormalize(self, x): + if self.affine: + x = x - self.affine_bias + x = x / (self.affine_weight + self.eps * self.eps) + x = x * self.stdev + if self.subtract_last: + x = x + self.last + else: + x = x + self.mean + return x diff --git a/ts_benchmark/baselines/olinear/layers/Transformer_EncDec.py b/ts_benchmark/baselines/olinear/layers/Transformer_EncDec.py new file mode 100644 index 00000000..810c3890 --- /dev/null +++ b/ts_benchmark/baselines/olinear/layers/Transformer_EncDec.py @@ -0,0 +1,95 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +import random +from typing import List + +class LinearEncoder(nn.Module): + def __init__(self, d_model, d_ff=None, CovMat=None, dropout=0.1, activation="relu", token_num=None, **kwargs): + super(LinearEncoder, self).__init__() + + d_ff = d_ff or 4 * d_model + self.d_model = d_model + self.d_ff = d_ff + self.CovMat = CovMat.unsqueeze(0) if CovMat is not None else None + self.token_num = token_num + + self.norm1 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + # attention --> linear + self.v_proj = nn.Linear(d_model, d_model) + self.out_proj = nn.Linear(d_model, d_model) + + init_weight_mat = torch.eye(self.token_num) * 1.0 + torch.randn(self.token_num, self.token_num) * 1.0 + self.weight_mat = nn.Parameter(init_weight_mat[None, :, :]) + + # self.bias = nn.Parameter(torch.zeros(1, 1, self.d_model)) + + self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1) + self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1) + self.activation = F.relu if activation == "relu" else F.gelu + self.norm2 = nn.LayerNorm(d_model) + + def forward(self, x, **kwargs): + # x.shape: b, l, d_model + values = self.v_proj(x) + + if self.CovMat is not None: + A = F.softmax(self.CovMat, dim=-1) + F.softplus(self.weight_mat) + else: + A = F.softplus(self.weight_mat) + + A = F.normalize(A, p=1, dim=-1) + A = self.dropout(A) + + new_x = A @ values # + self.bias + + x = x + self.dropout(self.out_proj(new_x)) + x = self.norm1(x) + + y = self.dropout(self.activation(self.conv1(x.transpose(-1, 1)))) + y = self.dropout(self.conv2(y).transpose(-1, 1)) + output = self.norm2(x + y) + + return output, None + +class Encoder_ori(nn.Module): + def __init__(self, attn_layers, conv_layers=None, norm_layer=None, one_output=False, CKA_flag=False): + super(Encoder_ori, self).__init__() + self.attn_layers = nn.ModuleList(attn_layers) + self.norm = norm_layer + self.one_output = one_output + self.CKA_flag = CKA_flag + if self.CKA_flag: + print('CKA is enabled...') + + def forward(self, x, attn_mask=None, tau=None, delta=None): + # x [B, nvars, D] + attns = [] + X0 = None # to make Pycharm happy + layer_len = len(self.attn_layers) + for i, attn_layer in enumerate(self.attn_layers): + x, attn = attn_layer(x, attn_mask=attn_mask, tau=tau, delta=delta) + attns.append(attn) + + if not self.training and self.CKA_flag and layer_len > 1: + if i == 0: + X0 = x + + if i == layer_len - 1 and random.uniform(0, 1) < 1e-1: + CudaCKA1 = CudaCKA(device=x.device) + cka_value = CudaCKA1.linear_CKA(X0.flatten(0, 1)[:1000], x.flatten(0, 1)[:1000]) + print(f'CKA: \t{cka_value:.3f}') + + if isinstance(x, tuple) or isinstance(x, List): + x = x[0] + + if self.norm is not None: + x = self.norm(x) + + if self.one_output: + return x + else: + return x, attns diff --git a/ts_benchmark/baselines/olinear/layers/__init__.py b/ts_benchmark/baselines/olinear/layers/__init__.py new file mode 100644 index 00000000..edc5e0b4 --- /dev/null +++ b/ts_benchmark/baselines/olinear/layers/__init__.py @@ -0,0 +1,4 @@ +__all__ = ["Encoder_ori", "LinearEncoder", "OLinearLayer","RevIN"] + +from ts_benchmark.baselines.olinear.layers.Transformer_EncDec import Encoder_ori, LinearEncoder +from ts_benchmark.baselines.olinear.layers.RevIN import RevIN \ No newline at end of file diff --git a/ts_benchmark/baselines/olinear/models/olinear_model.py b/ts_benchmark/baselines/olinear/models/olinear_model.py new file mode 100644 index 00000000..b2965986 --- /dev/null +++ b/ts_benchmark/baselines/olinear/models/olinear_model.py @@ -0,0 +1,146 @@ +import os +import torch +import torch.nn as nn +import numpy as np +from ts_benchmark.baselines.olinear.layers import RevIN, Encoder_ori, LinearEncoder + + +class Model(nn.Module): + def __init__(self, configs): + super(Model, self).__init__() + self.pred_len = configs.pred_len + self.enc_in = configs.enc_in # channels + self.seq_len = configs.seq_len + self.hidden_size = self.d_model = configs.d_model # hidden_size + self.d_ff = configs.d_ff # d_ff + + self.Q_chan_indep = configs.Q_chan_indep + + q_mat_dir = configs.Q_MAT_file if self.Q_chan_indep else configs.q_mat_file + if not os.path.isfile(q_mat_dir): + q_mat_dir = os.path.join(configs.root_path, q_mat_dir) + assert os.path.isfile(q_mat_dir) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + self.Q_mat = torch.from_numpy(np.load(q_mat_dir)).to(torch.float32).to(device) + + assert (self.Q_mat.ndim == 3 if self.Q_chan_indep else self.Q_mat.ndim == 2) + assert (self.Q_mat.shape[0] == self.enc_in if self.Q_chan_indep else self.Q_mat.shape[0] == self.seq_len) + + q_out_mat_dir = configs.Q_OUT_MAT_file if self.Q_chan_indep else configs.q_out_mat_file + if not os.path.isfile(q_out_mat_dir): + q_out_mat_dir = os.path.join(configs.root_path, q_out_mat_dir) + assert os.path.isfile(q_out_mat_dir) + self.Q_out_mat = torch.from_numpy(np.load(q_out_mat_dir)).to(torch.float32).to(device) + + assert (self.Q_out_mat.ndim == 3 if self.Q_chan_indep else self.Q_out_mat.ndim == 2) + assert (self.Q_out_mat.shape[0] == self.enc_in if self.Q_chan_indep else + self.Q_out_mat.shape[0] == self.pred_len) + + self.patch_len = configs.temp_patch_len + self.stride = configs.temp_stride + + # self.channel_independence = configs.channel_independence + self.embed_size = configs.embed_size # embed_size + self.embeddings = nn.Parameter(torch.randn(1, self.embed_size)) + + self.fc = nn.Sequential( + nn.Linear(self.pred_len * self.embed_size, self.d_ff), + nn.GELU(), + nn.Linear(self.d_ff, self.pred_len) + ) + + # for final input and output + self.revin_layer = RevIN(self.enc_in, affine=True) + self.dropout = nn.Dropout(configs.dropout) + + # ############# transformer related ######### + self.encoder = Encoder_ori( + [ + LinearEncoder( + d_model=configs.d_model, d_ff=configs.d_ff, CovMat=None, + dropout=configs.dropout, activation=configs.activation, token_num=self.enc_in, + ) for _ in range(configs.e_layers) + ], + norm_layer=nn.LayerNorm(configs.d_model), + one_output=True, + ) + self.ortho_trans = nn.Sequential( + nn.Linear(self.seq_len * self.embed_size, self.d_model), + self.encoder, + nn.Linear(self.d_model, self.pred_len * self.embed_size) + ) + + # learnable delta + self.delta1 = nn.Parameter(torch.zeros(1, self.enc_in, 1, self.seq_len)) + self.delta2 = nn.Parameter(torch.zeros(1, self.enc_in, 1, self.pred_len)) + + # dimension extension + def tokenEmb(self, x, embeddings): + if self.embed_size <= 1: + return x.transpose(-1, -2).unsqueeze(-1) + # x: [B, T, N] --> [B, N, T] + x = x.transpose(-1, -2) + x = x.unsqueeze(-1) + # B*N*T*1 x 1*D = B*N*T*D + return x * embeddings + + def Fre_Trans(self, x): + # [B, N, T, D] + B, N, T, D = x.shape + assert T == self.seq_len + # [B, N, D, T] + x = x.transpose(-1, -2) + + # orthogonal transformation + # [B, N, D, T] + if self.Q_chan_indep: + x_trans = torch.einsum('bndt,ntv->bndv', x, self.Q_mat.transpose(-1, -2)) + else: + x_trans = torch.einsum('bndt,tv->bndv', x, self.Q_mat.transpose(-1, -2)) + self.delta1 + # added on 25/1/30 + # x_trans = F.gelu(x_trans) + # [B, N, D, T] + assert x_trans.shape[-1] == self.seq_len + + # ########## transformer #### + x_trans = self.ortho_trans(x_trans.flatten(-2)).reshape(B, N, D, self.pred_len) + + # [B, N, D, tau]; orthogonal transformation + if self.Q_chan_indep: + x = torch.einsum('bndt,ntv->bndv', x_trans, self.Q_out_mat) + else: + x = torch.einsum('bndt,tv->bndv', x_trans, self.Q_out_mat) + self.delta2 + # added on 25/1/30 + # x = F.gelu(x) + + # [B, N, tau, D] + x = x.transpose(-1, -2) + return x + + def forward(self, x, x_mark_enc=None, x_dec=None, x_mark_dec=None, mask=None): + # x: [Batch, Input length, Channel] + B, T, N = x.shape + + # revin norm + x = self.revin_layer(x, mode='norm') + x_ori = x + + # ########### frequency (high-level) part ########## + # input fre fine-tuning + # [B, T, N] + # embedding x: [B, N, T, D] + x = self.tokenEmb(x_ori, self.embeddings) + # [B, N, tau, D] + x = self.Fre_Trans(x) + + # linear + # [B, N, tau*D] --> [B, N, dim] --> [B, N, tau] --> [B, tau, N] + out = self.fc(x.flatten(-2)).transpose(-1, -2) + + # dropout + out = self.dropout(out) + + # revin denorm + out = self.revin_layer(out, mode='denorm') + + return out diff --git a/ts_benchmark/baselines/olinear/olinear.py b/ts_benchmark/baselines/olinear/olinear.py new file mode 100644 index 00000000..4e1128c3 --- /dev/null +++ b/ts_benchmark/baselines/olinear/olinear.py @@ -0,0 +1,60 @@ +from ts_benchmark.baselines.deep_forecasting_model_base import DeepForecastingModelBase +from ts_benchmark.baselines.olinear.models.olinear_model import Model +from ts_benchmark.common.constant import FORECASTING_DATASET_PATH + +MODEL_HYPER_PARAMS = { + "pred_len": 96, + "seq_len": 96, + "enc_in": 7, + "d_model": 512, + "d_ff": 2048, + "Q_chan_indep": 0, + "Q_MAT_file": None, + "q_mat_file": None, + "q_out_mat_file": None, + "Q_OUT_MAT_file": None, + "root_path": FORECASTING_DATASET_PATH, + "data_path": "Covid-19.csv", + "temp_patch_len": 16, + "temp_stride": 8, + "embed_size": 8, + "dropout": 0.1, + "d_model": 512, + "d_ff": 2048, + "activation": "gelu", + "e_layers": 2, + "loss": "MSE", +} + + +class OLinear(DeepForecastingModelBase): + """ + OLinear adapter class for TFB baseline. + + Attributes: + model_name (str): Name of the model for identification purposes. + _init_model: Initializes an instance of the OLinear. + _adjust_lr:Adjusts the learning rate of the optimizer based on the current epoch and configuration. + _process: Executes the model's forward pass and returns the output. + """ + + def __init__(self, **kwargs): + super(OLinear, self).__init__(MODEL_HYPER_PARAMS, **kwargs) + if self.config.Q_chan_indep: + if self.config.Q_MAT_file is None or self.config.Q_OUT_MAT_file is None: + raise ValueError("Please set Q_MAT_file and Q_OUT_MAT_file") + else: + if self.config.q_mat_file is None or self.config.q_out_mat_file is None: + raise ValueError("Please set q_mat_file and q_out_mat_file") + + def _init_model(self): + return Model(self.config) + + @property + def model_name(self): + return "OLinear" + + def _process(self, input, target, input_mark, target_mark): + output = self.model(input) + + return {"output": output}