add olinear and four scripts

scripts/multivariate_forecast/Covid-19_script/OLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 24}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 24, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_24_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 36}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 36, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_36_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 48}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 48, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_48_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Covid-19.csv" --strategy-args '{"horizon": 60}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 60, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/covid/covid_60_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Covid-19/OLinear"

scripts/multivariate_forecast/ILI_script/OLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ILI.csv" --strategy-args '{"horizon": 24}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 24, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/ILI/ILI_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/ILI/ILI_24_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "ILI/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ILI.csv" --strategy-args '{"horizon": 36}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 36, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/ILI/ILI_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/ILI/ILI_36_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "ILI/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ILI.csv" --strategy-args '{"horizon": 48}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 48, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/ILI/ILI_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/ILI/ILI_48_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "ILI/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ILI.csv" --strategy-args '{"horizon": 60}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 60, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/ILI/ILI_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/ILI/ILI_60_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "ILI/OLinear"

scripts/multivariate_forecast/NASDAQ_script/OLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "nasdaq.csv" --strategy-args '{"horizon": 24}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 24, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_24_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "NASDAQ/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "nasdaq.csv" --strategy-args '{"horizon": 36}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 36, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_36_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "NASDAQ/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "nasdaq.csv" --strategy-args '{"horizon": 48}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 48, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_48_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "NASDAQ/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "nasdaq.csv" --strategy-args '{"horizon": 60}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 16, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 60, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_60_ratio0.70.npy", "q_out_mat_file": "dataset/Q_matrices/nasdaq/nasdaq_60_ratio0.70.npy", "seq_len": 60, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "NASDAQ/OLinear"

scripts/multivariate_forecast/Weather_script/OLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 96}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 96, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_96_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 192}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 192, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_192_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 336}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 336, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Weather.csv" --strategy-args '{"horizon": 720}' --model-name "olinear.OLinear" --model-hyper-params '{"batch_size": 32, "d_ff": 512, "d_model": 256, "dropout": 0.1, "e_layers": 2, "embed_size": 1, "horizon": 720, "lr": 0.0001, "num_epochs": 10, "q_mat_file": "dataset/Q_matrices/weather/weather_336_ratio0.7.npy", "q_out_mat_file": "dataset/Q_matrices/weather/weather_720_ratio0.7.npy", "seq_len": 336, "temp_patch_len": 16, "temp_stride": 8, "use_amp": 0}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Weather/OLinear"

ts_benchmark/baselines/olinear/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["OLinear"]
+
+from ts_benchmark.baselines.olinear.olinear import OLinear

ts_benchmark/baselines/olinear/layers/RevIN.py

@@ -0,0 +1,94 @@
+# code from https://github.com/ts-kim/RevIN, with minor modifications
+
+import torch
+import torch.nn as nn
+
+
+class RevIN(nn.Module):
+    def __init__(self, num_features: int, eps=1e-5, affine=True, subtract_last=False):
+        """
+        :param num_features: number of features or channels
+        :param eps: a value added for numerical stability
+        :param affine: if True, RevIN has learnable affine parameters
+        """
+        super(RevIN, self).__init__()
+        self.num_features = num_features
+        self.eps = eps
+        self.affine = affine
+        self.subtract_last = subtract_last
+        self.mask = None
+        if self.affine:
+            self._init_params()
+
+    def forward(self, x, mode: str, mask=None):
+        # x [b,l,n]
+        if mode == 'norm':
+            self._get_statistics(x, mask)
+            x = self._normalize(x, mask)
+        elif mode == 'denorm':
+            x = self._denormalize(x)
+        else:
+            raise NotImplementedError
+        return x
+
+    def _init_params(self):
+        # initialize RevIN params: (C,)
+        self.affine_weight = nn.Parameter(torch.ones(self.num_features))
+        self.affine_bias = nn.Parameter(torch.zeros(self.num_features))
+
+    def _get_statistics(self, x, mask=None):
+        self.mask = mask
+        dim2reduce = tuple(range(1, x.ndim - 1))
+        if self.subtract_last:
+            self.last = x[:, -1, :].unsqueeze(1)
+        else:
+            if mask is None:
+                self.mean = torch.mean(x, dim=dim2reduce, keepdim=True).detach()
+            else:
+                assert isinstance(mask, torch.Tensor)
+                # print(type(mask))
+                x = x.masked_fill(mask, 0)  # in case other values are filled
+                self.mean = (torch.sum(x, dim=1) / torch.sum(~mask, dim=1)).unsqueeze(1).detach()
+                # self.mean could be nan or inf
+                self.mean = torch.nan_to_num(self.mean, nan=0.0, posinf=0.0, neginf=0.0)
+
+        if mask is None:
+            self.stdev = torch.sqrt(torch.var(x, dim=dim2reduce, keepdim=True, unbiased=False) + self.eps).detach()
+        else:
+            self.stdev = (torch.sqrt(torch.sum((x - self.mean) ** 2, dim=1) / torch.sum(~mask, dim=1) + self.eps)
+                          .unsqueeze(1).detach())
+            self.stdev = torch.nan_to_num(self.stdev, nan=0.0, posinf=None, neginf=None)
+
+    def _normalize(self, x, mask=None):
+        self.mask = mask
+        if self.subtract_last:
+            x = x - self.last
+        else:
+            x = x - self.mean
+
+        x = x / self.stdev
+
+        # x should be zero, if values are masked
+        if mask is not None:
+            # forward fill
+            # x, mask2 = forward_fill(x, mask)
+            # x = x.masked_fill(mask2, 0)
+
+            # mean imputation
+            x = x.masked_fill(mask, 0)
+
+        if self.affine:
+            x = x * self.affine_weight
+            x = x + self.affine_bias
+        return x
+
+    def _denormalize(self, x):
+        if self.affine:
+            x = x - self.affine_bias
+            x = x / (self.affine_weight + self.eps * self.eps)
+        x = x * self.stdev
+        if self.subtract_last:
+            x = x + self.last
+        else:
+            x = x + self.mean
+        return x

ts_benchmark/baselines/olinear/layers/Transformer_EncDec.py

@@ -0,0 +1,102 @@
+import random
+from typing import List
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class LinearEncoder(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        d_ff=None,
+        CovMat=None,
+        dropout=0.1,
+        activation="relu",
+        token_num=None,
+        **kwargs,
+    ):
+        super(LinearEncoder, self).__init__()
+
+        d_ff = d_ff or 4 * d_model
+        self.d_model = d_model
+        self.d_ff = d_ff
+        self.CovMat = CovMat.unsqueeze(0) if CovMat is not None else None
+        self.token_num = token_num
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+        self.v_proj = nn.Linear(d_model, d_model)
+        self.out_proj = nn.Linear(d_model, d_model)
+
+        init_weight_mat = torch.eye(self.token_num) * 1.0 + torch.randn(
+            self.token_num, self.token_num
+        ) * 1.0
+        self.weight_mat = nn.Parameter(init_weight_mat[None, :, :])
+
+        self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
+        self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
+        self.activation = F.relu if activation == "relu" else F.gelu
+        self.norm2 = nn.LayerNorm(d_model)
+
+    def forward(self, x, **kwargs):
+        values = self.v_proj(x)
+
+        if self.CovMat is not None:
+            A = F.softmax(self.CovMat, dim=-1) + F.softplus(self.weight_mat)
+        else:
+            A = F.softplus(self.weight_mat)
+
+        A = F.normalize(A, p=1, dim=-1)
+        A = self.dropout(A)
+
+        new_x = A @ values
+
+        x = x + self.dropout(self.out_proj(new_x))
+        x = self.norm1(x)
+
+        y = self.dropout(self.activation(self.conv1(x.transpose(-1, 1))))
+        y = self.dropout(self.conv2(y).transpose(-1, 1))
+        output = self.norm2(x + y)
+
+        return output, None
+
+
+class Encoder_ori(nn.Module):
+    def __init__(
+        self,
+        attn_layers,
+        conv_layers=None,
+        norm_layer=None,
+        one_output=False,
+        CKA_flag=False,
+    ):
+        super(Encoder_ori, self).__init__()
+        self.attn_layers = nn.ModuleList(attn_layers)
+        self.norm = norm_layer
+        self.one_output = one_output
+        self.CKA_flag = CKA_flag
+
+    def forward(self, x, attn_mask=None, tau=None, delta=None):
+        attns = []
+        layer_len = len(self.attn_layers)
+        for i, attn_layer in enumerate(self.attn_layers):
+            x, attn = attn_layer(x, attn_mask=attn_mask, tau=tau, delta=delta)
+            attns.append(attn)
+
+            if not self.training and self.CKA_flag and layer_len > 1:
+                if i == layer_len - 1 and random.uniform(0, 1) < 0.0:
+                    pass
+
+        if isinstance(x, tuple) or isinstance(x, List):
+            x = x[0]
+
+        if self.norm is not None:
+            x = self.norm(x)
+
+        if self.one_output:
+            return x
+        else:
+            return x, attns

ts_benchmark/baselines/olinear/layers/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["Encoder_ori", "LinearEncoder"]
+
+from ts_benchmark.baselines.olinear.layers.Transformer_EncDec import Encoder_ori, LinearEncoder

ts_benchmark/baselines/olinear/model/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["OLinear"]
+
+from ts_benchmark.baselines.olinear.model.olinear_model import OLinear