Merge pull request #54 from LAMPSPUC/Add_assertions

Add assertions to improve input validation and prevent bugs

Author: andreramosfdc

Project.toml

@@ -1,7 +1,7 @@
 name = "StateSpaceLearning"
 uuid = "971c4b7c-2c4e-4bac-8525-e842df3cde7b"
 authors = ["andreramosfc <[email protected]>"]
-version = "1.4.1"
+version = "1.4.2"
 
 [deps]
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"

paper_tests/m4_test/m4_test.py

@@ -2,13 +2,17 @@
 import math
 import statsmodels.api as sm
 import numpy as np
+from prophet import Prophet
+from chronos import ChronosPipeline
+import torch
 
 df_train1 = pd.read_csv("paper_tests/m4_test/Monthly-train1.csv")
 df_train2 = pd.read_csv("paper_tests/m4_test/Monthly-train2.csv")
 df_train3 = pd.read_csv("paper_tests/m4_test/Monthly-train3.csv")
 df_train4 = pd.read_csv("paper_tests/m4_test/Monthly-train4.csv")
 df_train = pd.concat([df_train1, df_train2, df_train3, df_train4])
 m4_info = pd.read_csv("paper_tests/m4_test/M4-info.csv")
+
 df_test = pd.read_csv("paper_tests/m4_test/Monthly-test.csv")
 ssl_init_df = pd.read_csv("paper_tests/m4_test/init_SSL/SSL_aic_0.1_false.csv")
 
@@ -54,6 +58,7 @@ def evaluate_ss(input, sample_size, init, hyperparameters_inicialization):
     forecast_values = [x * (max_train - min_train) + min_train for x in normalized_forecast_values]
     return sMAPE(test, forecast_values), MASE(train, test, forecast_values)
 
+
 results      = []
 results_init = []
 for i in range(0, 48000):
@@ -85,4 +90,78 @@ def evaluate_ss(input, sample_size, init, hyperparameters_inicialization):
 df_mean_init = pd.DataFrame({'smape': [np.mean(smape_SS_init)], 'mase': [np.mean(mase_SS_init)], 'owa': [(np.mean(smape_SS_init) / NAIVE_sMAPE + np.mean(mase_SS_init) / NAIVE_MASE) / 2]})
 
 df_mean.to_csv('paper_tests/m4_test/metrics_results/SS_METRICS_RESULTS.csv')
-df_mean_init.to_csv('paper_tests/m4_test/metrics_results/SS_INIT_METRICS_RESULTS.csv')
+df_mean_init.to_csv('paper_tests/m4_test/metrics_results/SS_INIT_METRICS_RESULTS.csv')
+
+def evaluate_prophet(input):
+    train = input["train"]
+    test = input["test"]
+    timestamps = pd.date_range(start="2020-01-01", periods=len(train), freq='ME')
+    #add random seed 
+    df = pd.DataFrame({
+        'ds': timestamps,
+        'y': train
+    })
+    model = Prophet(interval_width=0.95)
+    model.fit(df)
+    future = pd.DataFrame({
+        'ds': (pd.date_range(start="2020-01-01", periods=len(train) + 18, freq='ME'))[len(train):]
+    })
+    model_forecast = model.predict(future)
+    prediction = model_forecast['yhat'].values
+    return sMAPE(test, prediction), MASE(train, test, prediction)
+
+def evaluate_chronos(input):
+    train = input["train"]
+    test = input["test"]
+    chronos_forecast = ChronosPipeline.from_pretrained(
+            f"amazon/chronos-t5-large",
+            device_map="mps"
+        ).predict(
+            context=torch.tensor(train),
+            prediction_length=18,
+            limit_prediction_length=False
+        )
+    prediction = np.quantile(chronos_forecast[0].numpy(), [0.5], axis=0)[0]
+    return sMAPE(test, prediction), MASE(train, test, prediction)
+
+smape_prophet_vec = []
+mase_prophet_vec = []
+smape_chronos_vec = []
+mase_chronos_vec = []
+for i in range(0, len(dict_vec)):
+    smape_prophet, mase_prophet = evaluate_prophet(dict_vec[i])
+    smape_prophet_vec.append(smape_prophet)
+    mase_prophet_vec.append(mase_prophet)
+    smape_chronos, mase_chronos = evaluate_chronos(dict_vec[i])
+    smape_chronos_vec.append(smape_chronos)
+    mase_chronos_vec.append(mase_chronos)
+    # 
+    print("Runningg series ", i)
+    if i % 1000 == 0:
+        print("Runningg series ", i)
+        smape_mean_prophet = np.mean(smape_prophet_vec)
+        smape_emean_chronos = np.mean(smape_chronos_vec)
+        mase_mean_prophet = np.mean(mase_prophet_vec)
+        mase_mean_chronos = np.mean(mase_chronos_vec)
+        print("Mean sMape Prophet: ", smape_mean_prophet)
+        print("Mean sMape Chronos: ", smape_emean_chronos)
+        print("Mean Mase Prophet: ", mase_mean_prophet)
+        print("Mean Mase Chronos: ", mase_mean_chronos)
+
+
+NAIVE_sMAPE = 14.427 #M4 Paper
+NAIVE_MASE = 1.063   #M4 Paper
+
+owa_prophet = (np.mean(smape_prophet_vec) / NAIVE_sMAPE + np.mean(mase_prophet_vec) / NAIVE_MASE) / 2
+owa_chronos = (np.mean(smape_chronos_vec) / NAIVE_sMAPE + np.mean(mase_chronos_vec) / NAIVE_MASE) / 2
+
+mean_mase_prophet = np.mean(mase_prophet_vec)
+mean_smape_prophet = np.mean(smape_prophet_vec)
+mean_mase_chronos = np.mean(mase_chronos_vec)
+mean_smape_chronos = np.mean(smape_chronos_vec)
+
+df_results_mean = pd.DataFrame({'smape': [mean_smape_prophet, mean_smape_chronos], 'mase': [mean_mase_prophet, mean_mase_chronos], 'owa': [owa_prophet, owa_chronos]})
+
+# save to csv
+
+df_results_mean.to_csv('paper_tests/m4_test/metrics_results/PROPHET_CHRONOS_METRICS_RESULTS.csv')

src/models/structural_model.jl

@@ -116,6 +116,19 @@ mutable struct StructuralModel <: StateSpaceLearningModel
         else
             cycle_matrix = Vector{Matrix}(undef, 0)
         end
+
+        if typeof(freq_seasonal) <: Vector
+            @assert all(freq_seasonal .> 0) "Seasonal period must be greater than 0"
+        end
+
+        if typeof(cycle_period) <: Vector
+            @assert all(cycle_period .>= 0) "Cycle period must be greater than or equal to 0"
+        end
+
+        if cycle_period == 0
+            @assert !stochastic_cycle "stochastic_cycle must be false if cycle_period is 0"
+        end
+
         X = create_X(
             level,
             stochastic_level,