From c5819e8ecf655ec19183f05fa205a2591912866d Mon Sep 17 00:00:00 2001 From: andre_ramos Date: Wed, 19 Feb 2025 15:24:47 -0300 Subject: [PATCH 1/2] Add assertions to improve input validation and prevent bugs --- Project.toml | 2 +- paper_tests/m4_test/m4_test.py | 81 +++++++++++++++++++++++++++++++++- src/models/structural_model.jl | 13 ++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index b937b4d..d9cc2ff 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StateSpaceLearning" uuid = "971c4b7c-2c4e-4bac-8525-e842df3cde7b" authors = ["andreramosfc "] -version = "1.4.1" +version = "1.4.2" [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/paper_tests/m4_test/m4_test.py b/paper_tests/m4_test/m4_test.py index d612816..82bd183 100644 --- a/paper_tests/m4_test/m4_test.py +++ b/paper_tests/m4_test/m4_test.py @@ -2,6 +2,9 @@ import math import statsmodels.api as sm import numpy as np +from prophet import Prophet +from chronos import ChronosPipeline +import torch df_train1 = pd.read_csv("paper_tests/m4_test/Monthly-train1.csv") df_train2 = pd.read_csv("paper_tests/m4_test/Monthly-train2.csv") @@ -9,6 +12,7 @@ df_train4 = pd.read_csv("paper_tests/m4_test/Monthly-train4.csv") df_train = pd.concat([df_train1, df_train2, df_train3, df_train4]) m4_info = pd.read_csv("paper_tests/m4_test/M4-info.csv") + df_test = pd.read_csv("paper_tests/m4_test/Monthly-test.csv") ssl_init_df = pd.read_csv("paper_tests/m4_test/init_SSL/SSL_aic_0.1_false.csv") @@ -54,6 +58,7 @@ def evaluate_ss(input, sample_size, init, hyperparameters_inicialization): forecast_values = [x * (max_train - min_train) + min_train for x in normalized_forecast_values] return sMAPE(test, forecast_values), MASE(train, test, forecast_values) + results = [] results_init = [] for i in range(0, 48000): @@ -85,4 +90,78 @@ def evaluate_ss(input, sample_size, init, hyperparameters_inicialization): df_mean_init = pd.DataFrame({'smape': [np.mean(smape_SS_init)], 'mase': [np.mean(mase_SS_init)], 'owa': [(np.mean(smape_SS_init) / NAIVE_sMAPE + np.mean(mase_SS_init) / NAIVE_MASE) / 2]}) df_mean.to_csv('paper_tests/m4_test/metrics_results/SS_METRICS_RESULTS.csv') -df_mean_init.to_csv('paper_tests/m4_test/metrics_results/SS_INIT_METRICS_RESULTS.csv') \ No newline at end of file +df_mean_init.to_csv('paper_tests/m4_test/metrics_results/SS_INIT_METRICS_RESULTS.csv') + +def evaluate_prophet(input): + train = input["train"] + test = input["test"] + timestamps = pd.date_range(start="2020-01-01", periods=len(train), freq='ME') + #add random seed + df = pd.DataFrame({ + 'ds': timestamps, + 'y': train + }) + model = Prophet(interval_width=0.95) + model.fit(df) + future = pd.DataFrame({ + 'ds': (pd.date_range(start="2020-01-01", periods=len(train) + 18, freq='ME'))[len(train):] + }) + model_forecast = model.predict(future) + prediction = model_forecast['yhat'].values + return sMAPE(test, prediction), MASE(train, test, prediction) + +def evaluate_chronos(input): + train = input["train"] + test = input["test"] + chronos_forecast = ChronosPipeline.from_pretrained( + f"amazon/chronos-t5-large", + device_map="mps" + ).predict( + context=torch.tensor(train), + prediction_length=18, + limit_prediction_length=False + ) + prediction = np.quantile(chronos_forecast[0].numpy(), [0.5], axis=0)[0] + return sMAPE(test, prediction), MASE(train, test, prediction) + +smape_prophet_vec = [] +mase_prophet_vec = [] +smape_chronos_vec = [] +mase_chronos_vec = [] +for i in range(0, len(dict_vec)): + smape_prophet, mase_prophet = evaluate_prophet(dict_vec[i]) + smape_prophet_vec.append(smape_prophet) + mase_prophet_vec.append(mase_prophet) + smape_chronos, mase_chronos = evaluate_chronos(dict_vec[i]) + smape_chronos_vec.append(smape_chronos) + mase_chronos_vec.append(mase_chronos) + # + print("Runningg series ", i) + if i % 1000 == 0: + print("Runningg series ", i) + smape_mean_prophet = np.mean(smape_prophet_vec) + smape_emean_chronos = np.mean(smape_chronos_vec) + mase_mean_prophet = np.mean(mase_prophet_vec) + mase_mean_chronos = np.mean(mase_chronos_vec) + print("Mean sMape Prophet: ", smape_mean_prophet) + print("Mean sMape Chronos: ", smape_emean_chronos) + print("Mean Mase Prophet: ", mase_mean_prophet) + print("Mean Mase Chronos: ", mase_mean_chronos) + + +NAIVE_sMAPE = 14.427 #M4 Paper +NAIVE_MASE = 1.063 #M4 Paper + +owa_prophet = (np.mean(smape_prophet_vec) / NAIVE_sMAPE + np.mean(mase_prophet_vec) / NAIVE_MASE) / 2 +owa_chronos = (np.mean(smape_chronos_vec) / NAIVE_sMAPE + np.mean(mase_chronos_vec) / NAIVE_MASE) / 2 + +mean_mase_prophet = np.mean(mase_prophet_vec) +mean_smape_prophet = np.mean(smape_prophet_vec) +mean_mase_chronos = np.mean(mase_chronos_vec) +mean_smape_chronos = np.mean(smape_chronos_vec) + +df_results_mean = pd.DataFrame({'smape': [mean_smape_prophet, mean_smape_chronos], 'mase': [mean_mase_prophet, mean_mase_chronos], 'owa': [owa_prophet, owa_chronos]}) + +# save to csv + +df_results_mean.to_csv('paper_tests/m4_test/metrics_results/PROPHET_CHRONOS_METRICS_RESULTS.csv') \ No newline at end of file diff --git a/src/models/structural_model.jl b/src/models/structural_model.jl index 7fe96e3..a3569c4 100644 --- a/src/models/structural_model.jl +++ b/src/models/structural_model.jl @@ -116,6 +116,19 @@ mutable struct StructuralModel <: StateSpaceLearningModel else cycle_matrix = Vector{Matrix}(undef, 0) end + + if typeof(freq_seasonal) <: Vector + @assert all(freq_seasonal .> 0) "Seasonal period must be greater than 0" + end + + if typeof(cycle_period) <: Vector + @assert all(cycle_period .>= 0) "Cycle period must be greater than or equal to 0" + end + + if cycle_period == 0 + @assert !stochastic_cycle "stochastic_cycle must be false if cycle_period is 0" + end + X = create_X( level, stochastic_level, From 701515ac15334284ec4819a6647caac3943ce87a Mon Sep 17 00:00:00 2001 From: andre_ramos Date: Wed, 19 Feb 2025 15:33:40 -0300 Subject: [PATCH 2/2] Add assertions to improve input validation and prevent bugs --- src/models/structural_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/structural_model.jl b/src/models/structural_model.jl index a3569c4..8200490 100644 --- a/src/models/structural_model.jl +++ b/src/models/structural_model.jl @@ -124,7 +124,7 @@ mutable struct StructuralModel <: StateSpaceLearningModel if typeof(cycle_period) <: Vector @assert all(cycle_period .>= 0) "Cycle period must be greater than or equal to 0" end - + if cycle_period == 0 @assert !stochastic_cycle "stochastic_cycle must be false if cycle_period is 0" end