Update model wrappers for v0.6.0 (#44)

Co-authored-by: azul <[email protected]>

Author: shchur

benchmarks/chronos_zeroshot/results/timecopilot.csv

@@ -0,0 +1,28 @@
+model_name,dataset_path,dataset_config,horizon,num_windows,initial_cutoff,window_step_size,min_context_length,max_context_length,seasonality,eval_metric,extra_metrics,quantile_levels,id_column,timestamp_column,target,generate_univariate_targets_from,known_dynamic_columns,past_dynamic_columns,static_columns,task_name,test_error,training_time_s,inference_time_s,dataset_fingerprint,trained_on_this_dataset,fev_version,MASE,WQL
+timecopilot,autogluon/chronos_datasets,monash_traffic,24,1,-24,24,-23,,24,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_traffic,0.7739645971400719,,101.55883938,a395745939b7b0f3,False,0.6.0,0.7739645971400719,0.2281103168740663
+timecopilot,autogluon/chronos_datasets,monash_australian_electricity,48,1,-48,48,-47,,48,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_australian_electricity,0.761278878663326,,87.18234509,c3fa76db06a6f37a,False,0.6.0,0.761278878663326,0.0354336959497435
+timecopilot,autogluon/chronos_datasets,ercot,24,1,-24,24,-23,,24,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],ercot,0.6833983448950258,,81.13162435999999,95b91121d95f89c8,False,0.6.0,0.6833983448950258,0.0206758714793285
+timecopilot,autogluon/chronos_datasets_extra,ETTm,24,1,-96,24,-23,,96,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,__ALL__,[],[],[],ETTm,0.6237841762093976,,92.28126142000002,8b02054712ef1d41,False,0.6.0,0.6237841762093976,0.0516134532630169
+timecopilot,autogluon/chronos_datasets_extra,ETTh,24,1,-24,24,-23,,24,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,__ALL__,[],[],[],ETTh,0.7625638462320636,,85.33888267200001,d61f7a559d253cb4,False,0.6.0,0.7625638462320636,0.0740316799327656
+timecopilot,autogluon/chronos_datasets,exchange_rate,30,1,-30,30,-29,,5,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],exchange_rate,1.981209557451495,,95.825562333,a8476b6fa723f5ed,False,0.6.0,1.981209557451495,0.0134032471152773
+timecopilot,autogluon/chronos_datasets,nn5,56,1,-56,56,-55,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],nn5,0.5556876939254136,,96.910774641,91434e4cabb44eab,False,0.6.0,0.5556876939254136,0.1453783915571264
+timecopilot,autogluon/chronos_datasets,monash_nn5_weekly,8,1,-8,8,-7,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_nn5_weekly,0.9072348440238922,,91.79065736799998,480088d6b89d7100,False,0.6.0,0.9072348440238922,0.0828638298840608
+timecopilot,autogluon/chronos_datasets,monash_weather,30,1,-30,30,-29,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_weather,0.7919336327531099,,147.496713613,980478e7349a0e49,False,0.6.0,0.7919336327531099,0.1284542879603267
+timecopilot,autogluon/chronos_datasets,monash_covid_deaths,30,1,-30,30,-29,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_covid_deaths,38.40480428649353,,99.978413443,6cdf94c2ac2fae09,False,0.6.0,38.40480428649353,0.0402622984596447
+timecopilot,autogluon/chronos_datasets,monash_fred_md,12,1,-12,12,-11,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_fred_md,0.5513709624273953,,68.48606713199999,a508fc699f3f3683,False,0.6.0,0.5513709624273953,0.0371942971225067
+timecopilot,autogluon/chronos_datasets,m4_quarterly,8,1,-8,8,-7,,4,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],m4_quarterly,1.1700822028966442,,204.034223728,97b166b18d38e710,False,0.6.0,1.1700822028966442,0.0741636274514156
+timecopilot,autogluon/chronos_datasets,m4_yearly,6,1,-6,6,-5,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],m4_yearly,3.4115641258577467,,88.85265404000003,020f5777849c9c62,False,0.6.0,3.4115641258577467,0.1169761730880261
+timecopilot,autogluon/chronos_datasets,dominick,8,1,-8,8,-7,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],dominick,0.8171339001717135,,582.5803540879999,ab1a8018332665ab,False,0.6.0,0.8171339001717135,0.3295555818386664
+timecopilot,autogluon/chronos_datasets,m5,28,1,-28,28,-27,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],m5,0.906674248517517,,584.7930052490001,78a5b1db66dfb76c,False,0.6.0,0.906674248517517,0.5536143922620141
+timecopilot,autogluon/chronos_datasets,monash_tourism_monthly,24,1,-24,24,-23,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_tourism_monthly,1.4292292732537968,,107.435675182,3ce75ace84bef572,False,0.6.0,1.4292292732537968,0.0790971941665618
+timecopilot,autogluon/chronos_datasets,monash_tourism_quarterly,8,1,-8,8,-7,,4,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_tourism_quarterly,1.6223408706515454,,90.725377077,f34c4184775eda7e,False,0.6.0,1.6223408706515454,0.0597801393650899
+timecopilot,autogluon/chronos_datasets,monash_tourism_yearly,4,1,-4,4,-3,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_tourism_yearly,3.5747123618200907,,74.45144080200001,7c477aee96f482c4,False,0.6.0,3.5747123618200907,0.1580192799957833
+timecopilot,autogluon/chronos_datasets,monash_car_parts,12,1,-12,12,-11,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_car_parts,0.8438107216883975,,116.798960247,1ba8c4719a51b23c,False,0.6.0,0.8438107216883975,0.977330257613002
+timecopilot,autogluon/chronos_datasets,monash_hospital,12,1,-12,12,-11,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_hospital,0.7715656244296779,,104.193838361,e364316542f2f5c7,False,0.6.0,0.7715656244296779,0.0536278408642718
+timecopilot,autogluon/chronos_datasets,monash_cif_2016,12,1,-12,12,-11,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_cif_2016,0.937178219129742,,70.38486716899999,89203c02fafca301,False,0.6.0,0.937178219129742,0.0128493017821839
+timecopilot,autogluon/chronos_datasets,monash_m1_yearly,6,1,-6,6,-5,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m1_yearly,4.028149050098825,,88.92860413,bd7efdf4c711a830,False,0.6.0,4.028149050098825,0.1400529738252418
+timecopilot,autogluon/chronos_datasets,monash_m1_quarterly,8,1,-8,8,-7,,4,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m1_quarterly,1.695922770506937,,78.39436832999999,5dd7170c16393209,False,0.6.0,1.695922770506937,0.0991189288385913
+timecopilot,autogluon/chronos_datasets,monash_m1_monthly,18,1,-18,18,-17,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m1_monthly,1.053624977666057,,82.058715941,88d42195e0f8e7cc,False,0.6.0,1.053624977666057,0.1332771768039729
+timecopilot,autogluon/chronos_datasets,monash_m3_monthly,18,1,-18,18,-17,,12,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m3_monthly,0.8286361654975168,,115.440668056,372532bc65a1f203,False,0.6.0,0.8286361654975168,0.0900856487113432
+timecopilot,autogluon/chronos_datasets,monash_m3_yearly,6,1,-6,6,-5,,1,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m3_yearly,2.759102155120125,,98.25139432900002,d607d91b11bbe4a1,False,0.6.0,2.759102155120125,0.1250363194133956
+timecopilot,autogluon/chronos_datasets,monash_m3_quarterly,8,1,-8,8,-7,,4,MASE,['WQL'],"[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]",id,timestamp,target,,[],[],[],monash_m3_quarterly,1.1839319005752151,,89.49035918899999,6401670c504aa82b,False,0.6.0,1.1839319005752151,0.0711619074323248

docs/tutorials/01-quickstart.ipynb

@@ -9,12 +9,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
     "import fev"
    ]
   },

examples/chronos/evaluate_model.py

@@ -26,56 +26,55 @@ def predict_with_model(
     torch_dtype: str = torch.bfloat16,
     num_samples: int = 20,
     seed: int = 123,
-) -> tuple[datasets.Dataset, float, dict]:
+) -> tuple[list[datasets.DatasetDict], float, dict]:
     pipeline = BaseChronosPipeline.from_pretrained(model_name, device_map=device_map, torch_dtype=torch_dtype)
-
-    past_data, future_data = task.get_input_data(trust_remote_code=True)
-    target = past_data.with_format("torch").cast_column(
-        task.target_column, datasets.Sequence(datasets.Value("float32"))
-    )[task.target_column]
-
-    quantile_levels = task.quantile_levels if task.quantile_levels is not None else []
-    quantiles_all = []
-    mean_all = []
-
     torch.manual_seed(seed)
-    start_time = time.monotonic()
-    for batch in tqdm(batchify(target, batch_size=batch_size), total=len(target) // batch_size):
-        kwargs = dict(
-            context=batch,
-            prediction_length=task.horizon,
-            limit_prediction_length=False,
-        )
-
-        if pipeline.forecast_type == ForecastType.SAMPLES:
-            kwargs.update(dict(num_samples=num_samples))
 
-        quantiles, mean = pipeline.predict_quantiles(
-            **kwargs,
-            # make sure to always compute the median prediction last
-            quantile_levels=quantile_levels + [0.5],
+    inference_time = 0.0
+    quantile_levels = task.quantile_levels.copy()
+    if 0.5 not in quantile_levels:
+        quantile_levels.append(0.5)
+
+    predictions_per_window = []
+    for window in task.iter_windows():
+        past_data, _ = fev.convert_input_data(window, adapter="datasets", as_univariate=True)
+        past_data = past_data.with_format("torch").cast_column("target", datasets.Sequence(datasets.Value("float32")))
+
+        quantiles_all = []
+        mean_all = []
+
+        start_time = time.monotonic()
+        for batch in batchify(past_data["target"], batch_size=batch_size):
+            quantiles, mean = pipeline.predict_quantiles(
+                context=batch,
+                prediction_length=task.horizon,
+                limit_prediction_length=False,
+                quantile_levels=quantile_levels,
+            )
+
+            quantiles_all.append(quantiles.numpy())
+            mean_all.append(mean.numpy())
+        inference_time += time.monotonic() - start_time
+
+        quantiles_np = np.concatenate(quantiles_all, axis=0)  # [num_items, horizon, num_quantiles]
+        mean_np = np.concatenate(mean_all, axis=0)  # [num_items, horizon]
+
+        if task.eval_metric in ["MSE", "RMSE", "RMSSE"]:
+            point_forecast = mean_np  # [num_items, horizon]
+        else:
+            # use median as the point forecast
+            point_forecast = quantiles_np[:, :, quantile_levels.index(0.5)]  # [num_items, horizon]
+        predictions_dict = {"predictions": point_forecast}
+
+        for idx, level in enumerate(task.quantile_levels):
+            predictions_dict[str(level)] = quantiles_np[:, :, idx]
+
+        predictions_per_window.append(
+            fev.utils.combine_univariate_predictions_to_multivariate(
+                datasets.Dataset.from_dict(predictions_dict), target_columns=task.target_columns
+            )
         )
 
-        quantiles_all.append(quantiles.numpy())
-        mean_all.append(mean.numpy())
-
-    inference_time = time.monotonic() - start_time
-
-    quantiles_np = np.concatenate(quantiles_all, axis=0)  # [num_items, horizon, num_quantiles]
-    mean_np = np.concatenate(mean_all, axis=0)  # [num_items, horizon]
-
-    if task.eval_metric in ["MSE", "RMSE", "RMSSE"]:
-        point_forecast = mean_np  # [num_items, horizon]
-    else:
-        # take the median from the last computed quantile
-        point_forecast = quantiles_np[:, :, -1]  # [num_items, horizon]
-
-    predictions_dict = {"predictions": point_forecast}
-
-    for idx, level in enumerate(quantile_levels):
-        predictions_dict[str(level)] = quantiles_np[:, :, idx]  # [num_items, horizon]
-
-    predictions = datasets.Dataset.from_dict(predictions_dict)
     extra_info = {
         "model_config": {
             "model_name": model_name,
@@ -86,7 +85,7 @@ def predict_with_model(
             "seed": seed,
         }
     }
-    return predictions, inference_time, extra_info
+    return predictions_per_window, inference_time, extra_info
 
 
 if __name__ == "__main__":

examples/chronos/requirements.txt

@@ -1,2 +1,2 @@
-chronos-forecasting==1.4.1
+chronos-forecasting==1.5.3
 torch==2.6.0

examples/moirai/evaluate_model.py

@@ -1,10 +1,12 @@
+import logging
 import time
+import warnings
 
 import datasets
 import numpy as np
 import pandas as pd
 import torch
-from uni2ts.model.moirai import MoiraiForecast, MoiraiModule
+from uni2ts.model.moirai2 import Moirai2Forecast, Moirai2Module
 
 import fev
 
@@ -13,58 +15,61 @@
 
 def predict_with_model(
     task: fev.Task,
-    model_name: str = "Salesforce/moirai-1.1-R-large",
-    context_length: int = 1024,
+    model_name: str = "Salesforce/moirai-2.0-R-small",
+    context_length: int = 500,
     batch_size: int = 128,
-    num_samples: int = 100,
-    device: str = "cuda",
+    device: str = "cpu",
     seed: int = 123,
-) -> tuple[datasets.Dataset, float, dict]:
-    _, prediction_dataset = fev.convert_input_data(task, "gluonts", trust_remote_code=True)
-
+) -> tuple[list[datasets.DatasetDict], float, dict]:
     torch.manual_seed(seed)
-    model = MoiraiForecast(
-        module=MoiraiModule.from_pretrained(model_name).to(device),
+    # Disable GluonTS warnings when accessing forecast.mean
+    gts_logger = logging.getLogger("gluonts")
+    gts_logger.setLevel(100)
+
+    model = Moirai2Forecast(
+        module=Moirai2Module.from_pretrained(model_name).to(device),
         prediction_length=task.horizon,
         context_length=context_length,
-        num_samples=num_samples,
         target_dim=1,
         feat_dynamic_real_dim=0,
         past_feat_dynamic_real_dim=0,
     )
     predictor = model.create_predictor(batch_size=batch_size)
 
-    start_time = time.monotonic()
-    samples = np.stack([f.samples for f in predictor.predict(prediction_dataset)])
-    inference_time = time.monotonic() - start_time
+    inference_time = 0.0
+    predictions_per_window = []
+    for window in task.iter_windows():
+        _, prediction_dataset = fev.convert_input_data(window, adapter="gluonts", as_univariate=True)
+        start_time = time.monotonic()
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            forecasts = list(predictor.predict(prediction_dataset))
+        inference_time += time.monotonic() - start_time
 
-    if task.eval_metric in ["MSE", "RMSE", "RMSSE"]:
-        point_forecast = np.mean(samples, axis=1)  # [num_items, horizon]
-    else:
-        point_forecast = np.median(samples, axis=1)  # [num_items, horizon]
-
-    predictions_dict = {"predictions": point_forecast}
-    if task.quantile_levels is not None:
+        predictions_dict = {"predictions": np.stack([f.mean for f in forecasts])}
         for q in task.quantile_levels:
-            predictions_dict[str(q)] = np.quantile(samples, q=q, axis=1)  # [num_items, horizon]
+            predictions_dict[str(q)] = np.stack([f.quantile(q) for f in forecasts])
+        predictions_per_window.append(
+            fev.utils.combine_univariate_predictions_to_multivariate(
+                datasets.Dataset.from_dict(predictions_dict), target_columns=task.target_columns
+            )
+        )
 
-    predictions = datasets.Dataset.from_dict(predictions_dict)
     extra_info = {
         "model_config": {
             "context_length": context_length,
             "model_name": model_name,
             "batch_size": batch_size,
-            "num_samples": num_samples,
             "device": device,
             "seed": seed,
         }
     }
 
-    return predictions, inference_time, extra_info
+    return predictions_per_window, inference_time, extra_info
 
 
 if __name__ == "__main__":
-    model_name = "Salesforce/moirai-1.1-R-base"
+    model_name = "Salesforce/moirai-2.0-R-small"
     num_tasks = 2  # replace with `num_tasks = None` to run on all tasks
 
     benchmark = fev.Benchmark.from_yaml(
@@ -85,4 +90,4 @@ def predict_with_model(
     # Show and save the results
     summary_df = pd.DataFrame(summaries)
     print(summary_df)
-    summary_df.to_csv(f"{model_name}.csv", index=False)
+    summary_df.to_csv("moirai-2.0.csv", index=False)

examples/moirai/requirements.txt

@@ -1,2 +1,2 @@
-uni2ts==1.2.0
-torch==2.6.0
+uni2ts @ git+https://github.com/SalesforceAIResearch/uni2ts@b472ef6dcf770b96a1b04bd98cd11e893058573a
+torch<2.5.0

examples/seasonal_naive/evaluate_model.py

@@ -8,23 +8,30 @@
 import fev
 
 
-def predict_with_model(task: fev.Task) -> tuple[datasets.Dataset, float, dict]:
-    _, prediction_dataset = fev.convert_input_data(task, "gluonts", trust_remote_code=True)
-
+def predict_with_model(task: fev.Task) -> tuple[list[datasets.DatasetDict], float, dict]:
     predictor = SeasonalNaivePredictor(prediction_length=task.horizon, season_length=task.seasonality)
 
-    start_time = time.monotonic()
-    forecast = np.stack([f.samples for f in predictor.predict(prediction_dataset)]).squeeze(1)  # [num_items, horizon]
-    inference_time = time.monotonic() - start_time
-
-    predictions_dict = {"predictions": forecast}
-    if task.quantile_levels is not None:
+    inference_time = 0.0
+    predictions_per_window = []
+    for window in task.iter_windows(trust_remote_code=True):
+        _, prediction_dataset = fev.convert_input_data(window, adapter="gluonts", as_univariate=True)
+        start_time = time.monotonic()
+        forecast = np.stack([f.samples for f in predictor.predict(prediction_dataset)]).squeeze(
+            1
+        )  # [num_items, horizon]
+        inference_time += time.monotonic() - start_time
+
+        predictions_dict = {"predictions": forecast}
         for q in task.quantile_levels:
             predictions_dict[str(q)] = forecast
 
-    predictions = datasets.Dataset.from_dict(predictions_dict)
+        predictions_per_window.append(
+            fev.combine_univariate_predictions_to_multivariate(
+                datasets.Dataset.from_dict(predictions_dict), target_columns=task.target_columns
+            )
+        )
 
-    return predictions, inference_time, {}
+    return predictions_per_window, inference_time, {}
 
 
 if __name__ == "__main__":