Skip to content

Commit 79088ed

Browse files
shchurshchur
authored
Rerun notebooks with v0.6.0 (#47)
1 parent d0130ba commit 79088ed

File tree

4 files changed

+135
-164
lines changed

4 files changed

+135
-164
lines changed

‎docs/tutorials/01-quickstart.ipynb‎

Lines changed: 117 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
},
1010
{
1111
"cell_type": "code",
12-
"execution_count": null,
12+
"execution_count": 1,
1313
"metadata": {},
1414
"outputs": [],
1515
"source": [
@@ -272,19 +272,19 @@
272272
" <tr style=\"text-align: right;\">\n",
273273
" <th></th>\n",
274274
" <th>model_name</th>\n",
275-
" <th>dataset_name</th>\n",
276275
" <th>dataset_path</th>\n",
277276
" <th>dataset_config</th>\n",
278277
" <th>horizon</th>\n",
279-
" <th>cutoff</th>\n",
280-
" <th>lead_time</th>\n",
281-
" <th>min_ts_length</th>\n",
278+
" <th>num_windows</th>\n",
279+
" <th>initial_cutoff</th>\n",
280+
" <th>window_step_size</th>\n",
281+
" <th>min_context_length</th>\n",
282282
" <th>max_context_length</th>\n",
283283
" <th>seasonality</th>\n",
284284
" <th>...</th>\n",
285-
" <th>multiple_target_columns</th>\n",
286285
" <th>past_dynamic_columns</th>\n",
287-
" <th>excluded_columns</th>\n",
286+
" <th>static_columns</th>\n",
287+
" <th>task_name</th>\n",
288288
" <th>test_error</th>\n",
289289
" <th>training_time_s</th>\n",
290290
" <th>inference_time_s</th>\n",
@@ -297,173 +297,166 @@
297297
" <tbody>\n",
298298
" <tr>\n",
299299
" <th>0</th>\n",
300-
" <td>auto_theta</td>\n",
301-
" <td>chronos_datasets_monash_m1_yearly</td>\n",
300+
" <td>seasonal_naive</td>\n",
302301
" <td>autogluon/chronos_datasets</td>\n",
303-
" <td>monash_m1_yearly</td>\n",
302+
" <td>monash_m1_quarterly</td>\n",
304303
" <td>8</td>\n",
304+
" <td>1</td>\n",
305305
" <td>-8</td>\n",
306+
" <td>8</td>\n",
306307
" <td>1</td>\n",
307-
" <td>9</td>\n",
308308
" <td>NaN</td>\n",
309-
" <td>1</td>\n",
309+
" <td>4</td>\n",
310310
" <td>...</td>\n",
311-
" <td>NaN</td>\n",
312311
" <td>[]</td>\n",
313312
" <td>[]</td>\n",
314-
" <td>4.241262</td>\n",
315-
" <td>NaN</td>\n",
316-
" <td>7.116432</td>\n",
317-
" <td>296cc3ca5975e847</td>\n",
313+
" <td>monash_m1_quarterly</td>\n",
314+
" <td>2.077537</td>\n",
315+
" <td>0.0</td>\n",
316+
" <td>1.687698</td>\n",
317+
" <td>5dd7170c16393209</td>\n",
318318
" <td>False</td>\n",
319-
" <td>0.2.1</td>\n",
320-
" <td>4.241262</td>\n",
319+
" <td>0.6.0</td>\n",
320+
" <td>2.077537</td>\n",
321321
" </tr>\n",
322322
" <tr>\n",
323323
" <th>1</th>\n",
324-
" <td>auto_theta</td>\n",
325-
" <td>chronos_datasets_monash_electricity_weekly</td>\n",
324+
" <td>ets</td>\n",
326325
" <td>autogluon/chronos_datasets</td>\n",
327-
" <td>monash_electricity_weekly</td>\n",
326+
" <td>monash_m1_quarterly</td>\n",
328327
" <td>8</td>\n",
329-
" <td>2013-01-01T00:00:00</td>\n",
330328
" <td>1</td>\n",
331-
" <td>9</td>\n",
332-
" <td>NaN</td>\n",
329+
" <td>-8</td>\n",
330+
" <td>8</td>\n",
333331
" <td>1</td>\n",
334-
" <td>...</td>\n",
335332
" <td>NaN</td>\n",
333+
" <td>4</td>\n",
334+
" <td>...</td>\n",
336335
" <td>[]</td>\n",
337336
" <td>[]</td>\n",
338-
" <td>1.428428</td>\n",
339-
" <td>NaN</td>\n",
340-
" <td>2.812927</td>\n",
341-
" <td>1bf59473dbf463a3</td>\n",
337+
" <td>monash_m1_quarterly</td>\n",
338+
" <td>1.660810</td>\n",
339+
" <td>0.0</td>\n",
340+
" <td>4.366176</td>\n",
341+
" <td>5dd7170c16393209</td>\n",
342342
" <td>False</td>\n",
343-
" <td>0.2.1</td>\n",
344-
" <td>1.428428</td>\n",
343+
" <td>0.6.0</td>\n",
344+
" <td>1.660810</td>\n",
345345
" </tr>\n",
346346
" <tr>\n",
347347
" <th>2</th>\n",
348-
" <td>auto_theta</td>\n",
349-
" <td>chronos_datasets_monash_electricity_weekly</td>\n",
348+
" <td>theta</td>\n",
350349
" <td>autogluon/chronos_datasets</td>\n",
351-
" <td>monash_electricity_weekly</td>\n",
350+
" <td>monash_m1_quarterly</td>\n",
352351
" <td>8</td>\n",
353-
" <td>2014-01-01T00:00:00</td>\n",
354352
" <td>1</td>\n",
355-
" <td>9</td>\n",
356-
" <td>NaN</td>\n",
353+
" <td>-8</td>\n",
354+
" <td>8</td>\n",
357355
" <td>1</td>\n",
358-
" <td>...</td>\n",
359356
" <td>NaN</td>\n",
357+
" <td>4</td>\n",
358+
" <td>...</td>\n",
360359
" <td>[]</td>\n",
361360
" <td>[]</td>\n",
362-
" <td>1.610647</td>\n",
363-
" <td>NaN</td>\n",
364-
" <td>6.573564</td>\n",
365-
" <td>1bf59473dbf463a3</td>\n",
361+
" <td>monash_m1_quarterly</td>\n",
362+
" <td>1.705247</td>\n",
363+
" <td>0.0</td>\n",
364+
" <td>0.125761</td>\n",
365+
" <td>5dd7170c16393209</td>\n",
366366
" <td>False</td>\n",
367-
" <td>0.2.1</td>\n",
368-
" <td>1.610647</td>\n",
367+
" <td>0.6.0</td>\n",
368+
" <td>1.705247</td>\n",
369369
" </tr>\n",
370370
" <tr>\n",
371371
" <th>3</th>\n",
372-
" <td>auto_arima</td>\n",
373-
" <td>chronos_datasets_monash_m1_yearly</td>\n",
372+
" <td>seasonal_naive</td>\n",
374373
" <td>autogluon/chronos_datasets</td>\n",
375-
" <td>monash_m1_yearly</td>\n",
374+
" <td>monash_electricity_weekly</td>\n",
375+
" <td>8</td>\n",
376+
" <td>2</td>\n",
377+
" <td>-16</td>\n",
376378
" <td>8</td>\n",
377-
" <td>-8</td>\n",
378379
" <td>1</td>\n",
379-
" <td>9</td>\n",
380380
" <td>NaN</td>\n",
381381
" <td>1</td>\n",
382382
" <td>...</td>\n",
383-
" <td>NaN</td>\n",
384383
" <td>[]</td>\n",
385384
" <td>[]</td>\n",
386-
" <td>3.993800</td>\n",
387-
" <td>NaN</td>\n",
388-
" <td>8.246975</td>\n",
389-
" <td>296cc3ca5975e847</td>\n",
385+
" <td>monash_electricity_weekly</td>\n",
386+
" <td>2.535526</td>\n",
387+
" <td>0.0</td>\n",
388+
" <td>1.175560</td>\n",
389+
" <td>b7cd1c9df3391815</td>\n",
390390
" <td>False</td>\n",
391-
" <td>0.2.1</td>\n",
392-
" <td>3.993800</td>\n",
391+
" <td>0.6.0</td>\n",
392+
" <td>2.535526</td>\n",
393393
" </tr>\n",
394394
" <tr>\n",
395395
" <th>4</th>\n",
396-
" <td>auto_arima</td>\n",
397-
" <td>chronos_datasets_monash_electricity_weekly</td>\n",
396+
" <td>ets</td>\n",
398397
" <td>autogluon/chronos_datasets</td>\n",
399398
" <td>monash_electricity_weekly</td>\n",
400399
" <td>8</td>\n",
401-
" <td>2013-01-01T00:00:00</td>\n",
400+
" <td>2</td>\n",
401+
" <td>-16</td>\n",
402+
" <td>8</td>\n",
402403
" <td>1</td>\n",
403-
" <td>9</td>\n",
404404
" <td>NaN</td>\n",
405405
" <td>1</td>\n",
406406
" <td>...</td>\n",
407-
" <td>NaN</td>\n",
408407
" <td>[]</td>\n",
409408
" <td>[]</td>\n",
410-
" <td>1.720373</td>\n",
411-
" <td>NaN</td>\n",
412-
" <td>23.514658</td>\n",
413-
" <td>1bf59473dbf463a3</td>\n",
409+
" <td>monash_electricity_weekly</td>\n",
410+
" <td>2.552429</td>\n",
411+
" <td>0.0</td>\n",
412+
" <td>3.755289</td>\n",
413+
" <td>b7cd1c9df3391815</td>\n",
414414
" <td>False</td>\n",
415-
" <td>0.2.1</td>\n",
416-
" <td>1.720373</td>\n",
415+
" <td>0.6.0</td>\n",
416+
" <td>2.552429</td>\n",
417417
" </tr>\n",
418418
" </tbody>\n",
419419
"</table>\n",
420-
"<p>5 rows × 26 columns</p>\n",
420+
"<p>5 rows × 28 columns</p>\n",
421421
"</div>"
422422
],
423423
"text/plain": [
424-
" model_name dataset_name \\\n",
425-
"0 auto_theta chronos_datasets_monash_m1_yearly \n",
426-
"1 auto_theta chronos_datasets_monash_electricity_weekly \n",
427-
"2 auto_theta chronos_datasets_monash_electricity_weekly \n",
428-
"3 auto_arima chronos_datasets_monash_m1_yearly \n",
429-
"4 auto_arima chronos_datasets_monash_electricity_weekly \n",
430-
"\n",
431-
" dataset_path dataset_config horizon \\\n",
432-
"0 autogluon/chronos_datasets monash_m1_yearly 8 \n",
433-
"1 autogluon/chronos_datasets monash_electricity_weekly 8 \n",
434-
"2 autogluon/chronos_datasets monash_electricity_weekly 8 \n",
435-
"3 autogluon/chronos_datasets monash_m1_yearly 8 \n",
436-
"4 autogluon/chronos_datasets monash_electricity_weekly 8 \n",
424+
" model_name dataset_path dataset_config \\\n",
425+
"0 seasonal_naive autogluon/chronos_datasets monash_m1_quarterly \n",
426+
"1 ets autogluon/chronos_datasets monash_m1_quarterly \n",
427+
"2 theta autogluon/chronos_datasets monash_m1_quarterly \n",
428+
"3 seasonal_naive autogluon/chronos_datasets monash_electricity_weekly \n",
429+
"4 ets autogluon/chronos_datasets monash_electricity_weekly \n",
437430
"\n",
438-
" cutoff lead_time min_ts_length max_context_length \\\n",
439-
"0 -8 1 9 NaN \n",
440-
"1 2013-01-01T00:00:00 1 9 NaN \n",
441-
"2 2014-01-01T00:00:00 1 9 NaN \n",
442-
"3 -8 1 9 NaN \n",
443-
"4 2013-01-01T00:00:00 1 9 NaN \n",
431+
" horizon num_windows initial_cutoff window_step_size min_context_length \\\n",
432+
"0 8 1 -8 8 1 \n",
433+
"1 8 1 -8 8 1 \n",
434+
"2 8 1 -8 8 1 \n",
435+
"3 8 2 -16 8 1 \n",
436+
"4 8 2 -16 8 1 \n",
444437
"\n",
445-
" seasonality ... multiple_target_columns past_dynamic_columns \\\n",
446-
"0 1 ... NaN [] \n",
447-
"1 1 ... NaN [] \n",
448-
"2 1 ... NaN [] \n",
449-
"3 1 ... NaN [] \n",
450-
"4 1 ... NaN [] \n",
438+
" max_context_length seasonality ... past_dynamic_columns static_columns \\\n",
439+
"0 NaN 4 ... [] [] \n",
440+
"1 NaN 4 ... [] [] \n",
441+
"2 NaN 4 ... [] [] \n",
442+
"3 NaN 1 ... [] [] \n",
443+
"4 NaN 1 ... [] [] \n",
451444
"\n",
452-
" excluded_columns test_error training_time_s inference_time_s \\\n",
453-
"0 [] 4.241262 NaN 7.116432 \n",
454-
"1 [] 1.428428 NaN 2.812927 \n",
455-
"2 [] 1.610647 NaN 6.573564 \n",
456-
"3 [] 3.993800 NaN 8.246975 \n",
457-
"4 [] 1.720373 NaN 23.514658 \n",
445+
" task_name test_error training_time_s inference_time_s \\\n",
446+
"0 monash_m1_quarterly 2.077537 0.0 1.687698 \n",
447+
"1 monash_m1_quarterly 1.660810 0.0 4.366176 \n",
448+
"2 monash_m1_quarterly 1.705247 0.0 0.125761 \n",
449+
"3 monash_electricity_weekly 2.535526 0.0 1.175560 \n",
450+
"4 monash_electricity_weekly 2.552429 0.0 3.755289 \n",
458451
"\n",
459452
" dataset_fingerprint trained_on_this_dataset fev_version MASE \n",
460-
"0 296cc3ca5975e847 False 0.2.1 4.241262 \n",
461-
"1 1bf59473dbf463a3 False 0.2.1 1.428428 \n",
462-
"2 1bf59473dbf463a3 False 0.2.1 1.610647 \n",
463-
"3 296cc3ca5975e847 False 0.2.1 3.993800 \n",
464-
"4 1bf59473dbf463a3 False 0.2.1 1.720373 \n",
453+
"0 5dd7170c16393209 False 0.6.0 2.077537 \n",
454+
"1 5dd7170c16393209 False 0.6.0 1.660810 \n",
455+
"2 5dd7170c16393209 False 0.6.0 1.705247 \n",
456+
"3 b7cd1c9df3391815 False 0.6.0 2.535526 \n",
457+
"4 b7cd1c9df3391815 False 0.6.0 2.552429 \n",
465458
"\n",
466-
"[5 rows x 26 columns]"
459+
"[5 rows x 28 columns]"
467460
]
468461
},
469462
"execution_count": 11,
@@ -483,16 +476,6 @@
483476
"execution_count": 12,
484477
"metadata": {},
485478
"outputs": [
486-
{
487-
"name": "stderr",
488-
"output_type": "stream",
489-
"text": [
490-
"/var/folders/dj/hj4wkwks7pd840zxndb25m9w0000gr/T/ipykernel_61496/4135076758.py:2: UserWarning: Columns ['known_dynamic_columns', 'min_context_length', 'static_columns'] are missing from summaries, filling them with None\n",
491-
" fev.leaderboard(summaries, baseline_model=\"seasonal_naive\")\n",
492-
"/var/folders/dj/hj4wkwks7pd840zxndb25m9w0000gr/T/ipykernel_61496/4135076758.py:2: UserWarning: Evaluation summaries contain results from fev < 0.6.0. Results may not be comparable due to breaking changes.\n",
493-
" fev.leaderboard(summaries, baseline_model=\"seasonal_naive\")\n"
494-
]
495-
},
496479
{
497480
"data": {
498481
"text/html": [
@@ -533,38 +516,29 @@
533516
" </thead>\n",
534517
" <tbody>\n",
535518
" <tr>\n",
536-
" <th>auto_theta</th>\n",
537-
" <td>0.125545</td>\n",
538-
" <td>0.666667</td>\n",
539-
" <td>NaN</td>\n",
540-
" <td>6.573564</td>\n",
519+
" <th>ets</th>\n",
520+
" <td>0.133483</td>\n",
521+
" <td>0.833333</td>\n",
541522
" <td>0.0</td>\n",
542-
" <td>0</td>\n",
543-
" </tr>\n",
544-
" <tr>\n",
545-
" <th>auto_arima</th>\n",
546-
" <td>0.112664</td>\n",
547-
" <td>0.666667</td>\n",
548-
" <td>NaN</td>\n",
549-
" <td>23.514658</td>\n",
523+
" <td>3.755289</td>\n",
550524
" <td>0.0</td>\n",
551525
" <td>0</td>\n",
552526
" </tr>\n",
553527
" <tr>\n",
554-
" <th>auto_ets</th>\n",
555-
" <td>0.048807</td>\n",
556-
" <td>0.444444</td>\n",
557-
" <td>NaN</td>\n",
558-
" <td>0.741776</td>\n",
528+
" <th>theta</th>\n",
529+
" <td>0.105932</td>\n",
530+
" <td>0.333333</td>\n",
531+
" <td>0.0</td>\n",
532+
" <td>0.125761</td>\n",
559533
" <td>0.0</td>\n",
560534
" <td>0</td>\n",
561535
" </tr>\n",
562536
" <tr>\n",
563537
" <th>seasonal_naive</th>\n",
564538
" <td>0.000000</td>\n",
565-
" <td>0.222222</td>\n",
566-
" <td>NaN</td>\n",
567-
" <td>0.004139</td>\n",
539+
" <td>0.333333</td>\n",
540+
" <td>0.0</td>\n",
541+
" <td>1.444558</td>\n",
568542
" <td>0.0</td>\n",
569543
" <td>0</td>\n",
570544
" </tr>\n",
@@ -575,17 +549,15 @@
575549
"text/plain": [
576550
" skill_score win_rate median_training_time_s \\\n",
577551
"model_name \n",
578-
"auto_theta 0.125545 0.666667 NaN \n",
579-
"auto_arima 0.112664 0.666667 NaN \n",
580-
"auto_ets 0.048807 0.444444 NaN \n",
581-
"seasonal_naive 0.000000 0.222222 NaN \n",
552+
"ets 0.133483 0.833333 0.0 \n",
553+
"theta 0.105932 0.333333 0.0 \n",
554+
"seasonal_naive 0.000000 0.333333 0.0 \n",
582555
"\n",
583556
" median_inference_time_s training_corpus_overlap num_failures \n",
584557
"model_name \n",
585-
"auto_theta 6.573564 0.0 0 \n",
586-
"auto_arima 23.514658 0.0 0 \n",
587-
"auto_ets 0.741776 0.0 0 \n",
588-
"seasonal_naive 0.004139 0.0 0 "
558+
"ets 3.755289 0.0 0 \n",
559+
"theta 0.125761 0.0 0 \n",
560+
"seasonal_naive 1.444558 0.0 0 "
589561
]
590562
},
591563
"execution_count": 12,

0 commit comments

Comments
 (0)