Fix backtest (#136)

* Update diagnostics and bump version

* Fix backtest

* add visualization forecast test

Author: guilhermebodin

Project.toml

@@ -1,7 +1,7 @@
 name = "ScoreDrivenModels"
 uuid = "4a87933e-d659-11e9-0e65-7f40dedd4a3a"
 authors = ["guilhermebodin <[email protected]>, raphaelsaavedra <[email protected]>"]
-version = "0.1.8"
+version = "0.1.9"
 
 [deps]
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"

src/backtest.jl

@@ -40,7 +40,7 @@ TODO
 """
 function backtest(gas::Model{<:Distribution, T}, y::Vector{T}, steps_ahead::Int, start_idx::Int;
                   S::Int = 10_000,
-                  initial_params::Matrix{T} = stationary_initial_params(gas),
+                  initial_params::Matrix{T} = DEFAULT_INITIAL_PARAM,
                   opt_method = NelderMead(gas, DEFAULT_NUM_SEEDS)) where T
     num_mle = length(y) - start_idx - steps_ahead
     b = Backtest(num_mle, steps_ahead)
@@ -49,8 +49,10 @@ function backtest(gas::Model{<:Distribution, T}, y::Vector{T}, steps_ahead::Int,
         gas_to_fit = deepcopy(gas)
         y_to_fit = y[1:start_idx - 1 + i]
         y_to_verify = y[start_idx + i:start_idx - 1 + i + steps_ahead]
-        ScoreDrivenModels.fit!(gas_to_fit, y_to_fit; initial_params=initial_params, opt_method=opt_method)
-        forec = forecast(y_to_fit, gas_to_fit, steps_ahead; S=S, initial_params=initial_params)
+        fit!(gas_to_fit, y_to_fit; initial_params=initial_params, opt_method=opt_method)
+        forec = initial_params !== DEFAULT_INITIAL_PARAM ? 
+                    forecast(y_to_fit, gas_to_fit, steps_ahead; S=S, initial_params=initial_params) :
+                    forecast(y_to_fit, gas_to_fit, steps_ahead; S=S)
         abs_errors = evaluate_abs_error(y_to_verify, forec.observation_forecast)
         crps_scores = evaluate_crps(y_to_verify, forec.observation_scenarios)
         b.abs_errors[:, i] = abs_errors

src/visualization/forecast.jl

@@ -1 +1,46 @@
-#TODO
+RecipesBase.@recipe function f(y::Vector, forec::Forecast)
+    n = length(y)
+    steps_ahead = length(forec.observation_forecast)
+    forec_idx = collect(n + 1: n + steps_ahead)
+    fillrange_color = :steelblue
+    # Plot the series
+    @series begin
+        seriestype := :path
+        seriescolor := "black"
+        return y
+    end
+    # Plot the forecast
+    @series begin
+        seriescolor := "blue"
+        return forec_idx, forec.observation_forecast
+    end
+    # Plot the prediction interval
+    q975 = Vector{Float64}(undef, steps_ahead)
+    q025 = Vector{Float64}(undef, steps_ahead)
+    q900 = Vector{Float64}(undef, steps_ahead)
+    q100 = Vector{Float64}(undef, steps_ahead)
+    for i in 1:steps_ahead
+        q975[i] = quantile(forec.observation_scenarios[i, :], 0.975)
+        q025[i] = quantile(forec.observation_scenarios[i, :], 0.025)
+        q900[i] = quantile(forec.observation_scenarios[i, :], 0.9)
+        q100[i] = quantile(forec.observation_scenarios[i, :], 0.1)
+    end
+    @series begin
+        seriestype := :path
+        seriescolor := fillrange_color
+        fillcolor := fillrange_color
+        fillalpha := 0.5
+        fillrange := q975
+        label := ""
+        return forec_idx, q025
+    end
+    @series begin
+        seriestype := :path
+        seriescolor := fillrange_color
+        fillcolor := fillrange_color
+        fillalpha := 0.5
+        fillrange := q900
+        label := ""
+        return forec_idx, q100
+    end
+end

test/test_visualization.jl

@@ -16,4 +16,11 @@
     bac = backtest(gas, simulation, 10, 4985)
     rec = RecipesBase.apply_recipe(Dict{Symbol, Any}(), bac, "name")
     @test length(rec) == 2
+
+    simulation = simulate_GAS_1_1(Normal, 0.0, ω, A, B, 1)
+    gas = ScoreDrivenModels.Model(1, 1, Normal, 0.0)
+    fit!(gas, simulation)
+    forec = forecast(simulation, gas, 50)
+    rec = RecipesBase.apply_recipe(Dict{Symbol, Any}(), simulation, forec)
+    @test length(rec) == 4
 end