Cv (#139)

* change from ANE to NIE

* change from backtest to cross_validation

Author: guilhermebodin

examples/cpichg.jl

@@ -43,7 +43,7 @@ gas_t = Model(1, 1, TDistLocationScale, 1.0; time_varying_params = [1])
 gas_t_1_2 = Model(1, 2, TDistLocationScale, 1.0; time_varying_params = [1])
 steps_ahead = 50
 first_idx = 150
-b_t = backtest(gas_t, y, steps_ahead, first_idx)
-b_t_1_2 = backtest(gas_t_1_2, y, steps_ahead, first_idx)
+b_t = cross_validation(gas_t, y, steps_ahead, first_idx)
+b_t_1_2 = cross_validation(gas_t_1_2, y, steps_ahead, first_idx)
 plot(b_t, "GAS(1, 1) Student t")
 plot!(b_t_1_2, "GAS(1, 2) Student t"; legend = :topleft)

examples/nie.jl

@@ -3,6 +3,7 @@ using Dates, DelimitedFiles, Plots, Random, ScoreDrivenModels
 # Define dates and load historical Affluent Natural Energy data
 dates = collect(Date(1961):Month(1):Date(2000, 12))
 y = vec(readdlm("../test/data/nie_northeastern.csv"))
+
 y_train = y[1:400]
 y_test = y[401:460]
 
@@ -26,9 +27,6 @@ plot(f)
 forec = forecast(y_train, gas, 60; S=1000, initial_params=initial_params)
 
 # Plot results
-plotly()
-plot(dates[401:460], forec.observation_scenarios, color="grey", w=0.05, label="", ylims=(0, 70));
-plot!(dates[360:460], y[360:460], label="NIE", color="black", xlabel="Months", ylabel="GWmed", legend=:topright);
+plot(dates[401:460], forec.observation_scenarios, color="grey", w=0.05, label="", ylims=(0, 70))
+plot!(dates[360:460], y[360:460], label="NIE", color="black", xlabel="Months", ylabel="GWmed", legend=:topright)
 plot!(dates[401:460], forec.observation_quantiles, label=["Quantiles" "" ""], color="red", line=:dash)
-
-plot(dates[1:400], y_train, label = "in-sample NIE", ylabel = "GWmed", xlabel = "Months", color = "black")

src/ScoreDrivenModels.jl

@@ -20,7 +20,7 @@ include("simulate.jl")
 include("diagnostics.jl")
 include("univariate_score_driven_recursion.jl")
 include("MLE.jl")
-include("backtest.jl")
+include("cross_validation.jl")
 include("prints.jl")
 
 # Optimization methods
@@ -47,6 +47,6 @@ include("distributions/weibull.jl")
 
 include("visualization/forecast.jl")
 include("visualization/residuals.jl")
-include("visualization/backtest.jl")
+include("visualization/cross_validation.jl")
 
 end

src/backtest.jl renamed to src/cross_validation.jl

@@ -1,11 +1,11 @@
-export backtest
+export cross_validation
 
-struct Backtest
+struct CrossValidation
     abs_errors::Matrix{Float64}
     mae::Vector{Float64}
     crps_scores::Matrix{Float64}
     mean_crps::Vector{Float64}
-    function Backtest(n::Int, steps_ahead::Int)
+    function CrossValidation(n::Int, steps_ahead::Int)
         abs_errors = Matrix{Float64}(undef, steps_ahead, n)
         crps_scores = Matrix{Float64}(undef, steps_ahead, n)
         mae = Vector{Float64}(undef, steps_ahead)
@@ -38,14 +38,14 @@ end
 """
 TODO
 """
-function backtest(gas::Model{<:Distribution, T}, y::Vector{T}, steps_ahead::Int, start_idx::Int;
+function cross_validation(gas::Model{<:Distribution, T}, y::Vector{T}, steps_ahead::Int, start_idx::Int;
                   S::Int = 10_000,
                   initial_params::Matrix{T} = DEFAULT_INITIAL_PARAM,
                   opt_method = NelderMead(gas, DEFAULT_NUM_SEEDS)) where T
     num_mle = length(y) - start_idx - steps_ahead
-    b = Backtest(num_mle, steps_ahead)
+    b = CrossValidation(num_mle, steps_ahead)
     for i in 1:num_mle
-        println("Backtest: step $i of $num_mle")
+        println("CrossValidation: step $i of $num_mle")
         gas_to_fit = deepcopy(gas)
         y_to_fit = y[1:start_idx - 1 + i]
         y_to_verify = y[start_idx + i:start_idx - 1 + i + steps_ahead]

src/visualization/backtest.jl renamed to src/visualization/cross_validation.jl

@@ -1,4 +1,4 @@
-RecipesBase.@recipe function f(b::ScoreDrivenModels.Backtest, name::String)
+RecipesBase.@recipe function f(b::ScoreDrivenModels.CrossValidation, name::String)
     xguide := "lead times"
     @series begin
         seriestype := :path

test/runtests.jl

@@ -11,5 +11,5 @@ include("test_initial_params.jl")
 include("test_diagnostics.jl")
 include("test_estimate.jl")
 include("test_simulate.jl")
-include("test_backtest.jl")
+include("test_cross_validation.jl")
 include("test_visualization.jl")

test/test_backtest.jl renamed to test/test_cross_validation.jl

@@ -1,8 +1,8 @@
-@testset "Backtest" begin
+@testset "CrossValidation" begin
     ω = [0.1, 0.1]
     A = [0.5 0; 0 0.5]
     B = [0.5 0; 0 0.5]
     simulation = simulate_GAS_1_1(Normal, 0.0, ω, A, B, 1)
     gas = ScoreDrivenModels.Model(1, 1, Normal, 0.0)
-    bac = backtest(gas, simulation, 10, 4985)
+    bac = cross_validation(gas, simulation, 10, 4985)
 end

test/test_visualization.jl

@@ -13,7 +13,7 @@
     B = [0.5 0; 0 0.5]
     simulation = simulate_GAS_1_1(Normal, 0.0, ω, A, B, 1)
     gas = ScoreDrivenModels.Model(1, 1, Normal, 0.0)
-    bac = backtest(gas, simulation, 10, 4985)
+    bac = cross_validation(gas, simulation, 10, 4985)
     rec = RecipesBase.apply_recipe(Dict{Symbol, Any}(), bac, "name")
     @test length(rec) == 2