Skip to content

Commit 5930d2a

Browse files
guilhermebodinguilhermebodin
authored
Added some benchmarks to work on (#40)
* added some benchmarks to work on * added utils.jl file * remove LOAD_PATH * update benchmarks
1 parent bb1c3a5 commit 5930d2a

File tree

7 files changed

+207
-115
lines changed

7 files changed

+207
-115
lines changed

‎bench/benchmark_estimate.jl‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
using ScoreDrivenModels, Distributions, BenchmarkTools, Random, Test
2+
3+
include("test/utils.jl")
4+
5+
scaling = 0.0
6+
ω = [0.1, 0.1]
7+
A = [0.2 0; 0 0.2]
8+
B = [0.2 0; 0 0.2]
9+
simulation = simulate_GAS_1_1(Beta, scaling, ω, A, B, 1)
10+
verbose = 0
11+
p = 1
12+
q = 1
13+
num_seeds = 3
14+
@benchmark begin
15+
gas = GAS($p, $q, $Beta, $scaling)
16+
opt_method = ScoreDrivenModels.LBFGS(gas, $num_seeds)
17+
estimate!(gas, $simulation; verbose = $verbose, opt_method = opt_method)
18+
end
19+
# BenchmarkTools.Trial:
20+
# memory estimate: 7.33 GiB
21+
# allocs estimate: 95576498
22+
# --------------
23+
# minimum time: 5.632 s (15.93% GC)
24+
# median time: 5.632 s (15.93% GC)
25+
# mean time: 5.632 s (15.93% GC)
26+
# maximum time: 5.632 s (15.93% GC)
27+
# --------------
28+
# samples: 1
29+
# evals/sample: 1
30+
31+
scaling = 0.0
32+
ω = [0.5, 0.5]
33+
A = [0.1 0; 0 0.1]
34+
B = [0.1 0; 0 0.1]
35+
simulation = simulate_GAS_1_1(LogNormal, scaling, ω, A, B, 4)
36+
verbose = 0
37+
p = 1
38+
q = 1
39+
num_seeds = 3
40+
@benchmark begin
41+
gas = GAS($p, $q, $LogNormal, $scaling)
42+
opt_method = ScoreDrivenModels.LBFGS(gas, $num_seeds)
43+
estimate!(gas, $simulation; verbose = $verbose, opt_method = opt_method)
44+
end
45+
# BenchmarkTools.Trial:
46+
# memory estimate: 8.63 GiB
47+
# allocs estimate: 112481074
48+
# --------------
49+
# minimum time: 5.418 s (19.27% GC)
50+
# median time: 5.418 s (19.27% GC)
51+
# mean time: 5.418 s (19.27% GC)
52+
# maximum time: 5.418 s (19.27% GC)
53+
# --------------
54+
# samples: 1
55+
# evals/sample: 1
56+
57+
58+
scaling = 0.5
59+
@benchmark begin
60+
gas = GAS($p, $q, $LogNormal, $scaling)
61+
opt_method = ScoreDrivenModels.LBFGS(gas, $num_seeds)
62+
estimate!(gas, $simulation; verbose = $verbose, opt_method = opt_method)
63+
end
64+
# BenchmarkTools.Trial:
65+
# memory estimate: 17.39 GiB
66+
# allocs estimate: 325680001
67+
# --------------
68+
# minimum time: 9.760 s (23.28% GC)
69+
# median time: 9.760 s (23.28% GC)
70+
# mean time: 9.760 s (23.28% GC)
71+
# maximum time: 9.760 s (23.28% GC)
72+
# --------------
73+
# samples: 1
74+
# evals/sample: 1
75+
76+
scaling = 1.0
77+
@benchmark begin
78+
gas = GAS($p, $q, $LogNormal, $scaling)
79+
opt_method = ScoreDrivenModels.LBFGS(gas, $num_seeds)
80+
estimate!(gas, $simulation; verbose = $verbose, opt_method = opt_method)
81+
end
82+
# BenchmarkTools.Trial:
83+
# memory estimate: 11.99 GiB
84+
# allocs estimate: 172023821
85+
# --------------
86+
# minimum time: 8.502 s (20.62% GC)
87+
# median time: 8.502 s (20.62% GC)
88+
# mean time: 8.502 s (20.62% GC)
89+
# maximum time: 8.502 s (20.62% GC)
90+
# --------------
91+
# samples: 1
92+
# evals/sample: 1

‎test/runtests.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ using Test, Random, HypothesisTests
44

55
const SDM = ScoreDrivenModels
66

7+
include("utils.jl")
78
include("test_recursion.jl")
89
include("test_links.jl")
910
include("test_distributions.jl")

‎test/test_diagnostics.jl‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,3 @@
1-
function normality_quantile_and_pearson_residuals(D, n::Int, lags::Int; seed::Int = 11)
2-
Random.seed!(seed)
3-
4-
gas = GAS(1, 1, D, 0.0)
5-
gas.ω .= [0.0; 0.0]
6-
gas.A[1][[1; 4]] .= 0.2
7-
gas.B[1][[1; 4]] .= 0.2
8-
9-
y, params = simulate(gas, n)
10-
quant_res = quantile_residuals(y, gas, [params[1]])
11-
pearson = pearson_residuals(y, gas, [params[1]])
12-
13-
# quantile residuals
14-
jb = JarqueBeraTest(quant_res)
15-
@test pvalue(jb) >= 0.05
16-
Ljb = LjungBoxTest(quant_res, lags)
17-
@test pvalue(Ljb) >= 0.05
18-
19-
# pearson
20-
Ljb = LjungBoxTest(pearson, lags)
21-
@test pvalue(Ljb) >= 0.05
22-
return
23-
end
24-
251
@testset "quantile and pearson residuals" begin
262
n = 1000
273
lags = 1

‎test/test_distributions.jl‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,3 @@
1-
function test_score_mean(D::Type{<:Distribution}; n::Int = 10^7, seed::Int = 10,
2-
atol::Float64 = 1e-3, rtol::Float64 = 1e-3)
3-
Random.seed!(seed)
4-
dist = D()
5-
pars = [params(dist)...]
6-
avg = zeros(SDM.num_params(D))
7-
for i = 1:n
8-
avg += SDM.score(rand(D(pars...)), D, pars)
9-
end
10-
avg ./= n
11-
@test avg ≈ zeros(SDM.num_params(D)) atol = atol rtol = rtol
12-
end
13-
14-
function test_loglik(D::Type{<:Distribution}; atol::Float64 = 1e-3, rtol::Float64 = 1e-3,
15-
seed::Int = 13, n::Int = 100)
16-
Random.seed!(seed)
17-
dist = D()
18-
y = rand(dist, n)
19-
pars = [vcat(params(dist)...) for _ in 1:n]
20-
log_lik = SDM.log_likelihood(D, y, pars, n)
21-
@test log_lik ≈ -loglikelihood(dist, y) atol = atol rtol = rtol
22-
return
23-
end
24-
251
@testset "Score" begin
262
for dist in SDM.DISTS
273
@testset "$dist" begin

‎test/test_estimate.jl‎

Lines changed: 1 addition & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,3 @@
1-
function simulate_GAS_1_1(D::Type{<:Distribution}, scaling::Float64, ω::Vector{T}, A::Matrix{T},
2-
B::Matrix{T}, seed::Int) where T
3-
Random.seed!(seed)
4-
gas = GAS(1, 1, D, scaling)
5-
6-
gas.ω = ω
7-
gas.A[1] = A
8-
gas.B[1] = B
9-
series, param = simulate(gas, 5000)
10-
11-
return series
12-
end
13-
14-
function simulate_GAS_1_12(D::Type{<:Distribution}, scaling::Float64)
15-
Random.seed!(123)
16-
v = [0.1, 0.1]
17-
gas = GAS([1, 12], [1, 12], D, scaling)
18-
19-
gas.ω = v
20-
gas.A[1] = convert(Matrix{Float64}, Diagonal(3*v))
21-
gas.A[12] = convert(Matrix{Float64}, Diagonal(-3*v))
22-
gas.B[1] = convert(Matrix{Float64}, Diagonal(3*v))
23-
gas.B[12] = convert(Matrix{Float64}, Diagonal(-3*v))
24-
25-
series, param = simulate(gas, 5000)
26-
27-
return series
28-
end
29-
30-
function test_coefficients_GAS_1_1(gas::GAS{D, T}, ω::Vector{T}, A::Matrix{T}, B::Matrix{T};
31-
atol = 1e-1, rtol = 1e-1) where {D <: Distribution, T}
32-
@test gas.ω[1] ≈ ω[1] atol = atol rtol = rtol
33-
@test gas.ω[2] ≈ ω[2] atol = atol rtol = rtol
34-
@test gas.A[1][1, 1] ≈ A[1, 1] atol = atol rtol = rtol
35-
@test gas.A[1][2, 2] ≈ A[2, 2] atol = atol rtol = rtol
36-
@test gas.B[1][1, 1] ≈ B[1, 1] atol = atol rtol = rtol
37-
@test gas.B[1][2, 2] ≈ B[2, 2] atol = atol rtol = rtol
38-
return
39-
end
40-
41-
function test_coefficients_GAS_1_12(gas::GAS{D, T}; atol = 1e-1, rtol = 1e-1) where {D <: Distribution, T}
42-
@test gas.ω[1] ≈ 0.1 atol = atol rtol = rtol
43-
@test gas.ω[2] ≈ 0.1 atol = atol rtol = rtol
44-
@test gas.A[1][1, 1] ≈ 0.3 atol = atol rtol = rtol
45-
@test gas.A[1][2, 2] ≈ 0.3 atol = atol rtol = rtol
46-
@test gas.A[12][1, 1] ≈ -0.3 atol = atol rtol = rtol
47-
@test gas.A[12][2, 2] ≈ -0.3 atol = atol rtol = rtol
48-
@test gas.B[1][1, 1] ≈ 0.3 atol = atol rtol = rtol
49-
@test gas.B[1][2, 2] ≈ 0.3 atol = atol rtol = rtol
50-
@test gas.B[12][1, 1] ≈ -0.3 atol = atol rtol = rtol
51-
@test gas.B[12][2, 2] ≈ -0.3 atol = atol rtol = rtol
52-
return
53-
end
54-
551
@testset "Estimate" begin
562
@testset "Beta" begin
573
ω = [0.1, 0.1]
@@ -100,7 +46,7 @@ end
10046
test_coefficients_GAS_1_1(gas, ω, A, B)
10147
end
10248
@testset "GAS([1, 12], [1, 12])" begin
103-
simulation = simulate_GAS_1_12(LogNormal, 0.0)
49+
simulation = simulate_GAS_1_12(LogNormal, 0.0, 123)
10450
gas = GAS([1, 12], [1, 12], LogNormal, 0.0)
10551
estimate!(gas, simulation; verbose = 1, opt_method = ScoreDrivenModels.LBFGS(gas, 3))
10652
test_coefficients_GAS_1_12(gas)

‎test/test_initial_params.jl‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,3 @@
1-
function test_dynamic(sigma::Float64, lags::Int; seed::Int = 12, atol::Float64 = 1e-1, rtol::Float64 = 1e-3, n::Int = 100)
2-
Random.seed!(seed)
3-
dist = Normal(0, sigma^2)
4-
obs_dynamic = kron(ones(n), collect(1:lags)) + rand(dist, n*lags)
5-
gas_lag_lag = GAS(lags, lags, Normal, 0.0)
6-
initial_params = dynamic_initial_params(obs_dynamic, gas_lag_lag)
7-
for i in eachindex(initial_params)
8-
@test initial_params[i][1] ≈ i atol = atol rtol = rtol
9-
@test initial_params[i][2] ≈ sigma atol = atol rtol = rtol
10-
end
11-
end
12-
131
@testset "Inital params" begin
142
@testset "stationary initial params" begin
153
gas_1_1 = GAS(1, 1, Normal, 0.0)

‎test/utils.jl‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
function test_score_mean(D::Type{<:Distribution}; n::Int = 10^7, seed::Int = 10,
2+
atol::Float64 = 1e-3, rtol::Float64 = 1e-3)
3+
Random.seed!(seed)
4+
dist = D()
5+
pars = [params(dist)...]
6+
avg = zeros(SDM.num_params(D))
7+
for i = 1:n
8+
avg += SDM.score(rand(D(pars...)), D, pars)
9+
end
10+
avg ./= n
11+
@test avg ≈ zeros(SDM.num_params(D)) atol = atol rtol = rtol
12+
end
13+
14+
function test_loglik(D::Type{<:Distribution}; atol::Float64 = 1e-3, rtol::Float64 = 1e-3,
15+
seed::Int = 13, n::Int = 100)
16+
Random.seed!(seed)
17+
dist = D()
18+
y = rand(dist, n)
19+
pars = [vcat(params(dist)...) for _ in 1:n]
20+
log_lik = SDM.log_likelihood(D, y, pars, n)
21+
@test log_lik ≈ -loglikelihood(dist, y) atol = atol rtol = rtol
22+
return
23+
end
24+
25+
function test_dynamic(sigma::Float64, lags::Int; seed::Int = 12, atol::Float64 = 1e-1, rtol::Float64 = 1e-3, n::Int = 100)
26+
Random.seed!(seed)
27+
dist = Normal(0, sigma^2)
28+
obs_dynamic = kron(ones(n), collect(1:lags)) + rand(dist, n*lags)
29+
gas_lag_lag = GAS(lags, lags, Normal, 0.0)
30+
initial_params = dynamic_initial_params(obs_dynamic, gas_lag_lag)
31+
for i in eachindex(initial_params)
32+
@test initial_params[i][1] ≈ i atol = atol rtol = rtol
33+
@test initial_params[i][2] ≈ sigma atol = atol rtol = rtol
34+
end
35+
end
36+
37+
function simulate_GAS_1_1(D::Type{<:Distribution}, scaling::Float64, ω::Vector{T}, A::Matrix{T},
38+
B::Matrix{T}, seed::Int) where T
39+
Random.seed!(seed)
40+
gas = GAS(1, 1, D, scaling)
41+
42+
gas.ω = ω
43+
gas.A[1] = A
44+
gas.B[1] = B
45+
series, param = simulate(gas, 5000)
46+
47+
return series
48+
end
49+
50+
function simulate_GAS_1_12(D::Type{<:Distribution}, scaling::Float64, seed::Int)
51+
Random.seed!(seed)
52+
v = [0.1, 0.1]
53+
gas = GAS([1, 12], [1, 12], D, scaling)
54+
55+
gas.ω = v
56+
gas.A[1] = convert(Matrix{Float64}, Diagonal(3*v))
57+
gas.A[12] = convert(Matrix{Float64}, Diagonal(-3*v))
58+
gas.B[1] = convert(Matrix{Float64}, Diagonal(3*v))
59+
gas.B[12] = convert(Matrix{Float64}, Diagonal(-3*v))
60+
61+
series, param = simulate(gas, 5000)
62+
63+
return series
64+
end
65+
66+
function test_coefficients_GAS_1_1(gas::GAS{D, T}, ω::Vector{T}, A::Matrix{T}, B::Matrix{T};
67+
atol = 1e-1, rtol = 1e-1) where {D <: Distribution, T}
68+
@test gas.ω[1] ≈ ω[1] atol = atol rtol = rtol
69+
@test gas.ω[2] ≈ ω[2] atol = atol rtol = rtol
70+
@test gas.A[1][1, 1] ≈ A[1, 1] atol = atol rtol = rtol
71+
@test gas.A[1][2, 2] ≈ A[2, 2] atol = atol rtol = rtol
72+
@test gas.B[1][1, 1] ≈ B[1, 1] atol = atol rtol = rtol
73+
@test gas.B[1][2, 2] ≈ B[2, 2] atol = atol rtol = rtol
74+
return
75+
end
76+
77+
function test_coefficients_GAS_1_12(gas::GAS{D, T}; atol = 1e-1, rtol = 1e-1) where {D <: Distribution, T}
78+
@test gas.ω[1] ≈ 0.1 atol = atol rtol = rtol
79+
@test gas.ω[2] ≈ 0.1 atol = atol rtol = rtol
80+
@test gas.A[1][1, 1] ≈ 0.3 atol = atol rtol = rtol
81+
@test gas.A[1][2, 2] ≈ 0.3 atol = atol rtol = rtol
82+
@test gas.A[12][1, 1] ≈ -0.3 atol = atol rtol = rtol
83+
@test gas.A[12][2, 2] ≈ -0.3 atol = atol rtol = rtol
84+
@test gas.B[1][1, 1] ≈ 0.3 atol = atol rtol = rtol
85+
@test gas.B[1][2, 2] ≈ 0.3 atol = atol rtol = rtol
86+
@test gas.B[12][1, 1] ≈ -0.3 atol = atol rtol = rtol
87+
@test gas.B[12][2, 2] ≈ -0.3 atol = atol rtol = rtol
88+
return
89+
end
90+
91+
function normality_quantile_and_pearson_residuals(D, n::Int, lags::Int; seed::Int = 11)
92+
Random.seed!(seed)
93+
94+
gas = GAS(1, 1, D, 0.0)
95+
gas.ω .= [0.0; 0.0]
96+
gas.A[1][[1; 4]] .= 0.2
97+
gas.B[1][[1; 4]] .= 0.2
98+
99+
y, params = simulate(gas, n)
100+
quant_res = quantile_residuals(y, gas, [params[1]])
101+
pearson = pearson_residuals(y, gas, [params[1]])
102+
103+
# quantile residuals
104+
jb = JarqueBeraTest(quant_res)
105+
@test pvalue(jb) >= 0.05
106+
Ljb = LjungBoxTest(quant_res, lags)
107+
@test pvalue(Ljb) >= 0.05
108+
109+
# pearson
110+
Ljb = LjungBoxTest(pearson, lags)
111+
@test pvalue(Ljb) >= 0.05
112+
return
113+
end

0 commit comments

Comments
 (0)