Skip to content

Commit a9b3333

Browse files
theogftheogf
committed
Bugs corrections
1 parent ac40268 commit a9b3333

File tree

10 files changed

+247
-25
lines changed

10 files changed

+247
-25
lines changed

‎dev/test_macro.jl‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
using AugmentedGaussianProcesses; const AGP = AugmentedGaussianProcesses
2+
using LinearAlgebra, Distributions, Plots
3+
using BenchmarkTools
4+
b = 2.0
5+
C()=1/(2b)
6+
g(y) = 0.0
7+
α(y) = y^2
8+
β(y) = 2*y
9+
γ(y) = 1.0
10+
φ(r) = exp(-sqrt(r)/b)
11+
∇φ(r) = -exp(-sqrt(r)/b)/(2*b*sqrt(r))
12+
ll(y,x) = 0.5*exp(0.5*y*x)*sech(0.5*sqrt(x^2))
13+
14+
##
15+
formula = :(p(y|x)=exp(0.5*y*x)*sech(0.5*sqrt(y^2 - 2*y*x + x^2)))
16+
# formula = :(p(y,x)=exp(0.5*y*x)*sech(0.5*sqrt(0.0 - 0.0*x + x^2)))
17+
formula.args[2].args[2].args
18+
19+
topargs = formula.args[2].args[2].args
20+
if topargs[1] == :*
21+
@show topargs[1]
22+
global CC = copy(topargs[2])
23+
popfirst!(topargs)
24+
popfirst!(topargs)
25+
else
26+
global CC = :0
27+
end
28+
args2 = topargs[1]
29+
if args2.args[1] == :exp
30+
gargs = args2.args[2]
31+
if gargs.args[1] == :*
32+
deleteat!(gargs.args,findfirst(isequal(:x),gargs.args))
33+
else
34+
@error "BAD BAD BAD"
35+
end
36+
global GG = copy(gargs)
37+
popfirst!(topargs)
38+
else
39+
global GG = :0
40+
end
41+
args3 = topargs[1]
42+
seq = string(args3)
43+
findh= r"\([^(]*\-.*x.*\+.*x \^ 2[^)]*"
44+
b = occursin(findh,seq)
45+
m = match(findh,seq).match
46+
alphar = r"[^(][^-]*"
47+
malpha = match(alphar,m).match[1:end-1]
48+
betar = r"- [^x]*x"
49+
mbeta = match(betar,m).match[3:end]
50+
gammar = r"\+ [^x]*x \^ 2"
51+
mgamma = match(gammar,m).match[3:end]
52+
53+
AA = :($malpha)
54+
BB = :($(mbeta[1:end-1]))
55+
GG = :($(mgamma == "x ^ 2" ? "1.0" : mgamma[1:end-5]))
56+
57+
loc = findfirst(findh,seq)
58+
newseq = seq[1:loc[1]-1]*"r"*seq[(loc[end]+1):end]
59+
PHI = :($newseq)
60+
##
61+
f_lap = :(p(y|x)=0.5/β * exp(- sqrt((y^2 - 2*y*f + f^2))/β))
62+
display.(AGP.@augmodel NewSuperLaplace Regression (p(y|x)=0.5/β * exp( y * x) * exp(- sqrt((sqrt(y^2) - exp(4.0*y)*x + 2.0*x^2))/β)) β)
63+
pdfstring = "(0.5 / β) * exp(2*y*x) * exp(-(sqrt((y ^ 2 - 2.0 * y * x) + 1.0*x ^ 2)) / β)"
64+
65+
Gstring
66+
##
67+
PhiHstring = match(Regex("(?<=$(AGP.correct_parenthesis(Gstringfull))x\\) \\* ).*"),pdfstring).match
68+
Hstring = match(r"(?<=\().+\-.*x.*\+.+x \^ 2(?=\))",PhiHstring).match
69+
locx = findfirst("x ^ 2",PhiHstring)
70+
count_parenthesis = 1
71+
locf = locx[1]
72+
while count_parenthesis != 0
73+
global locf = locf - 1
74+
println(PhiHstring[locf])
75+
if PhiHstring[locf] == ')'
76+
global count_parenthesis += 1
77+
elseif PhiHstring[locf] == '('
78+
global count_parenthesis -= 1
79+
end
80+
end
81+
Hstring = PhiHstring[(locf+1):locx[end]]
82+
83+
alphar = r"[^(][^-]*"
84+
alpha_string = match(alphar,Hstring).match[1:end-1]
85+
# betar = r"(?>=- )[^x]+(?= * x)"
86+
betar = r"(?<=- )[^x]+(?= * x)"
87+
mbeta = match(betar,Hstring).match
88+
while last(mbeta) == ' ' || last(mbeta) == '*'
89+
global mbeta = mbeta[1:end-1]
90+
end
91+
mbeta
92+
gammar = r"(?<=\+ )[^x]*(?=x \^ 2)"
93+
mgamma = match(gammar,m).match == "" ? "1.0" : match(gammar,m).match
94+
##
95+
findnext(isequal(')'),PhiHstring,locx[end])
96+
code = Meta.parse(PhiHstring)
97+
code.args
98+
99+
S = code.args[2].args[2].args[2].args[2].args[2].args
100+
S = code.args[2].args[2].args[2].args[2].args
101+
for args in S.args
102+
if args == :(x ^ 2)
103+
@show "BLAH"
104+
end
105+
end
106+
S = string(code.args[2].args[2])
107+
Hstring = match(r"(?<=\().*x \^ 2.*\-.*x.*\+.*(?=\))",S)
108+
109+
##
110+
111+
txt = AGP.@augmodel("NewLaplace","Regression",C,g,α,β,γ,φ,∇φ)
112+
113+
# NewLaplaceLikelihood() |> display
114+
N = 500
115+
σ = 1.0
116+
X = sort(rand(N,1),dims=1)
117+
K = kernelmatrix(X,RBFKernel(0.1))+1e-4*I
118+
L = Matrix(cholesky(K).L)
119+
y_true = rand(MvNormal(K))
120+
y = y_true+randn(length(y_true))*2
121+
p = scatter(X[:],y,lab="data")
122+
NewLaplaceLikelihood() |> display
123+
m = VGP(X,y,RBFKernel(0.5),NewLaplaceLikelihood(),AnalyticVI(),optimizer=false)
124+
train!(m,iterations=100)
125+
y_p, sig_p = proba_y(m,collect(0:0.01:1))
126+
127+
m2 = VGP(X,y,RBFKernel(0.5),LaplaceLikelihood(b),AnalyticVI(),optimizer=false)
128+
train!(m2,iterations=100)
129+
y_p2, sig_p2 = proba_y(m2,collect(0:0.01:1))
130+
131+
plot!(X,y_true,lab="truth")
132+
133+
plot!(collect(0:0.01:1),y_p,lab="Auto Laplace")
134+
plot!(collect(0:0.01:1),y_p2,lab="Classic Laplace") |> display
135+
136+
137+
# @btime train!($m,iterations=1)
138+
# @btime train!($m2,iterations=1)
139+
140+
###

‎dev/test_vstp.jl‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
using Plots
2+
using AugmentedGaussianProcesses
3+
using LinearAlgebra, Distributions
4+
5+
N = 10; l = 0.1
6+
N_grid = 500
7+
Xgrid = collect(range(-0.2,1.2,length=N_grid))
8+
X = rand(N,1)
9+
mse(y,y_pred) = norm(y-y_pred)
10+
ll(y,y_pred) =
11+
12+
K = kernelmatrix(X,RBFKernel(l))
13+
μ_0 = 0.0*ones(N)# sin.(X[:]).+1.0
14+
s = sortperm(X[:])
15+
y_true= rand(MvNormal(μ_0,Symmetric(K+1e-1I)))
16+
y = y_true + rand(TDist(3.0),N)
17+
plot(X[s],y[s])
18+
pk = plot()
19+
function cplot(model,iter)
20+
global pk
21+
p_y,sig_y = proba_y(model,Xgrid)
22+
p = scatter(X[s],y[s],lab="data")
23+
plot!(Xgrid,p_y,lab="Prediction")
24+
p = plot!(Xgrid,p_y+2*sqrt.(sig_y),fill=p_y-2*sqrt.(sig_y),lab="",linewidth=0.0,alpha=0.2)
25+
pk = scatter!(pk,[getlengthscales(model.kernel[1])],[getvariance(model.kernel[1])],xlabel="Lengthscale",ylabel="Variance",lab="",xlims=(1e-3,1.0),ylims=(0.1,2.0),xaxis=:log)
26+
display(plot(p,pk))
27+
end
28+
29+
model = VStP(X,y,RBFKernel(0.1),GaussianLikelihood(0.01),AnalyticVI(),100.0,verbose=0,optimizer=true)
30+
train!(model,iterations=500)#,callback=cplot)
31+
p_y = predict_y(model,Xgrid)
32+
# plot!(Xgrid,p_y,lab="")
33+
# plot!(X[s],model.μ₀[1][s],lab="")
34+
gpmodel = GP(X,y,RBFKernel(0.1),noise=0.01,verbose=0,optimizer=true)
35+
train!(gpmodel,iterations=500)#,callback=cplot)
36+
p_y = predict_y(model,Xgrid)
37+
# plot!(Xgrid,p_y,lab="")
38+
cplot(model,1)
39+
cplot(gpmodel,1)
40+
41+
##
42+
p_y,sig_y = proba_y(model,Xgrid)
43+
p = scatter(X[s],y[s],lab="data")
44+
p = plot!(Xgrid,p_y+2*sqrt.(sig_y),fill=p_y-2*sqrt.(sig_y),lab="",linewidth=0.0,alpha=0.2,color=1)
45+
p = plot!(Xgrid,p_y,lab="Prediction T Process",color=1)
46+
47+
p_ygp,sig_ygp = proba_y(gpmodel,Xgrid)
48+
p = plot!(Xgrid,p_ygp+2*sqrt.(sig_ygp),fill=p_ygp-2*sqrt.(sig_ygp),lab="",linewidth=0.0,alpha=0.2,color=2)
49+
p = plot!(Xgrid,p_ygp,lab="Prediction G Process",color=2)
50+
51+
pk = scatter([getlengthscales(model.kernel[1])],[getvariance(model.kernel[1])],xlabel="Lengthscale",ylabel="Variance",lab="Student-T Process",xlims=(1e-3,1.0),ylims=(0.1,20.0),yaxis=:log,xaxis=:log)
52+
scatter!(pk,[getlengthscales(gpmodel.kernel[1])],[getvariance(gpmodel.kernel[1])],lab="Gaussian Process",legend=:bottom)
53+
54+
plot(p,pk)

‎src/inference/MCVI.jl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ mutable struct MCIntegrationVI{T<:Real} <: NumericalVI{T}
1313
∇η₂::AbstractVector{AbstractArray}
1414
ν::AbstractVector{AbstractVector}
1515
λ::AbstractVector{AbstractVector}
16-
x::SubArray{T,2,Matrix{T},Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
16+
x::SubArray{T,2,Matrix{T}}#,Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
1717
y::LatentArray{SubArray}
1818
function MCIntegrationVI{T}(ϵ::T,nMC::Integer,nIter::Integer,optimizer::Opt,Stochastic::Bool,nSamplesUsed::Integer=1) where {T<:Real,Opt<:Optimizer}
1919
return new{T}(ϵ,nIter,[optimizer],[optimizer],nMC,Stochastic,1,nSamplesUsed)

‎src/inference/gibbssampling.jl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ mutable struct GibbsSampling{T<:Real} <: Inference{T}
2424
ρ::T #Stochastic Coefficient
2525
HyperParametersUpdated::Bool #To know if the inverse kernel matrix must updated
2626
sample_store::AbstractVector{AbstractVector{AbstractVector{T}}}
27-
x::SubArray{T,2,Matrix{T},Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
27+
x::SubArray{T,2,Matrix{T}}#,Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
2828
y::LatentArray{SubArray}
2929
function GibbsSampling{T}(nBurnin::Int,samplefrequency::Int,ϵ::T,nIter::Integer,Stochastic::Bool,nSamples::Integer,nSamplesUsed::Integer,MBIndices::AbstractVector,ρ::T,flag::Bool) where T
3030
return new{T}(nBurnin,samplefrequency,ϵ,nIter,Stochastic,nSamples,nSamplesUsed,MBIndices,ρ,flag)

‎src/inference/quadratureVI.jl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ mutable struct QuadratureVI{T<:Real} <: NumericalVI{T}
3131
∇η₂::LatentArray{Symmetric{T,Matrix{T}}}
3232
ν::LatentArray{Vector{T}} #Derivative -<dv/dx>_qn
3333
λ::LatentArray{Vector{T}} #Derivative <d²V/dx²>_qm
34-
x::SubArray{T,2,Matrix{T},Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
34+
x::SubArray{T,2,Matrix{T}}#,Tuple{UnitRange{Int64},Base.Slice{Base.OneTo{Int64}}},false}#SubArray{T,2,Matrix{T},Tuple{Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true}
3535
y::LatentArray{SubArray}
3636
function QuadratureVI{T}(ϵ::T,nPoints::Integer,nIter::Integer,optimizer::Optimizer,Stochastic::Bool,clipping::Real,nSamplesUsed::Integer=1) where T
3737
gh = gausshermite(nPoints)
@@ -103,8 +103,8 @@ function compute_grad_expectations!(model::SVGP{T,L,<:QuadratureVI}) where {T,L}
103103
k_correct = model.nLatent == 1 ? 1 : k
104104
μ = model.κ[k_correct]*model.μ[k]
105105
Σ = opt_diag(model.κ[k_correct]*model.Σ[k],model.κ[k_correct])
106-
for i in 1:model.nSample
107-
model.inference.ν[k][i], model.inference.λ[k][i] = grad_quad(model.likelihood, model.y[k][i], μ[i], Σ[i], model.inference)
106+
for i in 1:model.inference.nSamplesUsed
107+
model.inference.ν[k][i], model.inference.λ[k][i] = grad_quad(model.likelihood, model.inference.y[k][i], μ[i], Σ[i], model.inference)
108108
end
109109
end
110110
end

‎src/likelihood/bayesiansvm.jl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,17 @@ end
5555
function compute_proba(l::BayesianSVM{T},μ::Vector{T},σ²::Vector{T}) where {T<:Real}
5656
N = length(μ)
5757
pred = zeros(T,N)
58+
sig_pred = zeros(T,N)
5859
for i in 1:N
5960
if σ²[i] <= 0.0
6061
pred[i] = svmlikelihood(μ[i])
6162
else
6263
nodes = pred_nodes.*sqrt2.*sqrt.(σ²[i]).+μ[i]
6364
pred[i] = dot(pred_weights,svmlikelihood.(nodes))
65+
sig_pred[i] = dot(pred_weights,svmlikelihood.(nodes).^2)-pred[i]^2
6466
end
6567
end
66-
return pred
68+
return pred, sig_pred
6769
end
6870

6971
###############################################################################

‎src/likelihood/logistic.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ function compute_proba(l::LogisticLikelihood{T},μ::AbstractVector{T},σ²::Abst
5757
if σ²[i] <= 0.0
5858
pred[i] = logistic(μ[i])
5959
else
60+
nodes = pred_nodes.*sqrt2.*sqrt.(σ²[i]).+μ[i]
6061
pred[i] = dot(pred_weights,logistic.(nodes))
6162
sig_pred[i] = dot(pred_weights,logistic.(nodes).^2)-pred[i]^2
6263
nodes = pred_nodes.*sqrt2.*sqrt.(σ²[i]).+μ[i]

‎src/models/SVGP.jl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ function SVGP(X::AbstractArray{T1},y::AbstractArray{T2},kernel::Union{Kernel,Abs
112112

113113
likelihood = init_likelihood(likelihood,inference,nLatent,nSamplesUsed,nFeatures)
114114
inference = init_inference(inference,nLatent,nFeatures,nSample,nSamplesUsed)
115-
inference.x = view(X,:,:)
115+
inference.x = view(X,1:nSample,:)
116116
inference.y = view.(y,:)
117117

118118
model = SVGP{T1,TLikelihood,TInference,ArrayType{T1}}(X,y,

‎src/predictions.jl‎

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -149,43 +149,67 @@ function proba_y(model::AbstractGP,X_test::AbstractMatrix)
149149
compute_proba(model.likelihood,μ_f,Σ_f)
150150
end
151151

152-
function proba_y(model::VGP{T,<:MultiClassLikelihood{T},<:GibbsSampling{T}},X_test::AbstractMatrix{T};nSamples::Int=200) where {T}
152+
function proba_y(model::VGP{T,<:Union{<:RegressionLikelihood{T},<:ClassificationLikelihood{T}},<:GibbsSampling},X_test::AbstractMatrix{T};nSamples::Int=200) where {T<:Real}
153+
N_test = size(X_test,1)
153154
k_star = kernelmatrix.([X_test],[model.X],model.kernel)
154155
f = [[k_star[min(k,model.nPrior)]*model.invKnn[min(k,model.nPrior)]].*model.inference.sample_store[k] for k in 1:model.nLatent]
155156
k_starstar = kerneldiagmatrix.([X_test],model.kernel)
156157
K̃ = k_starstar .- opt_diag.(k_star.*model.invKnn,k_star) .+ [zeros(size(X_test,1)) for i in 1:model.nLatent]
157158
nf = length(model.inference.sample_store[1])
158-
proba = zeros(size(X_test,1),model.nLatent)
159-
labels = Array{Symbol}(undef,model.nLatent)
159+
proba = [zeros(size(X_test,1)) for i in 1:model.nLatent]
160+
sig_proba = [zeros(size(X_test,1)) for i in 1:model.nLatent]
160161
for i in 1:nf
161-
res = compute_proba(model.likelihood,getindex.(f,[i]),K̃,nSamples)
162-
if i == 1
163-
labels = names(res)
162+
for k in 1:model.nLatent
163+
proba[k], sig_proba[k] = (proba[k],sig_proba[k]) .+ compute_proba(model.likelihood, getindex.(f,[i])[k],K̃[k])
164164
end
165-
proba .+= Matrix(res)
166165
end
167-
return DataFrame(proba/nf,labels)
166+
if model.nLatent == 1
167+
return (proba[1]/nf, sig_proba[1]/nf)
168+
else
169+
return (proba./nf, sig_proba./nf)
170+
end
168171
end
169172

170-
function proba_y(model::VGP{T,<:ClassificationLikelihood,<:GibbsSampling},X_test::AbstractMatrix{T};nSamples::Int=200) where {T<:Real}
173+
function proba_y(model::VGP{T,<:MultiClassLikelihood{T},<:GibbsSampling{T}},X_test::AbstractMatrix{T};nSamples::Int=200) where {T}
171174
k_star = kernelmatrix.([X_test],[model.X],model.kernel)
172175
f = [[k_star[min(k,model.nPrior)]*model.invKnn[min(k,model.nPrior)]].*model.inference.sample_store[k] for k in 1:model.nLatent]
173176
k_starstar = kerneldiagmatrix.([X_test],model.kernel)
174177
K̃ = k_starstar .- opt_diag.(k_star.*model.invKnn,k_star) .+ [zeros(size(X_test,1)) for i in 1:model.nLatent]
175178
nf = length(model.inference.sample_store[1])
176-
proba = [zeros(size(X_test,1)) for i in 1:model.nLatent]
179+
proba = zeros(size(X_test,1),model.nLatent)
180+
labels = Array{Symbol}(undef,model.nLatent)
177181
for i in 1:nf
178-
proba .+= compute_proba(model.likelihood,getindex.(f,[i]),K̃)
179-
end
180-
if model.nLatent == 1
181-
return proba[1]/nf
182-
else
183-
return proba./nf
182+
res = compute_proba(model.likelihood,getindex.(f,[i]),K̃,nSamples)
183+
if i == 1
184+
labels = names(res)
185+
end
186+
proba .+= Matrix(res)
184187
end
188+
return DataFrame(proba/nf,labels)
185189
end
190+
#
191+
# function proba_y(model::VGP{T,<:ClassificationLikelihood,<:GibbsSampling},X_test::AbstractMatrix{T};nSamples::Int=200) where {T<:Real}
192+
# k_star = kernelmatrix.([X_test],[model.X],model.kernel)
193+
# f = [[k_star[min(k,model.nPrior)]*model.invKnn[min(k,model.nPrior)]].*model.inference.sample_store[k] for k in 1:model.nLatent]
194+
# k_starstar = kerneldiagmatrix.([X_test],model.kernel)
195+
# K̃ = k_starstar .- opt_diag.(k_star.*model.invKnn,k_star) .+ [zeros(size(X_test,1)) for i in 1:model.nLatent]
196+
# nf = length(model.inference.sample_store[1])
197+
# proba = [zeros(size(X_test,1)) for i in 1:model.nLatent]
198+
# sig_proba = [zeros(size(X_test,1)) for i in 1:model.nLatent]
199+
# for i in 1:nf
200+
# prob, sig_prob = compute_proba(model.likelihood,getindex.(f,[i]),K̃)
201+
# proba .+= prob
202+
# sig_proba .+= sig_prob
203+
# end
204+
# if model.nLatent == 1
205+
# return (proba[1]/nf, sig_proba[1]/nf)
206+
# else
207+
# return (proba./nf, sig_proba./nf)
208+
# end
209+
# end
186210

187211
function compute_proba(l::Likelihood{T},μ::AbstractVector{<:AbstractVector{T}},σ²::AbstractVector{<:AbstractVector{T}}) where {T<:Real}
188-
compute_proba.([l],μ,σ²)
212+
compute_proba.(l,μ,σ²)
189213
end
190214

191215
### TODO Think about a better solution (general multi-likelihood problem)

‎test/testingtools.jl‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ methods_implemented["PoissonLikelihood"] = ["AnalyticVI","AnalyticSVI"]
1111

1212
methods_implemented_VGP = deepcopy(methods_implemented)
1313
push!(methods_implemented_VGP["StudentTLikelihood"],"GibbsSampling")
14+
push!(methods_implemented_VGP["LaplaceLikelihood"],"GibbsSampling")
1415
push!(methods_implemented_VGP["LogisticLikelihood"],"GibbsSampling")
1516
push!(methods_implemented_VGP["LogisticSoftMaxLikelihood"],"GibbsSampling")
1617
push!(methods_implemented_VGP["HeteroscedasticLikelihood"],"AnalyticVI")
@@ -35,7 +36,7 @@ function testconv(model::AbstractGP,problem_type::String,X::AbstractArray,y::Abs
3536
if problem_type == "Regression"
3637
err=mean(abs.(y_pred-y))
3738
@info "Regression Error" err
38-
return err < 0.8
39+
return err < 1.5
3940
elseif problem_type == "Classification"
4041
err=mean(y_pred.!=y)
4142
@info "Classification Error" err

0 commit comments

Comments
 (0)