Merge pull request #112 from jbrea/lasso

fix scale_penalty_with_samples=true for proxgrad

Author: ablaom

src/fit/proxgrad.jl

@@ -21,7 +21,7 @@ function _fit(glr::GLR, solver::ProxGrad, X, y, scratch)
     # functions
     _f      = smooth_objective(glr, X, y; c=c)
     _fg!    = smooth_fg!(glr, X, y, scratch)
-    _prox!  = prox!(glr)
+    _prox!  = prox!(glr, size(X, 1))
     bt_cond = θ̂ ->
                 _f(θ̂) > fθ̄ + dot(θ̂ .- θ̄, ∇fθ̄) + sum(abs2.(θ̂ .- θ̄)) / (2η)
     # loop-related

src/glr/prox.jl

@@ -15,8 +15,8 @@
 # prox_{αr}(z) = sign(z)(abs(z) - αλ)₊
 # ------------------------------------
 
-function prox!(glr::GLR{<:Loss,<:Union{L1R,CompositePenalty}})
-    γ = getscale_l1(glr.penalty)
+function prox!(glr::GLR{<:Loss,<:Union{L1R,CompositePenalty}}, n)
+    γ = get_penalty_scale_l1(glr, n)
     (p, α, z) -> begin
         p .= soft_thresh.(z, α * γ)
         glr.fit_intercept && (glr.penalize_intercept || (p[end] = z[end]))

src/loss-penalty/utils.jl

@@ -21,3 +21,4 @@ getscale_l2(cp::CompositePenalty) = is_elnet(cp) ? cp |> get_l2 |> getscale :
 
 get_penalty_scale(glr, n) = getscale(glr.penalty) * ifelse(glr.scale_penalty_with_samples, float(n), 1.0)
 get_penalty_scale_l2(glr, n) = getscale_l2(glr.penalty) * ifelse(glr.scale_penalty_with_samples, float(n), 1.0)
+get_penalty_scale_l1(glr, n) = getscale_l1(glr.penalty) * ifelse(glr.scale_penalty_with_samples, float(n), 1.0)

test/fit/ols-ridge-lasso-elnet.jl

@@ -56,6 +56,12 @@ n, p = 500, 100
     @test nnz(θ_fista) == 12 # sparse
     @test nnz(θ_ista)  == 12
 
+    # scale_penalty_with_samples
+    lr_scaled = LassoRegression(λ/n; fit_intercept=false,
+                                     scale_penalty_with_samples = true)
+    θ_scaled = fit(lr_scaled, X, y)
+    @test θ_scaled ≈ θ_fista
+
     # with intercept
     lr1 = LassoRegression(λ, penalize_intercept=true,
                              scale_penalty_with_samples = false)