Merge branch 'main' of github.com:scikit-learn-contrib/skglm into internalize-jit-compilation

mathurinm · mathurinm · commit b6c39e30fe6c · 2025-04-15T11:35:57.000+02:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
 jobs:
     build_docs:
       docker:
-        - image: circleci/python:3.8.1-buster
+        - image: cimg/python:3.10
       steps:
         - checkout
         - run:
@@ -32,6 +32,13 @@ jobs:
             keys:
               - pip-cache
 
+        # Install Xvfb and related dependencies
+        - run:
+            name: Install Xvfb and dependencies
+            command: |
+              sudo apt-get update
+              sudo apt-get install -y xvfb
+
         - run:
             name: Spin up Xvfb
             command: |
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -15,5 +15,5 @@ Any detail to be able to relate the PR changes
 ### Checks before merging PR
 
 - [ ] added documentation for any new feature
-- [ ] added unittests
-- [ ] edited the [what's new](../doc/changes/whats_new.rst)(if applicable)
+- [ ] added unit tests
+- [ ] edited the [what's new](../doc/changes/whats_new.rst) (if applicable)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -13,20 +13,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: 3.8
+        python-version: "3.10"
     - name: Install package and testing tools
       run: |
         python -m pip install --upgrade pip
         pip install .
         pip install .[test]
     - name: Install other dependencies
       run: |
-        pip install celer
+        conda install rpy2 -c conda-forge
         pip install statsmodels cvxopt
-        pip install git+https://github.com/jolars/pyslope.git
+        pip install sortedl1
         # for testing Cox estimator
         pip install lifelines
         pip install pandas
diff --git a/README.md b/README.md
@@ -2,6 +2,7 @@
 
 <img src="doc/_static/images/logo.svg" alt="skglm logo" width="34%" align="center"/>
 
+
 ## A fast ⚡ and modular ⚒️ scikit-learn replacement for sparse GLMs
 
 ![build](https://github.com/scikit-learn-contrib/skglm/workflows/pytest/badge.svg)
@@ -19,6 +20,24 @@ You get to choose from ``skglm``'s already-made estimators or **customize your o
 
 Excited to have a tour on ``skglm`` [documentation](https://contrib.scikit-learn.org/skglm/)?
 
+# Cite
+
+``skglm`` is the result of perseverant research. It is licensed under [BSD 3-Clause](https://github.com/scikit-learn-contrib/skglm/blob/main/LICENSE). You are free to use it and if you do so, please cite
+
+```bibtex
+@inproceedings{skglm,
+    title     = {Beyond L1: Faster and better sparse models with skglm},
+    author    = {Q. Bertrand and Q. Klopfenstein and P.-A. Bannier and G. Gidel and M. Massias},
+    booktitle = {NeurIPS},
+    year      = {2022},
+}
+
+@article{moufad2023skglm,
+  title={skglm: improving scikit-learn for regularized Generalized Linear Models},
+  author={Moufad, Badr and Bannier, Pierre-Antoine and Bertrand, Quentin and Klopfenstein, Quentin and Massias, Mathurin},
+  year={2023}
+}
+```
 
 # Why ``skglm``?
 
@@ -108,18 +127,7 @@ You can also take our tutorial to learn how to create your own datafit and penal
 - **pull request**: you may have fixed a bug, added a features, or even fixed a small typo in the documentation, ... you can submit a [pull request](https://github.com/scikit-learn-contrib/skglm/pulls) and we will reach out to you asap.
 
 
-# Cite
 
-``skglm`` is the result of perseverant research. It is licensed under [BSD 3-Clause](https://github.com/scikit-learn-contrib/skglm/blob/main/LICENSE). You are free to use it and if you do so, please cite
-
-```bibtex
-@inproceedings{skglm,
-    title     = {Beyond L1: Faster and better sparse models with skglm},
-    author    = {Q. Bertrand and Q. Klopfenstein and P.-A. Bannier and G. Gidel and M. Massias},
-    booktitle = {NeurIPS},
-    year      = {2022},
-}
-```
 
 
 # Useful links
diff --git a/doc/api.rst b/doc/api.rst
@@ -68,6 +68,7 @@ Datafits
    Poisson
    Quadratic
    QuadraticGroup
+   QuadraticHessian
    QuadraticSVC
    WeightedQuadratic
 
@@ -102,4 +103,4 @@ Experimental
    PDCD_WS
    Pinball
    SqrtQuadratic
-   SqrtLasso
+   SqrtLasso
diff --git a/doc/changes/0.4.rst b/doc/changes/0.4.rst
@@ -1,12 +1,12 @@
 .. _changes_0_4:
 
-Version 0.4 (in progress)
+Version 0.4 (2023/04/08)
 -------------------------
 - Add :ref:`GroupLasso Estimator <skglm.GroupLasso>` (PR: :gh:`228`)
 - Add support and tutorial for positive coefficients to :ref:`Group Lasso Penalty <skglm.penalties.WeightedGroupL2>` (PR: :gh:`221`)
 - Check compatibility with datafit and penalty in solver (PR :gh:`137`)
 - Add support to weight samples in the quadratic datafit :ref:`Weighted Quadratic Datafit <skglm.datafit.WeightedQuadratic>` (PR: :gh:`258`)
-
+- Add support for ElasticNet regularization (`penalty="l1_plus_l2"`) to :ref:`SparseLogisticRegression <skglm.SparseLogisticRegression>` (PR: :gh:`244`)
 
 Version 0.3.1 (2023/12/21)
 --------------------------
diff --git a/doc/changes/0.5.rst b/doc/changes/0.5.rst
@@ -0,0 +1,4 @@
+.. _changes_0_5:
+
+Version 0.5 (in progress)
+-------------------------
diff --git a/doc/changes/whats_new.rst b/doc/changes/whats_new.rst
@@ -5,6 +5,8 @@ What's new
 
 .. currentmodule:: skglm
 
+.. include:: 0.5.rst
+
 .. include:: 0.4.rst
 
 .. include:: 0.3.rst
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,6 +22,16 @@ dependencies = [
 ]
 dynamic = ["version"]
 
+requires-python = ">=3.9"
+
+classifiers = [
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+]
 
 [tool.setuptools.dynamic]
 version = {attr = "skglm.__version__"}
@@ -38,6 +48,7 @@ test = [
     "flake8",
     "coverage",
     "numpydoc",
+    "celer",
 ]
 
 doc = [
@@ -55,3 +66,7 @@ doc = [
     "furo",
     "lifelines",
 ]
+
+
+[tool.setuptools]
+license-files = []
diff --git a/skglm/__init__.py b/skglm/__init__.py
@@ -1,4 +1,4 @@
-__version__ = '0.4dev'
+__version__ = '0.5dev'
 
 from skglm.estimators import (  # noqa F401
     Lasso, WeightedLasso, ElasticNet, MCPRegression, MultiTaskLasso, LinearSVC,
diff --git a/skglm/datafits/__init__.py b/skglm/datafits/__init__.py
@@ -1,6 +1,6 @@
 from .base import BaseDatafit, BaseMultitaskDatafit
 from .single_task import (Quadratic, QuadraticSVC, Logistic, Huber, Poisson, Gamma,
-                          Cox, WeightedQuadratic,)
+                          Cox, WeightedQuadratic, QuadraticHessian,)
 from .multi_task import QuadraticMultiTask
 from .group import QuadraticGroup, LogisticGroup
 
@@ -9,5 +9,6 @@
     BaseDatafit, BaseMultitaskDatafit,
     Quadratic, QuadraticSVC, Logistic, Huber, Poisson, Gamma, Cox,
     QuadraticMultiTask,
-    QuadraticGroup, LogisticGroup, WeightedQuadratic
+    QuadraticGroup, LogisticGroup, WeightedQuadratic,
+    QuadraticHessian
 ]
diff --git a/skglm/datafits/single_task.py b/skglm/datafits/single_task.py
@@ -239,6 +239,41 @@ def intercept_update_step(self, y, Xw):
         return np.sum(self.sample_weights * (Xw - y)) / self.sample_weights.sum()
 
 
+class QuadraticHessian(BaseDatafit):
+    r"""Quadratic datafit where we pass the Hessian A directly.
+
+    The datafit reads:
+
+    .. math:: 1 / 2 x^(\top) A x + \langle b, x \rangle
+
+    For a symmetric A. Up to a constant, it is the same as a Quadratic, with
+    :math:`A = 1 / (n_"samples") X^(\top)X` and :math:`b = - 1 / n_"samples" X^(\top)y`.
+    When the Hessian is available, this datafit is more efficient than using Quadratic.
+    """
+
+    def __init__(self):
+        pass
+
+    def get_spec(self):
+        pass
+
+    def params_to_dict(self):
+        return dict()
+
+    def get_lipschitz(self, A, b):
+        n_features = A.shape[0]
+        lipschitz = np.zeros(n_features, dtype=A.dtype)
+        for j in range(n_features):
+            lipschitz[j] = A[j, j]
+        return lipschitz
+
+    def gradient_scalar(self, A, b, w, Ax, j):
+        return Ax[j] + b[j]
+
+    def value(self, b, x, Ax):
+        return 0.5 * (x*Ax).sum() + (b*x).sum()
+
+
 @njit
 def sigmoid(x):
     """Vectorwise sigmoid."""
diff --git a/skglm/estimators.py b/skglm/estimators.py
@@ -10,7 +10,7 @@
 from sklearn.utils.validation import (check_is_fitted, check_array,
                                       check_consistent_length)
 from sklearn.linear_model._base import (
-    LinearModel, RegressorMixin,
+    RegressorMixin, LinearModel,
     LinearClassifierMixin, SparseCoefMixin, BaseEstimator
 )
 from sklearn.utils.extmath import softmax
@@ -20,7 +20,7 @@
 
 from skglm.solvers import AndersonCD, MultiTaskBCD, GroupBCD
 from skglm.datafits import (Cox, Quadratic, Logistic, QuadraticSVC,
-                            QuadraticMultiTask, QuadraticGroup)
+                            QuadraticMultiTask, QuadraticGroup,)
 from skglm.penalties import (L1, WeightedL1, L1_plus_L2, L2, WeightedGroupL2,
                              MCPenalty, WeightedMCPenalty, IndicatorBox, L2_1)
 from skglm.utils.data import grp_converter
@@ -301,7 +301,7 @@ def get_params(self, deep=False):
         return params
 
 
-class Lasso(LinearModel, RegressorMixin):
+class Lasso(RegressorMixin, LinearModel):
     r"""Lasso estimator based on Celer solver and primal extrapolation.
 
     The optimization objective for Lasso is:
@@ -448,7 +448,7 @@ def path(self, X, y, alphas, coef_init=None, return_n_iter=True, **params):
         return solver.path(X, y, datafit, penalty, alphas, coef_init, return_n_iter)
 
 
-class WeightedLasso(LinearModel, RegressorMixin):
+class WeightedLasso(RegressorMixin, LinearModel):
     r"""WeightedLasso estimator based on Celer solver and primal extrapolation.
 
     The optimization objective for WeightedLasso is:
@@ -611,7 +611,7 @@ def fit(self, X, y):
         return _glm_fit(X, y, self, Quadratic(), penalty, solver)
 
 
-class ElasticNet(LinearModel, RegressorMixin):
+class ElasticNet(RegressorMixin, LinearModel):
     r"""Elastic net estimator.
 
     The optimization objective for Elastic net is:
@@ -765,7 +765,7 @@ def fit(self, X, y):
                         L1_plus_L2(self.alpha, self.l1_ratio, self.positive), solver)
 
 
-class MCPRegression(LinearModel, RegressorMixin):
+class MCPRegression(RegressorMixin, LinearModel):
     r"""Linear regression with MCP penalty estimator.
 
     The optimization objective for MCPRegression is, with :math:`x >= 0`:
@@ -958,14 +958,28 @@ class SparseLogisticRegression(LinearClassifierMixin, SparseCoefMixin, BaseEstim
 
     The optimization objective for sparse Logistic regression is:
 
-    .. math:: 1 / n_"samples" sum_(i=1)^(n_"samples") log(1 + exp(-y_i x_i^T w))
-        + alpha ||w||_1
+    .. math::
+        1 / n_"samples" \sum_{i=1}^{n_"samples"} log(1 + exp(-y_i x_i^T w))
+        + tt"l1_ratio" xx alpha ||w||_1
+        + (1 - tt"l1_ratio") xx alpha/2 ||w||_2 ^ 2
+
+    By default, ``l1_ratio=1.0`` corresponds to Lasso (pure L1 penalty).
+    When ``0 < l1_ratio < 1``, the penalty is a convex combination of L1 and L2
+    (i.e., ElasticNet). ``l1_ratio=0.0`` corresponds to Ridge (pure L2), but note
+    that pure Ridge is not typically used with this class.
 
     Parameters
     ----------
     alpha : float, default=1.0
         Regularization strength; must be a positive float.
 
+    l1_ratio : float, default=1.0
+        The ElasticNet mixing parameter, with ``0 <= l1_ratio <= 1``.
+        Only used when ``penalty="l1_plus_l2"``.
+        For ``l1_ratio = 0`` the penalty is an L2 penalty.
+        ``For l1_ratio = 1`` it is an L1 penalty.
+        For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2.
+
     tol : float, optional
         Stopping criterion for the optimization.
 
@@ -1002,10 +1016,11 @@ class SparseLogisticRegression(LinearClassifierMixin, SparseCoefMixin, BaseEstim
         Number of subproblems solved to reach the specified tolerance.
     """
 
-    def __init__(self, alpha=1.0, tol=1e-4, max_iter=20, max_epochs=1_000, verbose=0,
-                 fit_intercept=True, warm_start=False):
+    def __init__(self, alpha=1.0, l1_ratio=1.0, tol=1e-4, max_iter=20, max_epochs=1_000,
+                 verbose=0, fit_intercept=True, warm_start=False):
         super().__init__()
         self.alpha = alpha
+        self.l1_ratio = l1_ratio
         self.tol = tol
         self.max_iter = max_iter
         self.max_epochs = max_epochs
@@ -1034,7 +1049,8 @@ def fit(self, X, y):
             max_iter=self.max_iter, max_pn_iter=self.max_epochs, tol=self.tol,
             fit_intercept=self.fit_intercept, warm_start=self.warm_start,
             verbose=self.verbose)
-        return _glm_fit(X, y, self, Logistic(), L1(self.alpha), solver)
+        return _glm_fit(X, y, self, Logistic(), L1_plus_L2(self.alpha, self.l1_ratio),
+                        solver)
 
     def predict_proba(self, X):
         """Probability estimates.
@@ -1364,7 +1380,7 @@ def fit(self, X, y):
         return self
 
 
-class MultiTaskLasso(LinearModel, RegressorMixin):
+class MultiTaskLasso(RegressorMixin, LinearModel):
     r"""MultiTaskLasso estimator.
 
     The optimization objective for MultiTaskLasso is:
@@ -1540,7 +1556,7 @@ def path(self, X, Y, alphas, coef_init=None, return_n_iter=False, **params):
         return solver.path(X, Y, datafit, penalty, alphas, coef_init, return_n_iter)
 
 
-class GroupLasso(LinearModel, RegressorMixin):
+class GroupLasso(RegressorMixin, LinearModel):
     r"""GroupLasso estimator based on Celer solver and primal extrapolation.
 
     The optimization objective for GroupLasso is:
diff --git a/skglm/experimental/pdcd_ws.py b/skglm/experimental/pdcd_ws.py
@@ -82,13 +82,17 @@ class PDCD_WS(BaseSolver):
     _datafit_required_attr = ('prox_conjugate',)
     _penalty_required_attr = ("prox_1d",)
 
-    def __init__(self, max_iter=1000, max_epochs=1000, dual_init=None,
-                 p0=100, tol=1e-6, verbose=False):
+    def __init__(
+        self, max_iter=1000, max_epochs=1000, dual_init=None, p0=100, tol=1e-6,
+        fit_intercept=False, warm_start=True, verbose=False
+    ):
         self.max_iter = max_iter
         self.max_epochs = max_epochs
         self.dual_init = dual_init
         self.p0 = p0
         self.tol = tol
+        self.fit_intercept = fit_intercept  # TODO not handled
+        self.warm_start = warm_start  # TODO not handled
         self.verbose = verbose
 
     def _solve(self, X, y, datafit, penalty, w_init=None, Xw_init=None):
diff --git a/skglm/experimental/tests/test_quantile_regression.py b/skglm/experimental/tests/test_quantile_regression.py
diff --git a/skglm/tests/test_datafits.py b/skglm/tests/test_datafits.py
diff --git a/skglm/tests/test_estimators.py b/skglm/tests/test_estimators.py
diff --git a/skglm/tests/test_penalties.py b/skglm/tests/test_penalties.py

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +.. _changes_0_5:
++
 +Version 0.5 (in progress)
 +-------------------------
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-__version__ = '0.4dev'`
	`1`	`+__version__ = '0.5dev'`
`2`	`2`
`3`	`3`	`from skglm.estimators import ( # noqa F401`
`4`	`4`	`Lasso, WeightedLasso, ElasticNet, MCPRegression, MultiTaskLasso, LinearSVC,`