Merge pull request #27 from DataboyUsen/main

feat/fix/doc: Add separate constraints + ReHLoss() for plqMF_Ridge + NMF example (#7)

Author: statmlben

doc/source/example.rst

@@ -16,7 +16,7 @@ Example Gallery
    examples/Path_solution.ipynb
    examples/Warm_start.ipynb
    examples/Sklearn_Mixin.ipynb   
-   examples/MF.ipynb  
+   examples/NMF.ipynb
 
 List of Examples
 ----------------
@@ -32,4 +32,4 @@ List of Examples
    examples/Path_solution.ipynb
    examples/Warm_start.ipynb
    examples/Sklearn_Mixin.ipynb 
-   examples/MF.ipynb  
+   examples/NMF.ipynb

doc/source/examples/MF.ipynb

@@ -26,17 +26,20 @@ Considering a User-Item-Rating triplet dataset :math:`(u, i, r_{ui})` derived fr
 
 .. math::
         \ \text{ s.t. } \ 
-        \mathbf{A} \begin{pmatrix} \alpha_u \\ \mathbf{p}_u \end{pmatrix} + \mathbf{b} \geq \mathbf{0},\ u = 1,\dots,n
+        \mathbf{A}_{\text{user}} \begin{pmatrix} \alpha_u \\ \mathbf{p}_u \end{pmatrix} + \mathbf{b}_{\text{user}} \geq \mathbf{0},\ u = 1,\dots,n
         \quad \text{and} \quad
-        \mathbf{A} \begin{pmatrix} \beta_i \\ \mathbf{q}_i \end{pmatrix} + \mathbf{b} \geq \mathbf{0},\ i = 1,\dots,m
+        \mathbf{A}_{\text{item}} \begin{pmatrix} \beta_i \\ \mathbf{q}_i \end{pmatrix} + \mathbf{b}_{\text{item}} \geq \mathbf{0},\ i = 1,\dots,m
 
 where
 
 - :math:`\text{PLQ}(\cdot , \cdot)` 
   is a convex piecewise linear-quadratic loss function. You can find built-in loss functions in the `Loss <./loss.rst>`_ section.
 
-- :math:`\mathbf{A}` is a :math:`d \times (k+1)` matrix and :math:`\mathbf{b}` is a :math:`d`-dimensional vector 
-  representing :math:`d` linear constraints. See `Constraints <./constraint.rst>`_ for more details.
+- :math:`\mathbf{A}_{\text{user}}` is a :math:`d \times (k+1)` matrix and :math:`\mathbf{b}_{\text{user}}` is a :math:`d`-dimensional vector 
+  representing :math:`d` linear constraints to user side parameters. See `Constraints <./constraint.rst>`_ for more details.
+
+- :math:`\mathbf{A}_{\text{item}}` is a :math:`d \times (k+1)` matrix and :math:`\mathbf{b}_{\text{item}}` is a :math:`d`-dimensional vector 
+  representing :math:`d` linear constraints to item side parameters. See `Constraints <./constraint.rst>`_ for more details.
 
 - :math:`\Omega`
   is a user-item collection that records all training data
@@ -93,11 +96,11 @@ Basic Usage
 
    # 3. Model Construction
    clf = plqMF_Ridge(
-       C=0.001,                        ## Regularization strength
-       rank=6,                         ## Latent factor dimension
-       loss={'name': 'mae'},           ## Use absolute loss
-       n_users=user_num,               ## Number of users
-       n_items=item_num,               ## Number of items
+       C=0.001,                             ## Regularization strength
+       rank=6,                              ## Latent factor dimension
+       loss={'name': 'mae'},                ## Use absolute loss
+       n_users=user_num,                    ## Number of users
+       n_items=item_num,                    ## Number of items
    )
    clf.fit(X_train, y_train)
 
@@ -118,19 +121,19 @@ Choosing different `loss functions <./loss.rst>`_ through :code:`loss`:
    clf_mse = plqMF_Ridge(
         C=0.001, 
         rank=6, 
-        loss={'name': 'mse'},          ## Choose square loss
+        loss={'name': 'mse'},               ## Choose square loss
         n_users=user_num, 
         n_items=item_num)
    
    # Hinge loss (suitable for binary data)
    clf_hinge = plqMF_Ridge(
         C=0.001, 
         rank=6, 
-        loss={'name': 'hinge'},        ## Choose hinge loss
+        loss={'name': 'hinge'},             ## Choose hinge loss
         n_users=user_num, 
         n_items=item_num)
 
-`Linear constraints <./constraint.rst>`_ can be applied via :code:`constraint`:
+`Linear constraints <./constraint.rst>`_ can be applied via :code:`constraint_user` and :code:`constraint_item`:
 
 .. code-block:: python
 
@@ -141,7 +144,8 @@ Choosing different `loss functions <./loss.rst>`_ through :code:`loss`:
         loss={'name': 'mae'},
         n_users=user_num, 
         n_items=item_num,
-        constraint=[{'name': '>=0'}]   ## Use nonnegative constraint
+        constraint_user=[{'name': '>=0'}],  ## Use nonnegative constraint
+        constraint_item=[{'name': '>=0'}]
     )
   
 The algorithm includes bias terms :math:`\mathbf{\alpha}` and :math:`\mathbf{\beta}` by default. To disable them, that is, :math:`\mathbf{\alpha} = \mathbf{0}` and :math:`\mathbf{\beta} = \mathbf{0}`, set: :code:`biased=False`:
@@ -155,7 +159,7 @@ The algorithm includes bias terms :math:`\mathbf{\alpha}` and :math:`\mathbf{\be
         loss={'name': 'mae'},
         n_users=user_num, 
         n_items=item_num,
-        biased=False                   ## Disable bias terms
+        biased=False                        ## Disable bias terms
     )
   
 Imposing different strengths of regularization on items/users through :code:`rho`:
@@ -169,7 +173,7 @@ Imposing different strengths of regularization on items/users through :code:`rho
         loss={'name': 'mae'},
         n_users=user_num, 
         n_items=item_num,
-        rho=0.7                        ## Add heavier penalties for user parameters
+        rho=0.7                             ## Add heavier penalties for user parameters
     )
 
 Parameter Tuning
@@ -182,7 +186,7 @@ The model complexity is mainly controlled by :code:`C` and :code:`rank`.
    
    for C_value in [0.0002, 0.001, 0.005]:
        clf = plqMF_Ridge(
-            C=C_value,                 ## Try different regularization strengths
+            C=C_value,                      ## Try different regularization strengths
             rank=6, 
             loss={'name': 'mae'},
             n_users=user_num, 
@@ -197,7 +201,7 @@ The model complexity is mainly controlled by :code:`C` and :code:`rank`.
    for rank_value in [4, 8, 12]:
        clf = plqMF_Ridge(
             C=0.001, 
-            rank=rank_value,           ## Try different latent factor dimensions
+            rank=rank_value,                ## Try different latent factor dimensions
             loss={'name': 'mae'},
             n_users=user_num, 
             n_items=item_num
@@ -221,4 +225,4 @@ Example
    :caption: Empirical Risk Minimization
    :name: rst-link-gallery
 
-   ../examples/MF.ipynb
+   ../examples/NMF.ipynb

doc/source/examples/NMF.ipynb

@@ -46,3 +46,4 @@ Related Examples
    :name: rst-link-gallery
 
    ../examples/FairSVM.ipynb
+   ../examples/NMF.ipynb

doc/source/tutorials/ReHLine_MF.rst

@@ -7,6 +7,7 @@
 from ._sklearn_mixin import plq_Ridge_Classifier, plq_Ridge_Regressor
 from ._mf_class import plqMF_Ridge
 from ._data import make_mf_dataset 
+from ._loss import ReHLoss
 
 __all__ = ("_BaseReHLine",
            "ReHLine_solver",

doc/source/tutorials/constraint.rst

@@ -6,6 +6,7 @@
 from sklearn.base import BaseEstimator
 from sklearn.utils.validation import _check_sample_weight
 from sklearn.exceptions import ConvergenceWarning
+from ._loss import ReHLoss
 from ._base import  (_BaseReHLine, ReHLine_solver,
                     _make_loss_rehline_param,  _make_constraint_rehline_param,
                     _cast_sample_bias, _cast_sample_weight)
@@ -32,9 +33,9 @@ class plqMF_Ridge(_BaseReHLine, BaseEstimator):
 
     .. math::
         \ \text{ s.t. } \ 
-        \mathbf{A} \begin{pmatrix} \alpha_u \\ \mathbf{p}_u \end{pmatrix} + \mathbf{b} \geq \mathbf{0},\ u = 1,\dots,n
+        \mathbf{A}_{\text{user}} \begin{pmatrix} \alpha_u \\ \mathbf{p}_u \end{pmatrix} + \mathbf{b}_{\text{user}} \geq \mathbf{0},\ u = 1,\dots,n
         \quad \text{and} \quad
-        \mathbf{A} \begin{pmatrix} \beta_i \\ \mathbf{q}_i \end{pmatrix} + \mathbf{b} \geq \mathbf{0},\ i = 1,\dots,m
+        \mathbf{A}_{\text{item}} \begin{pmatrix} \beta_i \\ \mathbf{q}_i \end{pmatrix} + \mathbf{b}_{\text{item}} \geq \mathbf{0},\ i = 1,\dots,m
         
     The function supports various loss functions, including:
         - 'hinge', 'svm' or 'SVM'
@@ -58,8 +59,12 @@ class plqMF_Ridge(_BaseReHLine, BaseEstimator):
     loss : dict
         A dictionary specifying the loss function parameters. 
     
-    constraint : list of dict
-        A list of dictionaries, where each dictionary represents a constraint.
+    constraint_user : list of dict
+        A list of dictionaries, where each dictionary represents a constraint to user side parameters.
+        Each dictionary must contain a 'name' key, which specifies the type of constraint.
+
+    constraint_item : list of dict
+        A list of dictionaries, where each dictionary represents a constraint to item side parameters.
         Each dictionary must contain a 'name' key, which specifies the type of constraint.
 
     biased : bool, default=True
@@ -156,7 +161,7 @@ class plqMF_Ridge(_BaseReHLine, BaseEstimator):
     decision_function(X)
         The decision function evaluated on the given dataset.
 
-    obj(X, y, loss))
+    obj(X, y)
         Compute the values of loss term and objective function.
 
     Notes
@@ -165,7 +170,8 @@ class plqMF_Ridge(_BaseReHLine, BaseEstimator):
 
     """
 
-    def __init__(self, n_users, n_items, loss, constraint=[], biased=True,
+    def __init__(self, n_users, n_items, loss, biased=True,
+                    constraint_user=[], constraint_item=[],  
                     rank=10, C=1.0, rho=0.5,
                     init_mean=0.0, init_sd=0.1, random_state=None,
                     max_iter=10000, tol=1e-4, shrink=1, trace_freq=100, 
@@ -189,7 +195,8 @@ def __init__(self, n_users, n_items, loss, constraint=[], biased=True,
         self.n_users = n_users 
         self.n_items = n_items 
         self.loss = loss
-        self.constraint = constraint
+        self.constraint_user = constraint_user
+        self.constraint_item = constraint_item
         self.biased = biased
         ## -----------------------------hyper perameters-----------------------------
         self.rank = rank
@@ -259,7 +266,7 @@ def fit(self, X, y, sample_weight=None):
 
 
         # CD algorithm
-        self.history[0] = self.obj(X, y, loss=self.loss)
+        self.history[0] = self.obj(X, y)
         for l in range(self.max_iter_CD):
             ## User side update
             for user in range(self.n_users):
@@ -286,7 +293,7 @@ def fit(self, X, y, sample_weight=None):
                 U, V, Tau, S, T = _make_loss_rehline_param(loss=self.loss, X=Q_tmp, y=y_tmp)
                 U_bias, V_bias, Tau_bias, S_bias,  T_bias = _cast_sample_bias(U, V, Tau, S, T, sample_bias=bias_tmp)
                 U_weight, V_weight, Tau_weight, S_weight, T_weight = _cast_sample_weight(U_bias, V_bias, Tau_bias, S_bias, T_bias, C=C_user, sample_weight=weight_tmp)
-                A, b = _make_constraint_rehline_param(constraint=self.constraint, X=Q_tmp, y=y_tmp)
+                A, b = _make_constraint_rehline_param(constraint=self.constraint_user, X=Q_tmp, y=y_tmp)
 
                 ### solve and update
                 result_tmp = ReHLine_solver(X=Q_tmp, 
@@ -337,7 +344,7 @@ def fit(self, X, y, sample_weight=None):
                 U, V, Tau, S, T = _make_loss_rehline_param(loss=self.loss, X=P_tmp, y=y_tmp)
                 U_bias, V_bias, Tau_bias, S_bias,  T_bias = _cast_sample_bias(U, V, Tau, S, T, sample_bias=bias_tmp)
                 U_weight, V_weight, Tau_weight, S_weight, T_weight = _cast_sample_weight(U_bias, V_bias, Tau_bias, S_bias, T_bias, C=C_item, sample_weight=weight_tmp)
-                A, b = _make_constraint_rehline_param(constraint=self.constraint, X=P_tmp, y=y_tmp)
+                A, b = _make_constraint_rehline_param(constraint=self.constraint_item, X=P_tmp, y=y_tmp)
 
                 ### solve and update
                 result_tmp = ReHLine_solver(X=P_tmp, 
@@ -364,7 +371,7 @@ def fit(self, X, y, sample_weight=None):
 
 
             ## Check convergence
-            self.history[l+1] = self.obj(X, y, loss=self.loss)
+            self.history[l+1] = self.obj(X, y)
             obj_diff = (self.history[l] - self.history[l+1])[1]
 
 
@@ -407,7 +414,7 @@ def decision_function(self, X):
 
 
 
-    def obj(self, X, y, loss):
+    def obj(self, X, y):
         """
         Compute the values of loss term and objective function.
         
@@ -418,9 +425,6 @@ def obj(self, X, y, loss):
 
         y : array-like of shape (n_ratings,)
             Actual rating values.
-
-        loss : dict
-            A dictionary specifying the loss function parameters. 
             
         Returns
         -------
@@ -441,28 +445,9 @@ def obj(self, X, y, loss):
             item_penalty = np.sum(self.Q ** 2) * (1 - self.rho) / self.n_items
             penalty = user_penalty + item_penalty
 
-        if (loss['name'] == 'mae') \
-            or (loss['name'] == 'MAE') \
-            or (loss['name'] == 'mean absolute error'):
-            loss_term =  np.sum( np.abs(self.decision_function(X) - y) )
-
-        elif (loss['name'] == 'MSE') \
-            or (loss['name'] == 'mse') \
-            or (loss['name'] == 'mean squared error'):
-            loss_term =  np.sum( (self.decision_function(X) - y) ** 2 )
-            
-        elif (loss['name'] == 'hinge') \
-            or (loss['name'] == 'svm') \
-            or (loss['name'] == 'SVM'):
-            loss_term = np.sum( np.maximum(0, 1 - y * self.decision_function(X)) )
-        
-        elif (loss['name'] == 'squared hinge') \
-            or (loss['name'] == 'squared svm') \
-            or (loss['name'] == 'squared SVM'):
-            loss_term = np.sum( np.maximum(0, 1 - y * self.decision_function(X)) ** 2 )
-
-        else:
-            raise ValueError(f"Unsupported loss function: {loss['name']}. "
-                            f"Supported losses are: 'mae', 'mse', 'hinge', 'squared hinge'")
+        y_pred = self.decision_function(X)
+        U, V, Tau, S, T = _make_loss_rehline_param(loss=self.loss, X=X, y=y)
+        loss = ReHLoss(U, V, S, T, Tau)
+        loss_term = loss(y_pred)
 
         return loss_term, self.C * loss_term + penalty

rehline/__init__.py

@@ -77,7 +77,7 @@ def evaluate_single_params(params):
             y_pred = model.decision_function(X_val)
             y_pred_classes = np.where(y_pred > 0, 1, -1)
             accuracy = accuracy_score(y_val, y_pred_classes)
-        score = model.obj(X_val, y_val, loss=model.loss)[0] / len(y_val)
+        score = model.obj(X_val, y_val)[0] / len(y_val)
         return {'params': param_dict, 'score': score, 'accuracy':accuracy}
 
 
@@ -94,11 +94,13 @@ def evaluate_single_params(params):
 
 ## Parameters to be selected
 param_grid = {
-    'constraint': [[], [{'name': '>=0'}]],
+    'constraint_user': [[], [{'name': '>=0'}]],
+    'constraint_item': [[], [{'name': '>=0'}]],
     'biased': [True, False],
     'rank': [5, 10],
     'C': [0.0006, 0.0002],
     'rho': [0.3, 0.6],
+    'tol': [0.01]
 }