Working on KPCovC, investigating minor errors

Author: rvasav26

examples/pcovc/PCovC-DecisionGraphForPaper.ipynb

@@ -334,7 +334,7 @@ def fit(self, X, Y, W=None):
             # Check if regressor is fitted; if not, fit with precomputed K
             # to avoid needing to compute the kernel a second time
             self.regressor_ = check_krr_fit(regressor, K, X, Y)
-            
+            print(self.regressor_.n_features_in_)
             W = self.regressor_.dual_coef_.reshape(self.n_samples_in_, -1)
             print(W.shape)
             # Use this instead of `self.regressor_.predict(K)`

examples/pcovc/PCovC-IrisDataset.ipynb

@@ -4,6 +4,7 @@
 import scipy.sparse as sp
 from scipy import linalg
 from scipy.sparse.linalg import svds
+from sklearn.calibration import LinearSVC
 from sklearn.decomposition._base import _BasePCA
 from sklearn.decomposition._pca import _infer_dimension
 from sklearn.exceptions import NotFittedError
@@ -20,58 +21,15 @@
 from sklearn.svm import SVC
 from sklearn.base import clone
 from copy import deepcopy
+from sklearn.metrics import accuracy_score
 
 from skmatter.preprocessing import KernelNormalizer
-from skmatter.utils import check_krr_fit, pcovr_kernel
-
-def check_cl_fit(classifier, X, y):
-    r"""
-    Checks that a (linear) classifier is fitted, and if not,
-    fits it with the provided data
-    :param regressor: sklearn-style classifier
-    :type classifier: object
-    :param X: feature matrix with which to fit the classifier
-        if it is not already fitted
-    :type X: array
-    :param y: target values with which to fit the classifier
-        if it is not already fitted
-    :type y: array
-    """
-    try:
-        check_is_fitted(classifier)
-        fitted_classifier = deepcopy(classifier)
-
-        # Check compatibility with X
-        fitted_classifier._validate_data(X, y, reset=False, multi_output=True)
-
-        # Check compatibility with y
-
-        # changed from if fitted_classifier.coef_.ndim != y.ndim:
-        # dimension of classifier coefficients is always 2, hence we don't need to check 
-        # for match with Y
-        if fitted_classifier.coef_.shape[1] != X.shape[1]:
-            raise ValueError(
-                "The classifier coefficients have a shape incompatible "
-                "with the supplied feature space. "
-                "The coefficients have shape %d and the features "
-                "have shape %d" % (fitted_classifier.coef_.shape, X.shape)
-            )
-        # LogisticRegression does not support multioutput, but RidgeClassifier does
-        elif y.ndim == 2:
-            if fitted_classifier.coef_.shape[0] != y.shape[1]:
-                raise ValueError(
-                    "The classifier coefficients have a shape incompatible "
-                    "with the supplied target space. "
-                    "The coefficients have shape %r and the targets "
-                    "have shape %r" % (fitted_classifier.coef_.shape, y.shape)
-                )
-
-    except NotFittedError:
-        fitted_classifier = clone(classifier)
-        fitted_classifier.fit(X, y)
-
-    return fitted_classifier
+from skmatter.utils import pcovr_kernel
 
+import sys
+sys.path.append('scikit-matter')
+from src.skmatter.utils._pcovc_utils import check_svc_fit
+from src.skmatter.utils._pcovr_utils import check_krr_fit
 
 class KernelPCovC(_BasePCA, LinearModel):
     r"""
@@ -251,7 +209,7 @@ def __init__(
         gamma="scale",
         degree=3,
         coef0=0.0,
-     #   kernel_params,
+        kernel_params=None,
         center=False,
         fit_inverse_transform=False,
         tol=1e-12,
@@ -272,7 +230,7 @@ def __init__(
         self.gamma = gamma
         self.degree = degree
         self.coef0 = coef0
-    #    self.kernel_params = kernel_params
+        self.kernel_params = kernel_params
 
         self.n_jobs = n_jobs
 
@@ -284,7 +242,7 @@ def _get_kernel(self, X, Y=None):
         sparse = sp.issparse(X)
 
         if callable(self.kernel):
-            params = {} #self.kernel_params or {}
+            params = self.kernel_params or {}
         else:
             #this is how BaseSVC has it:
             if self.gamma == "scale":
@@ -367,7 +325,7 @@ def fit(self, X, y, W=None):
         """
 
         if self.classifier not in ["precomputed", None] and not isinstance(
-            self.classifier, SVC
+            self.classifier, SVC #make sure that decision_function_shape is ONLY "ovr" otherwise this will impact Z's shape
         ):
             raise ValueError(
                 "classifier must be an instance of `SVC`"
@@ -433,12 +391,14 @@ def fit(self, X, y, W=None):
             # Check if classifier is fitted; if not, fit with precomputed K
             # to avoid needing to compute the kernel a second time
             classifier.probability = True
-            self.z_classifier_ = check_krr_fit(classifier, K, X, y) #Pkz as weights - fits on K, y
-            
-            Z = self.z_classifier_.predict_proba(K)
+            self.z_classifier_ = check_svc_fit(classifier, K, X, y) #Pkz as weights - fits on K, y
+            Z = self.z_classifier_.decision_function(K)
+
             # print(K.shape)
             # print("Z: "+str(Z.shape))
 
+            #problem is that with a prefitted classifeir on X, y, we are trying to refit it on K, y
+
             W = np.linalg.lstsq(K, Z, self.tol)[0]
             #W should have shape (samples, classes) since Z = K*W
             #(samples, classes) = (samples, samples)*(samples,classes)
@@ -457,12 +417,12 @@ def fit(self, X, y, W=None):
             # it will work on the particular X
             # of the KPCovR call. The dual coefficients are kept.
             # Can be bypassed if the classifier is pre-fitted.
-            try:
-                check_is_fitted(classifier)
-            except NotFittedError:
-                self.z_classifier_.set_params(**classifier.get_params())
-                self.z_classifier_.X_fit_ = self.X_fit_
-                self.z_classifier_._check_n_features(self.X_fit_, reset=True)
+            # try:
+            #     check_is_fitted(classifier)
+            # except NotFittedError:
+            #     self.z_classifier_.set_params(**classifier.get_params())
+            #     self.z_classifier_.X_fit_ = self.X_fit_
+            #     self.z_classifier_._check_n_features(self.X_fit_, reset=True)
         else:
             Z = y.copy()
             if W is None:
@@ -497,14 +457,30 @@ def fit(self, X, y, W=None):
         if self.fit_inverse_transform:
             self.ptx_ = self.pt__ @ X
 
-
         #self.classifier_ = check_cl_fit(classifier, K @ self.pkt_, y) # Extract weights to get Ptz
-        if self.classifier != "precomputed":
-            self.classifier_ = clone(classifier).fit(K @ self.pkt_, y)
-        else:
-            self.classifier_ = SVC().fit(K @ self.pkt_, y)
+        self.classifier_ = LinearSVC().fit(K @ self.pkt_, y)
+        # if self.classifier != "precomputed":
+        #     self.classifier_ = clone(classifier).fit(K @ self.pkt_, y)
+        # else:
+        #     self.classifier_ = SVC().fit(K @ self.pkt_, y)
         self.classifier_._validate_data(K @ self.pkt_, y, reset=False)
 
+        if isinstance(self.classifier_, MultiOutputClassifier):
+            self.ptz_ = np.hstack(
+                [est_.coef_.T for est_ in self.classifier_.estimators_]
+            )
+            self.pkz_ = self.pkt_ @ self.ptz_
+        else:
+            self.ptz_ = self.classifier_.coef_.T
+            self.pkz_ = self.pkt_ @ self.ptz_
+
+        if len(Y.shape) == 1:
+            self.pkz_ = self.pkz_.reshape(
+                X.shape[1],
+            )
+            self.ptz_ = self.ptz_.reshape(
+                self.n_components_,
+            )
 
         self.components_ = self.pkt_.T  # for sklearn compatibility
         return self
@@ -522,14 +498,13 @@ def decision_function(self, X=None, T=None):
             K = self._get_kernel(X, self.X_fit_)
             if self.center:
                 K = self.centerer_.transform(K)
-
-            return self.z_classifier_.predict_proba(K)
-            #return K @ self.pkz_
+                
+            return K @ self.pkz_
 
         else:
             T = check_array(T)
-            return self.classifier_.predict_proba(T)
-            #return T @ self.ptz_
+            return T @ self.ptz_
+            
 
     def predict(self, X=None, T=None):
         """Predicts class values from X or T."""
@@ -602,67 +577,35 @@ def inverse_transform(self, T):
 
         return T @ self.ptx_
 
-    def score(self, X, Y):
-        r"""
-        Computes the (negative) loss values for KernelPCovC on the given predictor and
-        response variables. The loss in :math:`\mathbf{K}`, as explained in
-        [Helfrecht2020]_ does not correspond to a traditional Gram loss
-        :math:`\mathbf{K} - \mathbf{TT}^T`. Indicating the kernel between set
-        A and B as :math:`\mathbf{K}_{AB}`,
-        the projection of set A as :math:`\mathbf{T}_A`, and with N and V as the
-        train and validation/test set, one obtains
+    def score(self, X, Y, sample_weight=None):
+        #taken from sklearn's LogisticRegression score() implementation:
+        r"""Return the mean accuracy on the given test data and labels.
 
-        .. math::
+        In multi-label classification, this is the subset accuracy
+        which is a harsh metric since you require for each sample that
+        each label set be correctly predicted.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Test samples.
 
-            \ell=\frac{\operatorname{Tr}\left[\mathbf{K}_{VV} - 2
-            \mathbf{K}_{VN} \mathbf{T}_N
-                (\mathbf{T}_N^T \mathbf{T}_N)^{-1} \mathbf{T}_V^T
-            +\mathbf{T}_V(\mathbf{T}_N^T \mathbf{T}_N)^{-1}  \mathbf{T}_N^T
-            \mathbf{K}_{NN} \mathbf{T}_N (\mathbf{T}_N^T \mathbf{T}_N)^{-1}
-            \mathbf{T}_V^T\right]}{\operatorname{Tr}(\mathbf{K}_{VV})}
+        Y : array-like of shape (n_samples,) or (n_samples, n_outputs)
+            True labels for `X`.
 
-        The negative loss is returned for easier use in sklearn pipelines, e.g., a
-        grid search, where methods named 'score' are meant to be maximized.
+        T : ndarray, shape (n_samples, n_components)
+            Projected data, where n_samples is the number of samples
+            and n_components is the number of components.
 
-        Arguments
-        ---------
-        X:              independent (predictor) variable
-        Y:              dependent (response) variable
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
 
         Returns
         -------
-        L:             Negative sum of the KPCA and KRR losses, with the KPCA loss
-                       determined by the reconstruction of the kernel
-
+        score : float
+            Mean accuracy of ``self.predict(X, T)`` w.r.t. `Y`.
         """
-
-        check_is_fitted(self, ["pkt_", "X_fit_"])
-
-        X = check_array(X)
-
-        K_NN = self._get_kernel(self.X_fit_, self.X_fit_)
-        K_VN = self._get_kernel(X, self.X_fit_)
-        K_VV = self._get_kernel(X)
-
-        if self.center:
-            K_NN = self.centerer_.transform(K_NN)
-            K_VN = self.centerer_.transform(K_VN)
-            K_VV = self.centerer_.transform(K_VV)
-
-        y = K_VN @ self.pkz_
-        Lkrr = np.linalg.norm(Y - y) ** 2 / np.linalg.norm(Y) ** 2
-
-        t_n = K_NN @ self.pkt_
-        t_v = K_VN @ self.pkt_
-
-        w = (
-            t_n
-            @ np.linalg.lstsq(t_n.T @ t_n, np.eye(t_n.shape[1]), rcond=self.tol)[0]
-            @ t_v.T
-        )
-        Lkpca = np.trace(K_VV - 2 * K_VN @ w + w.T @ K_VV @ w) / np.trace(K_VV)
-
-        return -sum([Lkpca, Lkrr])
+        return accuracy_score(Y, self.predict(X), sample_weight=sample_weight)
 
     def _decompose_truncated(self, mat):
         if not 1 <= self.n_components_ <= self.n_samples_in_:

examples/pcovc/test_notebook.ipynb

@@ -12,6 +12,8 @@
 from sklearn.calibration import column_or_1d
 from sklearn.naive_bayes import LabelBinarizer
 from sklearn.svm import LinearSVC
+from sklearn.svm import SVC
+
 from sklearn.multioutput import MultiOutputClassifier
 from sklearn.utils import check_array
 from sklearn.utils.validation import check_is_fitted
@@ -172,7 +174,7 @@ class PCovC(_BasePCov):
     Examples
     --------
     >>> import numpy as np
-    >>> from skmatter.decomposition import PCovc
+    >>> from skmatter.decomposition import PCovC
     >>> X = np.array([[-1, 0, -2, 3], [3, -2, 0, 1], [-3, 0, -1, -1], [1, 3, 0, -2]])
     >>> Y = np.array([[0], [1], [2], [0]])
     >>> pcovc = PCovC(mixing=0.1, n_components=2)
@@ -256,7 +258,8 @@ class likelihoods, :math:`{\mathbf{Z}}`.
                         LogisticRegressionCV,
                         SGDClassifier,
                         LinearSVC,
-                        MultiOutputClassifier,
+                        MultiOutputClassifier
+                        #check to see if all linear classifiers are here: Perceptron, LDA
                     ),
                 ),
             ]
@@ -284,13 +287,15 @@ class likelihoods, :math:`{\mathbf{Z}}`.
 
             else:
                 W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1)
-                Z = self.z_classifier_.decision_function(X).reshape(X.shape[0], -1) #computes Z this will throw an error since pxz and ptz aren't defined yet
+                Z = self.z_classifier_.decision_function(X).reshape(X.shape[0], -1) 
+                #computes Z this will throw an error since pxz and ptz aren't defined yet
 
         else:
             Z = y.copy()  
             if W is None:
                 W = np.linalg.lstsq(X, Z, self.tol)[0]  #W = weights for Pxz
-
+        # print("Z: "+str(Z[:4]))
+        # print("W: "+str(W[:4]))
         self._label_binarizer = LabelBinarizer(neg_label=-1, pos_label=1)
         Y = self._label_binarizer.fit_transform(y) #check if we need this
 
@@ -409,8 +414,8 @@ def inverse_transform(self, T):
         return super().inverse_transform(T)
 
     def decision_function(self, X=None, T=None):
-        print(self.pxz_.shape)
-        print(self.ptz_.shape)
+        # print(self.pxz_.shape)
+        # print(self.ptz_.shape)
 
         """Predicts confidence scores from X or T."""
         check_is_fitted(self, attributes=["_label_binarizer", "pxz_", "ptz_"])
@@ -420,19 +425,10 @@ def decision_function(self, X=None, T=None):
 
         if X is not None:
             X = check_array(X)
-            return self.z_classifier_.decision_function(X)
+            return X @ self.pxz_
         else:
             T = check_array(T)
-
-            return self.classifier_.decision_function(T)
-
-        # if X is not None:
-        #     X = check_array(X)
-        #     return X @ self.pxz_
-        # else:
-        #     T = check_array(T)
-
-        #     return T @ self.ptz_
+            return T @ self.ptz_
 
     def predict(self, X=None, T=None):
         """Predicts the property labels using classification on T."""
@@ -460,7 +456,7 @@ def transform(self, X=None):
         """
         return super().transform(X)
 
-    def score(self, X, Y, T=None, sample_weight=None):
+    def score(self, X, Y, sample_weight=None):
         #taken from sklearn's LogisticRegression score() implementation:
         r"""Return the mean accuracy on the given test data and labels.
 
@@ -488,4 +484,4 @@ def score(self, X, Y, T=None, sample_weight=None):
         score : float
             Mean accuracy of ``self.predict(X, T)`` w.r.t. `Y`.
         """
-        return accuracy_score(Y, self.predict(X, T), sample_weight=sample_weight)
+        return accuracy_score(Y, self.predict(X), sample_weight=sample_weight)