RPCA PCP not imputing

Author: Julien Roussel

examples/1_timeSeries.ipynb

@@ -4,14 +4,9 @@
 import numpy as np
 import pandas as pd
 
-from qolmat import logging as qlog
 from qolmat.benchmark import cross_validation, utils
 from qolmat.benchmark.missing_patterns import _HoleGenerator
 
-qlog.log_setup()
-logger = logging.getLogger(__name__)
-# logger.setLevel(logging.DEBUG)
-
 
 class Comparator:
     """
@@ -153,7 +148,6 @@ def compare(self, df: pd.DataFrame, verbose: bool = True):
         dict_errors = {}
 
         for name, imputer in self.dict_models.items():
-            logger.setLevel(logging.DEBUG)
             print(f"Tested model: {type(imputer).__name__}")
 
             search_params = self.search_params.get(name, {})

qolmat/benchmark/comparator.py

@@ -59,7 +59,18 @@ def get_params_scale(self, D):
         dict_params = {"mu": mu, "lam": lam}
         return dict_params
 
-    def decompose_rpca(self, D: NDArray, mu:float, lam: float) -> Tuple[NDArray, NDArray]:
+    def decompose_rpca(self, D: NDArray) -> Tuple[NDArray, NDArray]:
+        proj_D = utils.impute_nans(D, method="median")
+
+        params_scale = self.get_params_scale(proj_D)
+
+        mu = params_scale["mu"] if self.mu is None else self.mu
+        lam = params_scale["lam"] if self.lam is None else self.lam
+        Omega = ~np.isnan(D)
+
+        print("mu:", mu)
+        print("lam:", lam)
+        
         D_norm = np.linalg.norm(D, "fro")
 
         A = np.full_like(D, 0)
@@ -69,15 +80,15 @@ def decompose_rpca(self, D: NDArray, mu:float, lam: float) -> Tuple[NDArray, NDA
 
         for iteration in range(self.max_iter):
 
-            M = utils.svd_thresholding(D - A + Y/mu, 1/mu)
-            A = utils.soft_thresholding(D - M + Y/mu, lam/mu)
-            Y += mu * (D - M - A)
+            M = utils.svd_thresholding(proj_D - A + Y/mu, 1/mu)
+            A = utils.soft_thresholding(proj_D - M + Y/mu, lam/mu)
+            A[~Omega] = (proj_D - M)[~Omega]
+            Y += mu * (proj_D - M - A)
 
             error = np.linalg.norm(D - M - A, "fro")/D_norm
             errors[iteration] = error
 
             if error < self.tol:
-                print(iteration, ":", error, "vs", self.tol)
                 if self.verbose:
                     print(f"Converged in {iteration} iterations")
                 break
@@ -110,20 +121,8 @@ def fit_transform(
             Array of iterative errors
         """
         X = X.copy().T
-        D_init = self._prepare_data(X)
-        print("D_init")
-        print(D_init.shape)
-        proj_D = utils.impute_nans(D_init, method="median")
-
-        params_scale = self.get_params_scale(proj_D)
-
-        mu = params_scale["mu"] if self.mu is None else self.mu
-        lam = params_scale["lam"] if self.lam is None else self.lam
-
-        print("mu:", mu)
-        print("lam:", lam)
-
-        M, A = self.decompose_rpca(proj_D, mu, lam)
+        D = self._prepare_data(X)
+        M, A = self.decompose_rpca(D)
 
         # U, _, V = np.linalg.svd(M, full_matrices=False, compute_uv=True)
 

qolmat/imputations/rpca/pcp_rpca.py

@@ -136,27 +136,18 @@ df_data.values.size
 df_imputed = imputer_rpca.fit_transform(df_data)
 ```
 
-```python
-df_imputed.shape
-```
-
 ```python
 df_imputed.iloc[:365 * (df_imputed.size // 365)]
 ```
 
 ```python
-D = df_imputed.iloc[:365 * (df_imputed.size // 365)].values.reshape(365, -1)
+D = df_imputed.iloc[:365 * (df_imputed.size // 365)].values.reshape(-1, 365).T
 ```
 
 ```python
 plt.plot(D)
 ```
 
-```python
-plt.plot(df_data["TEMP"].loc[station], ".", color="black")
-plt.plot(df_imputed["TEMP"].loc[station])
-```
-
 This part is devoted to the imputation methods. The idea is to try different algorithms and compare them.
 
 <u>**Methods**</u>: