Serialization bugfixing (#35)

* serde error

* more tests

* improvements

* drop lgbm

* cleanup

Author: bcebere

src/hyperimpute/plugins/core/base_plugin.py

@@ -38,8 +38,6 @@ class Plugin(Serializable, metaclass=ABCMeta):
     def __init__(self) -> None:
         super().__init__()
 
-        self.drop_consts = []
-
     @staticmethod
     @abstractmethod
     def hyperparameter_space(*args: Any, **kwargs: Any) -> List[Params]:
@@ -122,11 +120,6 @@ def fit_predict(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> pd.DataFram
     def fit(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> Any:
         X = cast.to_dataframe(X)
 
-        for col in X.columns:
-            if len(X.loc[X[col].notna(), col].unique()) <= 1:
-                self.drop_consts.append(col)
-
-        X = X.drop(columns=self.drop_consts)
         self.columns = X.columns
         return self._fit(X, *args, **kwargs)
 
@@ -136,7 +129,6 @@ def _fit(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> "Plugin":
 
     def transform(self, X: pd.DataFrame) -> pd.DataFrame:
         X = cast.to_dataframe(X)
-        X = X.drop(columns=self.drop_consts)
         return pd.DataFrame(self._transform(X))
 
     @abstractmethod
@@ -145,7 +137,6 @@ def _transform(self, X: pd.DataFrame) -> pd.DataFrame:
 
     def predict(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> pd.DataFrame:
         X = cast.to_dataframe(X)
-        X = X.drop(columns=self.drop_consts)
         return pd.DataFrame(self._predict(X, *args, *kwargs))
 
     @abstractmethod

src/hyperimpute/plugins/prediction/classifiers/plugin_lgbm.py

@@ -41,7 +41,7 @@ class RandomForestPlugin(base.ClassifierPlugin):
     """
 
     criterions = ["gini", "entropy"]
-    features = ["auto", "sqrt", "log2"]
+    features = ["sqrt", "log2", None]
 
     def __init__(
         self,
@@ -97,11 +97,13 @@ def _fit(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> "RandomForestPlugi
         return self
 
     def _predict(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> pd.DataFrame:
+        X = np.asarray(X)
         return self.model.predict(X, *args, **kwargs)
 
     def _predict_proba(
         self, X: pd.DataFrame, *args: Any, **kwargs: Any
     ) -> pd.DataFrame:
+        X = np.asarray(X)
         return self.model.predict_proba(X, *args, **kwargs)
 
 

src/hyperimpute/plugins/prediction/classifiers/plugin_random_forest.py

@@ -41,7 +41,7 @@ class RandomForestRegressionPlugin(base.RegressionPlugin):
     """
 
     criterions = ["squared_error", "absolute_error", "poisson"]
-    features = ["auto", "sqrt", "log2"]
+    features = ["sqrt", "log2", None]
 
     def __init__(
         self,
@@ -103,6 +103,8 @@ def _fit(
         return self
 
     def _predict(self, X: pd.DataFrame, *args: Any, **kwargs: Any) -> pd.DataFrame:
+        X = np.asarray(X)
+
         return self.model.predict(X, *args, **kwargs)
 
 

src/hyperimpute/plugins/prediction/regression/plugin_lgbm_regressor.py

@@ -138,19 +138,42 @@ def version() -> str:
         return MAJOR_VERSION
 
 
+def _add_version(obj: Any) -> Any:
+    obj._serde_version = MAJOR_VERSION
+    return obj
+
+
+def _check_version(obj: Any) -> Any:
+    local_version = obj._serde_version
+
+    if not hasattr(obj, "_serde_version"):
+        raise RuntimeError("Missing serialization version")
+
+    if local_version != MAJOR_VERSION:
+        raise ValueError(
+            f"Serialized object mismatch. Current major version is {MAJOR_VERSION}, but the serialized object has version {local_version}."
+        )
+
+
 def save(model: Any) -> bytes:
+    _add_version(model)
     return cloudpickle.dumps(model)
 
 
 def load(buff: bytes) -> Any:
-    return cloudpickle.loads(buff)
+    obj = cloudpickle.loads(buff)
+    _check_version(obj)
+    return obj
 
 
 def save_to_file(path: Union[str, Path], model: Any) -> Any:
+    _add_version(model)
     with open(path, "wb") as f:
         return cloudpickle.dump(model, f)
 
 
 def load_from_file(path: Union[str, Path]) -> Any:
     with open(path, "rb") as f:
-        return cloudpickle.load(f)
+        obj = cloudpickle.load(f)
+        _check_version(obj)
+        return obj

src/hyperimpute/plugins/prediction/regression/plugin_random_forest_regressor.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.14"
+__version__ = "0.1.15"
 
 MAJOR_VERSION = ".".join(__version__.split(".")[:-1])
 MINOR_VERSION = __version__.split(".")[-1]