implemented more of sklearn step API

Author: JohnMount

build/lib/data_algebra/data_ops.py

@@ -114,6 +114,9 @@ def columns_used_from_sources(self, using=None):
         with the using columns (None means all)."""
         raise NotImplementedError("base method called")
 
+    def columns_produced(self):
+        return self.column_names.copy()
+
     def _clear_columns_currently_used(self):
         self.columns_currently_used = set()
         for si in self.sources:
@@ -300,6 +303,7 @@ def eval_pandas(self, data_map, *, eval_env=None, data_model=None, narrow=True):
         :param data_map: map from table names to data frames
         :param eval_env: environment to evaluate in
         :param data_model: adaptor to Pandas dialect
+        :param narrow logical, if True don't copy unexpected columns
         :return:
         """
 
@@ -759,6 +763,9 @@ def __init__(self, *, underlying, data_map):
         self.data_map = data_map.copy()
         self.underlying = underlying
 
+    def columns_produced(self):
+        return self.underlying.columns_produced()
+
     # execution
 
     def ex(self):

build/lib/data_algebra/data_ops_types.py

@@ -18,6 +18,9 @@ def transform(self, X, *, eval_env=None, data_model=None, narrow=True):
         apply self to data frame X, may or may not commute with composition
 
         :param X: input data frame
+        :param eval_env environment to look for symbols in
+        :param data_model implementation to use
+        :param narrow logical, if True don't copy unexpected columns
         :return: transformed dataframe
         """
         raise NotImplementedError("base class called")
@@ -28,6 +31,8 @@ def act_on(self, X, *, eval_env=None, data_model=None):
         apply self to data frame X, must commute with composition
 
         :param X: input data frame
+        :param eval_env environment to look for symbols in
+        :param data_model implementation to use
         :return: transformed dataframe
         """
         return self.transform(X=X, eval_env=eval_env, data_model=data_model, narrow=False)
@@ -81,6 +86,11 @@ def add(self, other):
         """
         return other.apply_to(self)
 
+    # info
+
+    def columns_produced(self):
+        raise NotImplementedError("base class called")
+
     # query generation
 
     def to_sql_implementation(self, db_model, *, using, temp_id_source):
@@ -130,3 +140,28 @@ def order_rows(self, columns, *, reverse=None, limit=None):
 
     def convert_records(self, record_map):
         raise NotImplementedError("base class called")
+
+    # sklearn step style interface
+
+    # noinspection PyPep8Naming, PyUnusedLocal
+    def fit(self, X, y=None):
+        pass
+
+    # noinspection PyPep8Naming, PyUnusedLocal
+    def fit_transform(self, X, y=None):
+        return self.transform(X)
+
+    # noinspection PyUnusedLocal
+    def get_feature_names(self, input_features=None):
+        return self.columns_produced()
+
+    # noinspection PyUnusedLocal,PyMethodMayBeStatic
+    def get_params(self, deep=False):
+        return dict()
+
+    def set_params(self, **params):
+        pass
+
+    # noinspection PyPep8Naming
+    def inverse_transform(self, X):
+        raise TypeError("data_algebra does not support inverse_transform")

build/lib/data_algebra/near_sql.py

@@ -13,7 +13,7 @@ def __init__(self, *, terms, quoted_query_name, temp_tables):
         self.temp_tables = temp_tables
 
     def to_sql(self, *, columns=None, force_sql=False, constants=None, db_model):
-        raise NotImplemented("base method called")
+        raise NotImplementedError("base method called")
 
     def summary(self):
         return {"quoted_query_name": self.quoted_query_name, "is_table": False}

coverage.txt

@@ -63,8 +63,8 @@ data_algebra/cdata.py                    232     75    68%
 data_algebra/cdata_impl.py                10      1    90%
 data_algebra/connected_components.py      49      1    98%
 data_algebra/data_model.py                29     13    55%
-data_algebra/data_ops.py                1253    268    79%
-data_algebra/data_ops_types.py            56     22    61%
+data_algebra/data_ops.py                1257    270    79%
+data_algebra/data_ops_types.py            70     29    59%
 data_algebra/data_ops_utils.py            34      6    82%
 data_algebra/data_pipe.py                189     50    74%
 data_algebra/db_model.py                 402     70    83%
@@ -79,7 +79,7 @@ data_algebra/test_util.py                126     18    86%
 data_algebra/util.py                      45      6    87%
 data_algebra/yaml.py                      95     11    88%
 ----------------------------------------------------------
-TOTAL                                   3909    879    78%
+TOTAL                                   3927    888    77%
 
 
-============================== 85 passed in 8.37s ==============================
+============================== 85 passed in 7.12s ==============================

data_algebra/data_ops.py

@@ -114,6 +114,9 @@ def columns_used_from_sources(self, using=None):
         with the using columns (None means all)."""
         raise NotImplementedError("base method called")
 
+    def columns_produced(self):
+        return self.column_names.copy()
+
     def _clear_columns_currently_used(self):
         self.columns_currently_used = set()
         for si in self.sources:
@@ -300,6 +303,7 @@ def eval_pandas(self, data_map, *, eval_env=None, data_model=None, narrow=True):
         :param data_map: map from table names to data frames
         :param eval_env: environment to evaluate in
         :param data_model: adaptor to Pandas dialect
+        :param narrow logical, if True don't copy unexpected columns
         :return:
         """
 
@@ -759,6 +763,9 @@ def __init__(self, *, underlying, data_map):
         self.data_map = data_map.copy()
         self.underlying = underlying
 
+    def columns_produced(self):
+        return self.underlying.columns_produced()
+
     # execution
 
     def ex(self):

data_algebra/data_ops_types.py

@@ -18,6 +18,9 @@ def transform(self, X, *, eval_env=None, data_model=None, narrow=True):
         apply self to data frame X, may or may not commute with composition
 
         :param X: input data frame
+        :param eval_env environment to look for symbols in
+        :param data_model implementation to use
+        :param narrow logical, if True don't copy unexpected columns
         :return: transformed dataframe
         """
         raise NotImplementedError("base class called")
@@ -28,6 +31,8 @@ def act_on(self, X, *, eval_env=None, data_model=None):
         apply self to data frame X, must commute with composition
 
         :param X: input data frame
+        :param eval_env environment to look for symbols in
+        :param data_model implementation to use
         :return: transformed dataframe
         """
         return self.transform(X=X, eval_env=eval_env, data_model=data_model, narrow=False)
@@ -81,6 +86,11 @@ def add(self, other):
         """
         return other.apply_to(self)
 
+    # info
+
+    def columns_produced(self):
+        raise NotImplementedError("base class called")
+
     # query generation
 
     def to_sql_implementation(self, db_model, *, using, temp_id_source):
@@ -130,3 +140,28 @@ def order_rows(self, columns, *, reverse=None, limit=None):
 
     def convert_records(self, record_map):
         raise NotImplementedError("base class called")
+
+    # sklearn step style interface
+
+    # noinspection PyPep8Naming, PyUnusedLocal
+    def fit(self, X, y=None):
+        pass
+
+    # noinspection PyPep8Naming, PyUnusedLocal
+    def fit_transform(self, X, y=None):
+        return self.transform(X)
+
+    # noinspection PyUnusedLocal
+    def get_feature_names(self, input_features=None):
+        return self.columns_produced()
+
+    # noinspection PyUnusedLocal,PyMethodMayBeStatic
+    def get_params(self, deep=False):
+        return dict()
+
+    def set_params(self, **params):
+        pass
+
+    # noinspection PyPep8Naming
+    def inverse_transform(self, X):
+        raise TypeError("data_algebra does not support inverse_transform")