work on custom functions (todo: sqlite equiv path)

Author: JohnMount

build/lib/data_algebra/custom_functions.py

@@ -109,11 +109,6 @@ def make_custom_function_map(data_model):
             pandas_formatter=lambda expr: ("@min(" + expr.args[0].to_pandas() + ")"),
             implementation=lambda x: [numpy.min(x)] * len(x),
         ),
-        CustomFunction(
-            name="fn",  # special case, user defined function
-            pandas_formatter=lambda expr: "@fn(" + expr.args[0].to_pandas() + ")",
-            implementation=None,
-        ),
     ]
     mp = {cf.name: cf for cf in custom_functions}
     return mp

build/lib/data_algebra/data_ops.py

@@ -36,24 +36,37 @@
 
 # wrap a single argument function as a user callable function in pipeline
 # used for custom aggregators
-def user_fn(fn, var, display_form=None):
+def user_fn(fn, fvars=None, *, display_form=None):
     if isinstance(fn, str):
         if display_form is None:
             display_form = fn
         fn = eval(fn)
     if not callable(fn):
         raise TypeError("expected fn to be callable")
+    if fvars is None:
+        fvars = []
+    if isinstance(fvars, str):
+        fn_args = [data_algebra.expr_rep.ColumnReference(view=None, column_name=fvars)]
+    else:
+        for v in fvars:
+            if not isinstance(v, str):
+                raise TypeError("Expect all vars names to be strings")
+        fn_args = [data_algebra.expr_rep.ColumnReference(view=None, column_name=v) for v in fvars]
+    qvars = [v.__repr__() for v in fvars]
     if display_form is None:
-        display_form = fn.__name__
-    if not isinstance(var, str):
-        raise TypeError("expected var to be str")
+        display_form = ('user_fn('
+                        + fn.__name__
+                        + ', ['
+                        + ', '.join(qvars)
+                        + '])')
     return data_algebra.expr_rep.FnTerm(
         fn,
-        fn_arg=data_algebra.expr_rep.ColumnReference(view=None, column_name=var),
+        fn_args=fn_args,
         display_form=display_form,
     )
 
 
+
 class ViewRepresentation(OperatorPlatform, ABC):
     """Structure to represent the columns of a query or a table.
        Abstract base class."""

build/lib/data_algebra/expr_rep.py

@@ -690,17 +690,11 @@ def to_python(self, *, want_inline_parens=False):
         return self.value.__repr__()
 
 
-class Name(str):
+class UnQuotedStr(str):
     def __init__(self, v):
         self.v = v
         str.__init__(v)
 
-    def is_equal(self, other):
-        # can't use == as that builds a larger expression
-        if not isinstance(other, Name):
-            return False
-        return self.v == other.v
-
     def str(self):
         return self.v
 
@@ -709,22 +703,29 @@ def __repr__(self):
 
 
 class FnTerm(Term):
-    def __init__(self, value, fn_arg=None, display_form=None, op=None):
+    # represent a function of columns
+    def __init__(self, value, fn_args=None, name=None, display_form=None, op=None):
         if not callable(value):
             raise TypeError("value type must be callable")
         self.value = value
+        if name is None:
+            name = value.__name__
+        self.name = name
         if display_form is None:
             display_form = value.__name__
         if op is None:
             op = value.__name__
         self.display_form = display_form
         self.op = op
-        if fn_arg is None:
+        if fn_args is None:
             self.args = []
         else:
-            if not isinstance(fn_arg, ColumnReference):
-                raise TypeError("Expected fn_arg to be None or a ColumnReference")
-            self.args = [fn_arg]
+            if isinstance(fn_args, ColumnReference):
+                fn_args = [fn_args]
+            for v in fn_args:
+                if not isinstance(v, ColumnReference):
+                    raise TypeError("Expected fn_args to be None or all ColumnReference")
+            self.args = fn_args
         Term.__init__(self)
 
     def is_equal(self, other):
@@ -749,7 +750,7 @@ def replace_view(self, view):
         return self
 
     def to_python(self, *, want_inline_parens=False):
-        return Name(self.display_form)
+        return UnQuotedStr(self.display_form)
 
 
 class ListTerm(Term):
@@ -1068,7 +1069,7 @@ def parse_assignments_in_context(ops, view, *, parse_env=None):
         if not isinstance(v, Term):
             if callable(v):
                 # k = f(k) implicit form
-                v = FnTerm(v, fn_arg=ColumnReference(view=view, column_name=k))
+                v = FnTerm(v, fn_args=[ColumnReference(view=view, column_name=k)])
             else:
                 v = _parse_by_eval(
                     source_str=v, data_def=mp, outter_environemnt=parse_env

build/lib/data_algebra/pandas_base.py

@@ -112,10 +112,22 @@ def extend_step(self, op, *, data_map, eval_env, narrow):
         standin_name = "_data_algebra_temp_g"  # name of an arbitrary input variable
         if not window_situation:
             for (k, opk) in op.ops.items():
-                op_src = opk.to_pandas()
-                res[k] = res.eval(
-                    op_src, local_dict=self.pandas_eval_env, global_dict=eval_env
-                )
+                if isinstance(opk, data_algebra.expr_rep.FnTerm):
+                    # res[k] = opk.value(*[res[nm.column_name] for nm in opk.args])
+                    pe = self.pandas_eval_env.copy()
+                    pe[opk.name] = opk.value
+                    op_src = ('@' + opk.name
+                              + '('
+                              + ', '.join([nm.column_name for nm in opk.args])
+                              + ')')
+                    res[k] = res.eval(
+                        op_src, local_dict=pe, global_dict=eval_env
+                    )
+                else:
+                    op_src = opk.to_pandas()
+                    res[k] = res.eval(
+                        op_src, local_dict=self.pandas_eval_env, global_dict=eval_env
+                    )
         else:
             # build up a sub-frame to work on
             col_list = [c for c in set(op.partition_by)]

coverage.txt

@@ -72,7 +72,7 @@ data_algebra/cdata_impl.py                10      1    90%
 data_algebra/connected_components.py      49      1    98%
 data_algebra/custom_functions.py          17      2    88%
 data_algebra/data_model.py                40     18    55%
-data_algebra/data_ops.py                1221    247    80%
+data_algebra/data_ops.py                1228    248    80%
 data_algebra/data_ops_types.py            74     33    55%
 data_algebra/data_ops_utils.py            34      6    82%
 data_algebra/data_pipe.py                189     50    74%
@@ -81,17 +81,17 @@ data_algebra/diagram.py                   56     44    21%
 data_algebra/env.py                       31      3    90%
 data_algebra/eval_model.py                18      2    89%
 data_algebra/expr.py                      20      4    80%
-data_algebra/expr_rep.py                 695    219    68%
+data_algebra/expr_rep.py                 697    217    69%
 data_algebra/flow_text.py                 17      0   100%
 data_algebra/modin_model.py               36      5    86%
 data_algebra/near_sql.py                 140     16    89%
-data_algebra/pandas_base.py              219     21    90%
+data_algebra/pandas_base.py              224     21    91%
 data_algebra/pandas_model.py              25     17    32%
 data_algebra/test_util.py                126     18    86%
 data_algebra/util.py                      28      4    86%
 data_algebra/yaml.py                      95     11    88%
 ----------------------------------------------------------
-TOTAL                                   4276    944    78%
+TOTAL                                   4290    943    78%
 
 
-============================= 99 passed in 14.21s ==============================
+============================= 99 passed in 13.80s ==============================

data_algebra/custom_functions.py

@@ -109,11 +109,6 @@ def make_custom_function_map(data_model):
             pandas_formatter=lambda expr: ("@min(" + expr.args[0].to_pandas() + ")"),
             implementation=lambda x: [numpy.min(x)] * len(x),
         ),
-        CustomFunction(
-            name="fn",  # special case, user defined function
-            pandas_formatter=lambda expr: "@fn(" + expr.args[0].to_pandas() + ")",
-            implementation=None,
-        ),
     ]
     mp = {cf.name: cf for cf in custom_functions}
     return mp

data_algebra/data_ops.py

@@ -36,24 +36,37 @@
 
 # wrap a single argument function as a user callable function in pipeline
 # used for custom aggregators
-def user_fn(fn, var, display_form=None):
+def user_fn(fn, fvars=None, *, display_form=None):
     if isinstance(fn, str):
         if display_form is None:
             display_form = fn
         fn = eval(fn)
     if not callable(fn):
         raise TypeError("expected fn to be callable")
+    if fvars is None:
+        fvars = []
+    if isinstance(fvars, str):
+        fn_args = [data_algebra.expr_rep.ColumnReference(view=None, column_name=fvars)]
+    else:
+        for v in fvars:
+            if not isinstance(v, str):
+                raise TypeError("Expect all vars names to be strings")
+        fn_args = [data_algebra.expr_rep.ColumnReference(view=None, column_name=v) for v in fvars]
+    qvars = [v.__repr__() for v in fvars]
     if display_form is None:
-        display_form = fn.__name__
-    if not isinstance(var, str):
-        raise TypeError("expected var to be str")
+        display_form = ('user_fn('
+                        + fn.__name__
+                        + ', ['
+                        + ', '.join(qvars)
+                        + '])')
     return data_algebra.expr_rep.FnTerm(
         fn,
-        fn_arg=data_algebra.expr_rep.ColumnReference(view=None, column_name=var),
+        fn_args=fn_args,
         display_form=display_form,
     )
 
 
+
 class ViewRepresentation(OperatorPlatform, ABC):
     """Structure to represent the columns of a query or a table.
        Abstract base class."""

data_algebra/expr_rep.py

@@ -690,17 +690,11 @@ def to_python(self, *, want_inline_parens=False):
         return self.value.__repr__()
 
 
-class Name(str):
+class UnQuotedStr(str):
     def __init__(self, v):
         self.v = v
         str.__init__(v)
 
-    def is_equal(self, other):
-        # can't use == as that builds a larger expression
-        if not isinstance(other, Name):
-            return False
-        return self.v == other.v
-
     def str(self):
         return self.v
 
@@ -709,22 +703,29 @@ def __repr__(self):
 
 
 class FnTerm(Term):
-    def __init__(self, value, fn_arg=None, display_form=None, op=None):
+    # represent a function of columns
+    def __init__(self, value, fn_args=None, name=None, display_form=None, op=None):
         if not callable(value):
             raise TypeError("value type must be callable")
         self.value = value
+        if name is None:
+            name = value.__name__
+        self.name = name
         if display_form is None:
             display_form = value.__name__
         if op is None:
             op = value.__name__
         self.display_form = display_form
         self.op = op
-        if fn_arg is None:
+        if fn_args is None:
             self.args = []
         else:
-            if not isinstance(fn_arg, ColumnReference):
-                raise TypeError("Expected fn_arg to be None or a ColumnReference")
-            self.args = [fn_arg]
+            if isinstance(fn_args, ColumnReference):
+                fn_args = [fn_args]
+            for v in fn_args:
+                if not isinstance(v, ColumnReference):
+                    raise TypeError("Expected fn_args to be None or all ColumnReference")
+            self.args = fn_args
         Term.__init__(self)
 
     def is_equal(self, other):
@@ -749,7 +750,7 @@ def replace_view(self, view):
         return self
 
     def to_python(self, *, want_inline_parens=False):
-        return Name(self.display_form)
+        return UnQuotedStr(self.display_form)
 
 
 class ListTerm(Term):
@@ -1068,7 +1069,7 @@ def parse_assignments_in_context(ops, view, *, parse_env=None):
         if not isinstance(v, Term):
             if callable(v):
                 # k = f(k) implicit form
-                v = FnTerm(v, fn_arg=ColumnReference(view=view, column_name=k))
+                v = FnTerm(v, fn_args=[ColumnReference(view=view, column_name=k)])
             else:
                 v = _parse_by_eval(
                     source_str=v, data_def=mp, outter_environemnt=parse_env

data_algebra/pandas_base.py

@@ -112,10 +112,22 @@ def extend_step(self, op, *, data_map, eval_env, narrow):
         standin_name = "_data_algebra_temp_g"  # name of an arbitrary input variable
         if not window_situation:
             for (k, opk) in op.ops.items():
-                op_src = opk.to_pandas()
-                res[k] = res.eval(
-                    op_src, local_dict=self.pandas_eval_env, global_dict=eval_env
-                )
+                if isinstance(opk, data_algebra.expr_rep.FnTerm):
+                    # res[k] = opk.value(*[res[nm.column_name] for nm in opk.args])
+                    pe = self.pandas_eval_env.copy()
+                    pe[opk.name] = opk.value
+                    op_src = ('@' + opk.name
+                              + '('
+                              + ', '.join([nm.column_name for nm in opk.args])
+                              + ')')
+                    res[k] = res.eval(
+                        op_src, local_dict=pe, global_dict=eval_env
+                    )
+                else:
+                    op_src = opk.to_pandas()
+                    res[k] = res.eval(
+                        op_src, local_dict=self.pandas_eval_env, global_dict=eval_env
+                    )
         else:
             # build up a sub-frame to work on
             col_list = [c for c in set(op.partition_by)]