rebuild and recheck

Author: JohnMount

build/lib/data_algebra/pandas_base.py

@@ -78,7 +78,8 @@ def table_step(self, op, *, data_map, narrow):
         res = df.loc[:, columns_using]
         res = res.reset_index(drop=True)
         # check type compatibility
-        if (op.column_types is not None) and (len(op.column_types) > 0):
+        if((op.column_types is not None) and (len(op.column_types) > 0)
+                and (res.shape[0] > 0) and (res.shape[1] > 0)):
             types_seen = data_algebra.util.guess_column_types(res)
             for c in columns_using:
                 assert data_algebra.util.compatible_types(
@@ -465,15 +466,21 @@ def concat_rows_step(self, op, *, data_map, narrow):
             data_map=data_map, data_model=self, narrow=narrow
         )
         if op.id_column is not None:
-            left[op.id_column] = op.a_name
-            right[op.id_column] = op.b_name
-        type_checks = data_algebra.util.check_columns_appear_compatible(left, right)
-        if type_checks is not None:
-            raise ValueError(f"concat: incompatible column types: {type_checks}")
+            if left.shape[0] > 0:
+                left[op.id_column] = op.a_name
+            else:
+                left[op.id_column] = []
+            if right.shape[0] > 0:
+                right[op.id_column] = op.b_name
+            else:
+                right[op.id_column] = []
         if left.shape[0] < 1:
             return right
         if right.shape[0] < 1:
             return left
+        type_checks = data_algebra.util.check_columns_appear_compatible(left, right)
+        if type_checks is not None:
+            raise ValueError(f"concat: incompatible column types: {type_checks}")
         # noinspection PyUnresolvedReferences
         res = self.pd.concat([left, right], axis=0, ignore_index=True, sort=False)
         res = res.reset_index(drop=True)

build/lib/data_algebra/util.py

@@ -1,3 +1,6 @@
+
+import datetime
+
 import numpy
 
 import data_algebra
@@ -73,7 +76,8 @@ def guess_carried_scalar_type(col):
     :return: type of first non-None entry, if any , else type(None)
     """
     ct = map_type_to_canonical(type(col))
-    if ct in {str, int, float, bool, type(None), numpy.int64, numpy.float64}:
+    if ct in {str, int, float, bool, type(None), numpy.int64, numpy.float64,
+              datetime.datetime, datetime.date, datetime.timedelta}:
         return ct
     if len(col) < 1:
         return type(None)
@@ -85,25 +89,27 @@ def guess_carried_scalar_type(col):
 
 def guess_column_types(d, *, columns=None):
     """
-    Guess column types as type of first non-missing value
+    Guess column types as type of first non-missing value.
+    Will not return series types, as some pandas data frames with non-trivial indexing report this type.
 
     :param d: pandas.DataFrame
     :param columns: list of columns to check, if None all columns are checked
-    :return: map of column names to guessed types
+    :return: map of column names to guessed types, empty dict if any column guess fails
     """
-    if d.shape[1] <= 0:
+    if (d.shape[0] <= 0) or (d.shape[1] <= 0):
         return dict()
     if columns is None:
-        columns = d.columns
+        columns = d.columns.copy()
     assert len(set(columns) - set(d.columns)) == 0
-    if d.shape[0] <= 0:
-        return {c: type(None) for c in columns}
+    if len(columns) <= 0:
+        return dict()
     res = dict()
     for c in columns:
-        res[c] = guess_carried_scalar_type(d[c])
-    if any([str(v).endswith('.Series\'>') for v in res.values()]):
-        # pandas.concat() poisons types with Series, don't allow that
-        return dict()
+        gt = guess_carried_scalar_type(d[c])
+        if (gt is None) or (not isinstance(gt, type)) or str(gt).endswith('.Series\'>'):
+            # pandas.concat() poisons types with Series, don't allow that
+            return dict()
+        res[c] = gt
     return res
 
 
@@ -129,7 +135,11 @@ def check_columns_appear_compatible(d_left, d_right, *, columns=None):
     assert len(set(columns) - set(d_left.columns)) == 0
     assert len(set(columns) - set(d_right.columns)) == 0
     left_types = data_algebra.util.guess_column_types(d_left, columns=columns)
+    if (left_types is None) or (len(left_types) <= 0):
+        return None
     right_types = data_algebra.util.guess_column_types(d_right, columns=columns)
+    if (right_types is None) or (len(right_types) <= 0):
+        return None
     mismatches = dict()
     for c in columns:
         if not compatible_types([left_types[c], right_types[c]]):

coverage.txt

@@ -111,14 +111,14 @@ data_algebra/expr_rep.py                 562    103    82%
 data_algebra/flow_text.py                 17      0   100%
 data_algebra/near_sql.py                 144      1    99%
 data_algebra/op_container.py             112     39    65%
-data_algebra/pandas_base.py              305     32    90%
+data_algebra/pandas_base.py              309     32    90%
 data_algebra/pandas_model.py              18      3    83%
 data_algebra/parse_by_lark.py            143     26    82%
 data_algebra/python3_lark.py               1      0   100%
 data_algebra/test_util.py                195     37    81%
-data_algebra/util.py                      80      6    92%
+data_algebra/util.py                      86      6    93%
 ----------------------------------------------------------
-TOTAL                                   4794    853    82%
+TOTAL                                   4804    853    82%
 
 
-============================= 210 passed in 17.18s =============================
+============================= 210 passed in 16.93s =============================