rebuild and retest

Author: JohnMount

build/lib/data_algebra/polars_model.py

@@ -20,11 +20,15 @@
 
 
 def _build_lit(v):
-    if isinstance(v, int):
+    if isinstance(v, bool):
+        # bools return true to isinstance int.
+        return pl.lit(v)
+    elif isinstance(v, int):
         # Polars defaults ints in constructor to Int64,
         # but ints in lit to Int32. Try to prevent type clashes
         return pl.lit(v, pl.Int64)
-    return pl.lit(v)
+    else:
+        return pl.lit(v)
 
 
 def _reduce_plus(*args):
@@ -632,6 +636,7 @@ def _extend_step(self, op: data_algebra.data_ops_types.OperatorPlatform, *, data
         res = res.with_columns(produced_columns)
         if len(temp_v_columns) > 0:
             res = res.select(op.columns_produced())
+        # get back to lazy type if needed
         if self.use_lazy_eval and isinstance(res, pl.DataFrame):
             res = res.lazy()
         return res
@@ -685,6 +690,14 @@ def _project_step(self, op: data_algebra.data_ops_types.OperatorPlatform, *, dat
         res = res.groupby(group_by).agg(produced_columns)
         if len(temp_v_columns) > 0:
             res = res.select(op.columns_produced())
+        if (op.group_by is None) or (len(op.group_by) == 0):
+            # see if we have a zero row result
+            if isinstance(res, pl.LazyFrame):
+                res = res.collect()
+            if res.shape[0] <= 0:
+                # make an all None frame
+                res = pl.DataFrame({c: [None] for c in res.columns}, columns=[(res.columns[j], res.dtypes[j]) for j in range(res.shape[1])])
+        # see if we need to convert to lazy type
         if self.use_lazy_eval and isinstance(res, pl.DataFrame):
             res = res.lazy()
         return res

build/lib/data_algebra/solutions.py

@@ -341,111 +341,6 @@ def last_observed_carried_forward(
     return ops
 
 
-def braid_data(
-    *,
-    d_state: ViewRepresentation,
-    d_event: ViewRepresentation,
-    order_by: Iterable[str],
-    partition_by: Optional[Iterable[str]] = None,
-    state_value_column_name: str,
-    event_value_column_names: Iterable[str],
-    source_id_column: str = "record_type",
-    state_row_mark: str = "state_row",
-    event_row_mark: str = "event_row",
-    stand_in_values: Dict,
-    locf_to_use_column_name: str = "locf_to_use",
-    locf_non_null_rank_column_name: str = "locf_non_null_rank",
-    locf_tiebreaker_column_name: str = "locf_tiebreaker",
-) -> ViewRepresentation:
-    """
-    Mix data from two sources, ordering by order_by columns and carrying forward observations
-    on d_state value column.
-
-    :param d_state: ViewRepresentation representation of state by order_by.
-    :param d_event: ViewRepresentation representation of events by order_by.
-    :param order_by: columns to order by (non empty list of column names)
-    :param partition_by: optional partitioning column names
-    :param state_value_column_name: column to copy from d_state and propagate forward
-    :param event_value_column_names: columns to copy from d_event
-    :param source_id_column: name for source identification column.
-    :param state_row_mark: source annotation of state rows.
-    :param event_row_mark: source annotation of event rows.
-    :param stand_in_values: dictionary stand in values to use for state_value_column_name and event_value_column_names
-            needed to get column types correct, replaced by None and not passed further.
-    :param locf_to_use_column_name: name for a temporary values column
-    :param locf_non_null_rank_column_name: name for a temporary values column
-    :param locf_tiebreaker_column_name: name for a temporary values column
-    :return: ops
-    """
-    assert isinstance(d_state, ViewRepresentation)
-    assert isinstance(d_event, ViewRepresentation)
-    assert not isinstance(order_by, str)
-    order_by = list(order_by)
-    assert len(order_by) > 0
-    if partition_by is not None:
-        assert not isinstance(partition_by, str)
-        partition_by = list(partition_by)
-    else:
-        partition_by = []
-    assert isinstance(state_value_column_name, str)
-    assert not isinstance(event_value_column_names, str)
-    event_value_column_names = list(event_value_column_names)
-    assert isinstance(source_id_column, str)
-    assert isinstance(state_row_mark, str)
-    assert isinstance(event_row_mark, str)
-    assert isinstance(locf_to_use_column_name, str)
-    assert isinstance(locf_non_null_rank_column_name, str)
-    assert isinstance(locf_tiebreaker_column_name, str)
-    assert isinstance(stand_in_values, dict)
-    together = (
-        d_state.extend({k: stand_in_values[k] for k in event_value_column_names})
-        .select_columns(
-            partition_by
-            + order_by
-            + [state_value_column_name]
-            + event_value_column_names
-        )
-        .concat_rows(
-            b=(
-                d_event.extend(
-                    {state_value_column_name: stand_in_values[state_value_column_name]}
-                ).select_columns(
-                    partition_by
-                    + order_by
-                    + [state_value_column_name]
-                    + event_value_column_names
-                )
-            ),
-            id_column=source_id_column,
-            a_name=state_row_mark,
-            b_name=event_row_mark,
-        )
-        # clear out stand-in values
-        .extend(
-            {
-                state_value_column_name: f'({source_id_column} == "{event_row_mark}").if_else(None, {state_value_column_name})'
-            }
-        )
-        .extend(
-            {
-                k: f'({source_id_column} == "{state_row_mark}").if_else(None, {k})'
-                for k in event_value_column_names
-            }
-        )
-    )
-    ops = last_observed_carried_forward(
-        together,
-        order_by=order_by,
-        partition_by=partition_by,
-        value_column_name=state_value_column_name,
-        selection_predicate="is_null()",
-        locf_to_use_column_name=locf_to_use_column_name,
-        locf_non_null_rank_column_name=locf_non_null_rank_column_name,
-        locf_tiebreaker_column_name=locf_tiebreaker_column_name,
-    )
-    return ops
-
-
 def rank_to_average(
     d: ViewRepresentation,
     *,

coverage.txt

@@ -11,19 +11,18 @@ tests/test_arith.py .                                                    [  3%]
 tests/test_arrow1.py ...                                                 [  4%]
 tests/test_bigquery_user_fns.py .                                        [  4%]
 tests/test_bitemporal_obs_agg_join.py .                                  [  4%]
-tests/test_braid.py .                                                    [  5%]
 tests/test_calc_warnings_errors.py .                                     [  5%]
 tests/test_cc.py ....                                                    [  6%]
 tests/test_cdata1.py ....                                                [  7%]
 tests/test_cdata_convenience.py .                                        [  7%]
 tests/test_cdata_example.py ..                                           [  8%]
 tests/test_cdata_tr6_example.py ....                                     [  9%]
 tests/test_cdata_wvpy_case.py .                                          [  9%]
-tests/test_coalesce.py .                                                 [ 10%]
+tests/test_coalesce.py .                                                 [  9%]
 tests/test_cols_used.py .                                                [ 10%]
 tests/test_compare_data_frames.py .                                      [ 10%]
 tests/test_complex_expr.py ....                                          [ 11%]
-tests/test_compound_where.py .                                           [ 12%]
+tests/test_compound_where.py .                                           [ 11%]
 tests/test_concat_rows.py ...                                            [ 12%]
 tests/test_container.py ..                                               [ 13%]
 tests/test_cross_product_join.py .                                       [ 13%]
@@ -32,59 +31,59 @@ tests/test_data_space.py ..                                              [ 17%]
 tests/test_date_stuff.py ..                                              [ 17%]
 tests/test_db_handle.py ..                                               [ 18%]
 tests/test_db_model.py .                                                 [ 18%]
-tests/test_degenerate_project.py .                                       [ 19%]
+tests/test_degenerate_project.py .                                       [ 18%]
 tests/test_drop_columns.py .                                             [ 19%]
 tests/test_dup_detection_example.py .                                    [ 19%]
 tests/test_eval_cache.py ...                                             [ 20%]
 tests/test_ex_examples.py ............                                   [ 23%]
 tests/test_example1.py ....                                              [ 24%]
-tests/test_exp.py .                                                      [ 25%]
+tests/test_exp.py .                                                      [ 24%]
 tests/test_exp_parens.py .                                               [ 25%]
 tests/test_expand_rows.py .                                              [ 25%]
-tests/test_expr_parse.py .                                               [ 26%]
+tests/test_expr_parse.py .                                               [ 25%]
 tests/test_expression_expectations.py ..                                 [ 26%]
 tests/test_expression_expectations_polars.py .                           [ 26%]
 tests/test_extend.py ..........                                          [ 29%]
 tests/test_extend_order.py .                                             [ 29%]
-tests/test_first_last.py .                                               [ 30%]
+tests/test_first_last.py .                                               [ 29%]
 tests/test_float_divide.py ..                                            [ 30%]
-tests/test_flow_text.py .                                                [ 31%]
+tests/test_flow_text.py .                                                [ 30%]
 tests/test_forbidden_calculation.py ..                                   [ 31%]
 tests/test_forbidden_ops.py ...                                          [ 32%]
 tests/test_free_fn.py .                                                  [ 32%]
-tests/test_get_methods_used.py .                                         [ 33%]
+tests/test_get_methods_used.py .                                         [ 32%]
 tests/test_ghost_col_issue.py .                                          [ 33%]
-tests/test_idioms.py .................                                   [ 38%]
+tests/test_idioms.py .................                                   [ 37%]
 tests/test_if_else.py .....                                              [ 39%]
 tests/test_if_else_return_type.py .                                      [ 39%]
-tests/test_incomplete_agg.py .                                           [ 40%]
+tests/test_incomplete_agg.py .                                           [ 39%]
 tests/test_join_check.py .                                               [ 40%]
-tests/test_join_conditions.py ...                                        [ 41%]
+tests/test_join_conditions.py ...                                        [ 40%]
 tests/test_join_effects.py .                                             [ 41%]
 tests/test_join_multi_key.py .                                           [ 41%]
-tests/test_join_opt.py .                                                 [ 42%]
+tests/test_join_opt.py .                                                 [ 41%]
 tests/test_join_variations.py .                                          [ 42%]
 tests/test_lark_parser.py .....                                          [ 43%]
 tests/test_locf.py .                                                     [ 43%]
-tests/test_logistic_example.py .                                         [ 44%]
+tests/test_logistic_example.py .                                         [ 43%]
 tests/test_mapv.py ..                                                    [ 44%]
-tests/test_math.py .                                                     [ 45%]
+tests/test_math.py .                                                     [ 44%]
 tests/test_method_catalog_issues.py ..                                   [ 45%]
 tests/test_minimum.py .                                                  [ 45%]
 tests/test_mod_fns.py ...                                                [ 46%]
-tests/test_multi_map.py .                                                [ 47%]
+tests/test_multi_map.py .                                                [ 46%]
 tests/test_narrow.py .                                                   [ 47%]
 tests/test_natural_join.py ....                                          [ 48%]
 tests/test_neg.py .                                                      [ 48%]
 tests/test_null_bad.py ....                                              [ 49%]
-tests/test_one_row_cdata_convert.py .                                    [ 50%]
+tests/test_one_row_cdata_convert.py .                                    [ 49%]
 tests/test_ops.py .                                                      [ 50%]
 tests/test_ops_eq.py .                                                   [ 50%]
-tests/test_or.py .....                                                   [ 52%]
+tests/test_or.py .....                                                   [ 51%]
 tests/test_order_limit.py .                                              [ 52%]
 tests/test_parens.py ..                                                  [ 52%]
-tests/test_parse.py ..........................                           [ 60%]
-tests/test_polars.py ......................                              [ 66%]
+tests/test_parse.py ..........................                           [ 59%]
+tests/test_polars.py .......................                             [ 66%]
 tests/test_project.py .....                                              [ 67%]
 tests/test_rank.py .                                                     [ 68%]
 tests/test_rank_to_average.py .                                          [ 68%]
@@ -161,14 +160,14 @@ data_algebra/op_container.py             127     49    61%   46-47, 63-71, 80-81
 data_algebra/pandas_base.py              696     68    90%   54, 68, 77, 87, 92, 101, 223, 225, 239, 242, 247, 252, 427, 467, 478, 503, 506, 511, 514, 516, 528, 534-541, 548, 582-587, 617, 621, 624, 626, 663, 717, 758, 775, 795, 813, 823, 838, 886, 894, 902, 917, 928, 940, 959, 974, 1002, 1017, 1054, 1071, 1074, 1085, 1113, 1145, 1154, 1181, 1198, 1210, 1265, 1279-1281
 data_algebra/pandas_model.py              19      2    89%   32-33
 data_algebra/parse_by_lark.py            164     24    85%   71, 93, 108, 129-130, 137, 161, 171, 185-186, 188, 200, 206, 213-217, 245, 253, 263-266
-data_algebra/polars_model.py             532     61    89%   130, 139, 189, 401, 417, 427, 434, 447-451, 459, 461, 486, 489, 494, 497, 546, 564, 580, 644, 660-662, 689, 697, 749, 764, 782, 800, 820, 832-834, 837, 842, 844, 851-863, 870, 875, 906, 935, 944, 972, 987, 999, 1063-1065, 1074-1075, 1077
+data_algebra/polars_model.py             539     60    89%   134, 143, 193, 405, 421, 431, 438, 451-455, 463, 465, 490, 493, 498, 501, 550, 568, 584, 649, 665-667, 710, 762, 777, 795, 813, 833, 845-847, 850, 855, 857, 864-876, 883, 888, 919, 948, 957, 985, 1000, 1012, 1076-1078, 1087-1088, 1090
 data_algebra/python3_lark.py               1      0   100%
-data_algebra/solutions.py                135      4    97%   63, 308, 389, 472
+data_algebra/solutions.py                112      3    97%   63, 308, 367
 data_algebra/sql_format_options.py        15      2    87%   61, 69
 data_algebra/test_util.py                333     62    81%   28-29, 104, 126, 136, 139, 143, 166, 169, 173, 175-178, 189, 246-247, 263-268, 272, 284, 286-294, 331, 333, 344, 352, 363, 370, 376, 388, 399, 413, 468, 472, 523-526, 528-531, 533-536, 538-541, 652-657, 662-663, 665
 data_algebra/util.py                     127     28    78%   26, 59-60, 63-64, 67-68, 71-72, 75-76, 79-80, 83-84, 87-88, 91-92, 95-96, 143, 165, 167, 182, 223, 227, 229
 --------------------------------------------------------------------
-TOTAL                                   6786    964    86%
+TOTAL                                   6770    962    86%
 
 
-======================= 357 passed in 821.88s (0:13:41) ========================
+======================= 357 passed in 778.36s (0:12:58) ========================