rebuild and retest

Author: JohnMount

build/lib/data_algebra/__init__.py

@@ -19,26 +19,37 @@
 
 import data_algebra.data_model
 import data_algebra.data_ops
+
 # import for easy access for package users
 from data_algebra.data_ops import (
-    describe_table,
-    descr,
-    data,
-    ex,
+    describe_table, 
+    descr, 
+    data, 
+    ex, 
     table
 )
 import data_algebra.view_representations
 from data_algebra.view_representations import (
     TableDescription,
     ViewRepresentation,
-    SQLNode
+    SQLNode,
 )
 import data_algebra.expr_rep
-from data_algebra.expr_rep import lit, col, d_, one
+from data_algebra.expr_rep import (
+    lit, col, d_, one
+)
 
 import data_algebra.data_schema
+import data_algebra.cdata
+from data_algebra.cdata import (
+    RecordMap, 
+    RecordSpecification,
+    pivot_blocks_to_rowrecs,
+    pivot_rowrecs_to_blocks,
+    pivot_specification,
+    unpivot_specification
+)
 
 import data_algebra.pandas_model
-
 # do this last so everything is loaded/define
 data_algebra.pandas_model.register_pandas_model()

build/lib/data_algebra/cdata.py

@@ -23,7 +23,7 @@ def _format_table(
     record_id_cols: Iterable[str],
     control_id_cols: Iterable[str],
     add_style: bool = True,
-):
+) -> str:
     local_data_model = data_algebra.data_model.lookup_data_model_for_dataframe(d)
     d = local_data_model.to_pandas(d)
     pd = data_algebra.data_model.lookup_data_model_for_dataframe(d).pd
@@ -52,12 +52,13 @@ def _format_table(
         d = d.style.set_properties(
             **{"background-color": "#FFE4C4"}, subset=record_id_col_pairs
         ).set_properties(**{"background-color": "#7FFFD4"}, subset=control_id_col_pairs)
-    return d
+    return d._repr_html_()
 
 
 class RecordSpecification:
     """
-    Class to represent a multi-row data record.
+    Class to represent a data record. 
+    For single row data records use None as the specification.
     """
 
     row_columns: List[str]  # columns when in row form
@@ -93,11 +94,22 @@ def __init__(
         control_table = local_data_model.clean_copy(control_table)
         if control_table.shape[0] < 1:
             raise ValueError("control table should have at least 1 row")
+        if control_table.shape[1] < 2:
+            raise ValueError("control table must have at least 2 columns")
         if len(control_table.columns) != len(set(control_table.columns)):
             raise ValueError("control table columns should be unique")
+        if control_table_keys is None:
+            if control_table.shape[0] > 1:
+                control_table_keys = [control_table.columns[0]]
+            else:
+                control_table_keys = []  # single row records don't need to be keyed
+        if isinstance(control_table_keys, str):
+            control_table_keys = [control_table_keys]
+        else:
+            control_table_keys = list(control_table_keys)
+        assert isinstance(control_table_keys, List)
         if strict:
             if control_table.shape[0] > 1:
-                assert control_table_keys is not None
                 assert len(control_table_keys) > 0
                 assert local_data_model.table_is_keyed_by_columns(
                     control_table, column_names=control_table_keys
@@ -106,14 +118,12 @@ def __init__(
         assert self.control_table.shape[0] > 0
         if record_keys is None:
             record_keys = []
-        if isinstance(record_keys, str):
+        elif isinstance(record_keys, str):
             record_keys = [record_keys]
-        self.record_keys = list(record_keys)
-        if control_table_keys is None:
-            if self.control_table.shape[0] > 1:
-                control_table_keys = [self.control_table.columns[0]]
-            else:
-                control_table_keys = []  # single row records don't need to be keyed
+        else:
+            record_keys = list(record_keys)
+        assert isinstance(record_keys, list)
+        self.record_keys = record_keys
         if isinstance(control_table_keys, str):
             control_table_keys = [control_table_keys]
         if self.control_table.shape[0] > 1:
@@ -182,12 +192,55 @@ def row_version(self, *, include_record_keys: bool = True) -> List[str]:
         :param include_record_keys: logical, if True include record keys as columns
         :return: column list
         """
+        assert isinstance(include_record_keys, bool)
         cols: List[str] = []
         if include_record_keys:
             cols = cols + self.record_keys
         cols = cols + self.content_keys
         return cols
+    
+    def row_record_form(self):
+        """
+        Return specification of matching row record form.
+        Note: prefer using None to specify row records specs.
+        """
+        local_data_model = data_algebra.data_model.lookup_data_model_for_dataframe(self.control_table)
+        row_vals = self.row_version(include_record_keys=False)
+        ct = local_data_model.data_frame({
+            k: [k] for k in row_vals
+        })
+        v_set = set(self.row_version(include_record_keys=False))
+        return RecordSpecification(
+            ct,
+            record_keys=self.record_keys,
+            control_table_keys=[],
+            strict=self.strict,
+            local_data_model=local_data_model,
+        )
+
+    def value_column_form(self, *, key_column_name: str = "measure", value_column_name: str = "value"):
+        """
+        Return specification of the matching value column form.
+        Note: for type safety prefer map_to_rows() to map_to_keyed_column().
 
+        :param key_column_name: name for additional keying column
+        :param value_column_name: name for value column
+        """
+        assert isinstance(key_column_name, str)
+        assert isinstance(value_column_name, str)
+        local_data_model = data_algebra.data_model.lookup_data_model_for_dataframe(self.control_table)
+        ct = local_data_model.data_frame({
+            key_column_name: self.row_version(include_record_keys=False),
+            value_column_name: self.row_version(include_record_keys=False),
+        })
+        return RecordSpecification(
+            ct,
+            record_keys=self.record_keys,
+            control_table_keys=[key_column_name],
+            strict=self.strict,
+            local_data_model=local_data_model,
+        )
+        
     def __repr__(self):
         s = (
             "data_algebra.cdata.RecordSpecification(\n"
@@ -248,7 +301,12 @@ def _repr_html_(self):
             + _str_list_to_html(self.control_table_keys)
             + "</li>\n"
             + "<li>control_table:<br>\n"
-            + self.control_table._repr_html_()
+            + _format_table(
+                self.control_table,
+                record_id_cols=self.record_keys,
+                control_id_cols=self.control_table_keys,
+                add_style=True,
+            )
             + "</li>\n"
             + "</ul>"
             + "</p>\n"
@@ -266,7 +324,8 @@ def map_to_rows(self):
 
         :return: RecordMap
         """
-
+        if self.control_table.shape[0] <= 1:
+            raise ValueError("already in row record format")
         return RecordMap(blocks_in=self, strict=self.strict)
 
     def map_from_rows(self):
@@ -275,8 +334,33 @@ def map_from_rows(self):
 
         :return: RecordMap
         """
-
+        if self.control_table.shape[0] <= 1:
+            raise ValueError("already in row record format")
         return RecordMap(blocks_out=self, strict=self.strict)
+    
+    def map_to_keyed_column(self, *, key_column_name: str = "measure", value_column_name: str = "value"):
+        """
+        Build a RecordMap mapping this RecordSpecification to a table
+        where only one column holds values.
+        Note: for type safety prefer map_to_rows() to map_to_keyed_column().
+
+        
+        :param key_column_name: name for additional keying column
+        :param value_column_name: name for value column
+        :return: Record map
+        """
+        return RecordMap(blocks_in=self, blocks_out=self.value_column_form(), strict=self.strict)
+    
+    def map_from_keyed_column(self, *, key_column_name: str = "measure", value_column_name: str = "value"):
+        """
+        Build a RecordMap mapping this RecordSpecification from a table
+        where only one column holds values.
+
+        :param key_column_name: name for additional keying column
+        :param value_column_name: name for value column
+        :return: Record map
+        """
+        return RecordMap(blocks_in=self.value_column_form(), blocks_out=self, strict=self.strict)
 
 
 class RecordMap(ShiftPipeAction):
@@ -309,16 +393,23 @@ def __init__(
             assert isinstance(blocks_in, RecordSpecification)
             if strict:
                 assert blocks_in.strict
+            if blocks_in.control_table.shape[0] <= 1:
+                blocks_in = None
+        if blocks_in is not None:
             ck = [k for k in blocks_in.content_keys if k is not None]
             if len(ck) != len(set(ck)):
                 raise ValueError("blocks_in can not have duplicate content keys")
+            if blocks_in.control_table.shape[0] <= 1:
+                raise ValueError("for row records use None specification")
         if blocks_out is not None:
             assert isinstance(blocks_out, RecordSpecification)
             if strict:
                 assert blocks_out.strict
+            if blocks_out.control_table.shape[0] <= 1:
+                blocks_out = None
         if (blocks_in is None) and (blocks_out is None):
             raise ValueError(
-                "At least one of blocks_in or blocks_out should not be None"
+                "At least one of blocks_in or blocks_out should not be None or a non-row record"
             )
         if (blocks_in is not None) and (blocks_out is not None):
             unknown = set(blocks_out.record_keys) - set(blocks_in.record_keys)
@@ -658,9 +749,9 @@ def _repr_html_(self):
         )
         s = (
             "RecordMap: transforming records of the form:<br>\n"
-            + example_input_formatted._repr_html_()
+            + example_input_formatted
             + "<br>\nto records of the form:<br>\n"
-            + example_output_formatted._repr_html_()
+            + example_output_formatted
         )
         return s
 

build/lib/data_algebra/data_schema.py

@@ -12,6 +12,7 @@
 have_polars = False
 try:
     import polars as pl  # conditional import
+
     have_polars = True
 except ModuleNotFoundError:
     pass
@@ -75,7 +76,9 @@ def non_null_types_in_frame(d) -> Dict[str, Optional[Set[Type]]]:
     # check that this is a data frame, so we can raise a legible error
     # instead of having an non-existent attribute error raising
     if not is_data_frame(d):
-        raise TypeError(f"expected a Pandas or Polars data frame, had {_type_name(type(d))}")
+        raise TypeError(
+            f"expected a Pandas or Polars data frame, had {_type_name(type(d))}"
+        )
     result = dict()
     for col_name in d.columns:
         types_seen = {type(vi) for vi in d[col_name] if not _is_null(vi)}
@@ -139,7 +142,9 @@ class SchemaRaises(SchemaBase):
     Raises TypeError on schema violations.
     """
 
-    def _check_data_frame_matches_schema(self, *, d, expected_type: Optional[Dict]) -> Optional[str]:
+    def _check_data_frame_matches_schema(
+        self, *, d, expected_type: Optional[Dict]
+    ) -> Optional[str]:
         assert isinstance(expected_type, (type(None), Dict))
         # check that this is a data frame, so we can raise a legible error
         # instead of having an non-existent attribute error raising
@@ -190,7 +195,9 @@ def _check_spec(
                 )
                 return f"expected type one of {type_names}, found type {_type_name(observed_type)}"
         elif isinstance(expected_type, dict):
-            schema_issue = self._check_data_frame_matches_schema(d=observed_value, expected_type=expected_type)
+            schema_issue = self._check_data_frame_matches_schema(
+                d=observed_value, expected_type=expected_type
+            )
             if schema_issue is not None:
                 return schema_issue
         else:

build/lib/data_algebra/expr_parse_fn.py

@@ -1,25 +1,35 @@
-
-
 """
 Parse expresion to data ops in a medium environment. Uses eval() (don't run on untrusted expressions).
+Used for confirming repr() is reversible.
 """
 
 
 import data_algebra
 import data_algebra.data_model
 import numpy as np  # for globals() in eval_da_ops()
-from data_algebra.data_ops import describe_table, table, descr, data  # for globals() in eval_da_ops()
-from data_algebra.view_representations import ViewRepresentation, TableDescription  # for globals() in eval_da_ops()
-pd = data_algebra.data_model.default_data_model().pd  # for globals() in eval_da_ops()
+from data_algebra.data_ops import (
+    describe_table,
+    table,
+    descr,
+    data,
+)  # for globals() in eval_da_ops()
+from data_algebra.view_representations import (
+    ViewRepresentation,
+    TableDescription,
+)  # for globals() in eval_da_ops()
 
-g_env = globals()
+pd = data_algebra.data_model.default_data_model().pd  # for globals() in eval_da_ops()
+g_env = {k: v for k, v in globals().items()}
 
 from typing import Any, Dict, Optional
 
 
-def eval_da_ops(ops_str: str, *, data_model_map: Optional[Dict[str, Any]]) -> ViewRepresentation:
+def eval_da_ops(
+    ops_str: str, *, data_model_map: Optional[Dict[str, Any]]
+) -> ViewRepresentation:
     """
     Parse ops_str into a ViewRepresentation. Uses eval() (don't run on untrusted expressions).
+    Used for confirming repr() is reversible.
 
     :param ops_str: text representation of a data algebra pipeline or expression.
     :param data_model_map: tables
@@ -39,4 +49,4 @@ def eval_da_ops(ops_str: str, *, data_model_map: Optional[Dict[str, Any]]) -> Vi
         # cdata uses this
     )
     assert isinstance(ops, ViewRepresentation)
-    return ops
+    return ops

build/lib/data_algebra/flow_text.py

@@ -5,6 +5,7 @@
 
 def flow_text(tokens, *, align_right=70, sep_width=1):
     """
+    Flow text around a margin for presentation.
 
     :param tokens: list or tuple of strings
     :param align_right: integer, right alignment margin

build/lib/data_algebra/fmt_python.py

@@ -1,4 +1,3 @@
-
 _have_black = False
 try:
     # noinspection PyUnresolvedReferences

build/lib/data_algebra/test_util.py

@@ -42,7 +42,9 @@
 global_test_result_cache: Optional[data_algebra.eval_cache.ResultCache] = None
 
 
-def _re_parse(ops: ViewRepresentation, *, data_model_map: Dict[str, Any]) -> ViewRepresentation:
+def _re_parse(
+    ops: ViewRepresentation, *, data_model_map: Dict[str, Any]
+) -> ViewRepresentation:
     """
     Return copy of object made by dumping to string via repr() and then evaluating that string.
     """