Merge pull request #403 from Crunch-io/rework-subtotals-188552693

[#188552693]: Rework subtotals

Author: ernestoarbitrio

src/cr/cube/matrix/measure.py

@@ -13,6 +13,7 @@
     OverlapSubtotals,
     PositiveTermSubtotals,
     SumSubtotals,
+    WaveDiffSubtotal,
 )
 from cr.cube.smoothing import Smoother
 from cr.cube.util import lazyproperty
@@ -799,7 +800,13 @@ def _subtotal_columns(self):
         """
         # --- do not propagate divide-by-zero warnings to stderr ---
         with np.errstate(divide="ignore", invalid="ignore"):
-            return self._count_blocks[0][1] / self._weighted_base_blocks[0][1]
+            default_value = self._count_blocks[0][1] / self._weighted_base_blocks[0][1]
+            return WaveDiffSubtotal.subtotal_columns(
+                self._weighted_cube_counts.column_bases,
+                self._weighted_cube_counts.counts,
+                default_value,
+                self._dimensions,
+            )
 
     @lazyproperty
     def _subtotal_rows(self):
@@ -809,7 +816,13 @@ def _subtotal_rows(self):
         """
         # --- do not propagate divide-by-zero warnings to stderr ---
         with np.errstate(divide="ignore", invalid="ignore"):
-            return self._count_blocks[1][0] / self._weighted_base_blocks[1][0]
+            default_value = self._count_blocks[1][0] / self._weighted_base_blocks[1][0]
+            return WaveDiffSubtotal.subtotal_rows(
+                self._weighted_cube_counts.column_bases,
+                self._weighted_cube_counts.counts,
+                default_value,
+                self._dimensions,
+            )
 
     @lazyproperty
     def _weighted_base_blocks(self):
@@ -1564,17 +1577,18 @@ def _column_bases(self):
         if self._second_order_measures.columns_squared_base.is_defined:
             weighted_blocks = self._second_order_measures.column_weighted_bases.blocks
             squared_blocks = self._second_order_measures.column_squared_bases.blocks
-            effective_blocks = [
-                [
-                    weighted_blocks[0][0] ** 2 / squared_blocks[0][0],
-                    weighted_blocks[0][1] ** 2 / squared_blocks[0][1],
-                ],
-                [
-                    weighted_blocks[1][0] ** 2 / squared_blocks[1][0],
-                    weighted_blocks[1][1] ** 2 / squared_blocks[1][1],
-                ],
-            ]
-            return effective_blocks
+            with np.errstate(divide="ignore", invalid="ignore"):
+                effective_blocks = [
+                    [
+                        weighted_blocks[0][0] ** 2 / squared_blocks[0][0],
+                        weighted_blocks[0][1] ** 2 / squared_blocks[0][1],
+                    ],
+                    [
+                        weighted_blocks[1][0] ** 2 / squared_blocks[1][0],
+                        weighted_blocks[1][1] ** 2 / squared_blocks[1][1],
+                    ],
+                ]
+                return effective_blocks
 
         unweighted_blocks = self._second_order_measures.column_unweighted_bases.blocks
         return unweighted_blocks
@@ -1845,6 +1859,14 @@ class _RowProportions(_BaseSecondOrderMeasure):
     contributed by the weighted count of each matrix cell.
     """
 
+    @lazyproperty
+    def _count_blocks(self):
+        return self._second_order_measures.weighted_counts.blocks
+
+    @lazyproperty
+    def _weighted_base_blocks(self):
+        return self._second_order_measures.row_weighted_bases.blocks
+
     @lazyproperty
     def blocks(self):
         """Nested list of the four 2D ndarray "blocks" making up this measure.
@@ -1854,8 +1876,8 @@ def blocks(self):
 
         Row-proportions are row comparable counts divided by the row weighted bases.
         """
-        count_blocks = self._second_order_measures.weighted_counts.blocks
-        weighted_base_blocks = self._second_order_measures.row_weighted_bases.blocks
+        count_blocks = self._count_blocks
+        weighted_base_blocks = self._weighted_base_blocks
 
         # --- do not propagate divide-by-zero warnings to stderr ---
         with np.errstate(divide="ignore", invalid="ignore"):
@@ -1864,16 +1886,36 @@ def blocks(self):
                     # --- base values ---
                     count_blocks[0][0] / weighted_base_blocks[0][0],
                     # --- inserted columns ---
-                    count_blocks[0][1] / weighted_base_blocks[0][1],
+                    self._inserted_columns,
                 ],
                 [
                     # --- inserted rows ---
-                    count_blocks[1][0] / weighted_base_blocks[1][0],
+                    self._inserted_rows,
                     # --- intersections ---
                     count_blocks[1][1] / weighted_base_blocks[1][1],
                 ],
             ]
 
+    @lazyproperty
+    def _inserted_rows(self):
+        default_value = self._count_blocks[1][0] / self._weighted_base_blocks[1][0]
+        return WaveDiffSubtotal.subtotal_rows(
+            self._weighted_cube_counts.row_bases,
+            self._weighted_cube_counts.counts,
+            default_value,
+            self._dimensions,
+        )
+
+    @lazyproperty
+    def _inserted_columns(self):
+        default_value = self._count_blocks[0][1] / self._weighted_base_blocks[0][1]
+        return WaveDiffSubtotal.subtotal_columns(
+            self._weighted_cube_counts.row_bases,
+            self._weighted_cube_counts.counts,
+            default_value,
+            self._dimensions,
+        )
+
 
 class _RowShareSum(_BaseSecondOrderMeasure):
     """Provides the row share of sum measure for a matrix.

src/cr/cube/matrix/subtotals.py

@@ -12,6 +12,7 @@
 
 import numpy as np
 
+from cr.cube.enums import DIMENSION_TYPE as DT
 from cr.cube.util import lazyproperty
 
 
@@ -370,6 +371,136 @@ def _subtotal_row(self, subtotal):
         return addend_sum - subtrahend_sum
 
 
+class WaveDiffSubtotal:
+    """Subtotal "blocks" created by adding and subtracting terms for wave differences.
+
+    This class handles a special case for wave differences when a CAT_DATE variable is
+    involved in the calculation.
+
+    A wave difference for a CAT_DATE variable is calculate subtracting at the
+    percentages level: (count1/base1) - (count2/base2).
+    """
+
+    def __init__(self, base_values, counts, default_insertions, dimensions):
+        self._base_values = base_values
+        self._counts = counts
+        self._default_insertions = default_insertions
+        self._dimensions = dimensions
+
+    @classmethod
+    def subtotal_columns(cls, base_values, counts, default_insertions, dimensions):
+        """Return (n_column_subtotals, n_base_rows) ndarray of subtotal columns."""
+        return cls(
+            base_values, counts, default_insertions, dimensions
+        )._subtotal_columns
+
+    @classmethod
+    def subtotal_rows(cls, base_values, counts, default_insertions, dimensions):
+        """Return (n_row_subtotals, n_base_cols) ndarray of subtotal rows.
+        Keyword arguments:
+        `diff_cols_nan` -- Overrides subtotal differences in the columns direction eg
+        for column bases (default False)
+        `diff_rows_nan` -- Overrides subtotal differences in the rows direction eg for
+        row bases (default False)
+        """
+        return cls(base_values, counts, default_insertions, dimensions)._subtotal_rows
+
+    @lazyproperty
+    def _column_subtotals(self):
+        """Sequence of _Subtotal object for each subtotal in columns-dimension."""
+        return self._dimensions[1].subtotals
+
+    def _multiple_subtrahends_or_addends(self, subtotal):
+        """Returns true if the subtotal has multiple addend or subtrahend terms."""
+        return any(subtotal.subtrahend_idxs) and (
+            len(subtotal.subtrahend_idxs) > 1 or len(subtotal.addend_idxs) > 1
+        )
+
+    def _nan_subtotals(self, axis):
+        """Generate an array filled with NaN values.
+
+        Matches the size of the specified axis of the base values.
+        """
+        return np.full(self._base_values.shape[axis], np.nan)
+
+    @lazyproperty
+    def _row_subtotals(self):
+        """Sequence of _Subtotal object for each subtotal in rows-dimension."""
+        return self._dimensions[0].subtotals
+
+    @lazyproperty
+    def _subtotal_rows(self):
+        """(n_row_subtotals, n_cols) ndarray of subtotal rows."""
+        subtotals = self._row_subtotals
+        n_cols = self._base_values.shape[1]
+        if len(subtotals) == 0:
+            return np.empty((0, n_cols))
+
+        return np.vstack(
+            [
+                self._subtotal_row(subtotal, default)
+                for subtotal, default in zip(subtotals, self._default_insertions)
+            ]
+        )
+
+    @lazyproperty
+    def _subtotal_columns(self):
+        """(n_rows, n_col_subtotals) matrix of subtotal columns."""
+        subtotals = self._column_subtotals
+        n_rows = self._base_values.shape[0]
+        if len(subtotals) == 0:
+            return np.empty((n_rows, 0))
+        return np.hstack(
+            [
+                self._subtotal_column(subtotal, default).reshape(n_rows, 1)
+                for subtotal, default in zip(subtotals, self._default_insertions.T)
+            ]
+        )
+
+    def _subtotal_column(self, subtotal, default):
+        """Return (n_rows,) ndarray of values for `subtotal` column."""
+        if self._dimensions[1].dimension_type == DT.CAT_DATE and any(
+            subtotal.subtrahend_idxs
+        ):
+            if self._multiple_subtrahends_or_addends(subtotal):
+                return self._nan_subtotals(axis=0)
+            base_addend_sum = np.sum(self._base_values[:, subtotal.addend_idxs], axis=1)
+            base_subtrahend_sum = np.sum(
+                self._base_values[:, subtotal.subtrahend_idxs], axis=1
+            )
+            counts_addend_sum = np.sum(self._counts[:, subtotal.addend_idxs], axis=1)
+            counts_subtrahend_sum = np.sum(
+                self._counts[:, subtotal.subtrahend_idxs], axis=1
+            )
+            return (counts_addend_sum / base_addend_sum) - (
+                counts_subtrahend_sum / base_subtrahend_sum
+            )
+
+        return default
+
+    def _subtotal_row(self, subtotal, default):
+        """Return (n_cols,) ndarray of values for `subtotal` row."""
+
+        if self._dimensions[0].dimension_type == DT.CAT_DATE and any(
+            subtotal.subtrahend_idxs
+        ):
+            if self._multiple_subtrahends_or_addends(subtotal):
+                return self._nan_subtotals(axis=1)
+            base_addend_sum = np.sum(self._base_values[subtotal.addend_idxs, :], axis=0)
+            base_subtrahend_sum = np.sum(
+                self._base_values[subtotal.subtrahend_idxs, :], axis=0
+            )
+            counts_addend_sum = np.sum(self._counts[subtotal.addend_idxs, :], axis=0)
+            counts_subtrahend_sum = np.sum(
+                self._counts[subtotal.subtrahend_idxs, :], axis=0
+            )
+            return (counts_addend_sum / base_addend_sum) - (
+                counts_subtrahend_sum / base_subtrahend_sum
+            )
+
+        return default
+
+
 class OverlapSubtotals(SumSubtotals):
     """Subtotal blocks used exclusively for the "overlap" cube measure.
 

src/cr/cube/stripe/insertion.py

@@ -12,6 +12,7 @@
 
 import numpy as np
 
+from cr.cube.enums import DIMENSION_TYPE as DT
 from cr.cube.util import lazyproperty
 
 
@@ -104,3 +105,64 @@ def _subtotal_value(self, subtotal):
         subtrahend_sum = np.sum(base_values[subtotal.subtrahend_idxs])
 
         return addend_sum - subtrahend_sum
+
+
+class WaveDiffSubtotals(_BaseSubtotals):
+    """Subtotal "blocks" created by adding and subtracting terms for wave differences.
+
+    This class handles a special case for wave differences when a CAT_DATE variable is
+    involved in the calculation.
+
+    A wave difference for a CAT_DATE variable is calculate subtracting at the
+    percentages level: (count1/base1) - (count2/base2).
+    """
+
+    def __init__(self, base_values, counts, default_values, rows_dimension):
+        super(WaveDiffSubtotals, self).__init__(base_values, rows_dimension)
+        self._counts = counts
+        self._default_values = default_values
+
+    @classmethod
+    def subtotal_values(cls, base_values, counts, default_values, rows_dimension):
+        """Return (n_row_subtotals,) ndarray of subtotal values."""
+        return cls(base_values, counts, default_values, rows_dimension)._subtotal_values
+
+    def _multiple_subtrahends_or_addends(self, subtotal):
+        """Returns true if the subtotal has multiple addend or subtrahend terms."""
+        return any(subtotal.subtrahend_idxs) and (
+            len(subtotal.subtrahend_idxs) > 1 or len(subtotal.addend_idxs) > 1
+        )
+
+    @lazyproperty
+    def _subtotal_values(self):
+        """(n_row_subtotals,) ndarray of subtotal values for stripe."""
+        subtotals = self._row_subtotals
+
+        if len(subtotals) == 0:
+            return np.array([])
+
+        if self._rows_dimension.dimension_type != DT.CAT_DATE:
+            return self._default_values
+
+        return np.array(
+            [
+                self._subtotal_value(subtotal, default)
+                for subtotal, default in zip(subtotals, self._default_values)
+            ]
+        )
+
+    def _subtotal_value(self, subtotal, default):
+        """Return scalar value of wafe diff `subtotal` row."""
+        if len(subtotal.subtrahend_idxs) > 0 and len(subtotal.addend_idxs) > 0:
+            if self._multiple_subtrahends_or_addends(subtotal):
+                return np.nan
+            base_values = self._base_values
+            counts = self._counts
+            base_addend_sum = np.sum(base_values[subtotal.addend_idxs])
+            base_subtrahend_sum = np.sum(base_values[subtotal.subtrahend_idxs])
+            counts_addend_sum = np.sum(counts[subtotal.addend_idxs])
+            counts_subtrahend_sum = np.sum(counts[subtotal.subtrahend_idxs])
+            return (counts_addend_sum / base_addend_sum) - (
+                counts_subtrahend_sum / base_subtrahend_sum
+            )
+        return default

src/cr/cube/stripe/measure.py

@@ -12,6 +12,7 @@
     NegativeTermSubtotals,
     PositiveTermSubtotals,
     SumSubtotals,
+    WaveDiffSubtotals,
 )
 from cr.cube.util import lazyproperty
 
@@ -628,7 +629,13 @@ def subtotal_values(self):
 
         # --- do not propagate divide-by-zero warnings to stderr ---
         with np.errstate(divide="ignore", invalid="ignore"):
-            return subtotal_values / weighted_table_base
+            default_value = subtotal_values / weighted_table_base
+            return WaveDiffSubtotals.subtotal_values(
+                self._weighted_cube_counts.bases,
+                self._weighted_cube_counts.counts,
+                default_value,
+                self._rows_dimension,
+            )
 
 
 class _UnweightedBases(_BaseSecondOrderMeasure):