Fix/hist new polars (#2374)

* enh hist to always include lowest bin

* fix hist(bin_count=…) when data is empty

* add shuffling to simple smoke tests

* add hist fastpaths for no data and minimal bins

* simplify hist hypothesis tests

* add coverage for unreachable code

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: camriddell

narwhals/_arrow/series.py

@@ -1065,58 +1065,20 @@ def hist(  # noqa: PLR0915
 
         def _hist_from_bin_count(bin_count: int):  # type: ignore[no-untyped-def] # noqa: ANN202
             d = pc.min_max(self.native)
-            lower, upper = d["min"], d["max"]
-            pa_float = pa.type_for_alias("float")
+            lower, upper = d["min"].as_py(), d["max"].as_py()
             if lower == upper:
-                range_: pa.Scalar[Any] = lit(1.0)
-                mid = lit(0.5)
-                width = pc.divide(range_, lit(bin_count))
-                lower = pc.subtract(lower, mid)
-                upper = pc.add(upper, mid)
-            else:
-                range_ = pc.subtract(upper, lower)
-                width = pc.divide(pc.cast(range_, pa_float), lit(float(bin_count)))
-
-            bin_proportions = pc.divide(pc.subtract(self.native, lower), width)
-            bin_indices = pc.floor(bin_proportions)
-
-            # shift bins so they are right-closed
-            bin_indices = pc.if_else(
-                pc.and_(
-                    pc.equal(bin_indices, bin_proportions),
-                    pc.greater(bin_indices, lit(0)),
-                ),
-                pc.subtract(bin_indices, lit(1)),
-                bin_indices,
-            )
-            possible = pa.Table.from_arrays(
-                [pa.Array.from_pandas(np.arange(bin_count, dtype="int64"))], ["values"]
-            )
-            counts = (  # count bin id occurrences
-                pa.Table.from_arrays(
-                    pc.value_counts(bin_indices).flatten(),
-                    names=["values", "counts"],
-                )
-                # nan values are implicitly dropped in value_counts
-                .filter(~pc.field("values").is_nan())
-                .cast(pa.schema([("values", pa.int64()), ("counts", pa.int64())]))
-                # align bin ids to all possible bin ids (populate in missing bins)
-                .join(possible, keys="values", join_type="right outer")
-                .sort_by("values")
-            )
-            # empty bin intervals should have a 0 count
-            counts_coalesce = cast(
-                "ArrowArray", pc.coalesce(counts.column("counts"), lit(0))
-            )
-            counts = counts.set_column(0, "counts", counts_coalesce)
-
-            # extract left/right side of the intervals
-            bin_left = pc.add(lower, pc.multiply(counts.column("values"), width))
-            bin_right = pc.add(bin_left, width)
-            return counts.column("counts"), bin_right
+                lower -= 0.5
+                upper += 0.5
+            bins = np.linspace(lower, upper, bin_count + 1)
+            return _hist_from_bins(bins)
 
         def _hist_from_bins(bins: Sequence[int | float]):  # type: ignore[no-untyped-def] # noqa: ANN202
             bin_indices = np.searchsorted(bins, self.native, side="left")
+            bin_indices = pc.if_else(  # lowest bin is inclusive
+                pc.equal(self.native, lit(bins[0])), 1, bin_indices
+            )
+
+            # align unique categories and counts appropriately
             obs_cats, obs_counts = np.unique(bin_indices, return_counts=True)
             obj_cats = np.arange(1, len(bins))
             counts = np.zeros_like(obj_cats)
@@ -1125,15 +1087,51 @@ def _hist_from_bins(bins: Sequence[int | float]):  # type: ignore[no-untyped-def
             bin_right = bins[1:]
             return counts, bin_right
 
+        counts: Sequence[int | float] | np.typing.ArrayLike
+        bin_right: Sequence[int | float] | np.typing.ArrayLike
+
+        data_count = pc.sum(
+            pc.invert(pc.or_(pc.is_nan(self.native), pc.is_null(self.native))).cast(
+                pa.uint8()
+            ),
+            min_count=0,
+        )
         if bins is not None:
             if len(bins) < 2:
                 counts, bin_right = [], []
+
+            elif data_count == pa.scalar(0, type=pa.uint64()):  # type:ignore[comparison-overlap]
+                counts = np.zeros(len(bins) - 1)
+                bin_right = bins[1:]
+
+            elif len(bins) == 2:
+                counts = [
+                    pc.sum(
+                        pc.and_(
+                            pc.greater_equal(self.native, lit(float(bins[0]))),
+                            pc.less_equal(self.native, lit(float(bins[1]))),
+                        ).cast(pa.uint8())
+                    )
+                ]
+                bin_right = [bins[-1]]
             else:
                 counts, bin_right = _hist_from_bins(bins)
 
         elif bin_count is not None:
             if bin_count == 0:
                 counts, bin_right = [], []
+            elif data_count == pa.scalar(0, type=pa.uint64()):  # type:ignore[comparison-overlap]
+                counts, bin_right = (
+                    np.zeros(bin_count),
+                    np.linspace(0, 1, bin_count + 1)[1:],
+                )
+            elif bin_count == 1:
+                d = pc.min_max(self.native)
+                lower, upper = d["min"], d["max"]
+                if lower == upper:
+                    counts, bin_right = [data_count], [pc.add(upper, pa.scalar(0.5))]
+                else:
+                    counts, bin_right = [data_count], [upper]
             else:
                 counts, bin_right = _hist_from_bin_count(bin_count)
 

narwhals/_pandas_like/series.py

@@ -966,34 +966,50 @@ def hist(
                 data["breakpoint"] = []
             data["count"] = []
             return PandasLikeDataFrame.from_native(ns.DataFrame(data), context=self)
-        elif self.native.count() < 1:
+
+        if self.native.count() < 1:
             if bins is not None:
                 data = {"breakpoint": bins[1:], "count": zeros(shape=len(bins) - 1)}
             else:
                 count = cast("int", bin_count)
-                data = {"breakpoint": linspace(0, 1, count), "count": zeros(shape=count)}
+                if bin_count == 1:
+                    data = {"breakpoint": [1.0], "count": [0]}
+                else:
+                    data = {
+                        "breakpoint": linspace(0, 1, count + 1)[1:],
+                        "count": zeros(shape=count),
+                    }
             if not include_breakpoint:
                 del data["breakpoint"]
             return PandasLikeDataFrame.from_native(ns.DataFrame(data), context=self)
 
-        elif bin_count is not None:  # use Polars binning behavior
+        if bin_count is not None:
+            # use Polars binning behavior
             lower, upper = self.native.min(), self.native.max()
-            pad_lowest_bin = False
             if lower == upper:
                 lower -= 0.5
                 upper += 0.5
-            else:
-                pad_lowest_bin = True
+
+            if bin_count == 1:
+                data = {
+                    "breakpoint": [upper],
+                    "count": [self.native.count()],
+                }
+                if not include_breakpoint:
+                    del data["breakpoint"]
+                return PandasLikeDataFrame.from_native(ns.DataFrame(data), context=self)
 
             bins = linspace(lower, upper, bin_count + 1)
-            if pad_lowest_bin and bins is not None:
-                bins[0] -= 0.001 * abs(bins[0]) if bins[0] != 0 else 0.001
             bin_count = None
 
         # pandas (2.2.*) .value_counts(bins=int) adjusts the lowest bin twice, result in improper counts.
         # pandas (2.2.*) .value_counts(bins=[...]) adjusts the lowest bin which should not happen since
         #   the bins were explicitly passed in.
-        categories = ns.cut(self.native, bins=bins if bin_count is None else bin_count)
+        categories = ns.cut(
+            self.native,
+            bins=bins if bin_count is None else bin_count,
+            include_lowest=True,  # Polars 1.27.0 always includes the lowest bin
+        )
         # modin (0.32.0) .value_counts(...) silently drops bins with empty observations, .reindex
         #   is necessary to restore these bins.
         result = categories.value_counts(dropna=True, sort=False).reindex(

narwhals/_polars/series.py

@@ -499,18 +499,26 @@ def hist(
                 data.append(pl.Series("breakpoint", [], dtype=pl.Float64))
             data.append(pl.Series("count", [], dtype=pl.UInt32))
             return PolarsDataFrame.from_native(pl.DataFrame(data), context=self)
-        elif (self._backend_version < (1, 15)) and self.native.count() < 1:
+
+        if self.native.count() < 1:
             data_dict: dict[str, Sequence[Any] | pl.Series]
             if bins is not None:
                 data_dict = {
                     "breakpoint": bins[1:],
                     "count": pl.zeros(n=len(bins) - 1, dtype=pl.Int64, eager=True),
                 }
-            elif bin_count is not None:
+            elif (bin_count is not None) and bin_count == 1:
+                data_dict = {"breakpoint": [1.0], "count": [0]}
+            elif (bin_count is not None) and bin_count > 1:
                 data_dict = {
-                    "breakpoint": pl.int_range(0, bin_count, eager=True) / bin_count,
+                    "breakpoint": pl.int_range(1, bin_count + 1, eager=True) / bin_count,
                     "count": pl.zeros(n=bin_count, dtype=pl.Int64, eager=True),
                 }
+            else:  # pragma: no cover
+                msg = (
+                    "congratulations, you entered unreachable code - please report a bug"
+                )
+                raise AssertionError(msg)
             if not include_breakpoint:
                 del data_dict["breakpoint"]
             return PolarsDataFrame.from_native(pl.DataFrame(data_dict), context=self)
@@ -519,25 +527,19 @@ def hist(
         # polars <1.5 with bin_count=...
         # returns bins that range from -inf to +inf and has bin_count + 1 bins.
         #   for compat: convert `bin_count=` call to `bins=`
-        if (
-            (self._backend_version < (1, 15))
-            and (bin_count is not None)
-            and (self.native.count() > 0)
+        if (self._backend_version < (1, 15)) and (
+            bin_count is not None
         ):  # pragma: no cover
             lower = cast("float", self.native.min())
             upper = cast("float", self.native.max())
-            pad_lowest_bin = False
             if lower == upper:
                 width = 1 / bin_count
                 lower -= 0.5
                 upper += 0.5
             else:
-                pad_lowest_bin = True
                 width = (upper - lower) / bin_count
 
             bins = (pl.int_range(0, bin_count + 1, eager=True) * width + lower).to_list()
-            if pad_lowest_bin:
-                bins[0] -= 0.001 * abs(bins[0]) if bins[0] != 0 else 0.001
             bin_count = None
 
         # Polars inconsistently handles NaN values when computing histograms
@@ -552,16 +554,22 @@ def hist(
             include_category=False,
             include_breakpoint=include_breakpoint,
         )
+
         if not include_breakpoint:
             df.columns = ["count"]
 
+        if self._backend_version < (1, 0) and include_breakpoint:
+            df = df.rename({"break_point": "breakpoint"})
+
         #  polars<1.15 implicitly adds -inf and inf to either end of bins
         if self._backend_version < (1, 15) and bins is not None:  # pragma: no cover
             r = pl.int_range(0, len(df))
             df = df.filter((r > 0) & (r < len(df) - 1))
 
-        if self._backend_version < (1, 0) and include_breakpoint:
-            df = df.rename({"break_point": "breakpoint"})
+        # polars<1.27 makes the lowest bin a left/right closed interval.
+        if self._backend_version < (1, 27) and bins is not None:
+            df[0, "count"] += (series == bins[0]).sum()
+
         return PolarsDataFrame.from_native(df, context=self)
 
     def to_polars(self: Self) -> pl.Series: