Skip to content

Commit 7b8761c

Browse files
authored
Fix rolling var bug (#62514)
1 parent f570026 commit 7b8761c

File tree

3 files changed

+82
-20
lines changed

3 files changed

+82
-20
lines changed

doc/source/whatsnew/v3.0.0.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1135,6 +1135,7 @@ Groupby/resample/rolling
11351135
- Bug in :meth:`Rolling.apply` for ``method="table"`` where column order was not being respected due to the columns getting sorted by default. (:issue:`59666`)
11361136
- Bug in :meth:`Rolling.apply` where the applied function could be called on fewer than ``min_period`` periods if ``method="table"``. (:issue:`58868`)
11371137
- Bug in :meth:`Series.resample` could raise when the date range ended shortly before a non-existent time. (:issue:`58380`)
1138+
- Bug in :meth:`Series.rolling.var` and :meth:`Series.rolling.std` where the end of window was not indexed correctly. (:issue:`47721`, :issue:`52407`, :issue:`54518`, :issue:`55343`)
11381139

11391140
Reshaping
11401141
^^^^^^^^^

pandas/_libs/window/aggregations.pyx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ def roll_var(const float64_t[:] values, ndarray[int64_t] start,
442442

443443
# Over the first window, observations can only be added
444444
# never removed
445-
if i == 0 or not is_monotonic_increasing_bounds or s >= end[i - 1]:
445+
if i == 0 or not is_monotonic_increasing_bounds or s < end[i]:
446446

447447
prev_value = values[s]
448448
num_consecutive_same_value = 0

pandas/tests/window/test_rolling.py

Lines changed: 80 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@
88

99
from pandas.compat import (
1010
IS64,
11-
is_platform_arm,
12-
is_platform_power,
13-
is_platform_riscv64,
1411
)
1512
from pandas.errors import Pandas4Warning
1613

@@ -1082,27 +1079,91 @@ def test_rolling_sem(frame_or_series):
10821079
tm.assert_series_equal(result, expected)
10831080

10841081

1085-
@pytest.mark.xfail(
1086-
is_platform_arm() or is_platform_power() or is_platform_riscv64(),
1087-
reason="GH 38921",
1088-
)
10891082
@pytest.mark.parametrize(
1090-
("func", "third_value", "values"),
1083+
("func", "values", "window", "ddof", "expected_values"),
10911084
[
1092-
("var", 1, [5e33, 0, 0.5, 0.5, 2, 0]),
1093-
("std", 1, [7.071068e16, 0, 0.7071068, 0.7071068, 1.414214, 0]),
1094-
("var", 2, [5e33, 0.5, 0, 0.5, 2, 0]),
1095-
("std", 2, [7.071068e16, 0.7071068, 0, 0.7071068, 1.414214, 0]),
1085+
("var", [99999999999999999, 1, 1, 2, 3, 1, 1], 2, 1, [5e33, 0, 0.5, 0.5, 2, 0]),
1086+
(
1087+
"std",
1088+
[99999999999999999, 1, 1, 2, 3, 1, 1],
1089+
2,
1090+
1,
1091+
[7.071068e16, 0, 0.7071068, 0.7071068, 1.414214, 0],
1092+
),
1093+
("var", [99999999999999999, 1, 2, 2, 3, 1, 1], 2, 1, [5e33, 0.5, 0, 0.5, 2, 0]),
1094+
(
1095+
"std",
1096+
[99999999999999999, 1, 2, 2, 3, 1, 1],
1097+
2,
1098+
1,
1099+
[7.071068e16, 0.7071068, 0, 0.7071068, 1.414214, 0],
1100+
),
1101+
(
1102+
"std",
1103+
[1.2e03, 1.3e17, 1.5e17, 1.995e03, 1.990e03],
1104+
2,
1105+
1,
1106+
[9.192388e16, 1.414214e16, 1.060660e17, 3.535534e00],
1107+
),
1108+
(
1109+
"var",
1110+
[
1111+
0.00000000e00,
1112+
0.00000000e00,
1113+
3.16188252e-18,
1114+
2.95781651e-16,
1115+
2.23153542e-51,
1116+
0.00000000e00,
1117+
0.00000000e00,
1118+
5.39943432e-48,
1119+
1.38206260e-73,
1120+
0.00000000e00,
1121+
],
1122+
3,
1123+
1,
1124+
[
1125+
3.33250036e-036,
1126+
2.88538519e-032,
1127+
2.88538519e-032,
1128+
2.91622617e-032,
1129+
1.65991678e-102,
1130+
9.71796366e-096,
1131+
9.71796366e-096,
1132+
9.71796366e-096,
1133+
],
1134+
),
1135+
(
1136+
"std",
1137+
[1, -1, 0, 1, 3, 2, -2, 10000000000, 1, 2, 0, -2, 1, 3, 0, 1],
1138+
6,
1139+
1,
1140+
[
1141+
1.41421356e00,
1142+
1.87082869e00,
1143+
4.08248290e09,
1144+
4.08248290e09,
1145+
4.08248290e09,
1146+
4.08248290e09,
1147+
4.08248290e09,
1148+
4.08248290e09,
1149+
1.72240142e00,
1150+
1.75119007e00,
1151+
1.64316767e00,
1152+
],
1153+
),
10961154
],
10971155
)
1098-
def test_rolling_var_numerical_issues(func, third_value, values):
1099-
# GH: 37051
1100-
ds = Series([99999999999999999, 1, third_value, 2, 3, 1, 1])
1101-
result = getattr(ds.rolling(2), func)()
1102-
expected = Series([np.nan] + values)
1103-
tm.assert_series_equal(result, expected)
1156+
def test_rolling_var_correctness(func, values, window, ddof, expected_values):
1157+
# GH: 37051, 42064, 54518, 52407, 47721
1158+
ts = Series(values)
1159+
result = getattr(ts.rolling(window=window), func)(ddof=ddof)
1160+
if result.last_valid_index():
1161+
result = result[
1162+
result.first_valid_index() : result.last_valid_index() + 1
1163+
].reset_index(drop=True)
1164+
expected = Series(expected_values)
1165+
tm.assert_series_equal(result, expected, atol=1e-55)
11041166
# GH 42064
1105-
# new `roll_var` will output 0.0 correctly
11061167
tm.assert_series_equal(result == 0, expected == 0)
11071168

11081169

0 commit comments

Comments
 (0)