API: date_range, timedelta_range infer unit from start/end/freq

Author: jbrockmendel

doc/source/whatsnew/v3.0.0.rst

@@ -741,6 +741,7 @@ Other API changes
   the dtype of the resulting Index (:issue:`60797`)
 - :class:`IncompatibleFrequency` now subclasses ``TypeError`` instead of ``ValueError``. As a result, joins with mismatched frequencies now cast to object like other non-comparable joins, and arithmetic with indexes with mismatched frequencies align (:issue:`55782`)
 - :class:`Series` "flex" methods like :meth:`Series.add` no longer allow passing a :class:`DataFrame` for ``other``; use the DataFrame reversed method instead (:issue:`46179`)
+- :func:`date_range` and :func:`timedelta_range` no longer default to ``unit="ns"``, instead will infer a unit from the ``start``, ``end``, and ``freq`` parameters. Explicitly specify a desired ``unit`` to override these (:issue:`59031`)
 - :meth:`CategoricalIndex.append` no longer attempts to cast different-dtype indexes to the caller's dtype (:issue:`41626`)
 - :meth:`ExtensionDtype.construct_array_type` is now a regular method instead of a ``classmethod`` (:issue:`58663`)
 - Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)

pandas/core/indexes/datetimes.py

@@ -25,6 +25,7 @@
     timezones,
     to_offset,
 )
+from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
 from pandas._libs.tslibs.offsets import prefix_mapping
 from pandas.errors import Pandas4Warning
 from pandas.util._decorators import (
@@ -883,7 +884,7 @@ def date_range(
     name: Hashable | None = None,
     inclusive: IntervalClosedType = "both",
     *,
-    unit: TimeUnit = "ns",
+    unit: TimeUnit | None = None,
     **kwargs,
 ) -> DatetimeIndex:
     """
@@ -922,8 +923,9 @@ def date_range(
         Name of the resulting DatetimeIndex.
     inclusive : {"both", "neither", "left", "right"}, default "both"
         Include boundaries; Whether to set each bound as closed or open.
-    unit : {'s', 'ms', 'us', 'ns'}, default 'ns'
+    unit : {'s', 'ms', 'us', 'ns', None}, default None
         Specify the desired resolution of the result.
+        If not specified, this is inferred from the 'start', 'end', and 'freq'
 
         .. versionadded:: 2.0.0
     **kwargs
@@ -1063,6 +1065,37 @@ def date_range(
     if freq is None and com.any_none(periods, start, end):
         freq = "D"
 
+    if unit is None:
+        # Infer the unit based on the inputs
+
+        if start is not None and end is not None:
+            start = Timestamp(start)
+            end = Timestamp(end)
+            if abbrev_to_npy_unit(start.unit) > abbrev_to_npy_unit(end.unit):
+                unit = start.unit
+            else:
+                unit = end.unit
+        elif start is not None:
+            start = Timestamp(start)
+            unit = start.unit
+        else:
+            end = Timestamp(end)
+            unit = end.unit
+
+        # Last we need to watch out for cases where the 'freq' implies a higher
+        #  unit than either start or end
+        if freq is not None:
+            freq = to_offset(freq)
+            creso = abbrev_to_npy_unit(unit)
+            if isinstance(freq, Tick):
+                if freq._creso > creso:
+                    unit = freq.base.freqstr
+            elif hasattr(freq, "offset") and freq.offset is not None:
+                # e.g. BDay with an offset
+                td = Timedelta(freq.offset)
+                if abbrev_to_npy_unit(td.unit) > creso:
+                    unit = td.unit
+
     dtarr = DatetimeArray._generate_range(
         start=start,
         end=end,

pandas/core/indexes/timedeltas.py

@@ -13,6 +13,7 @@
     Timedelta,
     to_offset,
 )
+from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
 from pandas.util._decorators import set_module
 
 from pandas.core.dtypes.common import (
@@ -252,7 +253,7 @@ def timedelta_range(
     name=None,
     closed=None,
     *,
-    unit: TimeUnit = "ns",
+    unit: TimeUnit | None = None,
 ) -> TimedeltaIndex:
     """
     Return a fixed frequency TimedeltaIndex with day as the default.
@@ -272,8 +273,9 @@ def timedelta_range(
     closed : str, default None
         Make the interval closed with respect to the given frequency to
         the 'left', 'right', or both sides (None).
-    unit : {'s', 'ms', 'us', 'ns'}, default 'ns'
+    unit : {'s', 'ms', 'us', 'ns', None}, default None
         Specify the desired resolution of the result.
+        If not specified, this is inferred from the 'start', 'end', and 'freq'
 
         .. versionadded:: 2.0.0
 
@@ -337,8 +339,39 @@ def timedelta_range(
     """
     if freq is None and com.any_none(periods, start, end):
         freq = "D"
-
     freq = to_offset(freq)
+
+    if com.count_not_none(start, end, periods, freq) != 3:
+        # This check needs to come before the `unit = start.unit` line below
+        raise ValueError(
+            "Of the four parameters: start, end, periods, "
+            "and freq, exactly three must be specified"
+        )
+
+    if unit is None:
+        # Infer the unit based on the inputs
+
+        if start is not None and end is not None:
+            start = Timedelta(start)
+            end = Timedelta(end)
+            if abbrev_to_npy_unit(start.unit) > abbrev_to_npy_unit(end.unit):
+                unit = start.unit
+            else:
+                unit = end.unit
+        elif start is not None:
+            start = Timedelta(start)
+            unit = start.unit
+        else:
+            end = Timedelta(end)
+            unit = end.unit
+
+        # Last we need to watch out for cases where the 'freq' implies a higher
+        #  unit than either start or end
+        if freq is not None:
+            creso = abbrev_to_npy_unit(unit)
+            if freq._creso > creso:
+                unit = freq.base.freqstr
+
     tdarr = TimedeltaArray._generate_range(
         start, end, periods, freq, closed=closed, unit=unit
     )

pandas/tests/apply/test_frame_apply.py

@@ -653,13 +653,15 @@ def test_apply_dict(df, dicts):
 
 def test_apply_non_numpy_dtype():
     # GH 12244
-    df = DataFrame({"dt": date_range("2015-01-01", periods=3, tz="Europe/Brussels")})
+    df = DataFrame(
+        {"dt": date_range("2015-01-01", periods=3, tz="Europe/Brussels", unit="ns")}
+    )
     result = df.apply(lambda x: x)
     tm.assert_frame_equal(result, df)
 
     result = df.apply(lambda x: x + pd.Timedelta("1day"))
     expected = DataFrame(
-        {"dt": date_range("2015-01-02", periods=3, tz="Europe/Brussels")}
+        {"dt": date_range("2015-01-02", periods=3, tz="Europe/Brussels", unit="ns")}
     )
     tm.assert_frame_equal(result, expected)
 
@@ -1425,7 +1427,7 @@ def test_nuiscance_columns():
             "A": [1, 2, 3],
             "B": [1.0, 2.0, 3.0],
             "C": ["foo", "bar", "baz"],
-            "D": date_range("20130101", periods=3),
+            "D": date_range("20130101", periods=3, unit="ns"),
         }
     )
 

pandas/tests/arithmetic/test_datetime64.py

@@ -589,7 +589,7 @@ def test_comparison_tzawareness_compat(self, comparison_op, box_with_array):
         op = comparison_op
         box = box_with_array
 
-        dr = date_range("2016-01-01", periods=6)
+        dr = date_range("2016-01-01", periods=6, unit="ns")
         dz = dr.tz_localize("US/Pacific")
 
         dr = tm.box_expected(dr, box)
@@ -637,7 +637,7 @@ def test_comparison_tzawareness_compat_scalars(self, comparison_op, box_with_arr
         # GH#18162
         op = comparison_op
 
-        dr = date_range("2016-01-01", periods=6)
+        dr = date_range("2016-01-01", periods=6, unit="ns")
         dz = dr.tz_localize("US/Pacific")
 
         dr = tm.box_expected(dr, box_with_array)
@@ -676,7 +676,7 @@ def test_scalar_comparison_tzawareness(
     ):
         op = comparison_op
         tz = tz_aware_fixture
-        dti = date_range("2016-01-01", periods=2, tz=tz)
+        dti = date_range("2016-01-01", periods=2, tz=tz, unit="ns")
 
         dtarr = tm.box_expected(dti, box_with_array)
         xbox = get_upcast_box(dtarr, other, True)
@@ -833,8 +833,8 @@ def test_dt64arr_add_timedeltalike_scalar(
         # GH#22005, GH#22163 check DataFrame doesn't raise TypeError
         tz = tz_naive_fixture
 
-        rng = date_range("2000-01-01", "2000-02-01", tz=tz)
-        expected = date_range("2000-01-01 02:00", "2000-02-01 02:00", tz=tz)
+        rng = date_range("2000-01-01", "2000-02-01", tz=tz, unit="ns")
+        expected = date_range("2000-01-01 02:00", "2000-02-01 02:00", tz=tz, unit="ns")
         if tz is not None:
             expected = expected._with_freq(None)
 
@@ -855,8 +855,8 @@ def test_dt64arr_sub_timedeltalike_scalar(
     ):
         tz = tz_naive_fixture
 
-        rng = date_range("2000-01-01", "2000-02-01", tz=tz)
-        expected = date_range("1999-12-31 22:00", "2000-01-31 22:00", tz=tz)
+        rng = date_range("2000-01-01", "2000-02-01", tz=tz, unit="ns")
+        expected = date_range("1999-12-31 22:00", "2000-01-31 22:00", tz=tz, unit="ns")
         if tz is not None:
             expected = expected._with_freq(None)
 
@@ -870,7 +870,7 @@ def test_dt64arr_sub_timedeltalike_scalar(
         tm.assert_equal(rng, expected)
 
     def test_dt64_array_sub_dt_with_different_timezone(self, box_with_array):
-        t1 = date_range("20130101", periods=3).tz_localize("US/Eastern")
+        t1 = date_range("20130101", periods=3, unit="ns").tz_localize("US/Eastern")
         t1 = tm.box_expected(t1, box_with_array)
         t2 = Timestamp("20130101").tz_localize("CET")
         tnaive = Timestamp(20130101)
@@ -897,11 +897,11 @@ def test_dt64_array_sub_dt_with_different_timezone(self, box_with_array):
             tnaive - t1
 
     def test_dt64_array_sub_dt64_array_with_different_timezone(self, box_with_array):
-        t1 = date_range("20130101", periods=3).tz_localize("US/Eastern")
+        t1 = date_range("20130101", periods=3, unit="ns").tz_localize("US/Eastern")
         t1 = tm.box_expected(t1, box_with_array)
-        t2 = date_range("20130101", periods=3).tz_localize("CET")
+        t2 = date_range("20130101", periods=3, unit="ns").tz_localize("CET")
         t2 = tm.box_expected(t2, box_with_array)
-        tnaive = date_range("20130101", periods=3)
+        tnaive = date_range("20130101", periods=3, unit="ns")
 
         result = t1 - t2
         expected = TimedeltaIndex(
@@ -928,7 +928,7 @@ def test_dt64arr_add_sub_td64_nat(self, box_with_array, tz_naive_fixture):
         # GH#23320 special handling for timedelta64("NaT")
         tz = tz_naive_fixture
 
-        dti = date_range("1994-04-01", periods=9, tz=tz, freq="QS")
+        dti = date_range("1994-04-01", periods=9, tz=tz, freq="QS", unit="ns")
         other = np.timedelta64("NaT")
         expected = DatetimeIndex(["NaT"] * 9, tz=tz).as_unit("ns")
 
@@ -947,11 +947,11 @@ def test_dt64arr_add_sub_td64_nat(self, box_with_array, tz_naive_fixture):
 
     def test_dt64arr_add_sub_td64ndarray(self, tz_naive_fixture, box_with_array):
         tz = tz_naive_fixture
-        dti = date_range("2016-01-01", periods=3, tz=tz)
+        dti = date_range("2016-01-01", periods=3, tz=tz, unit="ns")
         tdi = TimedeltaIndex(["-1 Day", "-1 Day", "-1 Day"])
         tdarr = tdi.values
 
-        expected = date_range("2015-12-31", "2016-01-02", periods=3, tz=tz)
+        expected = date_range("2015-12-31", "2016-01-02", periods=3, tz=tz, unit="ns")
 
         dtarr = tm.box_expected(dti, box_with_array)
         expected = tm.box_expected(expected, box_with_array)
@@ -961,7 +961,7 @@ def test_dt64arr_add_sub_td64ndarray(self, tz_naive_fixture, box_with_array):
         result = tdarr + dtarr
         tm.assert_equal(result, expected)
 
-        expected = date_range("2016-01-02", "2016-01-04", periods=3, tz=tz)
+        expected = date_range("2016-01-02", "2016-01-04", periods=3, tz=tz, unit="ns")
         expected = tm.box_expected(expected, box_with_array)
 
         result = dtarr - tdarr
@@ -991,7 +991,7 @@ def test_dt64arr_add_sub_td64ndarray(self, tz_naive_fixture, box_with_array):
     )
     def test_dt64arr_sub_dtscalar(self, box_with_array, ts):
         # GH#8554, GH#22163 DataFrame op should _not_ return dt64 dtype
-        idx = date_range("2013-01-01", periods=3)._with_freq(None)
+        idx = date_range("2013-01-01", periods=3, unit="ns")._with_freq(None)
         idx = tm.box_expected(idx, box_with_array)
 
         expected = TimedeltaIndex(["0 Days", "1 Day", "2 Days"])
@@ -1005,7 +1005,7 @@ def test_dt64arr_sub_dtscalar(self, box_with_array, ts):
         tm.assert_equal(result, -expected)
 
     def test_dt64arr_sub_timestamp_tzaware(self, box_with_array):
-        ser = date_range("2014-03-17", periods=2, freq="D", tz="US/Eastern")
+        ser = date_range("2014-03-17", periods=2, freq="D", tz="US/Eastern", unit="ns")
         ser = ser._with_freq(None)
         ts = ser[0]
 
@@ -1312,11 +1312,11 @@ def test_dti_add_tick_tzaware(self, tz_aware_fixture, box_with_array):
         # GH#21610, GH#22163 ensure DataFrame doesn't return object-dtype
         tz = tz_aware_fixture
         if tz == "US/Pacific":
-            dates = date_range("2012-11-01", periods=3, tz=tz)
+            dates = date_range("2012-11-01", periods=3, tz=tz, unit="ns")
             offset = dates + pd.offsets.Hour(5)
             assert dates[0] + pd.offsets.Hour(5) == offset[0]
 
-        dates = date_range("2010-11-01 00:00", periods=3, tz=tz, freq="h")
+        dates = date_range("2010-11-01 00:00", periods=3, tz=tz, freq="h", unit="ns")
         expected = DatetimeIndex(
             ["2010-11-01 05:00", "2010-11-01 06:00", "2010-11-01 07:00"],
             freq="h",
@@ -1604,7 +1604,9 @@ def test_dti_add_sub_nonzero_mth_offset(
     ):
         # GH 26258
         tz = tz_aware_fixture
-        date = date_range(start="01 Jan 2014", end="01 Jan 2017", freq="YS", tz=tz)
+        date = date_range(
+            start="01 Jan 2014", end="01 Jan 2017", freq="YS", tz=tz, unit="ns"
+        )
         date = tm.box_expected(date, box_with_array, False)
         mth = getattr(date, op)
         result = mth(offset)
@@ -1667,7 +1669,7 @@ def test_dt64_series_arith_overflow(self):
         # GH#12534, fixed by GH#19024
         dt = Timestamp("1700-01-31")
         td = Timedelta("20000 Days")
-        dti = date_range("1949-09-30", freq="100YE", periods=4)
+        dti = date_range("1949-09-30", freq="100YE", periods=4, unit="ns")
         ser = Series(dti)
         msg = "Overflow in int64 addition"
         with pytest.raises(OverflowError, match=msg):
@@ -1890,7 +1892,7 @@ def test_sub_single_tz(self, unit):
     def test_dt64tz_series_sub_dtitz(self):
         # GH#19071 subtracting tzaware DatetimeIndex from tzaware Series
         # (with same tz) raises, fixed by #19024
-        dti = date_range("1999-09-30", periods=10, tz="US/Pacific")
+        dti = date_range("1999-09-30", periods=10, tz="US/Pacific", unit="ns")
         ser = Series(dti)
         expected = Series(TimedeltaIndex(["0days"] * 10))
 
@@ -2040,7 +2042,7 @@ def test_dti_add_tdi(self, tz_naive_fixture):
         tz = tz_naive_fixture
         dti = DatetimeIndex([Timestamp("2017-01-01", tz=tz)] * 10)
         tdi = pd.timedelta_range("0 days", periods=10)
-        expected = date_range("2017-01-01", periods=10, tz=tz)
+        expected = date_range("2017-01-01", periods=10, tz=tz, unit="ns")
         expected = expected._with_freq(None)
 
         # add with TimedeltaIndex
@@ -2062,7 +2064,7 @@ def test_dti_iadd_tdi(self, tz_naive_fixture):
         tz = tz_naive_fixture
         dti = DatetimeIndex([Timestamp("2017-01-01", tz=tz)] * 10)
         tdi = pd.timedelta_range("0 days", periods=10)
-        expected = date_range("2017-01-01", periods=10, tz=tz)
+        expected = date_range("2017-01-01", periods=10, tz=tz, unit="ns")
         expected = expected._with_freq(None)
 
         # iadd with TimedeltaIndex
@@ -2088,7 +2090,7 @@ def test_dti_sub_tdi(self, tz_naive_fixture):
         tz = tz_naive_fixture
         dti = DatetimeIndex([Timestamp("2017-01-01", tz=tz)] * 10)
         tdi = pd.timedelta_range("0 days", periods=10)
-        expected = date_range("2017-01-01", periods=10, tz=tz, freq="-1D")
+        expected = date_range("2017-01-01", periods=10, tz=tz, freq="-1D", unit="ns")
         expected = expected._with_freq(None)
 
         # sub with TimedeltaIndex