feat: Remove redundant filtering, add tests for pyarrow trade filtering, use date utils for date to ts conversion

Author: mpagnoulle

freqtrade/data/history/datahandlers/featherdatahandler.py

@@ -143,7 +143,7 @@ def _trades_load(
 
             except (ImportError, AttributeError, ValueError) as e:
                 # Fallback: load entire file
-                logger.debug(f"Unable to use Arrow filtering, loading entire trades file: {e}")
+                logger.warning(f"Unable to use Arrow filtering, loading entire trades file: {e}")
                 tradesdata = read_feather(filename)
 
         return tradesdata

freqtrade/strategy/interface.py

@@ -41,7 +41,7 @@
 )
 from freqtrade.strategy.strategy_validation import StrategyResultValidator
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
-from freqtrade.util import dt_now
+from freqtrade.util import dt_now, dt_ts
 from freqtrade.wallets import Wallets
 
 
@@ -1770,29 +1770,14 @@ def _if_enabled_populate_trades(self, dataframe: DataFrame, metadata: dict) -> D
             pair = metadata["pair"]
             # Build timerange from dataframe date column
             if not dataframe.empty:
-                start_ts = int(dataframe["date"].iloc[0].timestamp() * 1000)
-                end_ts = int(dataframe["date"].iloc[-1].timestamp() * 1000)
+                start_ts = dt_ts(dataframe["date"].iloc[0])
+                end_ts = dt_ts(dataframe["date"].iloc[-1])
                 timerange = TimeRange("date", "date", startts=start_ts, stopts=end_ts)
             else:
                 timerange = None
 
             trades = self.dp.trades(pair=pair, copy=False, timerange=timerange)
 
-            # Apply additional filtering with buffer for faster backtesting
-            if not trades.empty and not dataframe.empty and "timestamp" in trades.columns:
-                # Add timeframe buffer to ensure complete candle coverage
-                timeframe_buffer = timeframe_to_seconds(self.config["timeframe"]) * 1000
-
-                # Create time bounds with buffer
-                time_start = start_ts - timeframe_buffer
-                time_end = end_ts + timeframe_buffer
-
-                # Filter trades within buffered timerange
-                trades_mask = (trades["timestamp"] >= time_start) & (
-                    trades["timestamp"] <= time_end
-                )
-                trades = trades.loc[trades_mask].reset_index(drop=True)
-
             cached_grouped_trades: DataFrame | None = self._cached_grouped_trades_per_pair.get(pair)
             dataframe, cached_grouped_trades = populate_dataframe_with_trades(
                 cached_grouped_trades, self.config, dataframe, trades

tests/data/test_datahandler.py

@@ -506,3 +506,70 @@ def test_get_datahandler(testdatadir):
     assert isinstance(dh, JsonGzDataHandler)
     dh1 = get_datahandler(testdatadir, "jsongz", dh)
     assert id(dh1) == id(dh)
+
+
+@pytest.fixture
+def feather_dh(testdatadir):
+    return FeatherDataHandler(testdatadir)
+
+
+@pytest.fixture
+def trades_full(feather_dh):
+    df = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT)
+    assert not df.empty
+    return df
+
+
+@pytest.fixture
+def timerange_full(trades_full):
+    # Pick a full-span window using actual timestamps
+    startts = int(trades_full["timestamp"].min())
+    stopts = int(trades_full["timestamp"].max())
+    return TimeRange("date", "date", startts=startts, stopts=stopts)
+
+
+@pytest.fixture
+def timerange_mid(trades_full):
+    # Pick a mid-range window using actual timestamps
+    mid_start = int(trades_full["timestamp"].iloc[len(trades_full) // 3])
+    mid_end = int(trades_full["timestamp"].iloc[(2 * len(trades_full)) // 3])
+    return TimeRange("date", "date", startts=mid_start, stopts=mid_end)
+
+
+def test_feather_trades_timerange_filter_fullspan(feather_dh, trades_full, timerange_full):
+    # Full-span filter should equal unfiltered
+    filtered = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_full)
+    assert_frame_equal(
+        trades_full.reset_index(drop=True), filtered.reset_index(drop=True), check_exact=True
+    )
+
+
+def test_feather_trades_timerange_filter_subset(feather_dh, trades_full, timerange_mid):
+    # Subset filter should be a subset of the full-span filter
+    subset = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_mid)
+    assert not subset.empty
+    assert subset["timestamp"].min() >= timerange_mid.startts
+    assert subset["timestamp"].max() <= timerange_mid.stopts
+    assert len(subset) < len(trades_full)
+
+
+def test_feather_trades_timerange_pushdown_fallback(
+    feather_dh, trades_full, timerange_mid, monkeypatch, caplog
+):
+    # Pushdown filter should fail, so fallback should load the entire file
+    import freqtrade.data.history.datahandlers.featherdatahandler as fdh
+
+    def raise_err(*args, **kwargs):
+        raise ValueError("fail")
+
+    # Mock the dataset loading to raise an error
+    monkeypatch.setattr(fdh.dataset, "dataset", raise_err)
+
+    with caplog.at_level("WARNING"):
+        out = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_mid)
+
+    assert len(out) == len(trades_full)
+    assert any(
+        "Unable to use Arrow filtering, loading entire trades file" in r.message
+        for r in caplog.records
+    )