feat: Only load trades needed for specified timerange

Author: mpagnoulle

freqtrade/data/dataprovider.py

@@ -498,7 +498,12 @@ def ohlcv(
             return DataFrame()
 
     def trades(
-        self, pair: str, timeframe: str | None = None, copy: bool = True, candle_type: str = ""
+        self,
+        pair: str,
+        timeframe: str | None = None,
+        copy: bool = True,
+        candle_type: str = "",
+        timerange: TimeRange | None = None,
     ) -> DataFrame:
         """
         Get candle (TRADES) data for the given pair as DataFrame
@@ -526,7 +531,7 @@ def trades(
                 self._config["datadir"], data_format=self._config["dataformat_trades"]
             )
             trades_df = data_handler.trades_load(
-                pair, self._config.get("trading_mode", TradingMode.SPOT)
+                pair, self._config.get("trading_mode", TradingMode.SPOT), timerange=timerange
             )
             return trades_df
 

freqtrade/data/history/datahandlers/featherdatahandler.py

@@ -1,6 +1,7 @@
 import logging
 
 from pandas import DataFrame, read_feather, to_datetime
+from pyarrow import dataset
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
@@ -116,17 +117,34 @@ def _trades_load(
     ) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
-        # TODO: respect timerange ...
         :param pair: Load trades for this pair
         :param trading_mode: Trading mode to use (used to determine the filename)
-        :param timerange: Timerange to load trades for - currently not implemented
+        :param timerange: Timerange to load trades for - filters data to this range if provided
         :return: Dataframe containing trades
         """
         filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
         if not filename.exists():
             return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 
-        tradesdata = read_feather(filename)
+        # Load trades data with optional timerange filtering
+        if timerange is None:
+            # No timerange filter - load entire file
+            logger.debug(f"Loading entire trades file for {pair}")
+            tradesdata = read_feather(filename)
+        else:
+            # Use Arrow dataset with predicate pushdown for efficient filtering
+            try:
+                dataset_reader = dataset.dataset(filename, format="feather")
+                time_filter = (dataset.field("timestamp") >= timerange.startts) & (
+                    dataset.field("timestamp") <= timerange.stopts
+                )
+                tradesdata = dataset_reader.to_table(filter=time_filter).to_pandas()
+                logger.debug(f"Loaded {len(tradesdata)} trades for {pair}")
+
+            except (ImportError, AttributeError, ValueError) as e:
+                # Fallback: load entire file
+                logger.debug(f"Unable to use Arrow filtering, loading entire trades file: {e}")
+                tradesdata = read_feather(filename)
 
         return tradesdata