Merge pull request freqtrade#12529 from mrpabloyeah/fix-backtesting-exception-when-no-data-is-available-for-a-pair

Fix backtesting exception when no data is available for a pair

Author: xmatthias

freqtrade/optimize/backtesting.py

@@ -126,6 +126,7 @@ def __init__(self, config: Config, exchange: Exchange | None = None) -> None:
 
         self.config["dry_run"] = True
         self.price_pair_prec: dict[str, Series] = {}
+        self.available_pairs: list[str] = []
         self.run_ids: dict[str, str] = {}
         self.strategylist: list[IStrategy] = []
         self.all_bt_content: dict[str, BacktestContentType] = {}
@@ -176,7 +177,8 @@ def __init__(self, config: Config, exchange: Exchange | None = None) -> None:
         self._validate_pairlists_for_backtesting()
 
         self.dataprovider.add_pairlisthandler(self.pairlists)
-        self.pairlists.refresh_pairlist()
+        self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
+        self.pairlists.refresh_pairlist(only_first=self.dynamic_pairlist)
 
         if len(self.pairlists.whitelist) == 0:
             raise OperationalException("No pair in whitelist.")
@@ -211,7 +213,6 @@ def __init__(self, config: Config, exchange: Exchange | None = None) -> None:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get("position_stacking", False)
         self.enable_protections: bool = self.config.get("enable_protections", False)
-        self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
         migrate_data(config, self.exchange)
 
         self.init_backtest()
@@ -335,10 +336,12 @@ def load_bt_data(self) -> tuple[dict[str, DataFrame], TimeRange]:
         self.progress.set_new_value(1)
         self._load_bt_data_detail()
         self.price_pair_prec = {}
+
         for pair in self.pairlists.whitelist:
             if pair in data:
                 # Load price precision logic
                 self.price_pair_prec[pair] = get_tick_size_over_time(data[pair])
+                self.available_pairs.append(pair)
         return data, self.timerange
 
     def _load_bt_data_detail(self) -> None:
@@ -1587,7 +1590,7 @@ def time_pair_generator(
             self.check_abort()
 
             if self.dynamic_pairlist and self.pairlists:
-                self.pairlists.refresh_pairlist()
+                self.pairlists.refresh_pairlist(pairs=self.available_pairs)
                 pairs = self.pairlists.whitelist
 
             # Reset open trade count for this candle

freqtrade/plugins/pairlistmanager.py

@@ -134,8 +134,20 @@ def short_desc(self) -> list[dict]:
     def _get_cached_tickers(self) -> Tickers:
         return self._exchange.get_tickers()
 
-    def refresh_pairlist(self) -> None:
-        """Run pairlist through all configured Pairlist Handlers."""
+    def refresh_pairlist(self, only_first: bool = False, pairs: list[str] | None = None) -> None:
+        """
+        Run pairlist through all configured Pairlist Handlers.
+
+        :param only_first: If True, only run the first PairList handler (the generator)
+            and skip all subsequent filters. Used during backtesting startup to ensure
+            historic data is loaded for the complete universe of pairs that the
+            generator can produce (even if later filters would reduce the list size).
+            Prevents missing data when a filter returns a variable number of pairs
+            across refresh cycles.
+        :param pairs: Optional list of pairs to intersect with the generated pairlist.
+            Only pairs present both in the generated list and this parameter are kept.
+            Used in backtesting to filter out pairs with no available data.
+        """
         # Tickers should be cached to avoid calling the exchange on each call.
         tickers: dict = {}
         if self._tickers_needed:
@@ -144,10 +156,15 @@ def refresh_pairlist(self) -> None:
         # Generate the pairlist with first Pairlist Handler in the chain
         pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
 
-        # Process all Pairlist Handlers in the chain
-        # except for the first one, which is the generator.
-        for pairlist_handler in self._pairlist_handlers[1:]:
-            pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
+        # Optional intersection with an explicit list of pairs (used in backtesting)
+        if pairs is not None:
+            pairlist = [p for p in pairlist if p in pairs]
+
+        if not only_first:
+            # Process all Pairlist Handlers in the chain
+            # except for the first one, which is the generator.
+            for pairlist_handler in self._pairlist_handlers[1:]:
+                pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
 
         # Validation against blacklist happens after the chain of Pairlist Handlers
         # to ensure blacklist is respected.

tests/optimize/test_backtesting.py

@@ -2772,7 +2772,7 @@ def test_time_pair_generator_open_trades_first(mocker, default_conf, dynamic_pai
     dummy_row = (end_date, 1.0, 1.1, 0.9, 1.0, 0, 0, 0, 0, None, None)
     data = {pair: [dummy_row] for pair in pairs}
 
-    def mock_refresh(self):
+    def mock_refresh(self, **kwargs):
         # Simulate shuffle
         self._whitelist = pairs[::-1]  # ['ETH/BTC', 'NEO/BTC', 'LTC/BTC', 'XRP/BTC']