Merge pull request freqtrade#12599 from freqtrade/fix/dynamic_funding_fees

Adjust to dynamic funding fees

Author: xmatthias

docs/commands/download-data.md

@@ -11,6 +11,7 @@ usage: freqtrade download-data [-h] [-v] [--no-color] [--logfile FILE] [-V]
                                [--data-format-ohlcv {json,jsongz,feather,parquet}]
                                [--data-format-trades {json,jsongz,feather,parquet}]
                                [--trading-mode {spot,margin,futures}]
+                               [--candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...]]
                                [--prepend]
 
 options:
@@ -50,6 +51,11 @@ options:
                         `feather`).
   --trading-mode, --tradingmode {spot,margin,futures}
                         Select Trading mode
+  --candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...]
+                        Select candle type to download. Defaults to the
+                        necessary candles for the selected trading mode (e.g.
+                        'spot' or ('futures', 'funding_rate' and 'mark') for
+                        futures).
   --prepend             Allow data prepending. (Data-appending is disabled)
 
 Common arguments:

docs/data-download.md

@@ -60,6 +60,7 @@ freqtrade download-data --exchange binance --pairs ".*/USDT"
 * Given starting points are ignored if data is already available, downloading only missing data up to today.
 * Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
 * To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
+* When downloading futures data (`--trading-mode futures` or a configuration specifying futures mode), freqtrade will automatically download the necessary candle types (e.g. `mark` and `funding_rate` candles) unless specified otherwise via `--candle-types`.
 
 ??? Note "Permission denied errors"
     If your configuration directory `user_data` was made by docker, you may get the following error:

docs/deprecated.md

@@ -98,3 +98,40 @@ Please use configuration based [log setup](advanced-setup.md#advanced-logging) i
 
 The edge module has been deprecated in 2023.9 and removed in 2025.6.
 All functionalities of edge have been removed, and having edge configured will result in an error.
+
+## Adjustment to dynamic funding rate handling
+
+With version 2025.12, the handling of dynamic funding rates has been adjusted to also support dynamic funding rates down to 1h funding intervals.
+As a consequence, the mark and funding rate timeframes have been changed to 1h for every supported futures exchange.
+
+As the timeframe for both mark and funding_fee candles has changed (usually from 8h to 1h) - already downloaded data will have to be adjusted or partially re-downloaded.
+You can either re-download everything (`freqtrade download-data [...] --erase` - :warning: can take a long time) - or download the updated data selectively.
+
+### Strategy
+
+Most strategies should not need adjustments to continue to work as expected - however, strategies using `@informative("8h", candle_type="funding_rate")` or similar will have to switch the timeframe to 1h.
+The same is true for `dp.get_pair_dataframe(metadata["pair"], "8h", candle_type="funding_rate")` - which will need to be switched to 1h.
+
+freqtrade will auto-adjust the timeframe and return `funding_rates` despite the wrongly given timeframe. It'll issue a warning - and may still break your strategy.
+
+### Selective data re-download
+
+The script below should serve as an example - you may need to adjust the timeframe and exchange to your needs!
+
+``` bash
+# Cleanup no longer needed data
+rm user_data/data/<exchange>/futures/*-mark-*
+rm user_data/data/<exchange>/futures/*-funding_rate-*
+
+# download new data (only required once to fix the mark and funding fee data)
+freqtrade download-data -t 1h --trading-mode futures --candle-types funding_rate mark [...] --timerange <full timerange you've got other data for>
+
+```
+
+The result of the above will be that your funding_rates and mark data will have the 1h timeframe.
+you can verify this with `freqtrade list-data --exchange <yourexchange> --show`.
+
+!!! Note "Additional arguments"
+    Additional arguments to the above commands may be necessary, like configuration files or explicit user_data if they deviate from the default.
+
+**Hyperliquid** is a special case now - which will no longer require 1h mark data - but will use regular candles instead (this data never existed and is identical to 1h futures candles). As we don't support download-data for hyperliquid (they don't provide historic data) - there won't be actions necessary for hyperliquid users.

freqtrade/commands/arguments.py

@@ -3,6 +3,7 @@
 """
 
 from argparse import ArgumentParser, Namespace, _ArgumentGroup
+from copy import deepcopy
 from functools import partial
 from pathlib import Path
 from typing import Any
@@ -174,6 +175,7 @@
     "dataformat_ohlcv",
     "dataformat_trades",
     "trading_mode",
+    "candle_types",
     "prepend_data",
 ]
 
@@ -348,7 +350,11 @@ def _parse_args(self) -> Namespace:
     def _build_args(self, optionlist: list[str], parser: ArgumentParser | _ArgumentGroup) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
-            parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
+            options = deepcopy(opt.kwargs)
+            help_text = options.pop("help", None)
+            if opt.fthelp and isinstance(opt.fthelp, dict) and hasattr(parser, "prog"):
+                help_text = opt.fthelp.get(parser.prog, help_text)
+            parser.add_argument(*opt.cli, dest=val, help=help_text, **options)
 
     def _build_subcommands(self) -> None:
         """

freqtrade/commands/cli_options.py

@@ -38,8 +38,14 @@ def check_int_nonzero(value: str) -> int:
 
 class Arg:
     # Optional CLI arguments
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, fthelp: dict[str, str] | None = None, **kwargs):
+        """
+        CLI Arguments - used to build subcommand parsers consistently.
+        :param fthelp: dict - fthelp per command - should be "freqtrade <command>": help_text
+            If not provided or not found, 'help' from kwargs is used instead.
+        """
         self.cli = args
+        self.fthelp = fthelp
         self.kwargs = kwargs
 
 
@@ -422,6 +428,14 @@ def __init__(self, *args, **kwargs):
     ),
     "candle_types": Arg(
         "--candle-types",
+        fthelp={
+            "freqtrade download-data": (
+                "Select candle type to download. "
+                "Defaults to the necessary candles for the selected trading mode "
+                "(e.g. 'spot' or ('futures', 'funding_rate' and 'mark') for futures)."
+            ),
+            "_": "Select candle type to convert. Defaults to all available types.",
+        },
         help="Select candle type to convert. Defaults to all available types.",
         choices=[c.value for c in CandleType],
         nargs="+",

freqtrade/data/converter/converter.py

@@ -38,7 +38,8 @@ def ohlcv_to_dataframe(
     cols = DEFAULT_DATAFRAME_COLUMNS
     df = DataFrame(ohlcv, columns=cols)
 
-    df["date"] = to_datetime(df["date"], unit="ms", utc=True)
+    # Floor date to seconds to account for exchange imprecisions
+    df["date"] = to_datetime(df["date"], unit="ms", utc=True).dt.floor("s")
 
     # Some exchanges return int values for Volume and even for OHLC.
     # Convert them since TA-LIB indicators used in the strategy assume floats

freqtrade/data/dataprovider.py

@@ -348,6 +348,22 @@ def get_required_startup(self, timeframe: str) -> int:
             )
         return total_candles
 
+    def __fix_funding_rate_timeframe(
+        self, pair: str, timeframe: str | None, candle_type: str
+    ) -> str | None:
+        if (
+            candle_type == CandleType.FUNDING_RATE
+            and (ff_tf := self.get_funding_rate_timeframe()) != timeframe
+        ):
+            # TODO: does this message make sense? might be pointless as funding fees don't
+            # have a timeframe
+            logger.warning(
+                f"{pair}, {timeframe} requested - funding rate timeframe not matching {ff_tf}."
+            )
+            return ff_tf
+
+        return timeframe
+
     def get_pair_dataframe(
         self, pair: str, timeframe: str | None = None, candle_type: str = ""
     ) -> DataFrame:
@@ -361,6 +377,7 @@ def get_pair_dataframe(
         :return: Dataframe for this pair
         :param candle_type: '', mark, index, premiumIndex, or funding_rate
         """
+        timeframe = self.__fix_funding_rate_timeframe(pair, timeframe, candle_type)
         if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
             # Get live OHLCV data.
             data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
@@ -620,3 +637,12 @@ def check_delisting(self, pair: str) -> datetime | None:
         except ExchangeError:
             logger.warning(f"Could not fetch market data for {pair}. Assuming no delisting.")
             return None
+
+    def get_funding_rate_timeframe(self) -> str:
+        """
+        Get the funding rate timeframe from exchange options
+        :return: Timeframe string
+        """
+        if self._exchange is None:
+            raise OperationalException(NO_EXCHANGE_EXCEPTION)
+        return self._exchange.get_option("funding_fee_timeframe")

freqtrade/data/history/datahandlers/idatahandler.py

@@ -397,6 +397,9 @@ def ohlcv_load(
         pairdf = self._ohlcv_load(
             pair, timeframe, timerange=timerange_startup, candle_type=candle_type
         )
+        if not pairdf.empty and candle_type == CandleType.FUNDING_RATE:
+            # Funding rate data is sometimes off by a couple of ms - floor to seconds
+            pairdf["date"] = pairdf["date"].dt.floor("s")
         if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
             return pairdf
         else:
@@ -508,8 +511,15 @@ def fix_funding_fee_timeframe(self, ff_timeframe: str):
         Applies to bybit and okx, where funding-fee and mark candles have different timeframes.
         """
         paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES)
+        ff_timeframe_s = timeframe_to_seconds(ff_timeframe)
+
         funding_rate_combs = [
-            f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe
+            f
+            for f in paircombs
+            if f[2] == CandleType.FUNDING_RATE
+            and f[1] != ff_timeframe
+            # Only allow smaller timeframes to move from smaller to larger timeframes
+            and timeframe_to_seconds(f[1]) < ff_timeframe_s
         ]
 
         if funding_rate_combs:

freqtrade/data/history/history_utils.py

@@ -353,6 +353,7 @@ def _download_pair_history(
 
 def refresh_backtest_ohlcv_data(
     exchange: Exchange,
+    *,
     pairs: list[str],
     timeframes: list[str],
     datadir: Path,
@@ -363,6 +364,7 @@ def refresh_backtest_ohlcv_data(
     data_format: str | None = None,
     prepend: bool = False,
     progress_tracker: CustomProgress | None = None,
+    candle_types: list[CandleType] | None = None,
     no_parallel_download: bool = False,
 ) -> list[str]:
     """
@@ -375,10 +377,44 @@ def refresh_backtest_ohlcv_data(
     pairs_not_available = []
     fast_candles: dict[PairWithTimeframe, DataFrame] = {}
     data_handler = get_datahandler(datadir, data_format)
-    candle_type = CandleType.get_default(trading_mode)
+    def_candletype = CandleType.SPOT if trading_mode != "futures" else CandleType.FUTURES
+    if trading_mode != "futures":
+        # Ignore user passed candle types for non-futures trading
+        timeframes_with_candletype = [(tf, def_candletype) for tf in timeframes]
+    else:
+        # Filter out SPOT candle type for futures trading
+        candle_types = (
+            [ct for ct in candle_types if ct != CandleType.SPOT] if candle_types else None
+        )
+        fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
+        tf_funding_rate = exchange.get_option("funding_fee_timeframe")
+        tf_mark = exchange.get_option("mark_ohlcv_timeframe")
+
+        if candle_types:
+            for ct in candle_types:
+                exchange.verify_candle_type_support(ct)
+            timeframes_with_candletype = [
+                (tf, ct)
+                for ct in candle_types
+                for tf in timeframes
+                if ct != CandleType.FUNDING_RATE
+            ]
+        else:
+            # Default behavior
+            timeframes_with_candletype = [(tf, def_candletype) for tf in timeframes]
+            timeframes_with_candletype.append((tf_mark, fr_candle_type))
+        if not candle_types or CandleType.FUNDING_RATE in candle_types:
+            # All exchanges need FundingRate for futures trading.
+            # The timeframe is aligned to the mark-price timeframe.
+            timeframes_with_candletype.append((tf_funding_rate, CandleType.FUNDING_RATE))
+    # Deduplicate list ...
+    timeframes_with_candletype = list(dict.fromkeys(timeframes_with_candletype))
+    logger.debug(
+        "Downloading %s.", ", ".join(f'"{tf} {ct}"' for tf, ct in timeframes_with_candletype)
+    )
+
     with progress_tracker as progress:
-        tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2
-        timeframe_task = progress.add_task("Timeframe", total=tf_length)
+        timeframe_task = progress.add_task("Timeframe", total=len(timeframes_with_candletype))
         pair_task = progress.add_task("Downloading data...", total=len(pairs))
 
         for pair in pairs:
@@ -389,7 +425,7 @@ def refresh_backtest_ohlcv_data(
                 pairs_not_available.append(f"{pair}: Pair not available on exchange.")
                 logger.info(f"Skipping pair {pair}...")
                 continue
-            for timeframe in timeframes:
+            for timeframe, candle_type in timeframes_with_candletype:
                 # Get fast candles via parallel method on first loop through per timeframe
                 # and candle type. Downloads all the pairs in the list and stores them.
                 # Also skips if only 1 pair/timeframe combination is scheduled for download.
@@ -416,7 +452,7 @@ def refresh_backtest_ohlcv_data(
                 # get the already downloaded pair candles if they exist
                 pair_candles = fast_candles.pop((pair, timeframe, candle_type), None)
 
-                progress.update(timeframe_task, description=f"Timeframe {timeframe}")
+                progress.update(timeframe_task, description=f"Timeframe {timeframe} {candle_type}")
                 logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.")
                 _download_pair_history(
                     pair=pair,
@@ -432,33 +468,6 @@ def refresh_backtest_ohlcv_data(
                     pair_candles=pair_candles,  # optional pass of dataframe of parallel candles
                 )
                 progress.update(timeframe_task, advance=1)
-            if trading_mode == "futures":
-                # Predefined candletype (and timeframe) depending on exchange
-                # Downloads what is necessary to backtest based on futures data.
-                tf_mark = exchange.get_option("mark_ohlcv_timeframe")
-                tf_funding_rate = exchange.get_option("funding_fee_timeframe")
-
-                fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price"))
-                # All exchanges need FundingRate for futures trading.
-                # The timeframe is aligned to the mark-price timeframe.
-                combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
-                for candle_type_f, tf in combs:
-                    logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.")
-                    _download_pair_history(
-                        pair=pair,
-                        datadir=datadir,
-                        exchange=exchange,
-                        timerange=timerange,
-                        data_handler=data_handler,
-                        timeframe=str(tf),
-                        new_pairs_days=new_pairs_days,
-                        candle_type=candle_type_f,
-                        erase=erase,
-                        prepend=prepend,
-                    )
-                    progress.update(
-                        timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}"
-                    )
 
             progress.update(pair_task, advance=1)
             progress.update(timeframe_task, description="Timeframe")
@@ -804,6 +813,7 @@ def download_data(
                 trading_mode=config.get("trading_mode", "spot"),
                 prepend=config.get("prepend_data", False),
                 progress_tracker=progress_tracker,
+                candle_types=config.get("candle_types"),
                 no_parallel_download=config.get("no_parallel_download", False),
             )
     finally:

freqtrade/exchange/bitget.py

@@ -35,7 +35,6 @@ class Bitget(Exchange):
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
     }
     _ft_has_futures: FtHas = {
-        "mark_ohlcv_timeframe": "4h",
         "funding_fee_candle_limit": 100,
         "has_delisting": True,
     }