diff --git a/docs/commands/download-data.md b/docs/commands/download-data.md index 35f15d19d0c..dd6474e12ff 100644 --- a/docs/commands/download-data.md +++ b/docs/commands/download-data.md @@ -11,6 +11,7 @@ usage: freqtrade download-data [-h] [-v] [--no-color] [--logfile FILE] [-V] [--data-format-ohlcv {json,jsongz,feather,parquet}] [--data-format-trades {json,jsongz,feather,parquet}] [--trading-mode {spot,margin,futures}] + [--candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...]] [--prepend] options: @@ -50,6 +51,11 @@ options: `feather`). --trading-mode, --tradingmode {spot,margin,futures} Select Trading mode + --candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...] + Select candle type to download. Defaults to the + necessary candles for the selected trading mode (e.g. + 'spot' or ('futures', 'funding_rate' and 'mark') for + futures). --prepend Allow data prepending. (Data-appending is disabled) Common arguments: diff --git a/docs/data-download.md b/docs/data-download.md index c2b104b1bf6..4339e98629e 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -60,6 +60,7 @@ freqtrade download-data --exchange binance --pairs ".*/USDT" * Given starting points are ignored if data is already available, downloading only missing data up to today. * Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. * To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +* When downloading futures data (`--trading-mode futures` or a configuration specifying futures mode), freqtrade will automatically download the necessary candle types (e.g. `mark` and `funding_rate` candles) unless specified otherwise via `--candle-types`. ??? Note "Permission denied errors" If your configuration directory `user_data` was made by docker, you may get the following error: diff --git a/docs/deprecated.md b/docs/deprecated.md index f6ef8768fd0..ce3bf3b95ab 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -98,3 +98,40 @@ Please use configuration based [log setup](advanced-setup.md#advanced-logging) i The edge module has been deprecated in 2023.9 and removed in 2025.6. All functionalities of edge have been removed, and having edge configured will result in an error. + +## Adjustment to dynamic funding rate handling + +With version 2025.12, the handling of dynamic funding rates has been adjusted to also support dynamic funding rates down to 1h funding intervals. +As a consequence, the mark and funding rate timeframes have been changed to 1h for every supported futures exchange. + +As the timeframe for both mark and funding_fee candles has changed (usually from 8h to 1h) - already downloaded data will have to be adjusted or partially re-downloaded. +You can either re-download everything (`freqtrade download-data [...] --erase` - :warning: can take a long time) - or download the updated data selectively. + +### Strategy + +Most strategies should not need adjustments to continue to work as expected - however, strategies using `@informative("8h", candle_type="funding_rate")` or similar will have to switch the timeframe to 1h. +The same is true for `dp.get_pair_dataframe(metadata["pair"], "8h", candle_type="funding_rate")` - which will need to be switched to 1h. + +freqtrade will auto-adjust the timeframe and return `funding_rates` despite the wrongly given timeframe. It'll issue a warning - and may still break your strategy. + +### Selective data re-download + +The script below should serve as an example - you may need to adjust the timeframe and exchange to your needs! + +``` bash +# Cleanup no longer needed data +rm user_data/data//futures/*-mark-* +rm user_data/data//futures/*-funding_rate-* + +# download new data (only required once to fix the mark and funding fee data) +freqtrade download-data -t 1h --trading-mode futures --candle-types funding_rate mark [...] --timerange + +``` + +The result of the above will be that your funding_rates and mark data will have the 1h timeframe. +you can verify this with `freqtrade list-data --exchange --show`. + +!!! Note "Additional arguments" + Additional arguments to the above commands may be necessary, like configuration files or explicit user_data if they deviate from the default. + +**Hyperliquid** is a special case now - which will no longer require 1h mark data - but will use regular candles instead (this data never existed and is identical to 1h futures candles). As we don't support download-data for hyperliquid (they don't provide historic data) - there won't be actions necessary for hyperliquid users. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ca3b2b42225..f84822ac80b 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -3,6 +3,7 @@ """ from argparse import ArgumentParser, Namespace, _ArgumentGroup +from copy import deepcopy from functools import partial from pathlib import Path from typing import Any @@ -174,6 +175,7 @@ "dataformat_ohlcv", "dataformat_trades", "trading_mode", + "candle_types", "prepend_data", ] @@ -348,7 +350,11 @@ def _parse_args(self) -> Namespace: def _build_args(self, optionlist: list[str], parser: ArgumentParser | _ArgumentGroup) -> None: for val in optionlist: opt = AVAILABLE_CLI_OPTIONS[val] - parser.add_argument(*opt.cli, dest=val, **opt.kwargs) + options = deepcopy(opt.kwargs) + help_text = options.pop("help", None) + if opt.fthelp and isinstance(opt.fthelp, dict) and hasattr(parser, "prog"): + help_text = opt.fthelp.get(parser.prog, help_text) + parser.add_argument(*opt.cli, dest=val, help=help_text, **options) def _build_subcommands(self) -> None: """ diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index c256c46f3f8..ebc22152353 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -38,8 +38,14 @@ def check_int_nonzero(value: str) -> int: class Arg: # Optional CLI arguments - def __init__(self, *args, **kwargs): + def __init__(self, *args, fthelp: dict[str, str] | None = None, **kwargs): + """ + CLI Arguments - used to build subcommand parsers consistently. + :param fthelp: dict - fthelp per command - should be "freqtrade ": help_text + If not provided or not found, 'help' from kwargs is used instead. + """ self.cli = args + self.fthelp = fthelp self.kwargs = kwargs @@ -422,6 +428,14 @@ def __init__(self, *args, **kwargs): ), "candle_types": Arg( "--candle-types", + fthelp={ + "freqtrade download-data": ( + "Select candle type to download. " + "Defaults to the necessary candles for the selected trading mode " + "(e.g. 'spot' or ('futures', 'funding_rate' and 'mark') for futures)." + ), + "_": "Select candle type to convert. Defaults to all available types.", + }, help="Select candle type to convert. Defaults to all available types.", choices=[c.value for c in CandleType], nargs="+", diff --git a/freqtrade/data/converter/converter.py b/freqtrade/data/converter/converter.py index 4331519c232..99301d291f1 100644 --- a/freqtrade/data/converter/converter.py +++ b/freqtrade/data/converter/converter.py @@ -38,7 +38,8 @@ def ohlcv_to_dataframe( cols = DEFAULT_DATAFRAME_COLUMNS df = DataFrame(ohlcv, columns=cols) - df["date"] = to_datetime(df["date"], unit="ms", utc=True) + # Floor date to seconds to account for exchange imprecisions + df["date"] = to_datetime(df["date"], unit="ms", utc=True).dt.floor("s") # Some exchanges return int values for Volume and even for OHLC. # Convert them since TA-LIB indicators used in the strategy assume floats diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index dfab29c086c..ed396ba8e8a 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -348,6 +348,22 @@ def get_required_startup(self, timeframe: str) -> int: ) return total_candles + def __fix_funding_rate_timeframe( + self, pair: str, timeframe: str | None, candle_type: str + ) -> str | None: + if ( + candle_type == CandleType.FUNDING_RATE + and (ff_tf := self.get_funding_rate_timeframe()) != timeframe + ): + # TODO: does this message make sense? might be pointless as funding fees don't + # have a timeframe + logger.warning( + f"{pair}, {timeframe} requested - funding rate timeframe not matching {ff_tf}." + ) + return ff_tf + + return timeframe + def get_pair_dataframe( self, pair: str, timeframe: str | None = None, candle_type: str = "" ) -> DataFrame: @@ -361,6 +377,7 @@ def get_pair_dataframe( :return: Dataframe for this pair :param candle_type: '', mark, index, premiumIndex, or funding_rate """ + timeframe = self.__fix_funding_rate_timeframe(pair, timeframe, candle_type) if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): # Get live OHLCV data. data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type) @@ -620,3 +637,12 @@ def check_delisting(self, pair: str) -> datetime | None: except ExchangeError: logger.warning(f"Could not fetch market data for {pair}. Assuming no delisting.") return None + + def get_funding_rate_timeframe(self) -> str: + """ + Get the funding rate timeframe from exchange options + :return: Timeframe string + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return self._exchange.get_option("funding_fee_timeframe") diff --git a/freqtrade/data/history/datahandlers/idatahandler.py b/freqtrade/data/history/datahandlers/idatahandler.py index 67ae386c6f3..9e7d9de3a61 100644 --- a/freqtrade/data/history/datahandlers/idatahandler.py +++ b/freqtrade/data/history/datahandlers/idatahandler.py @@ -397,6 +397,9 @@ def ohlcv_load( pairdf = self._ohlcv_load( pair, timeframe, timerange=timerange_startup, candle_type=candle_type ) + if not pairdf.empty and candle_type == CandleType.FUNDING_RATE: + # Funding rate data is sometimes off by a couple of ms - floor to seconds + pairdf["date"] = pairdf["date"].dt.floor("s") if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf else: @@ -508,8 +511,15 @@ def fix_funding_fee_timeframe(self, ff_timeframe: str): Applies to bybit and okx, where funding-fee and mark candles have different timeframes. """ paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES) + ff_timeframe_s = timeframe_to_seconds(ff_timeframe) + funding_rate_combs = [ - f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe + f + for f in paircombs + if f[2] == CandleType.FUNDING_RATE + and f[1] != ff_timeframe + # Only allow smaller timeframes to move from smaller to larger timeframes + and timeframe_to_seconds(f[1]) < ff_timeframe_s ] if funding_rate_combs: diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 1467497d658..13b7101cdf8 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -353,6 +353,7 @@ def _download_pair_history( def refresh_backtest_ohlcv_data( exchange: Exchange, + *, pairs: list[str], timeframes: list[str], datadir: Path, @@ -363,6 +364,7 @@ def refresh_backtest_ohlcv_data( data_format: str | None = None, prepend: bool = False, progress_tracker: CustomProgress | None = None, + candle_types: list[CandleType] | None = None, no_parallel_download: bool = False, ) -> list[str]: """ @@ -375,10 +377,44 @@ def refresh_backtest_ohlcv_data( pairs_not_available = [] fast_candles: dict[PairWithTimeframe, DataFrame] = {} data_handler = get_datahandler(datadir, data_format) - candle_type = CandleType.get_default(trading_mode) + def_candletype = CandleType.SPOT if trading_mode != "futures" else CandleType.FUTURES + if trading_mode != "futures": + # Ignore user passed candle types for non-futures trading + timeframes_with_candletype = [(tf, def_candletype) for tf in timeframes] + else: + # Filter out SPOT candle type for futures trading + candle_types = ( + [ct for ct in candle_types if ct != CandleType.SPOT] if candle_types else None + ) + fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) + tf_funding_rate = exchange.get_option("funding_fee_timeframe") + tf_mark = exchange.get_option("mark_ohlcv_timeframe") + + if candle_types: + for ct in candle_types: + exchange.verify_candle_type_support(ct) + timeframes_with_candletype = [ + (tf, ct) + for ct in candle_types + for tf in timeframes + if ct != CandleType.FUNDING_RATE + ] + else: + # Default behavior + timeframes_with_candletype = [(tf, def_candletype) for tf in timeframes] + timeframes_with_candletype.append((tf_mark, fr_candle_type)) + if not candle_types or CandleType.FUNDING_RATE in candle_types: + # All exchanges need FundingRate for futures trading. + # The timeframe is aligned to the mark-price timeframe. + timeframes_with_candletype.append((tf_funding_rate, CandleType.FUNDING_RATE)) + # Deduplicate list ... + timeframes_with_candletype = list(dict.fromkeys(timeframes_with_candletype)) + logger.debug( + "Downloading %s.", ", ".join(f'"{tf} {ct}"' for tf, ct in timeframes_with_candletype) + ) + with progress_tracker as progress: - tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2 - timeframe_task = progress.add_task("Timeframe", total=tf_length) + timeframe_task = progress.add_task("Timeframe", total=len(timeframes_with_candletype)) pair_task = progress.add_task("Downloading data...", total=len(pairs)) for pair in pairs: @@ -389,7 +425,7 @@ def refresh_backtest_ohlcv_data( pairs_not_available.append(f"{pair}: Pair not available on exchange.") logger.info(f"Skipping pair {pair}...") continue - for timeframe in timeframes: + for timeframe, candle_type in timeframes_with_candletype: # Get fast candles via parallel method on first loop through per timeframe # and candle type. Downloads all the pairs in the list and stores them. # Also skips if only 1 pair/timeframe combination is scheduled for download. @@ -416,7 +452,7 @@ def refresh_backtest_ohlcv_data( # get the already downloaded pair candles if they exist pair_candles = fast_candles.pop((pair, timeframe, candle_type), None) - progress.update(timeframe_task, description=f"Timeframe {timeframe}") + progress.update(timeframe_task, description=f"Timeframe {timeframe} {candle_type}") logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.") _download_pair_history( pair=pair, @@ -432,33 +468,6 @@ def refresh_backtest_ohlcv_data( pair_candles=pair_candles, # optional pass of dataframe of parallel candles ) progress.update(timeframe_task, advance=1) - if trading_mode == "futures": - # Predefined candletype (and timeframe) depending on exchange - # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange.get_option("mark_ohlcv_timeframe") - tf_funding_rate = exchange.get_option("funding_fee_timeframe") - - fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) - # All exchanges need FundingRate for futures trading. - # The timeframe is aligned to the mark-price timeframe. - combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) - for candle_type_f, tf in combs: - logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.") - _download_pair_history( - pair=pair, - datadir=datadir, - exchange=exchange, - timerange=timerange, - data_handler=data_handler, - timeframe=str(tf), - new_pairs_days=new_pairs_days, - candle_type=candle_type_f, - erase=erase, - prepend=prepend, - ) - progress.update( - timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}" - ) progress.update(pair_task, advance=1) progress.update(timeframe_task, description="Timeframe") @@ -804,6 +813,7 @@ def download_data( trading_mode=config.get("trading_mode", "spot"), prepend=config.get("prepend_data", False), progress_tracker=progress_tracker, + candle_types=config.get("candle_types"), no_parallel_download=config.get("no_parallel_download", False), ) finally: diff --git a/freqtrade/exchange/bitget.py b/freqtrade/exchange/bitget.py index 3351cda8857..f0da55cb0da 100644 --- a/freqtrade/exchange/bitget.py +++ b/freqtrade/exchange/bitget.py @@ -35,7 +35,6 @@ class Bitget(Exchange): "order_time_in_force": ["GTC", "FOK", "IOC", "PO"], } _ft_has_futures: FtHas = { - "mark_ohlcv_timeframe": "4h", "funding_fee_candle_limit": 100, "has_delisting": True, } diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 300344e1952..0184c6c42af 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -38,8 +38,6 @@ class Bybit(Exchange): } _ft_has_futures: FtHas = { "ohlcv_has_history": True, - "mark_ohlcv_timeframe": "4h", - "funding_fee_timeframe": "8h", "funding_fee_candle_limit": 200, "stoploss_on_exchange": True, "stoploss_order_types": {"limit": "limit", "market": "market"}, diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 628838db2b0..61206de9263 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -97,6 +97,9 @@ def _get_logging_mixin(): # 'fetchLeverageTiers', # Futures initialization # 'fetchMarketLeverageTiers', # Futures initialization # 'fetchOpenOrders', 'fetchClosedOrders', # 'fetchOrders', # Refinding balance... + # "fetchPremiumIndexOHLCV", # Futures additional data + # "fetchMarkOHLCV", # Futures additional data + # "fetchIndexOHLCV", # Futures additional data # ccxt.pro "watchOHLCV", ] diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 9c88a32bb96..6f8eab8208b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -153,8 +153,8 @@ class Exchange: "l2_limit_range_required": True, # Allow Empty L2 limit (kucoin) "l2_limit_upper": None, # Upper limit for L2 limit "mark_ohlcv_price": "mark", - "mark_ohlcv_timeframe": "8h", - "funding_fee_timeframe": "8h", + "mark_ohlcv_timeframe": "1h", + "funding_fee_timeframe": "1h", "ccxt_futures_name": "swap", "needs_trading_fees": False, # use fetch_trading_fees to cache fees "order_props_in_contracts": ["amount", "filled", "remaining"], @@ -2690,24 +2690,25 @@ def _build_ohlcv_dl_jobs( input_coroutines: list[Coroutine[Any, Any, OHLCVResponse]] = [] cached_pairs = [] for pair, timeframe, candle_type in set(pair_list): - invalid_funding = ( - candle_type == CandleType.FUNDING_RATE - and timeframe != self.get_option("funding_fee_timeframe") - ) + if candle_type == CandleType.FUNDING_RATE and timeframe != ( + ff_tf := self.get_option("funding_fee_timeframe") + ): + # TODO: does this message make sense? would docs be better? + # if any, this should be cached to avoid log spam! + logger.warning( + f"Wrong funding rate timeframe {timeframe} for pair {pair}, " + f"downloading {ff_tf} instead." + ) + timeframe = ff_tf invalid_timeframe = timeframe not in self.timeframes and candle_type in ( CandleType.SPOT, CandleType.FUTURES, ) - if invalid_timeframe or invalid_funding: - timeframes_ = ( - ", ".join(self.timeframes) - if candle_type != CandleType.FUNDING_RATE - else self.get_option("funding_fee_timeframe") - ) + if invalid_timeframe: logger.warning( f"Cannot download ({pair}, {timeframe}, {candle_type}) combination as this " f"timeframe is not available on {self.name}. Available timeframes are " - f"{timeframes_}." + f"{', '.join(self.timeframes)}." ) continue @@ -2744,7 +2745,11 @@ def _process_ohlcv_df( has_cache = cache and (pair, timeframe, c_type) in self._klines # in case of existing cache, fill_missing happens after concatenation ohlcv_df = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=not has_cache, drop_incomplete=drop_incomplete + ticks, + timeframe, + pair=pair, + fill_missing=not has_cache and c_type != CandleType.FUNDING_RATE, + drop_incomplete=drop_incomplete, ) # keeping parsed dataframe in cache if cache: @@ -2755,7 +2760,7 @@ def _process_ohlcv_df( concat([old, ohlcv_df], axis=0), timeframe, pair, - fill_missing=True, + fill_missing=c_type != CandleType.FUNDING_RATE, drop_incomplete=False, ) candle_limit = self.ohlcv_candle_limit(timeframe, self._config["candle_type_def"]) @@ -2890,9 +2895,10 @@ async def _async_get_candle_history( timeframe, candle_type=candle_type, since_ms=since_ms ) - if candle_type and candle_type not in (CandleType.SPOT, CandleType.FUTURES): - params.update({"price": candle_type.value}) if candle_type != CandleType.FUNDING_RATE: + if candle_type and candle_type not in (CandleType.SPOT, CandleType.FUTURES): + self.verify_candle_type_support(candle_type) + params.update({"price": str(candle_type)}) data = await self._api_async.fetch_ohlcv( pair, timeframe=timeframe, since=since_ms, limit=candle_limit, params=params ) @@ -2957,6 +2963,38 @@ async def _fetch_funding_rate_history( data = [[x["timestamp"], x["fundingRate"], 0, 0, 0, 0] for x in data] return data + def check_candle_type_support(self, candle_type: CandleType) -> bool: + """ + Check that the exchange supports the given candle type. + :param candle_type: CandleType to verify + :return: True if supported, False otherwise + """ + if candle_type == CandleType.FUNDING_RATE: + if not self.exchange_has("fetchFundingRateHistory"): + return False + elif candle_type not in (CandleType.SPOT, CandleType.FUTURES): + mapping = { + CandleType.MARK: "fetchMarkOHLCV", + CandleType.INDEX: "fetchIndexOHLCV", + CandleType.PREMIUMINDEX: "fetchPremiumIndexOHLCV", + CandleType.FUNDING_RATE: "fetchFundingRateHistory", + } + _method = mapping.get(candle_type, "fetchOHLCV") + if not self.exchange_has(_method): + return False + return True + + def verify_candle_type_support(self, candle_type: CandleType) -> None: + """ + Verify that the exchange supports the given candle type. + :param candle_type: CandleType to verify + :raises OperationalException: if the candle type is not supported + """ + if not self.check_candle_type_support(candle_type): + raise OperationalException( + f"Exchange {self._api.name} does not support fetching {candle_type} candles." + ) + # fetch Trade data stuff def needed_candle_for_trades_ms(self, timeframe: str, candle_type: CandleType) -> int: @@ -3808,8 +3846,16 @@ def combine_funding_and_mark( combined = mark_rates.merge( funding_rates, on="date", how="left", suffixes=["_mark", "_fund"] ) - combined["open_fund"] = combined["open_fund"].fillna(futures_funding_rate) - return combined[relevant_cols] + # Fill only leading missing funding rates so gaps stay untouched + first_valid_idx = combined["open_fund"].first_valid_index() + if first_valid_idx is None: + combined["open_fund"] = futures_funding_rate + else: + is_leading_na = (combined.index <= first_valid_idx) & combined[ + "open_fund" + ].isna() + combined.loc[is_leading_na, "open_fund"] = futures_funding_rate + return combined[relevant_cols].dropna() def calculate_funding_fees( self, diff --git a/freqtrade/exchange/hyperliquid.py b/freqtrade/exchange/hyperliquid.py index 212505b7285..f5e6fb7eef5 100644 --- a/freqtrade/exchange/hyperliquid.py +++ b/freqtrade/exchange/hyperliquid.py @@ -37,9 +37,9 @@ class Hyperliquid(Exchange): "stoploss_order_types": {"limit": "limit"}, "stoploss_blocks_assets": False, "stop_price_prop": "stopPrice", - "funding_fee_timeframe": "1h", "funding_fee_candle_limit": 500, "uses_leverage_tiers": False, + "mark_ohlcv_price": "futures", } _supported_trading_mode_margin_pairs: list[tuple[TradingMode, MarginMode]] = [ diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 1f8b3cc26d0..60efdf95487 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -35,7 +35,6 @@ class Kraken(Exchange): "trades_pagination_arg": "since", "trades_pagination_overlap": False, "trades_has_history": True, - "mark_ohlcv_timeframe": "4h", } _supported_trading_mode_margin_pairs: list[tuple[TradingMode, MarginMode]] = [ diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 51c1120100e..14f20a0bb3b 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -29,8 +29,6 @@ class Okx(Exchange): _ft_has: FtHas = { "ohlcv_candle_limit": 100, # Warning, special case with data prior to X months - "mark_ohlcv_timeframe": "4h", - "funding_fee_timeframe": "8h", "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, "trades_has_history": False, # Endpoint doesn't have a "since" parameter diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 80de520259d..4c5207e9338 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -2054,6 +2054,7 @@ def execute_trade_exit( exit_tag: str | None = None, ordertype: str | None = None, sub_trade_amt: float | None = None, + skip_custom_exit_price: bool = False, ) -> bool: """ Executes a trade exit for the given trade and limit @@ -2080,30 +2081,34 @@ def execute_trade_exit( ): exit_type = "stoploss" + order_type = ( + (ordertype or self.strategy.order_types[exit_type]) + if exit_check.exit_type != ExitType.EMERGENCY_EXIT + else self.strategy.order_types.get("emergency_exit", "market") + ) + # set custom_exit_price if available proposed_limit_rate = limit + custom_exit_price = limit + current_profit = trade.calc_profit_ratio(limit) - custom_exit_price = strategy_safe_wrapper( - self.strategy.custom_exit_price, default_retval=proposed_limit_rate - )( - pair=trade.pair, - trade=trade, - current_time=datetime.now(UTC), - proposed_rate=proposed_limit_rate, - current_profit=current_profit, - exit_tag=exit_reason, - ) + if order_type == "limit" and not skip_custom_exit_price: + custom_exit_price = strategy_safe_wrapper( + self.strategy.custom_exit_price, default_retval=proposed_limit_rate + )( + pair=trade.pair, + trade=trade, + current_time=datetime.now(UTC), + proposed_rate=proposed_limit_rate, + current_profit=current_profit, + exit_tag=exit_reason, + ) limit = self.get_valid_price(custom_exit_price, proposed_limit_rate) # First cancelling stoploss on exchange ... trade = self.cancel_stoploss_on_exchange(trade, allow_nonblocking=True) - order_type = ordertype or self.strategy.order_types[exit_type] - if exit_check.exit_type == ExitType.EMERGENCY_EXIT: - # Emergency exits (default to market!) - order_type = self.strategy.order_types.get("emergency_exit", "market") - amount = self._safe_exit_amount(trade, trade.pair, sub_trade_amt or trade.amount) time_in_force = self.strategy.order_time_in_force["exit"] diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fd78162dbb8..e3e340a1b76 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -374,6 +374,7 @@ def _load_bt_data_detail(self) -> None: timerange=self.timerange, startup_candles=0, fail_without_data=True, + fill_up_missing=False, data_format=self.config["dataformat_ohlcv"], candle_type=CandleType.FUNDING_RATE, ) diff --git a/freqtrade/rpc/api_server/api_download_data.py b/freqtrade/rpc/api_server/api_download_data.py index 6e446b063e4..b755474c9c1 100644 --- a/freqtrade/rpc/api_server/api_download_data.py +++ b/freqtrade/rpc/api_server/api_download_data.py @@ -63,6 +63,8 @@ def pairlists_evaluate( config_loc["timeframes"] = payload.timeframes config_loc["erase"] = payload.erase config_loc["download_trades"] = payload.download_trades + if payload.candle_types is not None: + config_loc["candle_types"] = payload.candle_types handleExchangePayload(payload, config_loc) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 21a93807c04..0dcd95edf0c 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -426,6 +426,7 @@ class ForceExitPayload(BaseModel): tradeid: str | int ordertype: OrderTypeValues | None = None amount: float | None = None + price: float | None = None class BlacklistPayload(BaseModel): @@ -506,6 +507,7 @@ class DownloadDataPayload(ExchangeModePayloadMixin, BaseModel): timerange: str | None = None erase: bool = False download_trades: bool = False + candle_types: list[str] | None = None @model_validator(mode="before") def check_mutually_exclusive(cls, values): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index dc9682b0963..39c8e982172 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -91,7 +91,9 @@ # 2.41: Add download-data endpoint # 2.42: Add /pair_history endpoint with live data # 2.43: Add /profit_all endpoint -API_VERSION = 2.43 +# 2.44: Add candle_types parameter to download-data endpoint +# 2.45: Add price to forceexit endpoint +API_VERSION = 2.45 # Public API, requires no auth. router_public = APIRouter() @@ -324,7 +326,9 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)): @router.post("/forcesell", response_model=ResultMsg, tags=["trading"]) def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)): ordertype = payload.ordertype.value if payload.ordertype else None - return rpc._rpc_force_exit(str(payload.tradeid), ordertype, amount=payload.amount) + return rpc._rpc_force_exit( + str(payload.tradeid), ordertype, amount=payload.amount, price=payload.price + ) @router.get("/blacklist", response_model=BlacklistResponse, tags=["info", "pairlist"]) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 22e457eef49..81bb52ce5c9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -940,7 +940,11 @@ def _rpc_reload_trade_from_exchange(self, trade_id: int) -> dict[str, str]: return {"status": "Reloaded from orders from exchange"} def __exec_force_exit( - self, trade: Trade, ordertype: str | None, amount: float | None = None + self, + trade: Trade, + ordertype: str | None, + amount: float | None = None, + price: float | None = None, ) -> bool: # Check if there is there are open orders trade_entry_cancelation_registry = [] @@ -964,8 +968,13 @@ def __exec_force_exit( # Order cancellation failed, so we can't exit. return False # Get current rate and execute sell - current_rate = self._freqtrade.exchange.get_rate( - trade.pair, side="exit", is_short=trade.is_short, refresh=True + + current_rate = ( + self._freqtrade.exchange.get_rate( + trade.pair, side="exit", is_short=trade.is_short, refresh=True + ) + if ordertype == "market" or price is None + else price ) exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT) order_type = ordertype or self._freqtrade.strategy.order_types.get( @@ -983,18 +992,28 @@ def __exec_force_exit( sub_amount = amount self._freqtrade.execute_trade_exit( - trade, current_rate, exit_check, ordertype=order_type, sub_trade_amt=sub_amount + trade, + current_rate, + exit_check, + ordertype=order_type, + sub_trade_amt=sub_amount, + skip_custom_exit_price=price is not None and ordertype == "limit", ) return True return False def _rpc_force_exit( - self, trade_id: str, ordertype: str | None = None, *, amount: float | None = None + self, + trade_id: str, + ordertype: str | None = None, + *, + amount: float | None = None, + price: float | None = None, ) -> dict[str, str]: """ Handler for forceexit . - Sells the given trade at current price + exits the given trade. Uses current price if price is None. """ if self._freqtrade.state == State.STOPPED: @@ -1024,7 +1043,7 @@ def _rpc_force_exit( logger.warning("force_exit: Invalid argument received") raise RPCException("invalid argument") - result = self.__exec_force_exit(trade, ordertype, amount) + result = self.__exec_force_exit(trade, ordertype, amount, price) Trade.commit() self._freqtrade.wallets.update() if not result: diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index ac9a2a1ad11..7c7edb973c1 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -104,8 +104,11 @@ def _create_and_merge_informative_pair( ): asset = inf_data.asset or "" timeframe = inf_data.timeframe + timeframe1 = inf_data.timeframe fmt = inf_data.fmt candle_type = inf_data.candle_type + if candle_type == CandleType.FUNDING_RATE: + timeframe1 = strategy.dp.get_funding_rate_timeframe() config = strategy.config @@ -132,10 +135,10 @@ def _create_and_merge_informative_pair( fmt = "{base}_{quote}_" + fmt # Informatives of other pairs inf_metadata = {"pair": asset, "timeframe": timeframe} - inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe, candle_type) + inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe1, candle_type) if inf_dataframe.empty: raise ValueError( - f"Informative dataframe for ({asset}, {timeframe}, {candle_type}) is empty. " + f"Informative dataframe for ({asset}, {timeframe1}, {candle_type}) is empty. " "Can't populate informative indicators." ) inf_dataframe = populate_indicators_fn(strategy, inf_dataframe, inf_metadata) @@ -163,7 +166,7 @@ def _create_and_merge_informative_pair( dataframe, inf_dataframe, strategy.timeframe, - timeframe, + timeframe1, ffill=inf_data.ffill, append_timeframe=False, date_column=date_column, diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a1e541bd170..40127275b22 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1801,10 +1801,10 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() - assert "Found 6 pair / timeframe combinations." in captured.out + assert "Found 5 pair / timeframe combinations." in captured.out assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) assert re.search(r"\n.* XRP/USDT:USDT .* 5m, 1h .* futures |\n", captured.out) - assert re.search(r"\n.* XRP/USDT:USDT .* 1h, 8h .* mark |\n", captured.out) + assert re.search(r"\n.* XRP/USDT:USDT .* 1h.* mark |\n", captured.out) args = [ "list-data", diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 6110d17b869..9d40bf5cbea 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -126,8 +126,7 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): ("XRP/USDT:USDT", "5m", "futures"), ("XRP/USDT:USDT", "1h", "futures"), ("XRP/USDT:USDT", "1h", "mark"), - ("XRP/USDT:USDT", "8h", "mark"), - ("XRP/USDT:USDT", "8h", "funding_rate"), + ("XRP/USDT:USDT", "1h", "funding_rate"), } paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index afbba3d5d52..d3bd5fb98e7 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -9,7 +9,7 @@ from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.util import dt_utc -from tests.conftest import EXMS, generate_test_data, get_patched_exchange +from tests.conftest import EXMS, generate_test_data, get_patched_exchange, log_has_re @pytest.mark.parametrize( @@ -185,6 +185,28 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): assert len(df) == 2 # ohlcv_history is limited to 2 rows now +def test_get_pair_dataframe_funding_rate(mocker, default_conf, ohlcv_history, caplog): + default_conf["runmode"] = RunMode.DRY_RUN + timeframe = "1h" + exchange = get_patched_exchange(mocker, default_conf) + candletype = CandleType.FUNDING_RATE + exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history + + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.DRY_RUN + assert ohlcv_history.equals( + dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type="funding_rate") + ) + msg = r".*funding rate timeframe not matching" + assert not log_has_re(msg, caplog) + + assert ohlcv_history.equals( + dp.get_pair_dataframe("UNITTEST/BTC", "5h", candle_type="funding_rate") + ) + assert log_has_re(msg, caplog) + + def test_available_pairs(mocker, default_conf, ohlcv_history): exchange = get_patched_exchange(mocker, default_conf) timeframe = default_conf["timeframe"] @@ -636,3 +658,21 @@ def test_check_delisting(mocker, default_conf_usdt): assert res == dt_utc(2025, 10, 2) assert delist_mock2.call_count == 1 + + +def test_get_funding_rate_timeframe(mocker, default_conf_usdt): + default_conf_usdt["trading_mode"] = "futures" + default_conf_usdt["margin_mode"] = "isolated" + exchange = get_patched_exchange(mocker, default_conf_usdt) + mock_get_option = mocker.spy(exchange, "get_option") + dp = DataProvider(default_conf_usdt, exchange) + + assert dp.get_funding_rate_timeframe() == "1h" + mock_get_option.assert_called_once_with("funding_fee_timeframe") + + +def test_get_funding_rate_timeframe_no_exchange(default_conf_usdt): + dp = DataProvider(default_conf_usdt, None) + + with pytest.raises(OperationalException, match=r"Exchange is not available to DataProvider."): + dp.get_funding_rate_timeframe() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 115e73192f1..59606d48569 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -534,18 +534,19 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No @pytest.mark.parametrize( - "trademode,callcount", + "trademode,callcount, callcount_parallel", [ - ("spot", 4), - ("margin", 4), - ("futures", 8), # Called 8 times - 4 normal, 2 funding and 2 mark/index calls + ("spot", 4, 2), + ("margin", 4, 2), + ("futures", 8, 4), # Called 8 times - 4 normal, 2 funding and 2 mark/index calls ], ) def test_refresh_backtest_ohlcv_data( - mocker, default_conf, markets, caplog, testdatadir, trademode, callcount + mocker, default_conf, markets, caplog, testdatadir, trademode, callcount, callcount_parallel ): caplog.set_level(logging.DEBUG) dl_mock = mocker.patch("freqtrade.data.history.history_utils._download_pair_history") + mocker.patch(f"{EXMS}.verify_candle_type_support", MagicMock()) def parallel_mock(pairs, timeframe, candle_type, **kwargs): return {(pair, timeframe, candle_type): DataFrame() for pair in pairs} @@ -573,14 +574,15 @@ def parallel_mock(pairs, timeframe, candle_type, **kwargs): ) # Called once per timeframe (as we return an empty dataframe) - assert parallel_mock.call_count == 2 + # called twice for spot/margin and 4 times for futures + assert parallel_mock.call_count == callcount_parallel assert dl_mock.call_count == callcount assert dl_mock.call_args[1]["timerange"].starttype == "date" assert log_has_re(r"Downloading pair ETH/BTC, .* interval 1m\.", caplog) if trademode == "futures": - assert log_has_re(r"Downloading pair ETH/BTC, funding_rate, interval 8h\.", caplog) - assert log_has_re(r"Downloading pair ETH/BTC, mark, interval 4h\.", caplog) + assert log_has_re(r"Downloading pair ETH/BTC, funding_rate, interval 1h\.", caplog) + assert log_has_re(r"Downloading pair ETH/BTC, mark, interval 1h\.", caplog) # Test with only one pair - no parallel download should happen 1 pair/timeframe combination # doesn't justify parallelization @@ -599,6 +601,24 @@ def parallel_mock(pairs, timeframe, candle_type, **kwargs): ) assert parallel_mock.call_count == 0 + if trademode == "futures": + dl_mock.reset_mock() + refresh_backtest_ohlcv_data( + exchange=ex, + pairs=[ + "ETH/BTC", + ], + timeframes=["5m", "1h"], + datadir=testdatadir, + timerange=timerange, + erase=False, + trading_mode=trademode, + no_parallel_download=True, + candle_types=["premiumIndex", "funding_rate"], + ) + assert parallel_mock.call_count == 0 + assert dl_mock.call_count == 3 # 2 timeframes premiumIndex + 1x funding_rate + def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): dl_mock = mocker.patch( diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a846cb3e6c8..5772a1c4a11 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2389,6 +2389,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ] ] exchange = get_patched_exchange(mocker, default_conf, exchange=exchange_name) + mocker.patch.object(exchange, "verify_candle_type_support") # Monkey-patch async function exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) @@ -2439,6 +2440,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf_usdt, caplog, candle_type) -> caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf_usdt) + mocker.patch.object(exchange, "verify_candle_type_support") exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pairs = [("IOTA/USDT", "5m", candle_type), ("XRP/USDT", "5m", candle_type)] @@ -2689,6 +2691,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach time_machine.move_to(start + timedelta(hours=99, minutes=30)) exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object(exchange, "verify_candle_type_support") exchange._set_startup_candle_count(default_conf) mocker.patch(f"{EXMS}.ohlcv_candle_limit", return_value=100) @@ -2837,6 +2840,29 @@ def test_refresh_ohlcv_with_cache(mocker, default_conf, time_machine) -> None: assert ohlcv_mock.call_args_list[0][0][0] == pairs +def test_refresh_latest_ohlcv_funding_rate(mocker, default_conf_usdt, caplog) -> None: + ohlcv = generate_test_data_raw("1h", 24, "2025-01-02 12:00:00+00:00") + funding_data = [{"timestamp": x[0], "fundingRate": x[1]} for x in ohlcv] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf_usdt) + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + exchange._api_async.fetch_funding_rate_history = get_mock_coro(funding_data) + + pairs = [ + ("IOTA/USDT:USDT", "8h", CandleType.FUNDING_RATE), + ("XRP/USDT:USDT", "1h", CandleType.FUNDING_RATE), + ] + # empty dicts + assert not exchange._klines + res = exchange.refresh_latest_ohlcv(pairs, cache=False) + + assert len(res) == len(pairs) + assert log_has_re(r"Wrong funding rate timeframe 8h for pair IOTA/USDT:USDT", caplog) + assert not log_has_re(r"Wrong funding rate timeframe 8h for pair XRP/USDT:USDT", caplog) + assert exchange._api_async.fetch_ohlcv.call_count == 0 + + @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ @@ -5342,11 +5368,12 @@ def test_combine_funding_and_mark( df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate) if futures_funding_rate is not None: - assert len(df) == 3 + assert len(df) == 2 assert df.iloc[0]["open_fund"] == funding_rate - assert df.iloc[1]["open_fund"] == futures_funding_rate - assert df.iloc[2]["open_fund"] == funding_rate - assert df["date"].to_list() == [prior2_date, prior_date, trade_date] + # assert df.iloc[1]["open_fund"] == futures_funding_rate + assert df.iloc[-1]["open_fund"] == funding_rate + # Mid-candle is dropped ... + assert df["date"].to_list() == [prior2_date, trade_date] else: assert len(df) == 2 assert df["date"].to_list() == [prior2_date, trade_date] @@ -5440,8 +5467,13 @@ def test__fetch_and_calculate_funding_fees( api_mock = MagicMock() api_mock.fetch_funding_rate_history = get_mock_coro(return_value=funding_rate_history) api_mock.fetch_ohlcv = get_mock_coro(return_value=mark_ohlcv) - type(api_mock).has = PropertyMock(return_value={"fetchOHLCV": True}) - type(api_mock).has = PropertyMock(return_value={"fetchFundingRateHistory": True}) + type(api_mock).has = PropertyMock( + return_value={ + "fetchFundingRateHistory": True, + "fetchMarkOHLCV": True, + "fetchOHLCV": True, + } + ) ex = get_patched_exchange(mocker, default_conf, api_mock, exchange=exchange) mocker.patch(f"{EXMS}.timeframes", PropertyMock(return_value=["1h", "4h", "8h"])) @@ -5485,8 +5517,13 @@ def test__fetch_and_calculate_funding_fees_datetime_called( api_mock.fetch_funding_rate_history = get_mock_coro( return_value=funding_rate_history_octohourly ) - type(api_mock).has = PropertyMock(return_value={"fetchOHLCV": True}) - type(api_mock).has = PropertyMock(return_value={"fetchFundingRateHistory": True}) + type(api_mock).has = PropertyMock( + return_value={ + "fetchFundingRateHistory": True, + "fetchMarkOHLCV": True, + "fetchOHLCV": True, + } + ) mocker.patch(f"{EXMS}.timeframes", PropertyMock(return_value=["4h", "8h"])) exchange = get_patched_exchange(mocker, default_conf, api_mock, exchange=exchange) d1 = datetime.strptime("2021-08-31 23:00:01 +0000", "%Y-%m-%d %H:%M:%S %z") @@ -6573,3 +6610,51 @@ def test_fetch_funding_rate(default_conf, mocker, exchange_name): with pytest.raises(DependencyException, match=r"Pair XRP/ETH not available"): exchange.fetch_funding_rate(pair="XRP/ETH") + + +def test_verify_candle_type_support(default_conf, mocker): + api_mock = MagicMock() + type(api_mock).has = PropertyMock( + return_value={ + "fetchFundingRateHistory": True, + "fetchIndexOHLCV": True, + "fetchMarkOHLCV": True, + "fetchPremiumIndexOHLCV": False, + } + ) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + # Should pass + exchange.verify_candle_type_support("futures") + exchange.verify_candle_type_support(CandleType.FUTURES) + exchange.verify_candle_type_support(CandleType.FUNDING_RATE) + exchange.verify_candle_type_support(CandleType.SPOT) + exchange.verify_candle_type_support(CandleType.MARK) + + # Should fail: + + with pytest.raises( + OperationalException, + match=r"Exchange .* does not support fetching premiumindex candles\.", + ): + exchange.verify_candle_type_support(CandleType.PREMIUMINDEX) + + type(api_mock).has = PropertyMock( + return_value={ + "fetchFundingRateHistory": False, + "fetchIndexOHLCV": False, + "fetchMarkOHLCV": False, + "fetchPremiumIndexOHLCV": True, + } + ) + for candle_type in [ + CandleType.FUNDING_RATE, + CandleType.INDEX, + CandleType.MARK, + ]: + with pytest.raises( + OperationalException, + match=rf"Exchange .* does not support fetching {candle_type.value} candles\.", + ): + exchange.verify_candle_type_support(candle_type) + exchange.verify_candle_type_support(CandleType.PREMIUMINDEX) diff --git a/tests/exchange_online/test_ccxt_compat.py b/tests/exchange_online/test_ccxt_compat.py index 073ce0a3813..d0dc715b989 100644 --- a/tests/exchange_online/test_ccxt_compat.py +++ b/tests/exchange_online/test_ccxt_compat.py @@ -270,11 +270,14 @@ def test_ccxt_fetch_ohlcv_startdate(self, exchange: EXCHANGE_FIXTURE_TYPE): assert exch.klines(pair_tf).iloc[-1]["date"] >= timeframe_to_prev_date(timeframe, now) assert exch.klines(pair_tf)["date"].astype(int).iloc[0] // 1e6 == since_ms - def _ccxt__async_get_candle_history(self, exchange, pair, timeframe, candle_type, factor=0.9): + def _ccxt__async_get_candle_history( + self, exchange, pair: str, timeframe: str, candle_type: CandleType, factor: float = 0.9 + ): timeframe_ms = timeframe_to_msecs(timeframe) + timeframe_ms_8h = timeframe_to_msecs("8h") now = timeframe_to_prev_date(timeframe, datetime.now(UTC)) - for offset in (360, 120, 30, 10, 5, 2): - since = now - timedelta(days=offset) + for offset_days in (360, 120, 30, 10, 5, 2): + since = now - timedelta(days=offset_days) since_ms = int(since.timestamp() * 1000) res = exchange.loop.run_until_complete( @@ -289,8 +292,15 @@ def _ccxt__async_get_candle_history(self, exchange, pair, timeframe, candle_type candles = res[3] candle_count = exchange.ohlcv_candle_limit(timeframe, candle_type, since_ms) * factor candle_count1 = (now.timestamp() * 1000 - since_ms) // timeframe_ms * factor - assert len(candles) >= min(candle_count, candle_count1), ( - f"{len(candles)} < {candle_count} in {timeframe}, Offset: {offset} {factor}" + # funding fees can be 1h or 8h - depending on pair and time. + candle_count2 = (now.timestamp() * 1000 - since_ms) // timeframe_ms_8h * factor + min_value = min( + candle_count, + candle_count1, + candle_count2 if candle_type == CandleType.FUNDING_RATE else candle_count1, + ) + assert len(candles) >= min_value, ( + f"{len(candles)} < {candle_count} in {timeframe} {offset_days=} {factor=}" ) # Check if first-timeframe is either the start, or start + 1 assert candles[0][0] == since_ms or (since_ms + timeframe_ms) @@ -309,6 +319,8 @@ def test_ccxt__async_get_candle_history(self, exchange: EXCHANGE_FIXTURE_TYPE): [ CandleType.FUTURES, CandleType.FUNDING_RATE, + CandleType.INDEX, + CandleType.PREMIUMINDEX, CandleType.MARK, ], ) @@ -322,6 +334,10 @@ def test_ccxt__async_get_candle_history_futures( timeframe = exchange._ft_has.get( "funding_fee_timeframe", exchange._ft_has["mark_ohlcv_timeframe"] ) + else: + # never skip funding rate! + if not exchange.check_candle_type_support(candle_type): + pytest.skip(f"Exchange does not support candle type {candle_type}") self._ccxt__async_get_candle_history( exchange, pair=pair, @@ -337,6 +353,7 @@ def test_ccxt_fetch_funding_rate_history(self, exchange_futures: EXCHANGE_FIXTUR timeframe_ff = exchange._ft_has.get( "funding_fee_timeframe", exchange._ft_has["mark_ohlcv_timeframe"] ) + timeframe_ff_8h = "8h" pair_tf = (pair, timeframe_ff, CandleType.FUNDING_RATE) funding_ohlcv = exchange.refresh_latest_ohlcv( @@ -350,14 +367,26 @@ def test_ccxt_fetch_funding_rate_history(self, exchange_futures: EXCHANGE_FIXTUR hour1 = timeframe_to_prev_date(timeframe_ff, this_hour - timedelta(minutes=1)) hour2 = timeframe_to_prev_date(timeframe_ff, hour1 - timedelta(minutes=1)) hour3 = timeframe_to_prev_date(timeframe_ff, hour2 - timedelta(minutes=1)) - val0 = rate[rate["date"] == this_hour].iloc[0]["open"] - val1 = rate[rate["date"] == hour1].iloc[0]["open"] - val2 = rate[rate["date"] == hour2].iloc[0]["open"] - val3 = rate[rate["date"] == hour3].iloc[0]["open"] + # Alternative 8h timeframe - funding fee timeframe is not stable. + h8_this_hour = timeframe_to_prev_date(timeframe_ff_8h) + h8_hour1 = timeframe_to_prev_date(timeframe_ff_8h, h8_this_hour - timedelta(minutes=1)) + h8_hour2 = timeframe_to_prev_date(timeframe_ff_8h, h8_hour1 - timedelta(minutes=1)) + h8_hour3 = timeframe_to_prev_date(timeframe_ff_8h, h8_hour2 - timedelta(minutes=1)) + row0 = rate.iloc[-1] + row1 = rate.iloc[-2] + row2 = rate.iloc[-3] + row3 = rate.iloc[-4] + + assert row0["date"] == this_hour or row0["date"] == h8_this_hour + assert row1["date"] == hour1 or row1["date"] == h8_hour1 + assert row2["date"] == hour2 or row2["date"] == h8_hour2 + assert row3["date"] == hour3 or row3["date"] == h8_hour3 # Test For last 4 hours # Avoids random test-failure when funding-fees are 0 for a few hours. - assert val0 != 0.0 or val1 != 0.0 or val2 != 0.0 or val3 != 0.0 + assert ( + row0["open"] != 0.0 or row1["open"] != 0.0 or row2["open"] != 0.0 or row3["open"] != 0.0 + ) # We expect funding rates to be different from 0.0 - or moving around. assert ( rate["open"].max() != 0.0 @@ -369,7 +398,10 @@ def test_ccxt_fetch_mark_price_history(self, exchange_futures: EXCHANGE_FIXTURE_ exchange, exchangename = exchange_futures pair = EXCHANGES[exchangename].get("futures_pair", EXCHANGES[exchangename]["pair"]) since = int((datetime.now(UTC) - timedelta(days=5)).timestamp() * 1000) - pair_tf = (pair, "1h", CandleType.MARK) + candle_type = CandleType.from_string( + exchange.get_option("mark_ohlcv_price", default=CandleType.MARK) + ) + pair_tf = (pair, "1h", candle_type) mark_ohlcv = exchange.refresh_latest_ohlcv([pair_tf], since_ms=since, drop_incomplete=False) diff --git a/tests/freqtradebot/test_freqtradebot.py b/tests/freqtradebot/test_freqtradebot.py index f8b790c4445..d08daa2f128 100644 --- a/tests/freqtradebot/test_freqtradebot.py +++ b/tests/freqtradebot/test_freqtradebot.py @@ -3092,7 +3092,7 @@ def test_execute_trade_exit_custom_exit_price( "exit_reason": "foo", "open_date": ANY, "close_date": ANY, - "close_rate": ANY, + "close_rate": 2.25, # the custom exit price "sub_trade": False, "cumulative_profit": 0.0, "stake_amount": pytest.approx(60), diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index b1d13756f6e..bbf55d2bc41 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -970,8 +970,8 @@ def custom_entry_price(proposed_rate, **kwargs): @pytest.mark.parametrize( "use_detail,exp_funding_fee, exp_ff_updates", [ - (True, -0.018054162, 10), - (False, -0.01780296, 6), + (True, -0.0180457882, 15), + (False, -0.0178000543, 12), ], ) def test_backtest_one_detail_futures( @@ -1081,8 +1081,8 @@ def custom_entry_price(proposed_rate, **kwargs): @pytest.mark.parametrize( "use_detail,entries,max_stake,ff_updates,expected_ff", [ - (True, 50, 3000, 55, -1.18038144), - (False, 6, 360, 11, -0.14679994), + (True, 50, 3000, 78, -1.17988972), + (False, 6, 360, 34, -0.14673681), ], ) def test_backtest_one_detail_futures_funding_fees( @@ -2382,13 +2382,12 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker, caplog, testda f"Using data directory: {testdatadir} ...", "Loading data from 2021-11-17 01:00:00 up to 2021-11-21 04:00:00 (4 days).", "Backtesting with data from 2021-11-17 21:00:00 up to 2021-11-21 04:00:00 (3 days).", - "XRP/USDT:USDT, funding_rate, 8h, data starts at 2021-11-18 00:00:00", - "XRP/USDT:USDT, mark, 8h, data starts at 2021-11-18 00:00:00", + "XRP/USDT:USDT, funding_rate, 1h, data starts at 2021-11-18 00:00:00", f"Running backtesting for Strategy {CURRENT_TEST_STRATEGY}", ] for line in exists: - assert log_has(line, caplog) + assert log_has(line, caplog), line captured = capsys.readouterr() assert "BACKTESTING REPORT" in captured.out diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index edfb4ff7574..768858aacd3 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1852,9 +1852,35 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets): Trade.rollback() trade = Trade.get_trades([Trade.id == 5]).first() + last_order = trade.orders[-1] + + assert last_order.side == "sell" + assert last_order.status == "closed" + assert last_order.order_type == "market" + assert last_order.amount == 23 assert pytest.approx(trade.amount) == 100 assert trade.is_open is True + # Test with explicit price + rc = client_post( + client, + f"{BASE_URI}/forceexit", + data={"tradeid": "5", "ordertype": "limit", "amount": 25, "price": 0.12345}, + ) + assert_response(rc) + assert rc.json() == {"result": "Created exit order for trade 5."} + Trade.rollback() + + trade = Trade.get_trades([Trade.id == 5]).first() + last_order = trade.orders[-1] + assert last_order.status == "closed" + assert last_order.order_type == "limit" + assert pytest.approx(last_order.safe_price) == 0.12345 + assert pytest.approx(last_order.amount) == 25 + + assert pytest.approx(trade.amount) == 75 + assert trade.is_open is True + rc = client_post(client, f"{BASE_URI}/forceexit", data={"tradeid": "5"}) assert_response(rc) assert rc.json() == {"result": "Created exit order for trade 5."} @@ -3250,6 +3276,7 @@ def test_api_download_data(botclient, mocker, tmp_path): body = { "pairs": ["ETH/BTC", "XRP/BTC"], "timeframes": ["5m"], + "candle_types": ["spot"], } # Fail, already running diff --git a/tests/testdata/futures/XRP_USDT_USDT-8h-funding_rate.feather b/tests/testdata/futures/XRP_USDT_USDT-1h-funding_rate.feather similarity index 100% rename from tests/testdata/futures/XRP_USDT_USDT-8h-funding_rate.feather rename to tests/testdata/futures/XRP_USDT_USDT-1h-funding_rate.feather diff --git a/tests/testdata/futures/XRP_USDT_USDT-8h-mark.feather b/tests/testdata/futures/XRP_USDT_USDT-8h-mark.feather deleted file mode 100644 index f4150269047..00000000000 Binary files a/tests/testdata/futures/XRP_USDT_USDT-8h-mark.feather and /dev/null differ diff --git a/tests/util/test_binance_mig.py b/tests/util/test_binance_mig.py index f700ff73a6d..db3d8b282b3 100644 --- a/tests/util/test_binance_mig.py +++ b/tests/util/test_binance_mig.py @@ -20,8 +20,8 @@ def test_binance_mig_data_conversion(default_conf_usdt, tmp_path, testdatadir): files = [ "-1h-mark.feather", "-1h-futures.feather", - "-8h-funding_rate.feather", - "-8h-mark.feather", + "-1h-funding_rate.feather", + "-1h-mark.feather", ] # Copy files to tmpdir and rename to old naming diff --git a/tests/util/test_funding_rate_migration.py b/tests/util/test_funding_rate_migration.py index 094ee15623f..69d603ed096 100644 --- a/tests/util/test_funding_rate_migration.py +++ b/tests/util/test_funding_rate_migration.py @@ -5,13 +5,13 @@ def test_migrate_funding_rate_timeframe(default_conf_usdt, tmp_path, testdatadir): copytree(testdatadir / "futures", tmp_path / "futures") - file_4h = tmp_path / "futures" / "XRP_USDT_USDT-4h-funding_rate.feather" - file_8h = tmp_path / "futures" / "XRP_USDT_USDT-8h-funding_rate.feather" + file_30m = tmp_path / "futures" / "XRP_USDT_USDT-30m-funding_rate.feather" + file_1h_fr = tmp_path / "futures" / "XRP_USDT_USDT-1h-funding_rate.feather" file_1h = tmp_path / "futures" / "XRP_USDT_USDT-1h-futures.feather" - file_8h.rename(file_4h) + file_1h_fr.rename(file_30m) assert file_1h.exists() - assert file_4h.exists() - assert not file_8h.exists() + assert file_30m.exists() + assert not file_1h_fr.exists() default_conf_usdt["datadir"] = tmp_path @@ -22,7 +22,7 @@ def test_migrate_funding_rate_timeframe(default_conf_usdt, tmp_path, testdatadir migrate_funding_fee_timeframe(default_conf_usdt, None) - assert not file_4h.exists() - assert file_8h.exists() + assert not file_30m.exists() + assert file_1h_fr.exists() # futures files is untouched. assert file_1h.exists()