From f8f21cfc9cd1a0f0f6905cb16d4901e778080b3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 06:50:54 +0100 Subject: [PATCH 1/7] fix: Improve error message for download-data edgecase hyperliquid doesn't provide historic data, neither klines nor trades. This made the error message missleading. closes #11270 --- freqtrade/data/history/history_utils.py | 16 +++++++++++----- tests/data/test_download_data.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index a5d5afc2402..164a9f34a4c 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -679,11 +679,17 @@ def download_data( ) else: if not exchange.get_option("ohlcv_has_history", True): - raise OperationalException( - f"Historic klines not available for {exchange.name}. " - "Please use `--dl-trades` instead for this exchange " - "(will unfortunately take a long time)." - ) + if not exchange.get_option("trades_has_history", True): + raise OperationalException( + f"Historic data not available for {exchange.name}. " + f"{exchange.name} does not support downloading trades or ohlcv data." + ) + else: + raise OperationalException( + f"Historic klines not available for {exchange.name}. " + "Please use `--dl-trades` instead for this exchange " + "(will unfortunately take a long time)." + ) migrate_data(config, exchange) pairs_not_available = refresh_backtest_ohlcv_data( exchange, diff --git a/tests/data/test_download_data.py b/tests/data/test_download_data.py index ef1d938d315..16a3f5559ec 100644 --- a/tests/data/test_download_data.py +++ b/tests/data/test_download_data.py @@ -102,3 +102,16 @@ def test_download_data_main_data_invalid(mocker): ) with pytest.raises(OperationalException, match=r"Historic klines not available for .*"): download_data_main(config) + + patch_exchange(mocker, exchange="hyperliquid") + mocker.patch(f"{EXMS}.get_markets", return_value={"ETH/USDC": {}}) + config2 = setup_utils_configuration({"exchange": "hyperliquid"}, RunMode.UTIL_EXCHANGE) + config2.update( + { + "days": 20, + "pairs": ["ETH/USDC", "XRP/USDC"], + "timeframes": ["5m", "1h"], + } + ) + with pytest.raises(OperationalException, match=r"Historic data not available for .*"): + download_data_main(config2) From f4bc956b1b56019247ae8285092317124dd4332a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 07:08:54 +0100 Subject: [PATCH 2/7] chore: fix type declaration --- freqtrade/optimize/optimize_reports/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 23119077bfb..23f4d76f73f 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -368,7 +368,7 @@ def generate_strategy_stats( :param market_change: float indicating the market change :return: Dictionary containing results per strategy and a strategy summary. """ - results: dict[str, DataFrame] = content["results"] + results: DataFrame = content["results"] if not isinstance(results, DataFrame): return {} config = content["config"] From 03a22186bb23260392c476d7b7b9fa7e1cf35cc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 07:12:54 +0100 Subject: [PATCH 3/7] chore: have backtest exit orders account for fee --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eae9c91a68f..b621b2d2638 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -855,7 +855,7 @@ def _exit_trade( amount=amount, filled=0, remaining=amount, - cost=amount * close_rate, + cost=amount * close_rate * (1 + self.fee), ft_order_tag=exit_reason, ) order._trade_bt = trade From 1d22cf98c860bcba4a71d82c8896770a769da7f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 07:14:40 +0100 Subject: [PATCH 4/7] feat: add cost to minimized order json output --- freqtrade/persistence/trade_model.py | 2 +- tests/optimize/test_backtesting.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0d5e7d20058..26e06e0239a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -270,6 +270,7 @@ def to_json(self, entry_side: str, minified: bool = False) -> dict[str, Any]: "order_filled_timestamp": dt_ts_none(self.order_filled_utc), "ft_is_entry": self.ft_order_side == entry_side, "ft_order_tag": self.ft_order_tag, + "cost": self.cost if self.cost else 0, } if not minified: resp.update( @@ -278,7 +279,6 @@ def to_json(self, entry_side: str, minified: bool = False) -> dict[str, Any]: "order_id": self.order_id, "status": self.status, "average": round(self.average, 8) if self.average else 0, - "cost": self.cost if self.cost else 0, "filled": self.filled, "is_open": self.ft_is_open, "order_date": ( diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 81d63cf64fa..5c6f189480f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -5,7 +5,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import ANY, MagicMock, PropertyMock import numpy as np import pandas as pd @@ -795,6 +795,7 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "order_filled_timestamp": 1517251200000, "ft_is_entry": True, "ft_order_tag": "", + "cost": ANY, }, { "amount": 0.00957442, @@ -803,6 +804,7 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "order_filled_timestamp": 1517265300000, "ft_is_entry": False, "ft_order_tag": "roi", + "cost": ANY, }, ], [ @@ -813,6 +815,7 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "order_filled_timestamp": 1517283000000, "ft_is_entry": True, "ft_order_tag": "", + "cost": ANY, }, { "amount": 0.0097064, @@ -821,6 +824,7 @@ def test_backtest_one(default_conf, mocker, testdatadir) -> None: "order_filled_timestamp": 1517285400000, "ft_is_entry": False, "ft_order_tag": "roi", + "cost": ANY, }, ], ], From 7f2e6966a6ebe6866b8c95825d7fc3841db119ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 18:18:10 +0100 Subject: [PATCH 5/7] fix: update total_volume calculation to actually reflect volume closes #11268 --- .../optimize/optimize_reports/optimize_reports.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 23f4d76f73f..32ef864e2db 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -108,7 +108,14 @@ def _generate_result_line( } -def generate_pair_metrics( +def calculate_trade_volume(trades_dict: list[dict[str, Any]]) -> float: + # Aggregate the total volume traded from orders.cost. + # Orders is a nested dictionary within the trades list. + + return sum(sum(order["cost"] for order in trade.get("orders", [])) for trade in trades_dict) + + +def generate_pair_metrics( # pairlist: list[str], stake_currency: str, starting_balance: float, @@ -431,8 +438,9 @@ def generate_strategy_stats( expectancy, expectancy_ratio = calculate_expectancy(results) backtest_days = (max_date - min_date).days or 1 + trades_dict = results.to_dict(orient="records") strat_stats = { - "trades": results.to_dict(orient="records"), + "trades": trades_dict, "locks": [lock.to_json() for lock in content["locks"]], "best_pair": best_pair, "worst_pair": worst_pair, @@ -444,7 +452,7 @@ def generate_strategy_stats( "total_trades": len(results), "trade_count_long": len(results.loc[~results["is_short"]]), "trade_count_short": len(results.loc[results["is_short"]]), - "total_volume": float(results["stake_amount"].sum()), + "total_volume": calculate_trade_volume(trades_dict), "avg_stake_amount": results["stake_amount"].mean() if len(results) > 0 else 0, "profit_mean": results["profit_ratio"].mean() if len(results) > 0 else 0, "profit_median": results["profit_ratio"].median() if len(results) > 0 else 0, From 4a7a51035dbb44b838c206c947f4f9d170e41f6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 18:56:33 +0100 Subject: [PATCH 6/7] docs: improve doc formatting --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 0ee5c4607e8..953fb5f8c87 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -145,7 +145,7 @@ To use BNFCR futures, you will have to have the following combination of setting The `stake_currency` setting defines the markets the bot will be operating in. This choice is really arbitrary. -On the exchange, you'll have to use "Multi-asset Mode" - and "Position Mode set to "One-way Mode". +On the exchange, you'll have to use "Multi-asset Mode" - and "Position Mode set to "One-way Mode". Freqtrade will check these settings on startup, but won't attempt to change them. ## Bingx From 9efa48f2a84b4ea9a4aa02853f55bcf5e03b663a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jan 2025 19:03:33 +0100 Subject: [PATCH 7/7] chore: ensure that logged message actually make sense --- freqtrade/freqai/freqai_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3b34f2d2a83..939039094b8 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -773,7 +773,7 @@ def _set_train_queue(self): """ current_pairlist = self.config.get("exchange", {}).get("pair_whitelist") if not self.dd.pair_dict: - logger.info("Set fresh train queue from whitelist. Queue: {current_pairlist}") + logger.info(f"Set fresh train queue from whitelist. Queue: {current_pairlist}") return deque(current_pairlist) best_queue = deque() @@ -789,7 +789,7 @@ def _set_train_queue(self): best_queue.appendleft(pair) logger.info( - "Set existing queue from trained timestamps. Best approximation queue: {best_queue}" + f"Set existing queue from trained timestamps. Best approximation queue: {best_queue}" ) return best_queue