Merge pull request freqtrade#11711 from freqtrade/feat/bt_historic_precision

Improve price precision logic and add significant digits calculation

Author: xmatthias

freqtrade/data/btanalysis/__init__.py

@@ -25,6 +25,7 @@
     trade_list_to_dataframe,
     update_backtest_metadata,
 )
+from .historic_precision import get_tick_size_over_time
 from .trade_parallelism import (
     analyze_trade_parallelism,
     evaluate_result_multi,

freqtrade/data/btanalysis/historic_precision.py

@@ -0,0 +1,27 @@
+from pandas import DataFrame, Series
+
+
+def get_tick_size_over_time(candles: DataFrame) -> Series:
+    """
+    Calculate the number of significant digits for candles over time.
+    It's using the Monthly maximum of the number of significant digits for each month.
+    :param candles: DataFrame with OHLCV data
+    :return: Series with the average number of significant digits for each month
+    """
+    # count the number of significant digits for the open and close prices
+    for col in ["open", "high", "low", "close"]:
+        candles[f"{col}_count"] = (
+            candles[col].round(14).astype(str).str.extract(r"\.(\d*[1-9])")[0].str.len()
+        )
+    candles["max_count"] = candles[["open_count", "close_count", "high_count", "low_count"]].max(
+        axis=1
+    )
+
+    candles1 = candles.set_index("date", drop=True)
+    # Group by month and calculate the average number of significant digits
+    monthly_count_avg1 = candles1["max_count"].resample("MS").max()
+    # monthly_open_count_avg
+    # convert monthly_open_count_avg from 5.0 to 0.00001, 4.0 to 0.0001, ...
+    monthly_open_count_avg = 1 / 10**monthly_count_avg1
+
+    return monthly_open_count_avg

freqtrade/optimize/backtesting.py

@@ -9,14 +9,18 @@
 from copy import deepcopy
 from datetime import datetime, timedelta
 
-from numpy import nan
-from pandas import DataFrame
+from numpy import isnan, nan
+from pandas import DataFrame, Series
 
 from freqtrade import constants
 from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongShort
 from freqtrade.data import history
-from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe
+from freqtrade.data.btanalysis import (
+    find_existing_backtest_stats,
+    get_tick_size_over_time,
+    trade_list_to_dataframe,
+)
 from freqtrade.data.converter import trim_dataframe, trim_dataframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.metrics import combined_dataframes_with_rel_mean
@@ -35,7 +39,7 @@
     price_to_precision,
     timeframe_to_seconds,
 )
-from freqtrade.exchange.exchange import Exchange
+from freqtrade.exchange.exchange import TICK_SIZE, Exchange
 from freqtrade.ft_types import (
     BacktestContentType,
     BacktestContentTypeIcomplete,
@@ -121,6 +125,7 @@ def __init__(self, config: Config, exchange: Exchange | None = None) -> None:
         self.order_id_counter: int = 0
 
         config["dry_run"] = True
+        self.price_pair_prec: dict[str, Series] = {}
         self.run_ids: dict[str, str] = {}
         self.strategylist: list[IStrategy] = []
         self.all_bt_content: dict[str, BacktestContentType] = {}
@@ -189,7 +194,6 @@ def __init__(self, config: Config, exchange: Exchange | None = None) -> None:
             self.fee = max(fee for fee in fees if fee is not None)
             logger.info(f"Using fee {self.fee:.4%} - worst case fee from exchange (lowest tier).")
         self.precision_mode = self.exchange.precisionMode
-        self.precision_mode_price = self.exchange.precision_mode_price
 
         if self.config.get("freqai_backtest_live_models", False):
             from freqtrade.freqai.utils import get_timerange_backtest_live_models
@@ -316,6 +320,11 @@ def load_bt_data(self) -> tuple[dict[str, DataFrame], TimeRange]:
 
         self.progress.set_new_value(1)
         self._load_bt_data_detail()
+        self.price_pair_prec = {}
+        for pair in self.pairlists.whitelist:
+            if pair in data:
+                # Load price precision logic
+                self.price_pair_prec[pair] = get_tick_size_over_time(data[pair])
         return data, self.timerange
 
     def _load_bt_data_detail(self) -> None:
@@ -385,6 +394,22 @@ def _load_bt_data_detail(self) -> None:
         else:
             self.futures_data = {}
 
+    def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float | None, int]:
+        """
+        Get pair precision at that moment in time
+        :param pair: Pair to get precision for
+        :param current_time: Time to get precision for
+        :return: tuple of price precision, precision_mode_price for the pair at that given time.
+        """
+        precision_series = self.price_pair_prec.get(pair)
+        if precision_series is not None:
+            precision = precision_series.asof(current_time)
+
+            if not isnan(precision):
+                # Force tick size if we define the precision
+                return precision, TICK_SIZE
+        return self.exchange.get_precision_price(pair), self.exchange.precision_mode_price
+
     def disable_database_use(self):
         disable_database_use(self.timeframe)
 
@@ -793,7 +818,7 @@ def _get_exit_for_signal(
                     )
                     if rate is not None and rate != close_rate:
                         close_rate = price_to_precision(
-                            rate, trade.price_precision, self.precision_mode_price
+                            rate, trade.price_precision, trade.precision_mode_price
                         )
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
@@ -926,6 +951,7 @@ def get_valid_price_and_stake(
         trade: LocalTrade | None,
         order_type: str,
         price_precision: float | None,
+        precision_mode_price: int,
     ) -> tuple[float, float, float, float]:
         if order_type == "limit":
             new_rate = strategy_safe_wrapper(
@@ -941,9 +967,7 @@ def get_valid_price_and_stake(
             # We can't place orders higher than current high (otherwise it'd be a stop limit entry)
             # which freqtrade does not support in live.
             if new_rate is not None and new_rate != propose_rate:
-                propose_rate = price_to_precision(
-                    new_rate, price_precision, self.precision_mode_price
-                )
+                propose_rate = price_to_precision(new_rate, price_precision, precision_mode_price)
             if direction == "short":
                 propose_rate = max(propose_rate, row[LOW_IDX])
             else:
@@ -1036,7 +1060,7 @@ def _enter_trade(
         pos_adjust = trade is not None and requested_rate is None
 
         stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0)
-        precision_price = self.exchange.get_precision_price(pair)
+        precision_price, precision_mode_price = self.get_pair_precision(pair, current_time)
 
         propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake(
             pair,
@@ -1049,6 +1073,7 @@ def _enter_trade(
             trade,
             order_type,
             precision_price,
+            precision_mode_price,
         )
 
         # replace proposed rate if another rate was requested
@@ -1124,7 +1149,7 @@ def _enter_trade(
                     amount_precision=precision_amount,
                     price_precision=precision_price,
                     precision_mode=self.precision_mode,
-                    precision_mode_price=self.precision_mode_price,
+                    precision_mode_price=precision_mode_price,
                     contract_size=contract_size,
                     orders=[],
                 )

tests/data/test_historic_precision.py

@@ -0,0 +1,92 @@
+# pragma pylint: disable=missing-docstring, C0103
+
+from datetime import timezone
+
+import pandas as pd
+from numpy import nan
+from pandas import DataFrame, Timestamp
+
+from freqtrade.data.btanalysis.historic_precision import get_tick_size_over_time
+
+
+def test_get_tick_size_over_time():
+    """
+    Test the get_tick_size_over_time function with predefined data
+    """
+    # Create test dataframe with different levels of precision
+    data = {
+        "date": [
+            Timestamp("2020-01-01 00:00:00", tz=timezone.utc),
+            Timestamp("2020-01-02 00:00:00", tz=timezone.utc),
+            Timestamp("2020-01-03 00:00:00", tz=timezone.utc),
+            Timestamp("2020-01-15 00:00:00", tz=timezone.utc),
+            Timestamp("2020-01-16 00:00:00", tz=timezone.utc),
+            Timestamp("2020-01-31 00:00:00", tz=timezone.utc),
+            Timestamp("2020-02-01 00:00:00", tz=timezone.utc),
+            Timestamp("2020-02-15 00:00:00", tz=timezone.utc),
+            Timestamp("2020-03-15 00:00:00", tz=timezone.utc),
+        ],
+        "open": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34],
+        "high": [1.23457, 1.235, 1.24, 1.3, 1.23456, 1.235, 2.3457, 2.34, 2.34],
+        "low": [1.23455, 1.233, 1.22, 1.1, 1.23456, 1.233, 2.3455, 2.34, 2.34],
+        "close": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34],
+        "volume": [100, 200, 300, 400, 500, 600, 700, 800, 900],
+    }
+
+    candles = DataFrame(data)
+
+    # Calculate significant digits
+    result = get_tick_size_over_time(candles)
+
+    # Check that the result is a pandas Series
+    assert isinstance(result, pd.Series)
+
+    # Check that we have three months of data (Jan, Feb and March 2020 )
+    assert len(result) == 3
+
+    # Before
+    assert result.asof("2019-01-01 00:00:00+00:00") is nan
+    # January should have 5 significant digits (based on 1.23456789 being the most precise value)
+    # which should be converted to 0.00001
+
+    assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001
+    assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001
+    assert result.asof("2020-02-25 00:00:00+00:00") == 0.0001
+    assert result.asof("2020-03-25 00:00:00+00:00") == 0.01
+    assert result.asof("2020-04-01 00:00:00+00:00") == 0.01
+    # Value far past the last date should be the last value
+    assert result.asof("2025-04-01 00:00:00+00:00") == 0.01
+
+    assert result.iloc[0] == 0.00001
+
+
+def test_get_tick_size_over_time_real_data(testdatadir):
+    """
+    Test the get_tick_size_over_time function with real data from the testdatadir
+    """
+    from freqtrade.data.history import load_pair_history
+
+    # Load some test data from the testdata directory
+    pair = "UNITTEST/BTC"
+    timeframe = "1m"
+
+    candles = load_pair_history(
+        datadir=testdatadir,
+        pair=pair,
+        timeframe=timeframe,
+    )
+
+    # Make sure we have test data
+    assert not candles.empty, "No test data found, cannot run test"
+
+    # Calculate significant digits
+    result = get_tick_size_over_time(candles)
+
+    assert isinstance(result, pd.Series)
+
+    # Verify that all values are between 0 and 1 (valid precision values)
+    assert all(result > 0)
+    assert all(result < 1)
+
+    assert all(result <= 0.0001)
+    assert all(result >= 0.00000001)

tests/optimize/test_backtesting.py

@@ -22,6 +22,7 @@
 from freqtrade.enums import CandleType, ExitType, RunMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_next_date, timeframe_to_prev_date
+from freqtrade.exchange.exchange_utils import DECIMAL_PLACES, TICK_SIZE
 from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, get_strategy_run_id
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import LocalTrade, Trade
@@ -348,6 +349,29 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     assert processed["UNITTEST/BTC"].equals(processed2["UNITTEST/BTC"])
 
 
+def test_get_pair_precision_bt(default_conf, mocker) -> None:
+    patch_exchange(mocker)
+    default_conf["timeframe"] = "30m"
+    backtesting = Backtesting(default_conf)
+    backtesting._set_strategy(backtesting.strategylist[0])
+    pair = "UNITTEST/BTC"
+    backtesting.pairlists._whitelist = [pair]
+    ex_mock = mocker.patch(f"{EXMS}.get_precision_price", return_value=1e-5)
+    data, timerange = backtesting.load_bt_data()
+    assert data
+
+    assert backtesting.get_pair_precision(pair, dt_utc(2018, 1, 1)) == (1e-8, TICK_SIZE)
+    assert ex_mock.call_count == 0
+    assert backtesting.get_pair_precision(pair, dt_utc(2017, 12, 15)) == (1e-8, TICK_SIZE)
+    assert ex_mock.call_count == 0
+
+    # Fallback to exchange logic
+    assert backtesting.get_pair_precision(pair, dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES)
+    assert ex_mock.call_count == 1
+    assert backtesting.get_pair_precision("ETH/BTC", dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES)
+    assert ex_mock.call_count == 2
+
+
 def test_backtest_abort(default_conf, mocker, testdatadir) -> None:
     patch_exchange(mocker)
     backtesting = Backtesting(default_conf)