backtesting library skips bars where any indicator has NA values (or don't place orders) #1339
Description
Expected behavior
backtesting library skips bars where any indicator has NA values. Here are a few solutions to handle this without using arbitrary warmup periods?
ok i use resample_apply for 100 bar (1D), current ohlcv use 30m timeframe, so be like that, 100 days at first of my ohlcvs don't any trades @@ cause backtesting.py skip or don't place order when have indicators na, but this maybe like conflic cause we don't really know that indicators was use or not for signal to place order
Code sample
from backtesting import Backtest, Strategy
from backtesting.lib import crossover, compute_stats, resample_apply
from backtesting.test import SMA
import pandas as pd
from utils.exporter import BacktestExporter
from utils.chart_generator import InteractiveChartGenerator
from utils.indicators import KC_BASIS, KC_LOWER, KC_UPPER
from utils.utils import fractions
from config import DataCrawlerConfig
import warnings
import logging
import numpy as np
import pandas_ta as ta
from talib import ATR, RSI, EMA, SMA,ADX, PLUS_DI, MINUS_DI
# Suppress DataFrame fragmentation warning
warnings.filterwarnings('ignore', category=pd.errors.PerformanceWarning)
# Configure logging
logging.basicConfig(level=getattr(logging, DataCrawlerConfig.LOG_LEVEL),
format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
def update_data_if_enabled():
"""Update OHLCV data if crawling is enabled in config"""
if not DataCrawlerConfig.should_update_data():
logger.info("Data crawling is disabled in config")
return
try:
from utils.data_crawler import update_data_files
logger.info("Updating OHLCV data...")
results = update_data_files(
data_dir=DataCrawlerConfig.DATA_DIRECTORY,
exchange_name=DataCrawlerConfig.EXCHANGE_NAME,
pairs=DataCrawlerConfig.DEFAULT_PAIRS,
timeframe=DataCrawlerConfig.DEFAULT_TIMEFRAME
)
# Log results
for filename, success in results.items():
status = "SUCCESS" if success else "FAILED"
logger.info(f"Data update for {filename}: {status}")
except ImportError:
logger.warning("ccxt not installed. Install with: pip install ccxt")
except Exception as e:
logger.error(f"Error updating data: {e}")
logger.info("Continuing with existing data...")
class BTCLongDCA(Strategy):
single_test_condition: str = ""
compound_volume: bool = True
leverage: int = 3
init_size_by_equity: float = 0.12 # 12%
step_size_percent: float = 0.12 # 12%
max_opentrades: int = 6
decrease_dca_with_atr: float = 2
delay_bar_per_order: int = 50
bar_to_turn_on_safe_mode: int = 800
atr_multiplier_tp: float = 5
init_min_tp: float = 0.01 # 1%
atr_multiplier_sl: float = 0.0
init_sl: float = 0.0
tfs = ['30min', '1h', '90min', '2h', '3h', '4h', '5h', '1D']
fibo = [0.618, 0.786, 0.5, 0.382, 0.236, 1.236, 1.382, 1.5, 1.786, 1.618]
frac = fractions['BTC']
initial_capital: float = 100_000 * frac
leverage = 5
def init(self):
df_ohlcv = pd.DataFrame({
'High': self.data.High,
'Low': self.data.Low,
'Close': self.data.Close,
'Open': self.data.Open,
'index': self.data.index
})
df_ohlcv.set_index('index', inplace=True)
for tf in self.tfs:
adx_13_70_data = resample_apply(tf, lambda df,l1,l2: ta.adx(df['High'], df['Low'], df['Close'], length = l1, signal_length=l2, tvmode=True), df_ohlcv, 13, 70, plot=False, overlay=False)
self.__setattr__(f'adx_70_{tf}', adx_13_70_data[0])
self.__setattr__(f'plus_di_13_{tf}', adx_13_70_data[2])
self.__setattr__(f'minus_di_13_{tf}', adx_13_70_data[3])
adx_50_20_data = resample_apply(tf, lambda df,l1,l2: ta.adx(df['High'], df['Low'], df['Close'], length=l1, signal_length=l2, tvmode= True), df_ohlcv, 50, 20, plot=False, overlay=False)
self.__setattr__(f'adx_20_{tf}', adx_50_20_data[0])
self.__setattr__(f'plus_di_50_{tf}', adx_50_20_data[2])
self.__setattr__(f'minus_di_50_{tf}', adx_50_20_data[3])
self.__setattr__(f'rsi_close_50_{tf}', resample_apply(tf , lambda df,l: ta.rsi(df['Close'], l, talib=True), df_ohlcv, 50, plot=False, overlay=False))
self.__setattr__(f'rsi_high_10_{tf}', resample_apply(tf , lambda df,l: ta.rsi(df['High'], l, talib=True), df_ohlcv, 10, plot=False, overlay=False))
self.__setattr__(f'rsi_high_14_{tf}', resample_apply(tf , lambda df,l: ta.rsi(df['High'], l, talib=True), df_ohlcv, 14, plot=False, overlay=False))
self.__setattr__(f'rsi_high_50_{tf}', resample_apply(tf , lambda df,l: ta.rsi(df['High'], l, talib=True), df_ohlcv, 50, plot=False, overlay=False))
self.__setattr__(f'sma_50_{tf}', resample_apply(tf, lambda df,l: ta.sma(df['Close'], l, talib=True), df_ohlcv, 50, plot=False, overlay=False))
self.__setattr__(f'sma_100_{tf}', resample_apply(tf, lambda df,l: ta.sma(df['Close'], l, talib=True), df_ohlcv, 100, plot=False, overlay=False))
self.__setattr__(f'sma_200_{tf}', resample_apply(tf, lambda df,l: ta.sma(df['Close'], l, talib=True), df_ohlcv, 200, plot=False, overlay=False))
self.__setattr__(f'ema_9_{tf}', resample_apply(tf, lambda df,l: ta.ema(df['Close'], l, talib=True), df_ohlcv, 9, plot=False, overlay=False))
self.__setattr__(f'ema_21_{tf}', resample_apply(tf, lambda df,l: ta.ema(df['Close'], l, talib=True), df_ohlcv, 21, plot=False, overlay=False))
self.__setattr__(f'ema_30_{tf}', resample_apply(tf, lambda df,l: ta.ema(df['Close'], l, talib=True), df_ohlcv, 30, plot=False, overlay=False))
self.__setattr__(f'ema_50_{tf}', resample_apply(tf, lambda df,l: ta.ema(df['Close'], l, talib=True), df_ohlcv, 50, plot=False, overlay=False))
self.__setattr__(f'ema_80_{tf}', resample_apply(tf, lambda df,l: ta.ema(df['Close'], l, talib=True), df_ohlcv, 80, plot=False, overlay=False))
self.__setattr__(f'atr_1_{tf}', resample_apply(tf, lambda df, l: ta.atr(df['High'], df['Low'], df['Close'], l, talib= True), df_ohlcv, 1, plot=False, overlay=False))
self.__setattr__(f'atr_6_{tf}', resample_apply(tf, lambda df, l: ta.atr(df['High'], df['Low'], df['Close'], l, talib= True), df_ohlcv, 6, plot=False, overlay=False))
self.__setattr__(f'atr_14_{tf}', resample_apply(tf, lambda df, l: ta.atr(df['High'], df['Low'], df['Close'], l, talib= True), df_ohlcv, 14, plot=False, overlay=False))
self.__setattr__(f'atr_50_{tf}', resample_apply(tf, lambda df, l: ta.atr(df['High'], df['Low'], df['Close'], l, talib= True), df_ohlcv, 50, plot=False, overlay=False))
kc_data_20_1 = resample_apply(tf, lambda df, l1,l2: ta.kc(df['High'], df['Low'], df['Close'], length = l1, scalar = l2, tr=True), df_ohlcv, 20, 1, plot=False, overlay=False)
self.__setattr__(f'kc_upper_20_1_{tf}', kc_data_20_1[2])
self.__setattr__(f'kc_basis_20_1_{tf}', kc_data_20_1[1])
self.__setattr__(f'kc_lower_20_1_{tf}', kc_data_20_1[0])
kc_data_50_2= resample_apply(tf, lambda df, l1,l2: ta.kc(df['High'], df['Low'], df['Close'], length = l1, scalar = l2, tr=True), df_ohlcv, 50, 2, plot=False, overlay=False)
self.__setattr__(f'kc_upper_50_2_{tf}', kc_data_50_2[2])
self.__setattr__(f'kc_basis_50_2_{tf}', kc_data_50_2[1])
self.__setattr__(f'kc_lower_50_2_{tf}', kc_data_50_2[0])
self.__setattr__(f'tsi_14_50{tf}', resample_apply(tf,lambda df, l1,l2: ta.tsi(df['Close'], l1, l2, talib=True), df_ohlcv, 14,50, plot=False, overlay=False))
self.__setattr__(f'tsi_7_21{tf}', resample_apply(tf,lambda df, l1,l2: ta.tsi(df['Close'], l1, l2, talib=True), df_ohlcv, 7, 21, plot=False, overlay=False))
self.__setattr__(f'open_{tf}', resample_apply(tf, lambda x:x, self.data.Open.s, plot=False, overlay=False))
self.__setattr__(f'high_{tf}', resample_apply(tf, lambda x:x, self.data.High.s, plot=False, overlay=False))
self.__setattr__(f'low_{tf}', resample_apply(tf, lambda x:x, self.data.Low.s, plot=False, overlay=False))
self.__setattr__(f'close_{tf}', resample_apply(tf, lambda x:x, self.data.Close.s, plot=False, overlay=False))
self.__setattr__(f'volume_{tf}', resample_apply(tf, lambda x:x, self.data.Volume.s, plot=False, overlay=False))
self.current_size_equity = self.init_size_by_equity
self.position_entry = [float('nan')] * len(self.data)
self.tp_price = [float('nan')] * len(self.data)
def get_current_position_entry(self, trades):
"""Calculate current position entry from open trades"""
if len(trades) == 0 or self.position.size == 0:
return None
total_size = sum([trade.size for trade in trades])
weighted_entry = sum([trade.entry_price * trade.size for trade in trades]) / total_size
return weighted_entry
def next(self):
just_place_order = False
bar_index = len(self.data) - 1
current_entry = self.get_current_position_entry(self.trades)
tp_price_value = max((1 + self.init_min_tp) * current_entry, current_entry + self.atr_multiplier_tp * self.atr_50_1h) if current_entry is not None else None
if current_entry is not None:
self.position_entry[bar_index] = current_entry
self.tp_price[bar_index] = tp_price_value
else:
self.position_entry[bar_index] = float('nan')
self.tp_price[bar_index] = float('nan')
entry_conds = [
crossover(self.rsi_high_14_30min, 20)
]
exit_conds = [] if current_entry is None else [
self.close_30min > current_entry + self.atr_50_1h and self.rsi_high_14_30min > 70
]
dca_conds = [] if current_entry is None else [
crossover(15,self.rsi_high_10_1h)
]
trade_conds = [
len(self.trades) < self.max_opentrades,
self.position.size == 0 or bar_index - self.trades[-1].entry_bar >= self.delay_bar_per_order,
self.position.size == 0 or self.close_30min <= self.position_entry[bar_index] - self.atr_50_1h * self.decrease_dca_with_atr
]
if all(trade_conds):
for i, cond in enumerate(entry_conds if self.position.size == 0 else dca_conds):
if not cond:
continue
sz = round(self.equity * self.leverage * self.current_size_equity / self.data.Close[-1])
self.buy(tag=f'entry-{"dca" if self.position.size != 0 else ""}{i}-{current_entry}-{self.current_size_equity:.4f}', size=sz)
just_place_order = True
self.current_size_equity *= 1 + self.step_size_percent
exit_price_ords = [ord for ord in self.orders if ord.tag == "exit-price"]
if self.position.size != 0 :
if len(exit_price_ords) == 0:
self.sell(tag="exit-price", size=self.position.size, limit=self.tp_price[bar_index])
else:
for ord in exit_price_ords:
ord.cancel()
self.sell(tag="exit-price", size=self.position.size, limit=self.tp_price[bar_index])
exit_price_ords = [ord for ord in self.orders if ord.tag == "exit-price"]
if self.position.size != 0:
for i, cond in enumerate(exit_conds):
if not cond:
continue
self.sell(tag=f"exit-cond", size=self.position.size)
try:
for ord in exit_price_ords:
ord.cancel()
except Exception as e:
print(e, self.orders, self.trades, self.position.size)
pass
if self.position.size == 0 and not just_place_order:
self.current_size_equity = self.init_size_by_equity
update_data_if_enabled()
df_30m = pd.read_csv('BTCUSD_30m_Coinbase.csv')
df_30m['Open time'] = pd.to_datetime(df_30m['Open time'])
df_30m.set_index('Open time', inplace=True)
df_30m = df_30m[['Open', 'High', 'Low', 'Close', 'Volume']]
df_30m = df_30m.loc[pd.Timestamp('2025-01-01'):pd.Timestamp('2025-12-23')]
df_30m = df_30m.dropna()
bt = Backtest(df_30m, BTCLongDCA, cash=BTCLongDCA.initial_capital, commission=0.001, exclusive_orders=False, trade_on_close=True, margin=1/BTCLongDCA.leverage, finalize_trades=True, )
stats = bt.run()
stats_with_config = compute_stats(stats=stats, risk_free_rate=0.02, data=df_30m)
stats['Sharpe Ratio'] = stats_with_config['Sharpe Ratio']
stats['Sortino Ratio'] = stats_with_config['Sortino Ratio']
exporter = BacktestExporter(stats, BTCLongDCA.frac, output_dir='.')
exporter.export_to_txt(filename='stats.txt')
chart_gen = InteractiveChartGenerator(df=df_30m, stats=stats, fraction=BTCLongDCA.frac,title="BTC Long DCA Strategy - Backtest Results")
if hasattr(stats._strategy, 'position_entry'):
chart_gen.add_indicator(name='Position Entry',data=pd.Series(stats._strategy.position_entry, index=df_30m.index),color="#C4C4C4", panel='main',line_width=2)
if hasattr(stats._strategy, 'tp_price'):
chart_gen.add_indicator(name='Take Profit',data=pd.Series(stats._strategy.tp_price, index=df_30m.index),color="#22FF00",panel='main',line_width=2)
chart_gen.add_indicator(name=stats._strategy.atr_50_1D.name,data=stats._strategy.atr_50_1D,color="#FF0000",panel='atr',line_width=2)
chart_gen.generate_html(output_path='backtest_chart.html', dark_theme=True)Actual behavior
self.__setattr__(f'sma_100_{tf}', resample_apply(tf, lambda df,l: ta.sma(df['Close'], l, talib=True), df_ohlcv, 100, plot=False, overlay=False))
Additional info, steps to reproduce, full crash traceback, screenshots
No response
Software versions
backtesting.__version__:pandas.__version__:numpy.__version__:bokeh.__version__:- OS: