Skip to content

Fix QuantBook Option/Future history api date range #8671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jhonabreul merged 2 commits into from
Apr 3, 2025
Merged

Fix QuantBook Option/Future history api date range #8671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5063fc1
Fix QuantBook Option/Future history api date range
jhonabreul Apr 2, 2025
392badf
Extended unit tests
jhonabreul Apr 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions Research/QuantBook.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ public OptionHistory OptionHistory(Symbol symbol, DateTime start, DateTime? end
// only add underlying if not present
AddIndex(symbol.Underlying.Value, resolutionToUseForUnderlying, fillForward: fillForward);
}
else if(symbol.Underlying.SecurityType == SecurityType.Future && symbol.Underlying.IsCanonical())
else if (symbol.Underlying.SecurityType == SecurityType.Future && symbol.Underlying.IsCanonical())
{
AddFuture(symbol.Underlying.ID.Symbol, resolutionToUseForUnderlying, fillForward: fillForward,
extendedMarketHours: extendedMarketHours);
Expand All @@ -435,13 +435,11 @@ public OptionHistory OptionHistory(Symbol symbol, DateTime start, DateTime? end
var allSymbols = new HashSet<Symbol>();
var optionFilterUniverse = new OptionFilterUniverse(option);

foreach (var date in QuantConnect.Time.EachTradeableDay(option, start, end.Value.AddDays(-1), extendedMarketHours))
foreach (var (date, chainData, underlyingData) in GetChainHistory<OptionUniverse>(option, start, end.Value, extendedMarketHours))
{
var universeData = GetChainHistory<OptionUniverse>(symbol, date, out var underlyingData);

if (underlyingData is not null)
{
optionFilterUniverse.Refresh(universeData, underlyingData, underlyingData.EndTime);
optionFilterUniverse.Refresh(chainData, underlyingData, underlyingData.EndTime);
allSymbols.UnionWith(option.ContractFilter.Filter(optionFilterUniverse).Select(x => x.Symbol));
}
}
Expand Down Expand Up @@ -499,13 +497,9 @@ public FutureHistory FutureHistory(Symbol symbol, DateTime start, DateTime? end
// canonical symbol, lets find the contracts
var future = Securities[symbol] as Future;

for (var date = start; date < end; date = date.AddDays(1))
foreach (var (date, chainData, underlyingData) in GetChainHistory<FutureUniverse>(future, start, end.Value, extendedMarketHours))
{
if (future.Exchange.DateIsOpen(date, extendedMarketHours))
{
var universeData = GetChainHistory<FutureUniverse>(future.Symbol, date, out _);
allSymbols.UnionWith(future.ContractFilter.Filter(new FutureFilterUniverse(universeData, date)).Select(x => x.Symbol));
}
allSymbols.UnionWith(future.ContractFilter.Filter(new FutureFilterUniverse(chainData, date)).Select(x => x.Symbol));
}
}
else
Expand Down Expand Up @@ -882,6 +876,20 @@ private IEnumerable<T> GetChainHistory<T>(Symbol canonicalSymbol, DateTime date,
return Enumerable.Empty<T>();
}

/// <summary>
/// Helper method to get option/future chain historical data for a given date range
/// </summary>
private IEnumerable<(DateTime Date, IEnumerable<T> ChainData, BaseData UnderlyingData)> GetChainHistory<T>(
Security security, DateTime start, DateTime end, bool extendedMarketHours)
where T : BaseChainUniverseData
{
foreach (var date in QuantConnect.Time.EachTradeableDay(security, start.Date, end.Date, extendedMarketHours))
{
var universeData = GetChainHistory<T>(security.Symbol, date, out var underlyingData);
yield return (date, universeData, underlyingData);
}
}

/// <summary>
/// Helper method to perform selection on the given data and filter it
/// </summary>
Expand Down
141 changes: 141 additions & 0 deletions Tests/Research/QuantBookHistoryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
using QuantConnect.Research;
using QuantConnect.Logging;
using QuantConnect.Data.Fundamental;
using System.Data;
using QuantConnect.Securities.Future;
using QuantConnect.Data;
using NodaTime;
using QuantConnect.Interfaces;
using QuantConnect.Data.UniverseSelection;

namespace QuantConnect.Tests.Research
{
Expand Down Expand Up @@ -179,6 +185,65 @@ public void CanonicalOptionIntradayQuantBookHistory()
}
}

private static TestCaseData[] CanonicalOptionIntradayHistoryTestCases
{
get
{
var twx = Symbol.Create("TWX", SecurityType.Equity, Market.USA);
var twxOption = Symbol.CreateCanonicalOption(twx);

return
[
new TestCaseData(twxOption, new DateTime(2014, 06, 05), (DateTime?)null),
new TestCaseData(twxOption, new DateTime(2014, 06, 05), new DateTime(2014, 06, 05)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05), new DateTime(2014, 06, 06)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05, 0, 0, 0), new DateTime(2014, 06, 05, 15, 0, 0)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 05, 15, 0, 0)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06, 10, 0, 0)),
new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06, 15, 0, 0))
];
}
}

[TestCaseSource(nameof(CanonicalOptionIntradayHistoryTestCases))]
public void CanonicalOptionIntradayQuantBookHistoryWithIntradayRange(Symbol canonicalOption, DateTime start, DateTime? end)
{
var quantBook = new QuantBook();
var historyProvider = new TestHistoryProvider(quantBook.HistoryProvider);
quantBook.SetHistoryProvider(historyProvider);
quantBook.SetStartDate((end ?? start).Date.AddDays(1));

var option = quantBook.AddSecurity(canonicalOption);
var history = quantBook.OptionHistory(canonicalOption, start, end, Resolution.Minute);

Assert.Greater(history.Count, 0);

var symbolsInHistory = history.SelectMany(slice => slice.AllData.Select(x => x.Symbol)).Distinct().ToList();
Assert.Greater(symbolsInHistory.Count, 1);

var underlying = symbolsInHistory.Where(x => x == canonicalOption.Underlying).ToList();
Assert.AreEqual(1, underlying.Count);

var contractsSymbols = symbolsInHistory.Where(x => x.SecurityType == canonicalOption.SecurityType).ToList();
Assert.Greater(contractsSymbols.Count, 1);

var expectedDates = new HashSet<DateTime> { start.Date };
if (end.HasValue && end.Value > end.Value.Date)
{
expectedDates.Add(end.Value.Date);
}

var dataDates = history.SelectMany(slice => slice.AllData.Where(x => contractsSymbols.Contains(x.Symbol)).Select(x => x.EndTime.Date)).ToHashSet();
CollectionAssert.AreEqual(expectedDates, dataDates);

// OptionUniverse must have been requested for all dates in the range
foreach (var date in Time.EachTradeableDay(option, start.Date, (end ?? start).Date))
{
Assert.AreEqual(1, historyProvider.HistoryRequests.Count(request => request.DataType == typeof(OptionUniverse) && request.EndTimeLocal == date));
}
}

[Test]
public void OptionContractQuantBookHistory()
{
Expand Down Expand Up @@ -301,6 +366,57 @@ public void CanonicalFutureIntradayQuantBookHistory(int maxFilter, int numberOfF
}
}

private static TestCaseData[] CanonicalFutureIntradayHistoryTestCases
{
get
{
var es = Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME);
return
[
new TestCaseData(es, new DateTime(2013, 10, 10), (DateTime?)null),
new TestCaseData(es, new DateTime(2013, 10, 10), new DateTime(2013, 10, 10)),
new TestCaseData(es, new DateTime(2013, 10, 10), new DateTime(2013, 10, 11)),
new TestCaseData(es, new DateTime(2013, 10, 10, 0, 0, 0), new DateTime(2013, 10, 10, 15, 0, 0)),
new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 10, 15, 0, 0)),
new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11)),
new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11, 10, 0, 0)),
new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11, 15, 0, 0))
];
}
}

[TestCaseSource(nameof(CanonicalFutureIntradayHistoryTestCases))]
public void CanonicalFutureIntradayQuantBookHistoryWithIntradayRange(Symbol canonicalFuture, DateTime start, DateTime? end)
{
var quantBook = new QuantBook();
var historyProvider = new TestHistoryProvider(quantBook.HistoryProvider);
quantBook.SetHistoryProvider(historyProvider);
quantBook.SetStartDate((end ?? start).Date.AddDays(1));
var future = quantBook.AddSecurity(canonicalFuture) as Future;
future.SetFilter(universe => universe);

var history = quantBook.FutureHistory(canonicalFuture, start, end, Resolution.Minute);
Assert.Greater(history.Count, 0);

var symbolsInHistory = history.SelectMany(slice => slice.AllData.Select(x => x.Symbol)).Distinct().ToList();
Assert.Greater(symbolsInHistory.Count, 1);

var expectedDates = new HashSet<DateTime> { start.Date };
if (end.HasValue && end.Value > end.Value.Date)
{
expectedDates.Add(end.Value.Date);
}

var dataDates = history.SelectMany(slice => slice.AllData.Select(x => x.EndTime.Date)).ToHashSet();
CollectionAssert.AreEqual(expectedDates, dataDates);

// FutureUniverse must have been requested for all dates in the range
foreach (var date in Time.EachTradeableDay(future, start.Date, (end ?? start).Date))
{
Assert.AreEqual(1, historyProvider.HistoryRequests.Count(request => request.DataType == typeof(FutureUniverse) && request.EndTimeLocal == date));
}
}

[Test]
public void FutureContractQuantBookHistory()
{
Expand Down Expand Up @@ -685,5 +801,30 @@ def getHistory():
Assert.IsFalse(pyHistory.HasAttr("data"));
}
}

private class TestHistoryProvider : HistoryProviderBase
{
private IHistoryProvider _provider;

public List<HistoryRequest> HistoryRequests { get; } = new();

public override int DataPointCount => _provider.DataPointCount;

public TestHistoryProvider(IHistoryProvider provider)
{
_provider = provider;
}

public override void Initialize(HistoryProviderInitializeParameters parameters)
{
}

public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
requests = requests.ToList();
HistoryRequests.AddRange(requests);
return _provider.GetHistory(requests, sliceTimeZone);
}
}
}
}
Loading