Disable extended market hours for universe history requests (#8670)

jhonabreul · web-flow · commit 68ec0b72b263 · 2025-04-02T14:11:38.000-03:00
* Disable extended market hours for universe history requests

* Exclude extended market hours for all chain universe history requests

* Disable extended market hours for chain universes

* Minor change

* Minor fix

* Minor change
diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs
@@ -2017,7 +2017,11 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool f
                             continuousContractSymbol.ID.Symbol,
                             continuousContractSymbol.ID.SecurityType,
                             security.Exchange.Hours);
-                        AddUniverse(new ContinuousContractUniverse(security, continuousUniverseSettings, LiveMode, new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
+                        AddUniverse(new ContinuousContractUniverse(security, continuousUniverseSettings, LiveMode,
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol,
+                                // We can use any data type here, since we are not going to use the data.
+                                // We just don't want to use the FutureUniverse type because it will force disable extended market hours
+                                objectType: typeof(Tick), extendedHours: extendedMarketHours)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }
@@ -2372,7 +2376,10 @@ public Option AddOptionContract(Symbol symbol, Resolution? resolution = null, bo
                     Resolution = underlyingConfigs.GetHighestResolution(),
                     ExtendedMarketHours = extendedMarketHours
                 };
-                universe = AddUniverse(new OptionContractUniverse(new SubscriptionDataConfig(configs.First(), symbol: universeSymbol), settings));
+                universe = AddUniverse(new OptionContractUniverse(new SubscriptionDataConfig(configs.First(),
+                    // We can use any data type here, since we are not going to use the data.
+                    // We just don't want to use the OptionUniverse type because it will force disable extended market hours
+                    symbol: universeSymbol, objectType: typeof(Tick), extendedHours: extendedMarketHours), settings));
             }
 
             // update the universe
@@ -3560,8 +3567,8 @@ private RestResponse SendBroadcast(string typeName, Dictionary<string, object> p
             {
                 payload["$type"] = typeName;
             }
-            return _api.BroadcastLiveCommand(Globals.OrganizationID, 
-                AlgorithmMode == AlgorithmMode.Live ? ProjectId : null, 
+            return _api.BroadcastLiveCommand(Globals.OrganizationID,
+                AlgorithmMode == AlgorithmMode.Live ? ProjectId : null,
                 payload);
         }
 
diff --git a/Common/Data/HistoryRequest.cs b/Common/Data/HistoryRequest.cs
@@ -18,6 +18,7 @@
 using NodaTime;
 using QuantConnect.Securities;
 using System.Collections.Generic;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data
 {
@@ -27,6 +28,7 @@ namespace QuantConnect.Data
     public class HistoryRequest : BaseDataRequest
     {
         private Resolution? _fillForwardResolution;
+        private bool _includeExtendedMarketHours;
 
         /// <summary>
         /// Gets the symbol to request data for
@@ -57,7 +59,17 @@ public Resolution? FillForwardResolution
         /// <summary>
         /// Gets whether or not to include extended market hours data, set to false for only normal market hours
         /// </summary>
-        public bool IncludeExtendedMarketHours { get; set; }
+        public bool IncludeExtendedMarketHours
+        {
+            get
+            {
+                return _includeExtendedMarketHours;
+            }
+            set
+            {
+                _includeExtendedMarketHours = value && LeanData.SupportsExtendedMarketHours(DataType);
+            }
+        }
 
         /// <summary>
         /// Gets the time zone of the time stamps on the raw input data
diff --git a/Common/Data/HistoryRequestFactory.cs b/Common/Data/HistoryRequestFactory.cs
@@ -15,6 +15,7 @@
 
 using System;
 using NodaTime;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
 using QuantConnect.Util;
@@ -166,8 +167,8 @@ public DateTime GetStartTimeAlgoTz(
             bool? extendedMarketHours = null)
         {
             var isExtendedMarketHours = false;
-            // hour resolution does no have extended market hours data
-            if (resolution != Resolution.Hour)
+            // hour resolution does no have extended market hours data. Same for chain universes
+            if (resolution != Resolution.Hour && LeanData.SupportsExtendedMarketHours(dataType))
             {
                 if (extendedMarketHours.HasValue)
                 {
diff --git a/Common/Data/SubscriptionDataConfig.cs b/Common/Data/SubscriptionDataConfig.cs
@@ -222,7 +222,7 @@ public SubscriptionDataConfig(Type objectType,
             Resolution = resolution;
             _sid = symbol.ID;
             Symbol = symbol;
-            ExtendedMarketHours = extendedHours;
+            ExtendedMarketHours = extendedHours && LeanData.SupportsExtendedMarketHours(Type);
             PriceScaleFactor = 1;
             IsInternalFeed = isInternalFeed;
             IsCustomData = isCustom;
diff --git a/Common/Util/LeanData.cs b/Common/Util/LeanData.cs
@@ -872,7 +872,8 @@ public static string GenerateZipFileName(string symbol, SecurityType securityTyp
 
             if (tickType == null)
             {
-                if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd) {
+                if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd)
+                {
                     tickType = TickType.Quote;
                 }
                 else
@@ -1589,24 +1590,34 @@ public static bool SetStrictEndTimes(IBaseData baseData, SecurityExchangeHours e
         /// <param name="fileName">File name extracted</param>
         /// <param name="entryName">Entry name extracted</param>
         public static void ParseKey(string key, out string fileName, out string entryName)
-         {
-             // Default scenario, no entryName included in key
-             entryName = null; // default to all entries
-             fileName = key;
-
-             if (key == null)
-             {
-                 return;
-             }
-
-             // Try extracting an entry name; Anything after a # sign
-             var hashIndex = key.LastIndexOf("#", StringComparison.Ordinal);
-             if (hashIndex != -1)
-             {
-                 entryName = key.Substring(hashIndex + 1);
-                 fileName = key.Substring(0, hashIndex);
-             }
-         }
+        {
+            // Default scenario, no entryName included in key
+            entryName = null; // default to all entries
+            fileName = key;
+
+            if (key == null)
+            {
+                return;
+            }
+
+            // Try extracting an entry name; Anything after a # sign
+            var hashIndex = key.LastIndexOf("#", StringComparison.Ordinal);
+            if (hashIndex != -1)
+            {
+                entryName = key.Substring(hashIndex + 1);
+                fileName = key.Substring(0, hashIndex);
+            }
+        }
+
+        /// <summary>
+        /// Helper method to determine if the specified data type supports extended market hours
+        /// </summary>
+        /// <param name="dataType">The data type</param>
+        /// <returns>Whether the specified data type supports extended market hours</returns>
+        public static bool SupportsExtendedMarketHours(Type dataType)
+        {
+            return !dataType.IsAssignableTo(typeof(BaseChainUniverseData));
+        }
 
         /// <summary>
         /// Helper method to aggregate ticks or bars into lower frequency resolutions
diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs
@@ -16,6 +16,7 @@
 
 using NUnit.Framework;
 using QuantConnect.Algorithm;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Cfd;
@@ -26,7 +27,6 @@
 using QuantConnect.Securities.Future;
 using QuantConnect.Securities.IndexOption;
 using QuantConnect.Securities.Option;
-using QuantConnect.Securities.Positions;
 using QuantConnect.Tests.Engine.DataFeeds;
 using System;
 using System.Collections.Generic;
@@ -124,7 +124,16 @@ public void ProperlyAddsFutureWithExtendedMarketHours(
             [ValueSource(nameof(FuturesTestCases))] Func<QCAlgorithm, Security> getFuture)
         {
             var future = _algo.AddFuture(Futures.Indices.VIX, Resolution.Minute, extendedMarketHours: extendedMarketHours);
-            Assert.That(_algo.SubscriptionManager.Subscriptions.Where(x => x.Symbol == future.Symbol).Select(x => x.ExtendedMarketHours),
+            var subscriptions = _algo.SubscriptionManager.Subscriptions.Where(x => x.Symbol == future.Symbol).ToList();
+
+            var universeSubscriptions = subscriptions.Where(x => x.Type == typeof(FutureUniverse)).ToList();
+            Assert.AreEqual(1, universeSubscriptions.Count);
+            // Universe does not support extended market hours
+            Assert.IsFalse(universeSubscriptions[0].ExtendedMarketHours);
+
+            var nonUniverseSubscriptions = subscriptions.Where(x => x.Type != typeof(FutureUniverse)).ToList();
+            Assert.Greater(nonUniverseSubscriptions.Count, 0);
+            Assert.That(nonUniverseSubscriptions.Select(x => x.ExtendedMarketHours),
                 Has.All.EqualTo(extendedMarketHours));
         }
 
diff --git a/Tests/Algorithm/AlgorithmChainsTests.cs b/Tests/Algorithm/AlgorithmChainsTests.cs
@@ -19,9 +19,7 @@
 using NUnit.Framework;
 using Python.Runtime;
 using QuantConnect.Algorithm;
-using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
 using QuantConnect.Tests.Engine.DataFeeds;
@@ -211,37 +209,65 @@ private static PyObject GetCanonicalSubDataFrame(PyObject dataFrame, Symbol symb
         private static IEnumerable<TestCaseData> GetOptionChainApisTestData()
         {
             var indexSymbol = Symbols.SPX;
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 23, 23, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 0, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 1, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 2, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 6, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 12, 0, 0));
-            yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 16, 0, 0));
-
             var equitySymbol = Symbols.GOOG;
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 0, 0, 0));
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 1, 0, 0));
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 2, 0, 0));
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 6, 0, 0));
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 12, 0, 0));
-            yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 16, 0, 0));
-
             var futureSymbol = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 6, 19));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 04, 23, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 0, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 1, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 2, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 6, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 12, 0, 0));
-            yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 16, 0, 0));
+
+            foreach (var withSecurityAdded in new[] { true, false })
+            {
+                var extendedMarketHoursCases = withSecurityAdded ? [true, false] : new[] { false };
+                foreach (var withExtendedMarketHours in extendedMarketHoursCases)
+                {
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 23, 23, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 0, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 1, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 2, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 6, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 12, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(indexSymbol, new DateTime(2015, 12, 24, 16, 0, 0), withSecurityAdded, withExtendedMarketHours);
+
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 0, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 1, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 2, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 6, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 12, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(equitySymbol, new DateTime(2015, 12, 24, 16, 0, 0), withSecurityAdded, withExtendedMarketHours);
+
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 04, 23, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 0, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 1, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 2, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 6, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 12, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 05, 16, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 0, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 1, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 2, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 6, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 12, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                    yield return new TestCaseData(futureSymbol, new DateTime(2020, 01, 06, 16, 0, 0), withSecurityAdded, withExtendedMarketHours);
+                }
+            }
         }
 
         [TestCaseSource(nameof(GetOptionChainApisTestData))]
-        public void OptionChainApisAreConsistent(Symbol symbol, DateTime dateTime)
+        public void OptionChainApisAreConsistent(Symbol symbol, DateTime dateTime, bool withSecurityAdded, bool withExtendedMarketHours)
         {
             _algorithm.SetDateTime(dateTime.ConvertToUtc(_algorithm.TimeZone));
 
+            if (withSecurityAdded)
+            {
+                if (symbol.SecurityType == SecurityType.Future)
+                {
+                    var future = _algorithm.AddFuture(symbol.ID.Symbol, extendedMarketHours: withExtendedMarketHours);
+                    _algorithm.AddFutureOption(future.Symbol);
+                    _algorithm.AddFutureContract(symbol, extendedMarketHours: withExtendedMarketHours);
+                }
+                else
+                {
+                    _algorithm.AddSecurity(symbol, extendedMarketHours: withExtendedMarketHours);
+                }
+            }
+
             var exchange  = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
             var chainFromAlgorithmApi = _algorithm.OptionChain(symbol).Select(x => x.Symbol).ToList();
             var chainFromChainProviderApi = _optionChainProvider.GetOptionContractList(symbol,
@@ -262,26 +288,30 @@ private static IEnumerable<TestCaseData> GetFutureChainApisTestData()
             {
                 foreach (var withFutureAdded in new[] { true, false })
                 {
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 06, 23, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 0, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 1, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 2, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 6, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 12, 0, 0), withFutureAdded);
-                    yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 16, 0, 0), withFutureAdded);
+                    var extendedMarketHoursCases = withFutureAdded ? [true, false] : new[] { false };
+                    foreach (var withExtendedMarketHours in extendedMarketHoursCases)
+                    {
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 06, 23, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 0, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 1, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 2, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 6, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 12, 0, 0), withFutureAdded, withExtendedMarketHours);
+                        yield return new TestCaseData(symbol, new DateTime(2013, 10, 07, 16, 0, 0), withFutureAdded, withExtendedMarketHours);
+                    }
                 }
             }
         }
 
         [TestCaseSource(nameof(GetFutureChainApisTestData))]
-        public void FuturesChainApisAreConsistent(Symbol symbol, DateTime dateTime, bool withFutureAdded)
+        public void FuturesChainApisAreConsistent(Symbol symbol, DateTime dateTime, bool withFutureAdded, bool withExtendedMarketHours)
         {
             _algorithm.SetDateTime(dateTime.ConvertToUtc(_algorithm.TimeZone));
 
             if (withFutureAdded)
             {
                 // It should work regardless of whether the future is added to the algorithm
-                _algorithm.AddFuture("ES");
+                _algorithm.AddFuture(symbol.ID.Symbol, extendedMarketHours: withExtendedMarketHours);
             }
 
             var exchange = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@`
`15`	`15`
`16`	`16`	`using System;`
`17`	`17`	`using NodaTime;`
	`18`	`+using QuantConnect.Data.UniverseSelection;`
`18`	`19`	`using QuantConnect.Interfaces;`
`19`	`20`	`using QuantConnect.Securities;`
`20`	`21`	`using QuantConnect.Util;`
`@@ -166,8 +167,8 @@ public DateTime GetStartTimeAlgoTz(`
`166`	`167`	`bool? extendedMarketHours = null)`
`167`	`168`	`{`
`168`	`169`	`var isExtendedMarketHours = false;`
`169`		`- // hour resolution does no have extended market hours data`
`170`		`- if (resolution != Resolution.Hour)`
	`170`	`+ // hour resolution does no have extended market hours data. Same for chain universes`
	`171`	`+ if (resolution != Resolution.Hour && LeanData.SupportsExtendedMarketHours(dataType))`
`171`	`172`	`{`
`172`	`173`	`if (extendedMarketHours.HasValue)`
`173`	`174`	`{`