Prevent fill-forwarding outside OpenInterest trading hours (#8662)

* Prevent fill-forwarding outside OpenInterest trading hours

* Solve review comments

* Fix unit test

* Resolve review comments

* Revert changes in FillForwardEnumerator

* Revert unrelated changes

Author: JosueNina

Algorithm.CSharp/OptionOpenInterestRegressionAlgorithm.cs

@@ -122,7 +122,7 @@ public override void OnOrderEvent(OrderEvent orderEvent)
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public int AlgorithmHistoryDataPoints => 146806;
+        public int AlgorithmHistoryDataPoints => 77028;
 
         /// <summary>
         /// Final status of the algorithm

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -165,7 +165,8 @@ private Subscription CreateSubscription(HistoryRequest request)
                 }
 
                 var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
-                reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
+                var exchange = GetSecurityExchange(security.Exchange, request.DataType, request.Symbol);
+                reader = new FillForwardEnumerator(reader, exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
             }
 
             // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire

Engine/HistoricalData/SynchronizingHistoryProvider.cs

@@ -35,6 +35,10 @@ namespace QuantConnect.Lean.Engine.HistoricalData
     /// </summary>
     public abstract class SynchronizingHistoryProvider : HistoryProviderBase
     {
+        /// <summary>
+        /// The market hours database
+        /// </summary>
+        protected static readonly MarketHoursDatabase MarketHours = MarketHoursDatabase.FromDataFolder();
         private int _dataPointCount;
 
         /// <summary>
@@ -116,6 +120,25 @@ protected IEnumerable<Slice> CreateSliceEnumerableFromSubscriptions(List<Subscri
             }
         }
 
+        /// <summary>
+        /// Retrieves the appropriate <see cref="SecurityExchange"/> based on the data type and symbol.
+        /// </summary>
+        /// <param name="exchange">The default exchange instance.</param>
+        /// <param name="dataType">The type of data being processed.</param>
+        /// <param name="symbol">The security symbol.</param>
+        /// <returns>The security exchange with appropriate market hours.</returns>
+        protected static SecurityExchange GetSecurityExchange(SecurityExchange exchange, Type dataType, Symbol symbol)
+        {
+            if (dataType == typeof(OpenInterest))
+            {
+                // Retrieve the original market hours, which include holidays and closed days.
+                var originalExchangeHours = MarketHours.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+                // Use the original market hours to prevent fill-forwarding on non-trading hours.
+                return new SecurityExchange(originalExchangeHours);
+            }
+            return exchange;
+        }
+
         /// <summary>
         /// Creates a subscription to process the history request
         /// </summary>
@@ -154,7 +177,8 @@ protected Subscription CreateSubscription(HistoryRequest request, IEnumerable<Ba
                 }
 
                 var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
-                reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, end, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
+                var exchange = GetSecurityExchange(security.Exchange, request.DataType, request.Symbol);
+                reader = new FillForwardEnumerator(reader, exchange, readOnlyRef, request.IncludeExtendedMarketHours, end, config.Increment, config.DataTimeZone, useDailyStrictEndTimes, request.DataType);
             }
 
             var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);

Tests/Algorithm/AlgorithmHistoryTests.cs

@@ -1749,7 +1749,7 @@ public void GetsHistoryWithGivenBarType(Language language)
                 Assert.AreEqual(132104, tickHistory.Count());
 
                 var openInterestHistory = algorithm.History<OpenInterest>(twxSymbol, twxHistoryStart, twxHistoryEnd);
-                Assert.AreEqual(1050, openInterestHistory.Count());
+                Assert.AreEqual(391, openInterestHistory.Count());
             }
             else
             {
@@ -1789,7 +1789,7 @@ def getOpenInterestHistory(algorithm, symbol, start, end):
                     Assert.AreEqual(132104, tickHistory.shape[0].As<int>());
 
                     dynamic openInterestHistory = getOpenInterestHistory(algorithm, twxSymbol, twxHistoryStart, twxHistoryEnd);
-                    Assert.AreEqual(1050, openInterestHistory.shape[0].As<int>());
+                    Assert.AreEqual(391, openInterestHistory.shape[0].As<int>());
                 }
             }
         }
@@ -3595,6 +3595,37 @@ public void HistoryHandlesSymbolChangedEventsCorrectly(bool useCreateSymbol)
             }
         }
 
+        [Test]
+        public void OpenInterestHistoryOnlyContainsDataDuringRegularTradingHours()
+        {
+            var start = new DateTime(2013, 12, 01);
+            _algorithm = GetAlgorithm(start);
+            _algorithm.SetEndDate(2013, 12, 31);
+
+            // Add ES (E-mini S&P 500)
+            var future = _algorithm.AddFuture("ES", Resolution.Daily, Market.CME);
+
+            var history = _algorithm.History<OpenInterest>(future.Symbol, new DateTime(2013, 10, 10), new DateTime(2013, 11, 01), Resolution.Daily).ToList();
+
+            /* Expected 16 trading days breakdown:
+                October 2013:
+                10(Thu), 11(Fri),
+                14(Mon), 15(Tue), 16(Wed), 17(Thu), 18(Fri),
+                21(Mon), 22(Tue), 23(Wed), 24(Thu), 25(Fri),
+                28(Mon), 29(Tue), 30(Wed), 31(Thu)
+            */
+            Assert.AreEqual(16, history.Count);
+
+            // Regular trading hours: Monday-Friday 9:30am-5:00pm ET
+            foreach (var data in history)
+            {
+                var date = data.EndTime;
+                var dayOfWeek = date.DayOfWeek;
+                Assert.AreNotEqual(DayOfWeek.Saturday, dayOfWeek);
+                Assert.AreNotEqual(DayOfWeek.Sunday, dayOfWeek);
+            }
+        }
+
         public class CustomUniverseData : BaseDataCollection
         {
             public decimal Weight { get; private set; }