Improve CrumbHelper and expand YahooClient tests

ooples · ooples · commit 780eabae1c08 · 2025-12-31T13:14:34.000-05:00
diff --git a/src/Helpers/CrumbHelper.cs b/src/Helpers/CrumbHelper.cs
@@ -1,10 +1,11 @@
-﻿// CrumbHelper.cs
+// CrumbHelper.cs
 //  Andrew Baylis
 //  Created: 29/10/2024
 
 #region using
 
 using System.Runtime.CompilerServices;
+using System.Net;
 
 #endregion
 
@@ -19,13 +20,19 @@ internal sealed class CrumbHelper
     private static CrumbHelper? _instance;
 
     internal static HttpMessageHandler? handler;
-    private List<string> cookies = [];
+    private static HttpClientHandler? _clientHandler;
 
     #endregion
 
+    static CrumbHelper()
+    {
+#if NET48
+        ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
+#endif
+    }
+
     private CrumbHelper()
     {
-        handler = GetClientHandler();
         Crumb = string.Empty;
     }
 
@@ -50,13 +57,19 @@ private static CrumbHelper Instance
 
     public static HttpClient GetHttpClient()
     {
-        HttpClient client = new(handler ?? GetClientHandler());
-        client.DefaultRequestHeaders.Add("Cookie", Instance.cookies);
-        client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
+        var h = handler ?? GetClientHandler();
+        HttpClient client = new(h, disposeHandler: false);
+
+        client.DefaultRequestHeaders.Add("User-Agent",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+            "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
         client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
         client.DefaultRequestHeaders.Add("Connection", "keep-alive");
         client.DefaultRequestHeaders.Add("Pragma", "no-cache");
         client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
+        client.DefaultRequestHeaders.Add("Accept",
+            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
 
         return client;
     }
@@ -73,9 +86,26 @@ public static async Task<CrumbHelper> GetInstance(bool setCrumb = true)
 
     private static HttpClientHandler GetClientHandler()
     {
-        return YahooClient.IsThrottled
-            ? new DownloadThrottleQueueHandler(40, TimeSpan.FromMinutes(1),4) //40 calls in a minute, no more than 4 simultaneously
+        if (_clientHandler != null) return _clientHandler;
+
+        var h = YahooClient.IsThrottled
+            ? new DownloadThrottleQueueHandler(40, TimeSpan.FromMinutes(1), 4) //40 calls in a minute, no more than 4 simultaneously
             : new HttpClientHandler();
+
+        if (h is HttpClientHandler httpClientHandler)
+        {
+            httpClientHandler.AllowAutoRedirect = true;
+            httpClientHandler.UseCookies = true; // Ensure cookies are used
+            // net48 doesn't support DecompressionMethods.All (introduced in .NET 7)
+#if NET48
+            httpClientHandler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
+#else
+            httpClientHandler.AutomaticDecompression = DecompressionMethods.All;
+#endif
+        }
+        
+        _clientHandler = h;
+        return h;
     }
 
     #endregion
@@ -84,22 +114,13 @@ private static HttpClientHandler GetClientHandler()
 
     public async Task SetCrumbAsync()
     {
-        var client = GetHttpClient();
-        var loginResponse = await client.GetAsync("https://login.yahoo.com/");
+        using var client = GetHttpClient();
+
+        using var crumbResponse = await client.GetAsync("https://query1.finance.yahoo.com/v1/test/getcrumb");
 
-        if (loginResponse.IsSuccessStatusCode)
+        if (crumbResponse.IsSuccessStatusCode)
         {
-            var login = await loginResponse.Content.ReadAsStringAsync();
-            if (loginResponse.Headers.TryGetValues("Set-Cookie", out var setCookie))
-            {
-                cookies = new List<string>(setCookie.Where(c => c.ToLower().IndexOf("domain=.yahoo.com") > 0));
-                var crumbResponse = await client.GetAsync("https://query1.finance.yahoo.com/v1/test/getcrumb");
-
-                if (crumbResponse.IsSuccessStatusCode)
-                {
-                    Crumb = await crumbResponse.Content.ReadAsStringAsync();
-                }
-            }
+            Crumb = await crumbResponse.Content.ReadAsStringAsync();
         }
 
         if (string.IsNullOrEmpty(Crumb))
@@ -112,11 +133,19 @@ public async Task SetCrumbAsync()
 
     #region Internal Methods
 
-    internal void Destroy()
+    internal static void Destroy()
     {
         _instance = null;
     }
 
+    internal static void Reset()
+    {
+        _instance = null;
+        handler = null;
+        _clientHandler?.Dispose();
+        _clientHandler = null;
+    }
+
     #endregion
 }
 
@@ -139,6 +168,13 @@ public DownloadThrottleQueueHandler(int maxPerPeriod, TimeSpan maxPeriod, int ma
         _throttleLoad = new SemaphoreSlim(maxParallel, maxParallel);
         _throttleRate = new SemaphoreSlim(maxPerPeriod, maxPerPeriod);
         _maxPeriod = maxPeriod;
+        AllowAutoRedirect = true;
+        // net48 doesn't support DecompressionMethods.All (introduced in .NET 7)
+#if NET48
+        AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
+#else
+        AutomaticDecompression = DecompressionMethods.All;
+#endif
     }
 
     #region Override Methods
@@ -164,4 +200,4 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
     }
 
     #endregion
-}
+}
diff --git a/src/Helpers/DownloadHelper.cs b/src/Helpers/DownloadHelper.cs
@@ -90,7 +90,7 @@ private static async Task<string> DownloadRawDataAsync(string urlString)
             }
             else
             {
-                (await CrumbHelper.GetInstance(false)).Destroy();
+                CrumbHelper.Destroy();
 
                 throw response.StatusCode switch
                 {
diff --git a/tests/TestConsoleApp/Program.cs b/tests/TestConsoleApp/Program.cs
@@ -12,6 +12,7 @@
 
     var historicalDataList = await yahooClient.GetHistoricalDataAsync(symbol, DataFrequency.Daily, startDate);
     var chartInfoList = await yahooClient.GetChartInfoAsync("GOOG", TimeRange._1Year, TimeInterval._1Day);
+    var data = await yahooClient.GetChartInfoAsync("CAPE", TimeRange._1Day, TimeInterval._1Day);
     Console.WriteLine();
     //var capitalGainList = await yahooClient.GetCapitalGainDataAsync(symbol, DataFrequency.Monthly, startDate);
     //var dividendList = await yahooClient.GetDividendDataAsync(symbol, DataFrequency.Weekly, startDate);
diff --git a/tests/UnitTests/Usings.cs b/tests/UnitTests/Usings.cs
@@ -3,3 +3,5 @@
 global using OoplesFinance.YahooFinanceAPI.Enums;
 global using FluentAssertions;
 global using System.Net.Http;
+
+[assembly: CollectionBehavior(DisableTestParallelization = true)]
diff --git a/tests/UnitTests/YahooClientTests.cs b/tests/UnitTests/YahooClientTests.cs
@@ -4,9 +4,10 @@
 
 namespace OoplesFinance.YahooFinanceAPI.Tests.Unit;
 
-public sealed class YahooClientTests
+public sealed class YahooClientTests : IDisposable
 {
     private readonly YahooClient _sut;
+    private readonly Mock<HttpMessageHandler> _mockHandler;
     private const string BadSymbol = "OOPLES";
     private const string GoodSymbol = "MSFT";
     private const string GoodFundSymbol = "VTSAX";
@@ -19,13 +20,119 @@ public sealed class YahooClientTests
 
     public YahooClientTests()
     {
+        _mockHandler = new Mock<HttpMessageHandler>();
+        OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.handler = _mockHandler.Object;
+        SetupMockResponses();
+
         _sut = new YahooClient();
         _startDate = DateTime.Now.AddMonths(-1);
         _emptySymbols = [];
         _tooManySymbols = Enumerable.Repeat(GoodSymbol, 255);
         _goodSymbols = Enumerable.Repeat(GoodSymbol, 20);
     }
 
+    private void SetupMockResponses()
+    {
+        // Crumb
+        _mockHandler.SetupRequest(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb")
+            .ReturnsResponse(HttpStatusCode.OK, "test_crumb");
+
+        // Chart / Historical (Fixed braces)
+        var chartJson = @"{""chart"":{""result"":[{""timestamp"":[1600000000],""indicators"":{""quote"":[{""close"":[100.0],""open"":[100.0],""high"":[100.0],""low"":[100.0],""volume"":[1000]}],""adjclose"":[{""adjclose"":[100.0]}]},""events"":{""dividends"":{""1600000000"":{""amount"":0.5,""date"":1600000000}},""splits"":{""1600000000"":{""numerator"":2,""denominator"":1,""splitRatio"":""2:1"",""date"":1600000000}}}}]}}";
+        
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains($"/v8/finance/chart/{GoodSymbol}"))
+            .ReturnsResponse(HttpStatusCode.OK, chartJson);
+
+        // Screener / Trending
+        var screenerJson = @"{""finance"":{""result"":[{""quotes"":[{""symbol"":""MSFT"",""language"":""en"",""region"":""US"",""typeDisp"":""Equity"",""quoteType"":""EQUITY""}]}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/finance/screener"))
+            .ReturnsResponse(HttpStatusCode.OK, screenerJson);
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/finance/trending"))
+            .ReturnsResponse(HttpStatusCode.OK, screenerJson);
+
+        // QuoteSummary (Stats)
+        var quoteSummaryJson = @"{
+            ""quoteSummary"": {
+                ""result"": [{
+                    ""defaultKeyStatistics"": { ""enterpriseValue"": { ""raw"": 100 } },
+                    ""summaryDetail"": { ""maxAge"": 1 },
+                    ""financialData"": { ""currentPrice"": { ""raw"": 100 } },
+                    ""insiderHolders"": { ""holders"": [{""name"":""Holder""}] },
+                    ""insiderTransactions"": { ""transactions"": [{""filerName"":""Filer""}] },
+                    ""institutionOwnership"": { ""ownershipList"": [{""organization"":""Org""}] },
+                    ""fundOwnership"": { ""ownershipList"": [{""organization"":""Fund""}] },
+                    ""majorDirectHolders"": { ""holders"": [{""holder"":""Major""}] },
+                    ""secFilings"": { ""filings"": [{""date"":""2020-01-01""}] },
+                    ""majorHoldersBreakdown"": { ""insidersPercentHeld"": { ""raw"": 0.1 } },
+                    ""esgScores"": { ""totalEsg"": { ""raw"": 10 } },
+                    ""recommendationTrend"": { ""trend"": [{""period"":""0m""}] },
+                    ""indexTrend"": { ""symbol"": ""SP500"", ""peRatio"": { ""raw"": 20 } },
+                    ""sectorTrend"": { ""symbol"": ""Sector"" },
+                    ""earningsTrend"": { ""trend"": [{""period"":""0q""}] },
+                    ""assetProfile"": { ""address1"": ""1 Way"" },
+                    ""fundProfile"": { ""family"": ""FundFamily"" },
+                    ""calendarEvents"": { ""earnings"": { ""earningsDate"": [{""raw"":1600000000}] } },
+                    ""earnings"": { ""financialsChart"": { ""yearly"": [{""date"":2020}], ""quarterly"": [{""date"":""3Q""}] } },
+                    ""balanceSheetHistory"": { ""balanceSheetStatements"": [{""cash"":{""raw"":100}}] },
+                    ""cashflowStatementHistory"": { ""cashflowStatements"": [{""netIncome"":{""raw"":100}}] },
+                    ""incomeStatementHistory"": { ""incomeStatementHistory"": [{""totalRevenue"":{""raw"":100}}] },
+                    ""earningsHistory"": { ""history"": [{""epsActual"":{""raw"":1}}] },
+                    ""quoteType"": { ""symbol"": ""MSFT"" },
+                    ""price"": { ""regularMarketPrice"": { ""raw"": 100 } },
+                    ""netSharePurchaseActivity"": { ""buyInfoCount"": { ""raw"": 10 } },
+                    ""incomeStatementHistoryQuarterly"": { ""incomeStatementHistory"": [{""totalRevenue"":{""raw"":100}}] },
+                    ""cashflowStatementHistoryQuarterly"": { ""cashflowStatements"": [{""netIncome"":{""raw"":100}}] },
+                    ""balanceSheetHistoryQuarterly"": { ""balanceSheetStatements"": [{""cash"":{""raw"":100}}] },
+                    ""upgradeDowngradeHistory"": { ""history"": [{""firm"":""Firm"",""action"":""Up""}] }
+                }]
+            }
+        }";
+        // Match GoodSymbol and GoodFundSymbol explicitly
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains($"/quoteSummary/{GoodSymbol}"))
+            .ReturnsResponse(HttpStatusCode.OK, quoteSummaryJson);
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains($"/quoteSummary/{GoodFundSymbol}"))
+            .ReturnsResponse(HttpStatusCode.OK, quoteSummaryJson);
+
+        // RealTimeQuote
+        var realTimeJson = @"{""quoteResponse"":{""result"":[{""symbol"":""MSFT"",""regularMarketPrice"":100.0}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/v7/finance/quote") && r.RequestUri.ToString().Contains(GoodSymbol))
+            .ReturnsResponse(HttpStatusCode.OK, realTimeJson);
+
+        // Market Summary
+        var marketSummaryJson = @"{""marketSummaryResponse"":{""result"":[{""symbol"":""^GSPC"",""regularMarketPrice"":{""raw"":100}}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/marketSummary"))
+            .ReturnsResponse(HttpStatusCode.OK, marketSummaryJson);
+
+        // AutoComplete
+        var autoCompleteJson = @"{""resultSet"":{""Result"":[{""symbol"":""MSFT""}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/autocomplete") && !r.RequestUri.ToString().Contains("string.Empty"))
+            .ReturnsResponse(HttpStatusCode.OK, autoCompleteJson);
+
+        // SparkChart
+        var sparkJson = @"{""spark"":{""result"":[{""symbol"":""MSFT"",""response"":[{""timestamp"":[1600000000],""indicators"":{""quote"":[{""close"":[100.0]}]}}]}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains("/finance/spark") && r.RequestUri.ToString().Contains(GoodSymbol))
+            .ReturnsResponse(HttpStatusCode.OK, sparkJson);
+
+        // Insights (Fixed shortTerm and added reports)
+        var insightsJson = @"{""finance"":{""result"":{""instrumentInfo"":{""technicalEvents"":{""shortTerm"":""Bearish""}},""reports"":[{""id"":""1"",""title"":""Report""}]}}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains($"/insights?symbol={GoodSymbol}"))
+            .ReturnsResponse(HttpStatusCode.OK, insightsJson);
+        
+        // Recommendations
+        var recommendJson = @"{""finance"":{""result"":[{""symbol"":""MSFT"",""recommendedSymbols"":[{""symbol"":""AAPL""}]}]}}";
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains($"/recommendationsbysymbol/{GoodSymbol}"))
+            .ReturnsResponse(HttpStatusCode.OK, recommendJson);
+
+        // Bad Symbol Handling - Return 404 which usually triggers empty/null parsing or exception in Helpers
+        _mockHandler.SetupRequest(HttpMethod.Get, r => r.RequestUri.ToString().Contains(BadSymbol))
+            .ReturnsResponse(HttpStatusCode.NotFound, "{}");
+    }
+
+    public void Dispose()
+    {
+        OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.Reset();
+    }
+
     [Fact]
     public async Task GetHistoricalData_ThrowsException_WhenNoSymbolIsFound()
     {
@@ -1460,10 +1567,10 @@ public async Task GetRealTimeQuotes_ThrowsException_WhenNoSymbolIsFound()
         // Arrange
 
         // Act
-        var result = await _sut.GetRealTimeQuotesAsync(BadSymbol);
+        var result = async () => await _sut.GetRealTimeQuotesAsync(BadSymbol);
 
         // Assert
-        result.Should().BeNull();
+        await result.Should().ThrowAsync<InvalidOperationException>().WithMessage("Requested Information Not Available On Yahoo Finance");
     }
 
     [Fact]
@@ -2442,22 +2549,15 @@ public async Task CreateCrumbHelpInstance_ReturnsValidCrumb()
     public async Task CreateCrumbHelpInstance_ThrowsException_WhenFetchCrumbFailed()
     {
         // Arrange
-        var mockHandler = new Mock<HttpMessageHandler>(MockBehavior.Default);
-        mockHandler.SetupRequest(HttpMethod.Get, "https://login.yahoo.com/")
-            .ReturnsJsonResponse(HttpStatusCode.OK, "");
+        var mockHandler = new Mock<HttpMessageHandler>();
         mockHandler.SetupRequest(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb")
-            .ReturnsJsonResponse(HttpStatusCode.OK, "");
+            .ReturnsResponse(HttpStatusCode.NotFound, "");
+        OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.handler = mockHandler.Object;
 
         // Act
-        Helpers.CrumbHelper.handler = mockHandler.Object;
-        using var client = Helpers.CrumbHelper.GetHttpClient();
         var ex = await Record.ExceptionAsync(()=>Helpers.CrumbHelper.GetInstance(true));
-
+        
         // Assert
         ex.Should().NotBeNull();
-        ex.Message.Should().Be("Failed to get crumb");
-
-        OoplesFinance.YahooFinanceAPI.Helpers.CrumbHelper.handler = new HttpClientHandler();
-        
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ private static async Task<string> DownloadRawDataAsync(string urlString)`
`90`	`90`	`}`
`91`	`91`	`else`
`92`	`92`	`{`
`93`		`- (await CrumbHelper.GetInstance(false)).Destroy();`
	`93`	`+ CrumbHelper.Destroy();`
`94`	`94`
`95`	`95`	`throw response.StatusCode switch`
`96`	`96`	`{`