From 5ba05b02c0dfc9b44ea8508994b25000c25598c1 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 11:13:05 -0800 Subject: [PATCH 1/6] fix flakiness due to failure to configure HTTPS endpoint due to lack of certificate --- .../B2CWebAppCallsWebApiLocally.cs | 4 +-- .../WebAppUiTests/TestingWebAppLocally.cs | 6 ++-- .../E2E Tests/WebAppUiTests/UiTestHelpers.cs | 36 +++++++++++-------- .../WebAppCallsApiCallsGraphLocally.cs | 16 ++++----- .../TestConstants.cs | 3 +- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs index a8f58f4e6..c5449e88a 100644 --- a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs @@ -55,7 +55,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { {"ASPNETCORE_ENVIRONMENT", "Development"}, {"AzureAdB2C__ClientSecret", clientSecret}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + TodoListClientPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + TodoListClientPort} }; // Get email and password from keyvault. @@ -92,7 +92,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { try { - await page.GotoAsync(TC.LocalhostUrl + TodoListClientPort); + await page.GotoAsync(TC.LocalhostHttpUrl + TodoListClientPort); break; } catch (PlaywrightException ex) diff --git a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs index 4959fa83f..1895d1131 100644 --- a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs @@ -21,7 +21,7 @@ namespace WebAppUiTests; [CollectionDefinition(nameof(UiTestNoParallelization), DisableParallelization = true)] public class TestingWebAppLocally : IClassFixture { - private const string UrlString = "https://localhost:5001/MicrosoftIdentity/Account/signin"; + private const string UrlString = "http://localhost:5001/MicrosoftIdentity/Account/signin"; private const string TraceFileClassName = "TestingWebAppLocally"; private const string TraceFileClassNameCiam = "TestingWebAppLocallyCiam"; private readonly ITestOutputHelper _output; @@ -47,8 +47,8 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordAsync } [Theory] - [InlineData("https://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority - [InlineData("https://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority + [InlineData("http://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority + [InlineData("http://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority [SupportedOSPlatform("windows")] public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailWithCiamPasswordAsync(string authority) { diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 15ceff437..1b1c8a7ca 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -181,26 +181,32 @@ public static Process StartProcessLocally( } else { - // Log the output and error streams - process.OutputDataReceived += (sender, e) => - { - output.WriteLine($"{process.Id} "); - output.WriteLine(e?.Data ?? "null output data received."); - }; - - process.ErrorDataReceived += (sender, e) => - { - output.WriteLine($"{process.Id} "); - output.WriteLine(e?.Data ?? "null error data received."); - }; - - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); + LogOutputErrorStreams(process, output); return process; } } + /// + /// Logs the output and error streams of a process. + /// + /// + /// + private static void LogOutputErrorStreams(Process process, ITestOutputHelper output) + { + process.OutputDataReceived += (sender, e) => + { + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); + }; + process.ErrorDataReceived += (sender, e) => + { + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); + }; + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + } + /// /// Builds the path to the process's directory /// diff --git a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs index 073848bf2..2916fca3b 100644 --- a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs @@ -49,7 +49,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds var grpcEnvVars = new Dictionary { {"ASPNETCORE_ENVIRONMENT", "Development"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + GrpcPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + GrpcPort} }; var serviceEnvVars = new Dictionary { @@ -59,7 +59,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds var clientEnvVars = new Dictionary { {"ASPNETCORE_ENVIRONMENT", "Development"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + TodoListClientPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + TodoListClientPort} }; Dictionary? processes = null; @@ -109,7 +109,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds // Sign out _output.WriteLine("Starting web app sign-out flow."); await page.GetByRole(AriaRole.Link, new() { Name = "Sign out" }).ClickAsync(); - await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostUrl + TodoListClientPort + SignOutPageUriPath, _output); + await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostHttpUrl + TodoListClientPort + SignOutPageUriPath, _output); _output.WriteLine("Web app sign out successful."); // Sign in again using Todo List button @@ -179,8 +179,8 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } [Theory] - [InlineData("https://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority - [InlineData("https://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority + [InlineData("http://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority + [InlineData("http://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority [SupportedOSPlatform("windows")] public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds_CallsDownStreamApiWithCiamAsync(string authority) { @@ -198,7 +198,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds {"AzureAd__ClientId", "b244c86f-ed88-45bf-abda-6b37aa482c79"}, {"AzureAd__Authority", authority}, {"DownstreamApi__Scopes__0", "api://634de702-3173-4a71-b336-a4fab786a479/.default"}, - {TC.KestrelEndpointEnvVar, TC.HttpsStarColon + WebAppCiamPort} + {TC.KestrelEndpointEnvVar, TC.HttpStarColon + WebAppCiamPort} }; Dictionary? processes = null; @@ -245,7 +245,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds // Sign out _output.WriteLine("Starting web app sign-out flow."); await page.GetByRole(AriaRole.Link, new() { Name = "Sign out" }).ClickAsync(); - await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostUrl + WebAppCiamPort + SignOutPageUriPath, _output); + await UiTestHelpers.PerformSignOut_MicrosoftIdFlowAsync(page, email, TC.LocalhostHttpUrl + WebAppCiamPort + SignOutPageUriPath, _output); _output.WriteLine("Web app sign out successful."); // Sign in again using Todo List button @@ -341,7 +341,7 @@ private async Task NavigateToWebAppAsync(IBrowserContext context, uint po { try { - await page.GotoAsync(TC.LocalhostUrl + port); + await page.GotoAsync(TC.LocalhostHttpUrl + port); break; } catch (PlaywrightException ex) diff --git a/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs b/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs index 50a612196..d9fe04f34 100644 --- a/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs +++ b/tests/Microsoft.Identity.Web.Test.Common/TestConstants.cs @@ -175,7 +175,8 @@ public static class TestConstants public const string PasswordText = "Password"; public const string TodoTitle1 = "Testing create todo item"; public const string TodoTitle2 = "Testing edit todo item"; - public const string LocalhostUrl = @"https://localhost:"; + public const string LocalhostHttpsUrl = @"https://localhost:"; + public const string LocalhostHttpUrl = @"http://localhost:"; public const string KestrelEndpointEnvVar = "Kestrel:Endpoints:Http:Url"; public const string HttpStarColon = "http://*:"; public const string HttpsStarColon = "https://*:"; From 993c4a9c0a552e257503c66ce26ab1eab06539d6 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 11:28:40 -0800 Subject: [PATCH 2/6] close std out/error on the processes now that we're logging them --- .../E2E Tests/WebAppUiTests/UiTestHelpers.cs | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 1b1c8a7ca..ee68b527b 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -173,7 +173,7 @@ public static Process StartProcessLocally( { Thread.Sleep(1000 * currentAttempt++); // linear backoff process = Process.Start(processStartInfo); - } while (currentAttempt++ <= maxRetries && ProcessIsAlive(process)); + } while (currentAttempt++ <= maxRetries && !ProcessIsAlive(process)); if (process == null) { @@ -181,32 +181,19 @@ public static Process StartProcessLocally( } else { - LogOutputErrorStreams(process, output); + process.OutputDataReceived += (sender, e) => + { + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); + }; + process.ErrorDataReceived += (sender, e) => + { + output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); + }; return process; } } - /// - /// Logs the output and error streams of a process. - /// - /// - /// - private static void LogOutputErrorStreams(Process process, ITestOutputHelper output) - { - process.OutputDataReceived += (sender, e) => - { - output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); - }; - process.ErrorDataReceived += (sender, e) => - { - output.WriteLine(e?.Data == null ? "null output data received." : $"{process.Id} {e.Data}"); - }; - - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - } - /// /// Builds the path to the process's directory /// @@ -271,6 +258,10 @@ public static void KillProcessTrees(Queue processQueue) { processQueue.Enqueue(child); } + _ = currentProcess.WaitForExitAsync(); + currentProcess.StandardOutput.Close(); + currentProcess.StandardError.Close(); + currentProcess.Kill(); currentProcess.Close(); } From e7ebc1b9171497155cc8059c93d5a92c44e28258 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 16:21:11 -0800 Subject: [PATCH 3/6] trial https for webapp tests --- tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs index 1895d1131..4959fa83f 100644 --- a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs @@ -21,7 +21,7 @@ namespace WebAppUiTests; [CollectionDefinition(nameof(UiTestNoParallelization), DisableParallelization = true)] public class TestingWebAppLocally : IClassFixture { - private const string UrlString = "http://localhost:5001/MicrosoftIdentity/Account/signin"; + private const string UrlString = "https://localhost:5001/MicrosoftIdentity/Account/signin"; private const string TraceFileClassName = "TestingWebAppLocally"; private const string TraceFileClassNameCiam = "TestingWebAppLocallyCiam"; private readonly ITestOutputHelper _output; @@ -47,8 +47,8 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordAsync } [Theory] - [InlineData("http://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority - [InlineData("http://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority + [InlineData("https://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority + [InlineData("https://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority [SupportedOSPlatform("windows")] public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailWithCiamPasswordAsync(string authority) { From a3538ff88ada5fb270738ce3fe5c872fa646ec4a Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 17:07:17 -0800 Subject: [PATCH 4/6] https for webapp test trial --- .../WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs index 2916fca3b..0da504ff2 100644 --- a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs @@ -179,8 +179,8 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } [Theory] - [InlineData("http://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority - [InlineData("http://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority + [InlineData("https://MSIDLABCIAM6.ciamlogin.com")] // CIAM authority + [InlineData("https://login.msidlabsciam.com/fe362aec-5d43-45d1-b730-9755e60dc3b9/v2.0/")] // CIAM CUD Authority [SupportedOSPlatform("windows")] public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds_CallsDownStreamApiWithCiamAsync(string authority) { @@ -233,6 +233,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } page = await NavigateToWebAppAsync(context, WebAppCiamPort); + _output.WriteLine("WebApp HTML: " + await page.InnerHTMLAsync("html")); // Initial sign in _output.WriteLine("Starting web app sign-in flow."); @@ -258,7 +259,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } catch (Exception ex) { - //Adding guid incase of multiple test runs. This will allow screenshots to be matched to their appropriet test runs. + //Adding guid incase of multiple test runs. This will allow screenshots to be matched to their appropriate test runs. var guid = Guid.NewGuid().ToString(); try { From 9d7dc47e0ce40fa45f47f2b756ece433903aaccb Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 17:08:41 -0800 Subject: [PATCH 5/6] rm log for testing purposes only --- tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs index 0da504ff2..341cd9302 100644 --- a/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/WebAppCallsApiCallsGraphLocally.cs @@ -233,7 +233,6 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds } page = await NavigateToWebAppAsync(context, WebAppCiamPort); - _output.WriteLine("WebApp HTML: " + await page.InnerHTMLAsync("html")); // Initial sign in _output.WriteLine("Starting web app sign-in flow."); From a776bd940292be562d551140017d3a4aa60937ac Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Wed, 8 Jan 2025 18:23:16 -0800 Subject: [PATCH 6/6] await the process.WaitForExitAsync call --- tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs | 2 +- tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs index c5449e88a..066d082cb 100644 --- a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs @@ -167,7 +167,7 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() Queue processes = new Queue(); if (serviceProcess != null) { processes.Enqueue(serviceProcess); } if (clientProcess != null) { processes.Enqueue(clientProcess); } - UiTestHelpers.KillProcessTrees(processes); + await UiTestHelpers.KillProcessTreesAsync(processes); // Stop tracing and export it into a zip archive. string path = UiTestHelpers.GetTracePath(_testAssemblyPath, TraceFileName); diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index ee68b527b..6b99eb886 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -245,7 +245,7 @@ public static string GetTracePath(string testAssemblyLocation, string traceName) /// /// queue of parent processes [SupportedOSPlatform("windows")] - public static void KillProcessTrees(Queue processQueue) + public static async Task KillProcessTreesAsync(Queue processQueue) { Process currentProcess; while (processQueue.Count > 0) @@ -258,7 +258,7 @@ public static void KillProcessTrees(Queue processQueue) { processQueue.Enqueue(child); } - _ = currentProcess.WaitForExitAsync(); + await currentProcess.WaitForExitAsync(); currentProcess.StandardOutput.Close(); currentProcess.StandardError.Close();