Re-use the Blazor app for stress runs (#21078)

* Re-use the Blazor app for stress runs

Our current stress runs re-used the perf tests which recreated the blazor app on each run. Perf runs are meant to run
and be done, however, we want stress apps to be long lasting to capture things like memory leaks.

This change creates a fork in the tests to support stress runs that re-use the app between runs.

Author: pranavkm

src/Components/WebAssembly/DebugProxy/src/Microsoft.AspNetCore.Components.WebAssembly.DebugProxy.csproj

@@ -13,6 +13,7 @@
     <!-- This is so that we add the FrameworkReference to Microsoft.AspNetCore.App -->
     <UseLatestAspNetCoreReference>true</UseLatestAspNetCoreReference>
     <MicrosoftAspNetCoreAppVersion>3.1.0</MicrosoftAspNetCoreAppVersion>
+    <NoWarn>$(NoWarn);CS0649</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResult.cs

@@ -8,6 +8,16 @@ class BenchmarkResult
         public List<BenchmarkScenarioResult> ScenarioResults { get; set; }
 
         /// <summary>Downloaded application size in bytes</summary>
-        public long DownloadSize { get; set; }
+        public long? DownloadSize { get; set; }
+
+        /// <summary>WASM memory usage</summary>
+        public long? WasmMemory { get; set; }
+
+        // See https://developer.mozilla.org/en-US/docs/Web/API/Performance/memory
+        /// <summary>JS memory usage</summary>
+        public long? UsedJSHeapSize { get; set; }
+
+        /// <summary>JS memory usage</summary>
+        public long? TotalJSHeapSize { get; set; }
     }
-}
+}

src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs

@@ -28,15 +28,21 @@ public static async Task<int> Main(string[] args)
         {
             // This cancellation token manages the timeout for the stress run.
             // By default the driver executes and reports a single Benchmark run. For stress runs,
-            // we'll pass in the duration to execute the runs in milliseconds. This will cause this driver
+            // we'll pass in the duration to execute the runs in seconds. This will cause this driver
             // to repeat executions for the duration specified.
             var stressRunCancellation = CancellationToken.None;
             var isStressRun = false;
             if (args.Length > 0)
             {
                 if (!int.TryParse(args[0], out var stressRunSeconds))
                 {
-                    Console.Error.WriteLine("Usage Driver <stress-run-duration>");
+                    Console.Error.WriteLine("Usage Driver <stress-run-duration-seconds>");
+                    return 1;
+                }
+
+                if (stressRunSeconds < 1)
+                {
+                    Console.Error.WriteLine("Stress run duration must be a positive integer.");
                     return 1;
                 }
 
@@ -53,31 +59,30 @@ public static async Task<int> Main(string[] args)
             // This write is required for the benchmarking infrastructure.
             Console.WriteLine("Application started.");
 
-            using var browser = await Selenium.CreateBrowser(default);
+            using var browser = await Selenium.CreateBrowser(default, captureBrowserMemory: isStressRun);
             using var testApp = StartTestApp();
             using var benchmarkReceiver = StartBenchmarkResultReceiver();
             var testAppUrl = GetListeningUrl(testApp);
+            if (isStressRun)
+            {
+                testAppUrl += "/stress.html";
+            }
+
             var receiverUrl = GetListeningUrl(benchmarkReceiver);
             Console.WriteLine($"Test app listening at {testAppUrl}.");
 
             var firstRun = true;
+            var timeForEachRun = TimeSpan.FromMinutes(3);
+
+            var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated";
+            browser.Url = launchUrl;
+            browser.Navigate();
+
             do
             {
                 BenchmarkResultTask = new TaskCompletionSource<BenchmarkResult>();
-                var timeForEachRun = TimeSpan.FromMinutes(3);
                 using var runCancellationToken = new CancellationTokenSource(timeForEachRun);
-                runCancellationToken.Token.Register(() => BenchmarkResultTask.TrySetException(new TimeoutException($"Timed out after {timeForEachRun}")));
-
-                if (firstRun)
-                {
-                    var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated";
-                    browser.Url = launchUrl;
-                    browser.Navigate();
-                }
-                else
-                {
-                    browser.FindElementById("runAll").Click();
-                }
+                using var registration = runCancellationToken.Token.Register(() => BenchmarkResultTask.TrySetException(new TimeoutException($"Timed out after {timeForEachRun}")));
 
                 var results = await BenchmarkResultTask.Task;
 
@@ -96,21 +101,75 @@ private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult, bo
         {
             // Sample of the the format: https://github.com/aspnet/Benchmarks/blob/e55f9e0312a7dd019d1268c1a547d1863f0c7237/src/Benchmarks/Program.cs#L51-L67
             var output = new BenchmarkOutput();
-            output.Metadata.Add(new BenchmarkMetadata
+
+
+            if (benchmarkResult.DownloadSize != null)
             {
-                Source = "BlazorWasm",
-                Name = "blazorwasm/download-size",
-                ShortDescription = "Download size (KB)",
-                LongDescription = "Download size (KB)",
-                Format = "n2",
-            });
+                output.Metadata.Add(new BenchmarkMetadata
+                {
+                    Source = "BlazorWasm",
+                    Name = "blazorwasm/download-size",
+                    ShortDescription = "Download size (KB)",
+                    LongDescription = "Download size (KB)",
+                    Format = "n2",
+                });
 
-            output.Measurements.Add(new BenchmarkMeasurement
+                output.Measurements.Add(new BenchmarkMeasurement
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Name = "blazorwasm/download-size",
+                    Value = ((float)benchmarkResult.DownloadSize) / 1024,
+                });
+            }
+
+            if (benchmarkResult.WasmMemory != null)
             {
-                Timestamp = DateTime.UtcNow,
-                Name = "blazorwasm/download-size",
-                Value = ((float)benchmarkResult.DownloadSize) / 1024,
-            });
+                output.Metadata.Add(new BenchmarkMetadata
+                {
+                    Source = "BlazorWasm",
+                    Name = "blazorwasm/wasm-memory",
+                    ShortDescription = "Memory (KB)",
+                    LongDescription = "WASM reported memory (KB)",
+                    Format = "n2",
+                });
+
+                output.Measurements.Add(new BenchmarkMeasurement
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Name = "blazorwasm/wasm-memory",
+                    Value = ((float)benchmarkResult.WasmMemory) / 1024,
+                });
+
+                output.Metadata.Add(new BenchmarkMetadata
+                {
+                    Source = "BlazorWasm",
+                    Name = "blazorwasm/js-usedjsheapsize",
+                    ShortDescription = "UsedJSHeapSize",
+                    LongDescription = "JS used heap size"
+                });
+
+                output.Measurements.Add(new BenchmarkMeasurement
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Name = "blazorwasm/js-usedjsheapsize",
+                    Value = benchmarkResult.UsedJSHeapSize,
+                });
+
+                output.Metadata.Add(new BenchmarkMetadata
+                {
+                    Source = "BlazorWasm",
+                    Name = "blazorwasm/js-totaljsheapsize",
+                    ShortDescription = "TotalJSHeapSize",
+                    LongDescription = "JS total heap size"
+                });
+
+                output.Measurements.Add(new BenchmarkMeasurement
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Name = "blazorwasm/js-totaljsheapsize",
+                    Value = benchmarkResult.TotalJSHeapSize,
+                });
+            }
 
             // Information about the build that this was produced from
             output.Metadata.Add(new BenchmarkMetadata

src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs

@@ -54,7 +54,7 @@ private static async ValueTask<Uri> WaitForServerAsync(int port, CancellationTok
             throw new Exception($"Unable to connect to selenium-server at {uri}");
         }
 
-        public static async Task<RemoteWebDriver> CreateBrowser(CancellationToken cancellationToken)
+        public static async Task<RemoteWebDriver> CreateBrowser(CancellationToken cancellationToken, bool captureBrowserMemory = false)
         {
             var uri = await WaitForServerAsync(SeleniumPort, cancellationToken);
 
@@ -65,6 +65,11 @@ public static async Task<RemoteWebDriver> CreateBrowser(CancellationToken cancel
                 options.AddArgument("--headless");
             }
 
+            if (captureBrowserMemory)
+            {
+                options.AddArgument("--enable-precise-memory-info");
+            }
+
             options.SetLoggingPreference(LogType.Browser, LogLevel.All);
 
             var attempt = 0;

src/Components/benchmarkapps/Wasm.Performance/Driver/Wasm.Performance.Driver.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>

src/Components/benchmarkapps/Wasm.Performance/TestApp/WasmMemory.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.JSInterop;
+
+namespace Wasm.Performance.TestApp
+{
+    public static class WasmMemory
+    {
+        [JSInvokable]
+        public static long GetTotalMemory() => GC.GetTotalMemory(forceFullCollection: true);
+    }
+}

src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/index.js

@@ -11,15 +11,14 @@ new HtmlUI('E2E Performance', '#display');
 if (location.href.indexOf('#automated') !== -1) {
   (async function() {
     const query = new URLSearchParams(window.location.search);
-    const group = query.get('group');
     const resultsUrl = query.get('resultsUrl');
 
     console.log('Calculating download size...');
     const downloadSize = await getBlazorDownloadSize();
     console.log('Download size: ', downloadSize);
 
     const scenarioResults = [];
-    groups.filter(g => !group || g.name === group).forEach(g => g.runAll());
+    groups.forEach(g => g.runAll());
 
     onBenchmarkEvent(async (status, args) => {
       switch (status) {

src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/jsonHandling.js

@@ -2,7 +2,7 @@ import { group, benchmark, setup, teardown } from './lib/minibench/minibench.js'
 import { BlazorApp } from './util/BlazorApp.js';
 import { receiveEvent } from './util/BenchmarkEvents.js';
 import { setInputValue } from './util/DOM.js';
-import { largeJsonToDeserialize, largeObjectToSerialize } from './jsonHandlingData.js';
+import { largeJsonToDeserialize, largeObjectToSerialize, benchmarkJson } from './jsonHandlingData.js';
 
 group('JSON handling', () => {
   let app;
@@ -71,17 +71,3 @@ group('JSON handling', () => {
     }
   });
 });
-
-async function benchmarkJson(app, buttonSelector, resultSelector, expectedResult) {
-  const appDocument = app.window.document;
-  appDocument.querySelector('#reset-all').click();
-
-  let nextRenderCompletion = receiveEvent('Finished JSON processing');
-  appDocument.querySelector(buttonSelector).click();
-  await nextRenderCompletion;
-
-  const resultElem = appDocument.querySelector(resultSelector);
-  if (resultElem.textContent != expectedResult.toString()) {
-    throw new Error(`Incorrect result: ${resultElem.textContent}`);
-  }
-}

src/Components/benchmarkapps/Wasm.Performance/TestApp/wwwroot/benchmarks/jsonHandlingData.js

@@ -0,0 +1,44 @@
+import { group, benchmark, setup, teardown } from './lib/minibench/minibench.js';
+import { benchmarkJson} from './jsonHandlingData.js';
+import { BlazorStressApp } from './util/BlazorStressApp.js';
+
+group('JSON handling', () => {
+  let app;
+
+  setup(() => {
+    app = BlazorStressApp.instance;
+    app.navigateTo('json');
+  });
+
+  benchmark('Serialize 1kb', () =>
+    benchmarkJson(app, '#serialize-small', '#serialized-length', 935), {
+      descriptor: {
+        name: 'blazorwasm/jsonserialize-1kb',
+        description: 'Serialize JSON 1kb - Time in ms'
+      }
+    });
+
+  benchmark('Serialize 340kb', () =>
+   benchmarkJson(app, '#serialize-large', '#serialized-length', 339803), {
+     descriptor: {
+       name: 'blazorwasm/jsonserialize-340kb',
+       description: 'Serialize JSON 340kb - Time in ms'
+     }
+   });
+
+  benchmark('Deserialize 1kb', () =>
+    benchmarkJson(app, '#deserialize-small', '#deserialized-count', 5), {
+      descriptor: {
+        name: 'blazorwasm/jsondeserialize-1kb',
+        description: 'Deserialize JSON 1kb - Time in ms'
+      }
+    });
+
+  benchmark('Deserialize 340kb', () =>
+   benchmarkJson(app, '#deserialize-large', '#deserialized-count', 1365), {
+     descriptor: {
+       name: 'blazorwasm/jsondeserialize-340kb',
+       description: 'Deserialize JSON 340kb - Time in ms'
+     }
+   });
+});