Enable verbose logging of when a test starts and stops to see which test(s) freeze in GHA

Author: lukebakken

.github/workflows/wf_build-and-test.yaml

@@ -35,7 +35,7 @@ jobs:
         run: ${{ github.workspace }}\.ci\windows\gha-setup.ps1
       - name: Test
         timeout-minutes: 25
-        run: dotnet test ${{ github.workspace }}\Build.csproj --no-restore --no-build --logger 'console;verbosity=detailed'
+        run: dotnet test ${{ github.workspace }}\Build.csproj --environment=RABBITMQ_CLIENT_TESTS_VERBOSE=true --no-restore --no-build --logger 'console;verbosity=detailed'
       - name: Check for errors in RabbitMQ logs
         run: ${{ github.workspace }}\.ci\windows\gha-log-check.ps1
       - name: Maybe upload RabbitMQ logs
@@ -63,7 +63,7 @@ jobs:
         run: ${{ github.workspace }}/.ci/ubuntu/cluster/gha-setup.sh
       - name: Test
         timeout-minutes: 25
-        run: dotnet test ${{ github.workspace }}/Build.csproj --no-restore --no-build --logger "console;verbosity=detailed"
+        run: dotnet test ${{ github.workspace }}/Build.csproj --environment=RABBITMQ_CLIENT_TESTS_VERBOSE=true --no-restore --no-build --logger "console;verbosity=detailed"
       - name: Check for errors in RabbitMQ logs
         run: ${{ github.workspace}}/.ci/ubuntu/cluster/gha-logs.sh check
       - name: Collect RabbitMQ logs (on failure)

Tests/Consumer/ConsumerPauseTests.cs

@@ -13,11 +13,15 @@
 
 namespace Tests.Consumer;
 
-public class ConsumerPauseTests(ITestOutputHelper testOutputHelper) : IntegrationTest(testOutputHelper), IDisposable
+public class ConsumerPauseTests(ITestOutputHelper testOutputHelper) : IntegrationTest(testOutputHelper)
 {
     private readonly HttpApiClient _httpApiClient = new();
 
-    public void Dispose() => _httpApiClient.Dispose();
+    public override Task DisposeAsync()
+    {
+        _httpApiClient.Dispose();
+        return base.DisposeAsync();
+    }
 
     [Fact]
     public async Task PauseShouldStopMessageArrivalUnpauseShouldResumeIt()

Tests/IntegrationTest.cs

@@ -50,8 +50,11 @@ public IntegrationTest(ITestOutputHelper testOutputHelper,
         _testDisplayName = InitTestDisplayName();
         _containerId = $"{_testDisplayName}:{Now}";
 
-        // TODO only if verbose
-        // testOutputHelper.WriteLine($"Running test: {_testDisplayName}");
+        if (SystemUtils.IsVerbose)
+        {
+            _testOutputHelper.WriteLine("{0} [DEBUG] [START] {1}", DateTime.Now, _testDisplayName);
+        }
+
         _connectionSettingBuilder = InitConnectionSettingsBuilder();
     }
 
@@ -80,8 +83,11 @@ public virtual async Task InitializeAsync()
 
     public virtual async Task DisposeAsync()
     {
-        // TODO only if verbose
-        // _testOutputHelper.WriteLine($"Disposing test: {_testDisplayName}");
+        if (SystemUtils.IsVerbose)
+        {
+            _testOutputHelper.WriteLine("{0} [DEBUG] [END] {1}", DateTime.Now, _testDisplayName);
+        }
+
         if (_management is not null && _management.State == State.Open)
         {
             try

Tests/Recovery/CustomPublisherConsumerRecoveryTests.cs

@@ -12,7 +12,8 @@
 
 namespace Tests.Recovery;
 
-public class CustomPublisherConsumerRecoveryTests(ITestOutputHelper testOutputHelper) : IntegrationTest(testOutputHelper, false)
+public class CustomPublisherConsumerRecoveryTests(ITestOutputHelper testOutputHelper)
+    : IntegrationTest(testOutputHelper, setupConnectionAndManagement: false)
 {
     /// <summary>
     /// The consumer and the publisher should not restart if the recovery is disabled

Tests/Rpc/RecoveryRPCTests.cs

@@ -1,3 +1,7 @@
+// This source code is dual-licensed under the Apache License, version
+// 2.0, and the Mozilla Public License, version 2.0.
+// Copyright (c) 2017-2024 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+
 using System;
 using System.Threading;
 using System.Threading.Tasks;
@@ -9,10 +13,14 @@
 namespace Tests.Rpc
 {
     public class RecoveryRpcTests(ITestOutputHelper testOutputHelper)
+        : IntegrationTest(testOutputHelper, setupConnectionAndManagement: false)
     {
         [Fact]
         public async Task RpcServerAndClientShouldRecoverAfterKillConnection()
         {
+            Assert.Null(_connection);
+            Assert.Null(_management);
+
             string containerId = $"rpc-server-client-recovery-{DateTime.Now}";
             IConnection connection = await AmqpConnection.CreateAsync(ConnectionSettingBuilder.Create()
                 .ContainerId(containerId).RecoveryConfiguration(RecoveryConfiguration.Create().Topology(true)).Build());
@@ -56,7 +64,7 @@ public async Task RpcServerAndClientShouldRecoverAfterKillConnection()
                 }
                 catch (Exception e)
                 {
-                    testOutputHelper.WriteLine($"Error sending message: {e.Message}");
+                    _testOutputHelper.WriteLine($"Error sending message: {e.Message}");
                     await Task.Delay(700);
                 }
 

Tests/SystemUtils.cs

@@ -11,12 +11,14 @@
 
 namespace Tests;
 
+// TODO this could be merged into IntegrationTest
 public static class SystemUtils
 {
     const string DefaultRabbitMqHost = "localhost";
     private static readonly HttpApiClient s_httpApiClient = new();
     private static readonly string s_rabbitMqHost = InitRabbitMqHost();
     private static readonly bool s_isRunningInCI = InitIsRunningInCI();
+    private static readonly bool s_isVerbose = InitIsVerbose();
     private static readonly bool s_isCluster;
     private static readonly ushort s_clusterSize;
     private static readonly TimeSpan s_initialDelaySpan = TimeSpan.FromMilliseconds(100);
@@ -25,6 +27,7 @@ public static class SystemUtils
 
     public static string RabbitMqHost => s_rabbitMqHost;
     public static bool IsRunningInCI => s_isRunningInCI;
+    public static bool IsVerbose => s_isVerbose;
     public static bool IsCluster => s_isCluster;
     public static ushort ClusterSize => s_clusterSize;
 
@@ -307,4 +310,14 @@ private static bool InitIsRunningInCI()
 
         return false;
     }
+
+    private static bool InitIsVerbose()
+    {
+        if (bool.TryParse(Environment.GetEnvironmentVariable("RABBITMQ_CLIENT_TESTS_VERBOSE"), out bool isVerbose))
+        {
+            return isVerbose;
+        }
+
+        return false;
+    }
 }

build.ps1

@@ -1,8 +1,12 @@
 [CmdletBinding(PositionalBinding=$false)]
 param(
-    [switch]$RunTests
+    [switch]$RunTests,
+    [switch]$RunTestsUntilFailure
 )
 
+New-Variable -Name verbose -Option Constant `
+    -Value ($VerbosePreference -ne 'SilentlyContinue')
+
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version Latest
 $PSNativeCommandUseErrorActionPreference = $true
@@ -18,17 +22,30 @@ New-Variable -Name build_csproj_file -Option Constant `
     -Value (Join-Path -Path $PSScriptRoot -ChildPath 'Build.csproj')
 
 dotnet build $build_csproj_file
-Write-Host "Done building." -ForegroundColor "Green"
 
-if ($RunTests) {
-    Write-Host "Running tests: Build.csproj traversal (all frameworks)" -ForegroundColor "Magenta"
-    dotnet test $build_csproj_file --no-build --logger 'console;verbosity=detailed'
-    if ($LastExitCode -ne 0)
+Write-Host "[INFO] done building." -ForegroundColor "Green"
+
+if ($RunTests -or $RunTestsUntilFailure)
+{
+    Do
     {
-        Write-Host "Error with tests, aborting build." -Foreground "Red"
-        Exit 1
-    }
-    Write-Host "Tests passed!" -ForegroundColor "Green"
+        Write-Host "Running tests: Build.csproj traversal (all frameworks)" -ForegroundColor "Magenta"
+        if ($verbose)
+        {
+            dotnet test $build_csproj_file --environment=RABBITMQ_CLIENT_TESTS_VERBOSE=true --no-build --logger 'console;verbosity=detailed'
+        }
+        else
+        {
+            dotnet test $build_csproj_file --no-build --logger 'console;verbosity=detailed'
+        }
+        if ($LASTEXITCODE -ne 0)
+        {
+            Write-Host "[ERROR] tests errored, exiting" -Foreground "Red"
+            Exit 1
+        }
+        else
+        {
+            Write-Host "[INFO] tests passed" -ForegroundColor "Green"
+        }
+    } While ($RunTestsUntilFailure)
 }
-
-Write-Host "Done."