Skip to content

Commit f1fac38

Browse files
WhitWaldoWhitWaldo
authored
Workflow: Make resilient to either/or loading + workflow reconnections (#1680)
* Added reconnect loop to workflow startup * Added the ability to have the test harness load resources and the app in either order to test load order --------- Signed-off-by: Whit Waldo <[email protected]>
1 parent 95e1168 commit f1fac38

File tree

8 files changed

+642
-137
lines changed

8 files changed

+642
-137
lines changed

‎src/Dapr.Testcontainers/Common/Testing/DaprTestApplicationBuilder.cs‎

Lines changed: 104 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public sealed class DaprTestApplicationBuilder(BaseHarness harness)
2929
{
3030
private Action<WebApplicationBuilder>? _configureServices;
3131
private Action<WebApplication>? _configureApp;
32+
private bool _shouldLoadResourcesFirst = true;
3233

3334
/// <summary>
3435
/// Configures services for the test application.
@@ -48,39 +49,121 @@ public DaprTestApplicationBuilder ConfigureApp(Action<WebApplication> configure)
4849
return this;
4950
}
5051

52+
/// <summary>
53+
/// Configures the startup order of Dapr resources and the application.
54+
/// </summary>
55+
/// <param name="shouldLoadResourcesFirst">
56+
/// If true (default), Dapr container starts before the app. If false, the
57+
/// app starts before the Dapr container.
58+
/// </param>
59+
public DaprTestApplicationBuilder WithDaprStartupOrder(bool shouldLoadResourcesFirst)
60+
{
61+
_shouldLoadResourcesFirst = shouldLoadResourcesFirst;
62+
return this;
63+
}
64+
5165
/// <summary>
5266
/// Builds and starts the test application and harness.
5367
/// </summary>
5468
/// <returns></returns>
5569
public async Task<DaprTestApplication> BuildAndStartAsync()
5670
{
57-
await harness.InitializeAsync();
58-
5971
WebApplication? app = null;
60-
if (_configureServices is not null || _configureApp is not null)
72+
73+
if (_shouldLoadResourcesFirst)
6174
{
62-
var builder = WebApplication.CreateBuilder();
63-
64-
// Configure Dapr endpoints via in-memory configuration instead of environment variables
65-
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
75+
// Load the harness and resources, then the app
76+
await harness.InitializeAsync();
77+
78+
if (_configureServices is not null || _configureApp is not null)
6679
{
67-
{ "DAPR_HTTP_ENDPOINT", $"http://127.0.0.1:{harness.DaprHttpPort}" },
68-
{ "DAPR_GRPC_ENDPOINT", $"http://127.0.0.1:{harness.DaprGrpcPort}" }
69-
});
70-
71-
builder.Logging.ClearProviders();
72-
builder.Logging.AddSimpleConsole();
73-
builder.WebHost.UseUrls($"http://0.0.0.0:{harness.AppPort}");
74-
75-
_configureServices?.Invoke(builder);
80+
app = CreateApp();
81+
await app.StartAsync();
82+
}
7683

77-
app = builder.Build();
78-
79-
_configureApp?.Invoke(app);
84+
return new DaprTestApplication(harness, app);
85+
}
86+
87+
// App-first: start app, then start resources
88+
// If daprd cannot bind the chosen ports, restart the app with new ports
89+
const int maxAttempts = 5;
90+
Exception? lastError = null;
91+
92+
for (var attempt = 1; attempt <= maxAttempts; attempt++)
93+
{
94+
WebApplication? attemptApp = null;
95+
96+
try
97+
{
98+
// Pre-assign prots for the app knows where Dapr will be (avoid collisions)
99+
var httpPort = PortUtilities.GetAvailablePort();
100+
var grpcPort = PortUtilities.GetAvailablePort();
101+
while (grpcPort == httpPort)
102+
grpcPort = PortUtilities.GetAvailablePort();
103+
104+
var appPort = PortUtilities.GetAvailablePort();
105+
while (appPort == httpPort || appPort == grpcPort)
106+
appPort = PortUtilities.GetAvailablePort();
107+
108+
harness.SetPorts(httpPort, grpcPort);
109+
harness.SetAppPort(appPort);
110+
111+
// Load the app (configuration/services/pipeline), but delay StartAsync until daprd is up
112+
if (_configureServices is not null || _configureApp is not null)
113+
{
114+
attemptApp = CreateApp();
115+
await attemptApp.StartAsync();
116+
}
117+
118+
await harness.InitializeAsync();
119+
120+
return new DaprTestApplication(harness, attemptApp);
121+
}
122+
catch (Exception ex)
123+
{
124+
lastError = ex;
80125

81-
await app.StartAsync();
126+
if (attemptApp is not null)
127+
{
128+
try
129+
{
130+
await attemptApp.StopAsync();
131+
}
132+
finally
133+
{
134+
await attemptApp.DisposeAsync();
135+
}
136+
}
137+
138+
// Try again with a frest set of ports
139+
}
82140
}
83141

84-
return new DaprTestApplication(harness, app);
142+
throw new InvalidOperationException(
143+
$"Failed to start app-first Dapr test application after {maxAttempts} attempts.", lastError);
144+
}
145+
146+
private WebApplication CreateApp()
147+
{
148+
var builder = WebApplication.CreateBuilder();
149+
150+
// Configure Dapr endpoints via in-memory configuration instead of environment variables
151+
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
152+
{
153+
{ "DAPR_HTTP_ENDPOINT", $"http://127.0.0.1:{harness.DaprHttpPort}" },
154+
{ "DAPR_GRPC_ENDPOINT", $"http://127.0.0.1:{harness.DaprGrpcPort}" }
155+
});
156+
157+
builder.Logging.ClearProviders();
158+
builder.Logging.AddSimpleConsole();
159+
builder.WebHost.UseUrls($"http://0.0.0.0:{harness.AppPort}");
160+
161+
_configureServices?.Invoke(builder);
162+
163+
var app = builder.Build();
164+
165+
_configureApp?.Invoke(app);
166+
167+
return app;
85168
}
86169
}

‎src/Dapr.Testcontainers/Containers/Dapr/DaprdContainer.cs‎

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
using System;
1515
using System.Collections.Generic;
16+
using System.Net.Sockets;
1617
using System.Threading;
1718
using System.Threading.Tasks;
1819
using Dapr.Testcontainers.Common;
@@ -48,6 +49,9 @@ public sealed class DaprdContainer : IAsyncStartable
4849
/// </summary>
4950
public int GrpcPort { get; private set; }
5051

52+
private readonly int? _requestedHttpPort;
53+
private readonly int? _requestedGrpcPort;
54+
5155
/// <summary>
5256
/// The hostname to locate the Dapr runtime on in the shared Docker network.
5357
/// </summary>
@@ -62,8 +66,22 @@ public sealed class DaprdContainer : IAsyncStartable
6266
/// <param name="network">The shared Docker network to connect to.</param>
6367
/// <param name="placementHostAndPort">The hostname and port of the Placement service.</param>
6468
/// <param name="schedulerHostAndPort">The hostname and port of the Scheduler service.</param>
65-
public DaprdContainer(string appId, string componentsHostFolder, DaprRuntimeOptions options, INetwork network, HostPortPair? placementHostAndPort = null, HostPortPair? schedulerHostAndPort = null)
69+
/// <param name="daprHttpPort">The host HTTP port to bind to.</param>
70+
/// <param name="daprGrpcPort">The host gRPC port to bind to.</param>
71+
public DaprdContainer(
72+
string appId,
73+
string componentsHostFolder,
74+
DaprRuntimeOptions options,
75+
INetwork network,
76+
HostPortPair? placementHostAndPort = null,
77+
HostPortPair? schedulerHostAndPort = null,
78+
int? daprHttpPort = null,
79+
int? daprGrpcPort = null
80+
)
6681
{
82+
_requestedHttpPort = daprHttpPort;
83+
_requestedGrpcPort = daprGrpcPort;
84+
6785
const string componentsPath = "/components";
6886
var cmd =
6987
new List<string>
@@ -102,28 +120,89 @@ public DaprdContainer(string appId, string componentsHostFolder, DaprRuntimeOpti
102120
cmd.Add("");
103121
}
104122

105-
_container = new ContainerBuilder()
123+
var containerBuilder = new ContainerBuilder()
106124
.WithImage(options.RuntimeImageTag)
107125
.WithName(_containerName)
108126
.WithLogger(ConsoleLogger.Instance)
109127
.WithCommand(cmd.ToArray())
110128
.WithNetwork(network)
111129
.WithExtraHost(ContainerHostAlias, "host-gateway")
112-
.WithPortBinding(InternalHttpPort, assignRandomHostPort: true)
113-
.WithPortBinding(InternalGrpcPort, assignRandomHostPort: true)
114130
.WithBindMount(componentsHostFolder, componentsPath, AccessMode.ReadOnly)
115131
.WithWaitStrategy(Wait.ForUnixContainer()
116-
.UntilMessageIsLogged("Internal gRPC server is running"))
132+
.UntilMessageIsLogged("Internal gRPC server is running"));
117133
//.UntilMessageIsLogged(@"^dapr initialized. Status: Running. Init Elapsed "))
118-
.Build();
134+
135+
containerBuilder = daprHttpPort is not null ? containerBuilder.WithPortBinding(containerPort: InternalHttpPort, hostPort: daprHttpPort.Value) : containerBuilder.WithPortBinding(port: InternalHttpPort, assignRandomHostPort: true);
136+
containerBuilder = daprGrpcPort is not null ? containerBuilder.WithPortBinding(containerPort: InternalGrpcPort, hostPort: daprGrpcPort.Value) : containerBuilder.WithPortBinding(port: InternalGrpcPort, assignRandomHostPort: true);
137+
138+
_container = containerBuilder.Build();
119139
}
120140

121141
/// <inheritdoc />
122142
public async Task StartAsync(CancellationToken cancellationToken = default)
123143
{
124144
await _container.StartAsync(cancellationToken);
125-
HttpPort = _container.GetMappedPublicPort(InternalHttpPort);
126-
GrpcPort = _container.GetMappedPublicPort(InternalGrpcPort);
145+
146+
var mappedHttpPort = _container.GetMappedPublicPort(InternalHttpPort);
147+
var mappedGrpcPort = _container.GetMappedPublicPort(InternalGrpcPort);
148+
149+
if (_requestedHttpPort is not null && mappedHttpPort != _requestedHttpPort.Value)
150+
{
151+
throw new InvalidOperationException(
152+
$"Dapr HTTP port mapping mismatch. Requested {_requestedHttpPort.Value}, but Docker mapped {mappedHttpPort}");
153+
}
154+
155+
if (_requestedGrpcPort is not null && mappedGrpcPort != _requestedGrpcPort.Value)
156+
{
157+
throw new InvalidOperationException(
158+
$"Dapr gRPC port mapping mismatch. Requested {_requestedGrpcPort.Value}, but Docker mapped {mappedGrpcPort}");
159+
}
160+
161+
HttpPort = mappedHttpPort;
162+
GrpcPort = mappedGrpcPort;
163+
164+
// The container log wait strategy can fire before the host port is actually accepting connections
165+
// (especially on Windows). Ensure the ports are reachable from the test process.
166+
await WaitForTcpPortAsync("127.0.0.1", HttpPort, TimeSpan.FromSeconds(30), cancellationToken);
167+
await WaitForTcpPortAsync("127.0.0.1", GrpcPort, TimeSpan.FromSeconds(30), cancellationToken);
168+
}
169+
170+
private static async Task WaitForTcpPortAsync(
171+
string host,
172+
int port,
173+
TimeSpan timeout,
174+
CancellationToken cancellationToken)
175+
{
176+
var start = DateTimeOffset.UtcNow;
177+
Exception? lastError = null;
178+
179+
while (DateTimeOffset.UtcNow - start < timeout)
180+
{
181+
cancellationToken.ThrowIfCancellationRequested();
182+
183+
try
184+
{
185+
using var client = new TcpClient();
186+
var connectTask = client.ConnectAsync(host, port);
187+
188+
var completed = await Task.WhenAny(connectTask,
189+
Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken));
190+
if (completed == connectTask)
191+
{
192+
// Will throw if connect failed
193+
await connectTask;
194+
return;
195+
}
196+
}
197+
catch (Exception ex) when (ex is SocketException or InvalidOperationException)
198+
{
199+
lastError = ex;
200+
}
201+
202+
await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
203+
}
204+
205+
throw new TimeoutException($"Timed out waiting for TCP port {host}:{port} to accept connections.", lastError);
127206
}
128207

129208
/// <inheritdoc />

0 commit comments

Comments
 (0)