Skip to content

Commit 6bb5f75

Browse files
brettsambrettsam
committed
adding retries to race tests
1 parent 2050598 commit 6bb5f75

File tree

2 files changed

+134
-72
lines changed

2 files changed

+134
-72
lines changed

‎test/WebJobs.Script.Tests.Integration/Host/SecretsRepositoryTests.cs‎

Lines changed: 110 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using Microsoft.WebJobs.Script.Tests;
2222
using WebJobs.Script.Tests;
2323
using Xunit;
24+
using Xunit.Abstractions;
2425

2526
namespace Microsoft.Azure.WebJobs.Script.Tests
2627
{
@@ -30,10 +31,13 @@ public class SecretsRepositoryTests : IClassFixture<SecretsRepositoryTests.Fixtu
3031
private readonly string functionName = "Test_test";
3132
public static string KeyName = "Te!@#st!1-te_st";
3233

33-
public SecretsRepositoryTests(SecretsRepositoryTests.Fixture fixture)
34+
private ITestOutputHelper _output;
35+
36+
public SecretsRepositoryTests(SecretsRepositoryTests.Fixture fixture, ITestOutputHelper outputHelper)
3437
{
3538
Utility.ColdStartDelayMS = 50;
3639
_fixture = fixture;
40+
_output = outputHelper;
3741
}
3842

3943
public enum SecretsRepositoryType
@@ -378,35 +382,39 @@ public async Task BlobRepository_WriteAsync_DoesNot_ClearBlobContents(SecretsRep
378382
using (var directory = new TempDirectory())
379383
{
380384
await _fixture.TestInitialize(repositoryType, directory.Path);
385+
string testFunctionName = "host";
386+
var target = _fixture.GetNewSecretRepository();
381387

382-
ScriptSecrets testSecrets = new HostSecrets()
388+
async Task RunTest()
383389
{
384-
MasterKey = new Key("master", "test"),
385-
FunctionKeys = new List<Key>() { new Key(KeyName, "test") },
386-
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
387-
};
388-
389-
string testFunctionName = "host";
390+
HostSecrets testSecrets = new HostSecrets()
391+
{
392+
MasterKey = new Key("master", "test"),
393+
FunctionKeys = new List<Key>(),
394+
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
395+
};
390396

391-
var target = _fixture.GetNewSecretRepository();
397+
// make the payload larger to guarantee the race condition
398+
for (int i = 0; i < 1000; i++)
399+
{
400+
testSecrets.FunctionKeys.Add(new Key(KeyName + i, "test"));
401+
}
402+
// Set up initial secrets.
403+
await _fixture.WriteSecret(testFunctionName, testSecrets);
392404

393-
// Set up initial secrets.
394-
await _fixture.WriteSecret(testFunctionName, testSecrets);
405+
// Perform a write and read similtaneously. Previously, our usage of OpenWriteAsync
406+
// would erase the content of the blob while writing, resulting in null secrets from the
407+
// read.
408+
Task writeTask = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
409+
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
395410

396-
// Perform a write and read similtaneously. Previously, our usage of OpenWriteAsync
397-
// would erase the content of the blob while writing, resulting in null secrets from the
398-
// read.
399-
Task writeTask = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
400-
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
411+
await writeTask;
401412

402-
await writeTask;
413+
Assert.NotNull(secretsContent);
414+
}
403415

404-
Assert.Equal(secretsContent.MasterKey.Name, "master");
405-
Assert.Equal(secretsContent.MasterKey.Value, "test");
406-
Assert.Equal(secretsContent.FunctionKeys[0].Name, KeyName);
407-
Assert.Equal(secretsContent.FunctionKeys[0].Value, "test");
408-
Assert.Equal(secretsContent.SystemKeys[0].Name, KeyName);
409-
Assert.Equal(secretsContent.SystemKeys[0].Value, "test");
416+
// this is a race so it may not fire every time; try several times
417+
await TestHelpers.RetryFailedTest(RunTest, 10, _output);
410418
}
411419
}
412420

@@ -418,40 +426,50 @@ public async Task BlobRepository_SimultaneousWrites_Throws_PreconditionFailed(Se
418426
using (var directory = new TempDirectory())
419427
{
420428
await _fixture.TestInitialize(repositoryType, directory.Path);
429+
var target = _fixture.GetNewSecretRepository();
430+
string testFunctionName = "host";
421431

422-
HostSecrets testSecrets = new HostSecrets()
432+
async Task RunTest()
423433
{
424-
MasterKey = new Key("master", "test"),
425-
FunctionKeys = new List<Key>() { new Key(KeyName, "test") },
426-
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
427-
};
434+
HostSecrets testSecrets = new HostSecrets()
435+
{
436+
MasterKey = new Key("master", "test"),
437+
FunctionKeys = new List<Key>(),
438+
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
439+
};
428440

429-
string testFunctionName = "host";
441+
// make the payload larger to guarantee the race condition
442+
for (int i = 0; i < 1000; i++)
443+
{
444+
testSecrets.FunctionKeys.Add(new Key(KeyName + i, "test"));
445+
}
430446

431-
var target = _fixture.GetNewSecretRepository();
447+
// Set up initial secrets.
448+
await _fixture.WriteSecret(testFunctionName, testSecrets);
449+
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
450+
Assert.Equal("test", secretsContent.FunctionKeys.First().Value);
432451

433-
// Set up initial secrets.
434-
await _fixture.WriteSecret(testFunctionName, testSecrets);
435-
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
436-
Assert.Equal("test", secretsContent.FunctionKeys.Single().Value);
452+
testSecrets.FunctionKeys.First().Value = "changed";
437453

438-
testSecrets.FunctionKeys.Single().Value = "changed";
454+
// Simultaneous writes will result in one of the writes being discarded due to
455+
// non-matching ETag.
456+
Task writeTask1 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
457+
Task writeTask2 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
439458

440-
// Simultaneous writes will result in one of the writes being discarded due to
441-
// non-matching ETag.
442-
Task writeTask1 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
443-
Task writeTask2 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
459+
var ex = await Assert.ThrowsAsync<RequestFailedException>(() => Task.WhenAll(writeTask1, writeTask2));
444460

445-
var ex = await Assert.ThrowsAsync<RequestFailedException>(() => Task.WhenAll(writeTask1, writeTask2));
461+
// Ensure the write went through.
462+
secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
463+
Assert.Equal("changed", secretsContent.FunctionKeys.First().Value);
446464

447-
// Ensure the write went through.
448-
secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
449-
Assert.Equal("changed", secretsContent.FunctionKeys.Single().Value);
465+
Assert.Equal("ConditionNotMet", ex.ErrorCode);
466+
Assert.Equal(412, ex.Status);
467+
Assert.True(writeTask1.IsCompletedSuccessfully || writeTask2.IsCompletedSuccessfully,
468+
"One of the write operations should have completed successfully.");
469+
}
450470

451-
Assert.Equal("ConditionNotMet", ex.ErrorCode);
452-
Assert.Equal(412, ex.Status);
453-
Assert.True(writeTask1.IsCompletedSuccessfully || writeTask2.IsCompletedSuccessfully,
454-
"One of the write operations should have completed successfully.");
471+
// this is a race so it may not fire every time; try several times
472+
await TestHelpers.RetryFailedTest(RunTest, 10, _output);
455473
}
456474
}
457475

@@ -463,37 +481,47 @@ public async Task BlobRepository_SimultaneousCreates_Throws_Conflict(SecretsRepo
463481
using (var directory = new TempDirectory())
464482
{
465483
await _fixture.TestInitialize(repositoryType, directory.Path);
484+
string testFunctionName = "host";
485+
var target = _fixture.GetNewSecretRepository();
466486

467-
HostSecrets testSecrets = new HostSecrets()
487+
async Task RunTest()
468488
{
469-
MasterKey = new Key("master", "test"),
470-
FunctionKeys = new List<Key>() { new Key(KeyName, "test") },
471-
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
472-
};
489+
HostSecrets testSecrets = new HostSecrets()
490+
{
491+
MasterKey = new Key("master", "test"),
492+
FunctionKeys = new List<Key>(),
493+
SystemKeys = new List<Key>() { new Key(KeyName, "test") }
494+
};
473495

474-
string testFunctionName = "host";
496+
// make the payload larger to guarantee the race condition
497+
for (int i = 0; i < 1000; i++)
498+
{
499+
testSecrets.FunctionKeys.Add(new Key(KeyName + i, "test"));
500+
}
475501

476-
var target = _fixture.GetNewSecretRepository();
502+
// Ensure nothing is there.
503+
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
504+
Assert.Null(secretsContent);
477505

478-
// Ensure nothing is there.
479-
HostSecrets secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
480-
Assert.Null(secretsContent);
506+
// Simultaneous creates will result in one of the writes being discarded due to
507+
// non-matching ETag.
508+
Task writeTask1 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
509+
Task writeTask2 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
481510

482-
// Simultaneous creates will result in one of the writes being discarded due to
483-
// non-matching ETag.
484-
Task writeTask1 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
485-
Task writeTask2 = target.WriteAsync(ScriptSecretsType.Host, testFunctionName, testSecrets);
511+
var ex = await Assert.ThrowsAsync<RequestFailedException>(() => Task.WhenAll(writeTask1, writeTask2));
486512

487-
var ex = await Assert.ThrowsAsync<RequestFailedException>(() => Task.WhenAll(writeTask1, writeTask2));
513+
// Ensure the write went through.
514+
secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
515+
Assert.Equal("test", secretsContent.FunctionKeys.First().Value);
488516

489-
// Ensure the write went through.
490-
secretsContent = await target.ReadAsync(ScriptSecretsType.Host, testFunctionName) as HostSecrets;
491-
Assert.Equal("test", secretsContent.FunctionKeys.Single().Value);
517+
Assert.Equal("BlobAlreadyExists", ex.ErrorCode);
518+
Assert.Equal(409, ex.Status);
519+
Assert.True(writeTask1.IsCompletedSuccessfully || writeTask2.IsCompletedSuccessfully,
520+
"One of the write operations should have completed successfully.");
521+
}
492522

493-
Assert.Equal("BlobAlreadyExists", ex.ErrorCode);
494-
Assert.Equal(409, ex.Status);
495-
Assert.True(writeTask1.IsCompletedSuccessfully || writeTask2.IsCompletedSuccessfully,
496-
"One of the write operations should have completed successfully.");
523+
// this is a race so it may not fire every time; try several times
524+
await TestHelpers.RetryFailedTest(RunTest, 10, _output);
497525
}
498526
}
499527

@@ -504,10 +532,16 @@ public Fixture()
504532
TestSiteName = "Test_test";
505533
var configuration = TestHelpers.GetTestConfiguration();
506534
BlobConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
535+
507536
KeyVaultConnectionString = configuration.GetWebJobsConnectionString(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultConnectionString);
508537
KeyVaultName = configuration.GetWebJobsConnectionString(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultName);
509-
AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider(KeyVaultConnectionString);
510-
KeyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback));
538+
539+
if (KeyVaultConnectionString is not null && KeyVaultName is not null)
540+
{
541+
AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider(KeyVaultConnectionString);
542+
KeyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback));
543+
}
544+
511545
Environment = new TestEnvironment();
512546
AzureStorageProvider = TestHelpers.GetAzureStorageProvider(configuration);
513547
}
@@ -557,7 +591,11 @@ public async Task TestInitialize(SecretsRepositoryType repositoryType, string se
557591

558592
await ClearAllBlobSecrets();
559593
ClearAllFileSecrets();
560-
await ClearAllKeyVaultSecrets();
594+
595+
if (KeyVaultClient != null)
596+
{
597+
await ClearAllKeyVaultSecrets();
598+
}
561599

562600
LoggerProvider = new TestLoggerProvider();
563601
var loggerFactory = new LoggerFactory();

‎test/WebJobs.Script.Tests.Shared/TestHelpers.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
using Microsoft.Extensions.Hosting;
2424
using Microsoft.Extensions.Options;
2525
using Microsoft.WebJobs.Script.Tests;
26+
using Xunit.Abstractions;
2627

2728
namespace Microsoft.Azure.WebJobs.Script.Tests
2829
{
@@ -117,6 +118,29 @@ public static async Task Await(Func<Task<bool>> condition, int timeout = 60 * 10
117118
}
118119
}
119120

121+
public static async Task RetryFailedTest(Func<Task> test, int retries, ITestOutputHelper output = null)
122+
{
123+
for (int i = 0; i < retries; i++)
124+
{
125+
try
126+
{
127+
await test();
128+
break;
129+
}
130+
catch (Exception ex)
131+
{
132+
if (i == retries - 1)
133+
{
134+
throw;
135+
}
136+
else
137+
{
138+
output.WriteLine($"Test failed attempt {i + 1} of {retries}. Retrying. Exception message: {ex.ToString()}");
139+
}
140+
}
141+
}
142+
}
143+
120144
public static async Task<string> WaitForBlobAndGetStringAsync(CloudBlockBlob blob, Func<string> userMessageCallback = null)
121145
{
122146
await WaitForBlobAsync(blob, userMessageCallback: userMessageCallback);

0 commit comments

Comments
 (0)