diff --git a/CromwellOnAzure.sln b/CromwellOnAzure.sln index 561d4dc7..067d6600 100644 --- a/CromwellOnAzure.sln +++ b/CromwellOnAzure.sln @@ -32,6 +32,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonUtilities", "src\ga4g EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonUtilities.Tests", "src\ga4gh-tes\src\CommonUtilities.Tests\CommonUtilities.Tests.csproj", "{E9B266DA-528E-44D5-B934-7C352E0E3BDE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tes.Repository", "src\ga4gh-tes\src\Tes.Repository\Tes.Repository.csproj", "{515A4905-0522-4C72-BC18-41BE6A3BE880}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "build-push-acr", "src\ga4gh-tes\src\build-push-acr\build-push-acr.csproj", "{42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}" EndProject Global @@ -80,6 +82,10 @@ Global {E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Debug|Any CPU.Build.0 = Debug|Any CPU {E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Release|Any CPU.ActiveCfg = Release|Any CPU {E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Release|Any CPU.Build.0 = Release|Any CPU + {515A4905-0522-4C72-BC18-41BE6A3BE880}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {515A4905-0522-4C72-BC18-41BE6A3BE880}.Debug|Any CPU.Build.0 = Debug|Any CPU + {515A4905-0522-4C72-BC18-41BE6A3BE880}.Release|Any CPU.ActiveCfg = Release|Any CPU + {515A4905-0522-4C72-BC18-41BE6A3BE880}.Release|Any CPU.Build.0 = Release|Any CPU {42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Debug|Any CPU.Build.0 = Debug|Any CPU {42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/src/Common/UtilityExtensions.cs b/src/Common/UtilityExtensions.cs deleted file mode 100644 index 2b59d089..00000000 --- a/src/Common/UtilityExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Generic; - -namespace Common -{ - public static class UtilityExtensions - { - #region AddRange - public static void AddRange(this IList list, IEnumerable values) - { - foreach (var value in values) - { - list.Add(value); - }; - } - - //public static void AddRange(this IDictionary dictionary, IDictionary values) - //{ - // foreach (var value in values) - // { - // dictionary.Add(value); - // }; - //} - #endregion - } -} diff --git a/src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs b/src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs index 7f5b66bf..09acbe07 100644 --- a/src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs +++ b/src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs @@ -144,7 +144,7 @@ private TriggerHostedService SetCromwellOnAzureEnvironment(string accountAuthori } var logger = new Mock>().Object; - var postgreSqlOptions = new Mock>().Object; + var postgreSqlOptions = new Mock>().Object; var cromwellApiClient = new Mock().Object; var repository = new Mock>().Object; var storageUtility = new Mock(); diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 35fe491c..871a8226 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -30,10 +30,7 @@ public class IntegrationTests [TestMethod] public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { - const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; - const string workflowFriendlyName = $"mutect2"; - - await RunIntegrationTestAsync(new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) }); + await RunIntegrationTestAsync([("https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json", "mutect2")]); } /// @@ -44,7 +41,8 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() [TestMethod] public async Task RunAllCommonWorkflowsWaitTilDoneAsync() { - var workflowTriggerFiles = new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { + await RunIntegrationTestAsync( + [ ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.b37.trigger.json", "preprocessing-b37"), ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.hg38.trigger.json", "preprocessing-hg38"), ("https://raw.githubusercontent.com/microsoft/gatk4-genome-processing-pipeline-azure/main-azure/WholeGenomeGermlineSingleSample.trigger.json", "germline"), @@ -53,9 +51,8 @@ public async Task RunAllCommonWorkflowsWaitTilDoneAsync() ("https://raw.githubusercontent.com/microsoft/gatk4-cnn-variant-filter-azure/main-azure/cram2filtered.trigger.json", "cram-to-filtered"), ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/interleaved-fastq-to-paired-fastq.trigger.json", "fastq-to-paired"), ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/paired-fastq-to-unmapped-bam.trigger.json", "paired-fastq-to-unmapped-bam"), - ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/cram-to-bam.trigger.json", "cram-to-bam") }; - - await RunIntegrationTestAsync(workflowTriggerFiles); + ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/cram-to-bam.trigger.json", "cram-to-bam") + ]); } /// @@ -89,7 +86,7 @@ public async Task RunGlobTestWdlAsync() var n = DateTime.UtcNow; var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}"; var triggerFileBlobName = $"new/globtest-{date}.json"; - string triggerJson = System.Text.Json.JsonSerializer.Serialize(workflowTrigger).Replace(@"\r\n\", @"\n"); + var triggerJson = System.Text.Json.JsonSerializer.Serialize(workflowTrigger).Replace(@"\r\n\", @"\n"); container = blobServiceClient.GetBlobContainerClient("workflows"); await container.GetBlobClient(triggerFileBlobName).UploadAsync(BinaryData.FromString(triggerJson), true); } @@ -164,7 +161,7 @@ public async Task DeleteOldBatchPoolsAsync() var cutoffTime = DateTime.UtcNow.Subtract(maxAge); var pools = await batchClient.PoolOperations.ListPools().ToListAsync(); - int count = 0; + var count = 0; foreach (var pool in pools) { @@ -201,7 +198,7 @@ public void CountCompletedWorkflowsTest() Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); } - private async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles) + private static async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles) { // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ @@ -217,20 +214,20 @@ private async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, stri Console.WriteLine($"Found path: {path}"); var lines = await File.ReadAllLinesAsync(path); - string storageAccountName = lines[0].Trim(); - string workflowsContainerSasToken = lines[1].Trim('"'); + var storageAccountName = lines[0].Trim(); + var workflowsContainerSasToken = lines[1].Trim('"'); - int countOfEachWorkflowToRun = 1; + var countOfEachWorkflowToRun = 1; if (lines.Length > 2) { - int.TryParse(lines[2].Trim('"'), out countOfEachWorkflowToRun); + _ = int.TryParse(lines[2].Trim('"'), out countOfEachWorkflowToRun); } await StartWorkflowsAsync(countOfEachWorkflowToRun, triggerFiles, storageAccountName, waitTilDone: true, workflowsContainerSasToken); } - private async Task StartWorkflowsAsync( + private static async Task StartWorkflowsAsync( int countOfEachWorkflowToRun, List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles, string storageAccountName, @@ -265,9 +262,9 @@ private async Task StartWorkflowsAsync( using var httpClient = new HttpClient(); var blobNames = new List(); - foreach (var triggerFile in triggerFiles) + foreach (var (triggerFileBlobUrl, workflowFriendlyName) in triggerFiles) { - var triggerFileJson = await (await httpClient.GetAsync(triggerFile.triggerFileBlobUrl)).Content.ReadAsStringAsync(); + var triggerFileJson = await (await httpClient.GetAsync(triggerFileBlobUrl)).Content.ReadAsStringAsync(); // 2. Start the workflows by uploading new trigger files var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; @@ -276,7 +273,7 @@ private async Task StartWorkflowsAsync( for (var i = 1; i <= countOfEachWorkflowToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json - var blobName = $"new/{triggerFile.workflowFriendlyName}-{i:D4}-of-{countOfEachWorkflowToRun:D4}-{date}.json"; + var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfEachWorkflowToRun:D4}-{date}.json"; blobNames.Add(blobName); await workflowsContainer.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } @@ -289,7 +286,7 @@ private async Task StartWorkflowsAsync( } } - private async Task> ListContainerBlobNamesAsync(BlobContainerClient container) + private static async Task> ListContainerBlobNamesAsync(BlobContainerClient container) { var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); var existingBlobNames = new List(); @@ -304,12 +301,12 @@ private async Task> ListContainerBlobNamesAsync(BlobContainerClient return existingBlobNames; } - private int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) + private static int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) { - return GetWorkflowsByState(originalBlobNames, currentBlobNames, state).Count(); + return GetWorkflowsByState(originalBlobNames, currentBlobNames, state).Count; } - private List GetWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) + private static List GetWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) { var stateString = state.ToString().ToLowerInvariant(); @@ -341,10 +338,10 @@ public void GetWorkflowsByStateTest() Assert.IsTrue(succeeded.Single() == currentBlobNames.Skip(1).First()); } - private async Task WaitTilAllWorkflowsInTerminalStateAsync(BlobContainerClient container, List originalBlobNames, DateTime startTime) + private static async Task WaitTilAllWorkflowsInTerminalStateAsync(BlobContainerClient container, List originalBlobNames, DateTime startTime) { - int succeededCount = 0; - int failedCount = 0; + var succeededCount = 0; + var failedCount = 0; var sw = Stopwatch.StartNew(); diff --git a/src/TriggerService.Tests/ProcessAbortRequestTests.cs b/src/TriggerService.Tests/ProcessAbortRequestTests.cs index c12f0cf8..fbfcd335 100644 --- a/src/TriggerService.Tests/ProcessAbortRequestTests.cs +++ b/src/TriggerService.Tests/ProcessAbortRequestTests.cs @@ -95,7 +95,7 @@ public async Task FailedAbortRequestFileGetsMovedToFailedSubdirectory() ApplicationInsightsAccountName = "fakeappinsights" }); - var postgreSqlOptions = new Mock>().Object; + var postgreSqlOptions = new Mock>().Object; var storageUtility = new Mock(); storageUtility diff --git a/src/TriggerService.Tests/ProcessNewWorkflowTests.cs b/src/TriggerService.Tests/ProcessNewWorkflowTests.cs index 140f7684..cd157761 100644 --- a/src/TriggerService.Tests/ProcessNewWorkflowTests.cs +++ b/src/TriggerService.Tests/ProcessNewWorkflowTests.cs @@ -16,6 +16,7 @@ using Newtonsoft.Json; using Tes.Models; using Tes.Repository; +using Tes.Repository.Models; namespace TriggerService.Tests { diff --git a/src/TriggerService.Tests/TriggerEngineTests.cs b/src/TriggerService.Tests/TriggerEngineTests.cs index 0d12a40c..3d9f236c 100644 --- a/src/TriggerService.Tests/TriggerEngineTests.cs +++ b/src/TriggerService.Tests/TriggerEngineTests.cs @@ -69,7 +69,7 @@ public async Task TriggerEngineRunsAndOnlyLogsAvailabilityOncePerSystemUponAvail MainRunIntervalMilliseconds = 25 }); - var postgreSqlOptions = new Mock>().Object; + var postgreSqlOptions = new Mock>().Object; var cromwellApiClient = new Mock().Object; var tesTaskRepository = new Mock>().Object; var azureStorage = new Mock(); diff --git a/src/TriggerService.Tests/UpdateWorkflowStatusTests.cs b/src/TriggerService.Tests/UpdateWorkflowStatusTests.cs index f6de3eb0..405e8f3e 100644 --- a/src/TriggerService.Tests/UpdateWorkflowStatusTests.cs +++ b/src/TriggerService.Tests/UpdateWorkflowStatusTests.cs @@ -423,7 +423,7 @@ static string ShardString(int shard) => DefaultStorageAccountName = "fakestorage", ApplicationInsightsAccountName = "fakeappinsights" }); - var postgreSqlOptions = new Mock>().Object; + var postgreSqlOptions = new Mock>().Object; var storageUtility = new Mock(); storageUtility diff --git a/src/TriggerService/TriggerService.cs b/src/TriggerService/TriggerService.cs index 3a402afa..8a8d3de9 100644 --- a/src/TriggerService/TriggerService.cs +++ b/src/TriggerService/TriggerService.cs @@ -84,12 +84,12 @@ await Host.CreateDefaultBuilder() { serviceCollection.Configure(hostBuilderContext.Configuration.GetSection(CromwellApiClientOptions.CromwellApiClientOptionsSectionName)); serviceCollection.Configure(hostBuilderContext.Configuration.GetSection(TriggerServiceOptions.TriggerServiceOptionsSectionName)); - serviceCollection.Configure(hostBuilderContext.Configuration.GetSection(PostgreSqlOptions.GetConfigurationSectionName("Tes"))); + serviceCollection.Configure(hostBuilderContext.Configuration.GetSection(Tes.Repository.Models.PostgreSqlOptions.GetConfigurationSectionName("Tes"))); serviceCollection.AddSingleton(azureCloudConfig); serviceCollection.AddSingleton(); // Cache is not needed for this usage of the TesTask repository. Ensure that the middleware doesn't provide it one. serviceCollection.AddSingleton, TesTaskPostgreSqlRepository>(sp => new( - sp.GetRequiredService>(), + sp.GetRequiredService>(), sp.GetRequiredService(), sp.GetRequiredService>())); serviceCollection.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, configuration)); diff --git a/src/TriggerService/TriggerService.csproj b/src/TriggerService/TriggerService.csproj index e6e5739a..71a75dc8 100644 --- a/src/TriggerService/TriggerService.csproj +++ b/src/TriggerService/TriggerService.csproj @@ -39,7 +39,7 @@ - + diff --git a/src/deploy-cromwell-on-azure.Tests/deploy-cromwell-on-azure.Tests.csproj b/src/deploy-cromwell-on-azure.Tests/deploy-cromwell-on-azure.Tests.csproj index 20dcbde0..de1d489a 100644 --- a/src/deploy-cromwell-on-azure.Tests/deploy-cromwell-on-azure.Tests.csproj +++ b/src/deploy-cromwell-on-azure.Tests/deploy-cromwell-on-azure.Tests.csproj @@ -3,6 +3,8 @@ net8.0 false + true + None diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index d2bb58e7..f55fba00 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -1182,7 +1182,7 @@ private async Task BuildPushAcrAsync(Dictionary settings, string } else { - canReturnEarly = sameVersionUpgrade && !tesUpgraded && !triggerServiceUpgraded && !string.IsNullOrEmpty(settings["ActualTesImageName"]) && !string.IsNullOrEmpty(settings["ActualTriggerServiceImageName"]); + canReturnEarly = !acrRequested && sameVersionUpgrade && !tesUpgraded && !triggerServiceUpgraded && !string.IsNullOrEmpty(settings["ActualTesImageName"]) && !string.IsNullOrEmpty(settings["ActualTriggerServiceImageName"]); } if (cromwellUpgraded || string.IsNullOrEmpty(settings["ActualCromwellImageName"])) diff --git a/src/deploy-cromwell-on-azure/scripts/helm/values-template.yaml b/src/deploy-cromwell-on-azure/scripts/helm/values-template.yaml index a5162f83..e1e953a8 100644 --- a/src/deploy-cromwell-on-azure/scripts/helm/values-template.yaml +++ b/src/deploy-cromwell-on-azure/scripts/helm/values-template.yaml @@ -19,7 +19,6 @@ config: contentMD5: "False" batchScheduling: usePreemptibleVmsOnly: RUNTIME_PARAMETER - useLegacyAutopools: "False" poolRotationForcedDays: "7" taskMaxWallClockTimeDays: "7" prefix: RUNTIME_PARAMETER diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 44860004..96f24b2e 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 448600041330bf9891f1fe4dca939ed99631acb1 +Subproject commit 96f24b2e3a0670627927f354b23ee50937c41d96