Specify workflow URLs as a URL string to Cromwell instead of downloading

src/CromwellApiClient/CromwellApiClient.cs

@@ -56,8 +56,7 @@ public async Task<PostAbortResponse> PostAbortAsync(Guid id)
             => await PostAsync<PostAbortResponse>($"/{id}/abort", id);
 
         public async Task<PostWorkflowResponse> PostWorkflowAsync(
-            string workflowSourceFilename,
-            byte[] workflowSourceData,
+            string workflowUrl,
             List<string> workflowInputsFilename,
             List<byte[]> workflowInputsData,
             string workflowOptionsFilename = null,
@@ -66,30 +65,28 @@ public async Task<PostWorkflowResponse> PostWorkflowAsync(
             byte[] workflowDependenciesData = null)
         {
             var files = AccumulatePostFiles(
-                workflowSourceFilename,
-                workflowSourceData,
                 workflowInputsFilename,
                 workflowInputsData,
                 workflowOptionsFilename,
                 workflowOptionsData,
                 workflowDependenciesFilename,
                 workflowDependenciesData);
-            return await PostAsync<PostWorkflowResponse>(string.Empty, files);
+
+            var parameters = new List<KeyValuePair<string, string>> { 
+                new KeyValuePair<string, string>("workflowUrl", workflowUrl) };
+
+            return await PostAsync<PostWorkflowResponse>(string.Empty, files, parameters);
         }
 
         internal static List<FileToPost> AccumulatePostFiles(
-            string workflowSourceFilename,
-            byte[] workflowSourceData,
             List<string> workflowInputsFilename,
             List<byte[]> workflowInputsData,
             string workflowOptionsFilename = null,
             byte[] workflowOptionsData = null,
             string workflowDependenciesFilename = null,
             byte[] workflowDependenciesData = null)
         {
-            var files = new List<FileToPost> {
-                new(workflowSourceFilename, workflowSourceData, "workflowSource", removeTabs: true)
-            };
+            var files = new List<FileToPost>();
 
             for (var i = 0; i < workflowInputsFilename.Count; i++)
             {
@@ -241,23 +238,30 @@ private async Task<T> PostAsync<T>(string path, string body)
             }
         }
 
-        private async Task<T> PostAsync<T>(string path, IEnumerable<FileToPost> files)
+        private async Task<T> PostAsync<T>(string path, IEnumerable<FileToPost> files, IEnumerable<KeyValuePair<string, string>> parameters = null)
         {
             HttpResponseMessage response = null;
             var url = string.Empty;
 
             try
             {
                 url = GetApiUrl(path);
-                var content = new MultipartFormDataContent();
+                using var formContent = new MultipartFormDataContent(Guid.NewGuid().ToString());
+                formContent.Headers.ContentType.MediaType = "multipart/form-data";
+
+                foreach (var parameter in parameters)
+                {
+                    formContent.Add(new StringContent(parameter.Value), parameter.Key);
+                }
 
                 foreach (var file in files)
                 {
-                    var contentPart = new ByteArrayContent(file.Data);
-                    content.Add(contentPart, file.ParameterName, file.Filename);
+                    formContent.Add(new ByteArrayContent(file.Data), file.ParameterName, file.Filename);
                 }
 
-                response = await httpClient.PostAsync(url, content);
+                var req = formContent.ToString();
+
+                response = await httpClient.PostAsync(url, formContent);
                 response.EnsureSuccessStatusCode();
                 return await response.Content.ReadAsAsync<T>();
             }

src/CromwellApiClient/ICromwellApiClient.cs

@@ -11,7 +11,7 @@ public interface ICromwellApiClient
     {
         string GetUrl();
         Task<PostWorkflowResponse> PostWorkflowAsync(
-            string workflowSourceFilename, byte[] workflowSourceData,
+            string workflowUrl,
             List<string> workflowInputsFilename, List<byte[]> workflowInputsData,
             string workflowOptionsFilename = null, byte[] workflowOptionsData = null,
             string workflowDependenciesFilename = null, byte[] workflowDependenciesData = null);

src/TriggerService.Tests/CromwellApiClientTests.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using TriggerService;
+
+namespace CromwellApiClient.Tests
+{
+    [TestClass]
+    public class CromwellApiClientTests
+    {
+        [Ignore]
+        [TestMethod]
+        public async Task PostWorkflowAsyncTest()
+        {
+            var options = new Mock<IOptions<CromwellApiClientOptions>>();
+
+            options.Setup(o => o.Value).Returns(new CromwellApiClientOptions()
+            {
+                BaseUrl = "http://cromwell"
+            }); ;
+
+            var cromwellApiClient = new CromwellApiClient(options.Object);
+            var workflowInputsFilenames = new List<string> { "inputs.json" };
+            var workflowInputsDatas = new List<byte[]> { Encoding.UTF8.GetBytes("{}") };
+
+            await cromwellApiClient.PostWorkflowAsync(
+                "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.wdl",
+                workflowInputsFilenames,
+                workflowInputsDatas);
+        }
+    }
+}

src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs

@@ -335,8 +335,6 @@ private async Task ExecuteTriggerFileTest(string triggerFileContent, int inputFi
 
         private void VerifyTriggerFileProcessing(ProcessedTriggerInfo processedTriggerInfo, int inputFilesCount)
         {
-            Assert.AreEqual(azureName, processedTriggerInfo.WorkflowSource.Filename, "comparing azureName to workflowSourceFilename");
-            AssertBytesEqual(processedTriggerInfo.WorkflowSource.Data, httpClientData, "workflowSourceData");
             AssertNamesEqual(processedTriggerInfo.WorkflowInputs.Select(a => a.Filename).ToList(), inputFilesCount, azureName, "workflowInputsFilenames");
             AssertBytesEqual(processedTriggerInfo.WorkflowInputs.Select(a => a.Data).ToList(), inputFilesCount, httpClientData, "workflowInputsData");
             AssertExtraDataNull(processedTriggerInfo);
@@ -385,8 +383,6 @@ private static void AssertBytesEqual(byte[] data, byte[] expectedData, string da
 
         private static List<CromwellApiClient.CromwellApiClient.FileToPost> RetrievePostFiles(ProcessedTriggerInfo processedTriggerInfo)
             => CromwellApiClient.CromwellApiClient.AccumulatePostFiles(
-                processedTriggerInfo.WorkflowSource.Filename,
-                processedTriggerInfo.WorkflowSource.Data,
                 processedTriggerInfo.WorkflowInputs.Select(a => a.Filename).ToList(),
                 processedTriggerInfo.WorkflowInputs.Select(a => a.Data).ToList(),
                 processedTriggerInfo.WorkflowOptions.Filename,
@@ -397,28 +393,24 @@ private static void AssertBytesEqual(byte[] data, byte[] expectedData, string da
         private static void VerifyPostFiles(ProcessedTriggerInfo processedTriggerInfo)
         {
             var files = RetrievePostFiles(processedTriggerInfo);
-
-            Assert.AreEqual(processedTriggerInfo.WorkflowInputs.Count + 1, files.Count, "unexpected number of files");
-
-            Assert.AreEqual("workflowSource", files[0].ParameterName, $"unexpected ParameterName for the 0th file");
-            Assert.AreEqual(processedTriggerInfo.WorkflowSource.Filename, files[0].Filename, $"unexpected Filename for the 0th file");
-            AssertBytesEqual(processedTriggerInfo.WorkflowSource.Data, files[0].Data, "files[0].Data");
+
+            Assert.AreEqual(processedTriggerInfo.WorkflowInputs.Count, files.Count, "unexpected number of files");
 
             for (var i = 0; i < processedTriggerInfo.WorkflowInputs.Count; i++)
             {
                 var ip1 = i + 1;
 
                 if (i == 0)
                 {
-                    Assert.AreEqual("workflowInputs", files[ip1].ParameterName, $"unexpected ParameterName for file #{ip1}");
+                    Assert.AreEqual("workflowInputs", files[i].ParameterName, $"unexpected ParameterName for file #{i}");
                 }
                 else
                 {
-                    Assert.AreEqual("workflowInputs_" + ip1, files[ip1].ParameterName, $"unexpected ParameterName for file #{ip1}");
+                    Assert.AreEqual("workflowInputs_" + ip1, files[i].ParameterName, $"unexpected ParameterName for file #{i}");
                 }
 
-                Assert.AreEqual(processedTriggerInfo.WorkflowInputs[i].Filename, files[ip1].Filename, $"unexpected Filename for file #{ip1}");
-                AssertBytesEqual(processedTriggerInfo.WorkflowInputs[i].Data, files[ip1].Data, $"files[{ip1}].Data");
+                Assert.AreEqual(processedTriggerInfo.WorkflowInputs[i].Filename, files[i].Filename, $"unexpected Filename for file #{i}");
+                AssertBytesEqual(processedTriggerInfo.WorkflowInputs[i].Data, files[i].Data, $"files[{i}].Data");
             }
         }
     }

src/TriggerService.Tests/ProcessNewWorkflowTests.cs

@@ -31,7 +31,7 @@ public async Task NewWorkflowsAreMovedToInProgressSubdirectory()
             var cromwellApiClient = new Mock<ICromwellApiClient>();
 
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Returns(Task.FromResult(new PostWorkflowResponse { Id = workflowId }));
 
             var (newTriggerName, newTriggerContent) = await ProcessNewWorkflowAsync(cromwellApiClient.Object);
@@ -46,7 +46,7 @@ public async Task NewWorkflowsThatFailToPostToCromwellAreMovedToFailedSubdirecto
             var exceptionMessage = "Error submitting new workflow";
 
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Throws(new Exception(exceptionMessage));
 
             var (newTriggerName, newTriggerContent) = await ProcessNewWorkflowAsync(cromwellApiClient.Object);
@@ -136,7 +136,7 @@ public async Task NewWorkflowsThatFailToParseAsJsonAreAnotatedAndMovedToFailedSu
 
             var triesToPost = false;
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Callback(() => triesToPost = true)
                 .Throws(new Exception("Should never get here."));
 

src/TriggerService.Tests/TriggerService.Tests.csproj

@@ -29,6 +29,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Common\Common.csproj" />
+    <ProjectReference Include="..\CromwellApiClient\CromwellApiClient.csproj" />
     <ProjectReference Include="..\ga4gh-tes\src\Tes\Tes.csproj" />
     <ProjectReference Include="..\TriggerService\TriggerService.csproj" />
   </ItemGroup>

src/TriggerService/ProcessedTriggerInfo.cs

@@ -7,15 +7,15 @@ namespace CromwellApiClient
 {
     public class ProcessedTriggerInfo
     {
-        public ProcessedWorkflowItem WorkflowSource { get; private set; }
+        public string WorkflowUrl { get; private set; }
         public List<ProcessedWorkflowItem> WorkflowInputs { get; private set; }
         public ProcessedWorkflowItem WorkflowOptions { get; private set; }
         public ProcessedWorkflowItem WorkflowDependencies { get; private set; }
 
-        public ProcessedTriggerInfo(ProcessedWorkflowItem workflowSource, List<ProcessedWorkflowItem> workflowInputs,
+        public ProcessedTriggerInfo(string workflowUrl, List<ProcessedWorkflowItem> workflowInputs,
             ProcessedWorkflowItem workflowOptions, ProcessedWorkflowItem workflowDependencies)
         {
-            this.WorkflowSource = workflowSource;
+            this.WorkflowUrl = workflowUrl;
             this.WorkflowInputs = workflowInputs;
             this.WorkflowOptions = workflowOptions;
             this.WorkflowDependencies = workflowDependencies;

src/TriggerService/TriggerHostedService.cs

@@ -125,7 +125,7 @@ public async Task ExecuteNewWorkflowsAsync()
                     var processedTriggerInfo = await ProcessBlobTrigger(blobTriggerJson);
 
                     var response = await cromwellApiClient.PostWorkflowAsync(
-                        processedTriggerInfo.WorkflowSource.Filename, processedTriggerInfo.WorkflowSource.Data,
+                        processedTriggerInfo.WorkflowUrl,
                         processedTriggerInfo.WorkflowInputs.Select(a => a.Filename).ToList(),
                         processedTriggerInfo.WorkflowInputs.Select(a => a.Data).ToList(),
                         processedTriggerInfo.WorkflowOptions.Filename, processedTriggerInfo.WorkflowOptions.Data,
@@ -165,7 +165,10 @@ internal async Task<ProcessedTriggerInfo> ProcessBlobTrigger(string blobTriggerJ
                 throw new ArgumentNullException(nameof(Workflow.WorkflowUrl), "must specify a WorkflowUrl in the Trigger File");
             }
 
-            var workflowSource = await GetBlobFileNameAndData(triggerInfo.WorkflowUrl);
+            if (!Uri.TryCreate(triggerInfo.WorkflowUrl, UriKind.Absolute, out var _))
+            {
+                throw new ArgumentException("The WorkflowUrl was not a valid URI");
+            }
 
             if (triggerInfo.WorkflowInputsUrl is not null)
             {
@@ -183,7 +186,7 @@ internal async Task<ProcessedTriggerInfo> ProcessBlobTrigger(string blobTriggerJ
             var workflowOptions = await GetBlobFileNameAndData(triggerInfo.WorkflowOptionsUrl);
             var workflowDependencies = await GetBlobFileNameAndData(triggerInfo.WorkflowDependenciesUrl);
 
-            return new ProcessedTriggerInfo(workflowSource, workflowInputs, workflowOptions, workflowDependencies);
+            return new ProcessedTriggerInfo(triggerInfo.WorkflowUrl, workflowInputs, workflowOptions, workflowDependencies);
         }
 
         public async Task UpdateWorkflowStatusesAsync()