Scm build package (#4432)

Author: maiqbal11

src/WebJobs.Script.WebHost/Management/InstanceManager.cs

@@ -16,6 +16,7 @@
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Microsoft.WindowsAzure.Storage.Blob;
 using Newtonsoft.Json;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.Management
@@ -130,7 +131,16 @@ public async Task<string> ValidateContext(HostAssignmentContext assignmentContex
             HttpResponseMessage response = null;
             try
             {
-                var zipUrl = assignmentContext.ZipUrl;
+                RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
+                _logger.LogInformation($"Will be using {pkgContext.EnvironmentVariableName} app setting as zip url");
+
+                if (pkgContext.IsScmRunFromPackage())
+                {
+                    // Not user assigned so limit validation
+                    return null;
+                }
+
+                var zipUrl = pkgContext.Url;
                 if (!string.IsNullOrEmpty(zipUrl))
                 {
                     // make sure the zip uri is valid and accessible
@@ -193,16 +203,35 @@ private async Task ApplyContext(HostAssignmentContext assignmentContext)
             _logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)");
             assignmentContext.ApplyAppSettings(_environment);
 
-            // We need to get the non-PlaceholderMode script path so we can unzip to the correct location.
+            // We need to get the non-PlaceholderMode script Path so we can unzip to the correct location.
             // This asks the factory to skip the PlaceholderMode check when configuring options.
             var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder);
+            RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
 
-            var zipPath = assignmentContext.ZipUrl;
+            var zipPath = pkgContext.Url;
             if (!string.IsNullOrEmpty(zipPath))
             {
                 // download zip and extract
                 var zipUri = new Uri(zipPath);
-                var filePath = await DownloadAsync(zipUri);
+                string filePath = null;
+
+                if (pkgContext.IsScmRunFromPackage())
+                {
+                    bool blobExists = await pkgContext.BlobExistsAsync(_logger);
+                    if (blobExists)
+                    {
+                        filePath = await DownloadAsync(zipUri);
+                    }
+                    else
+                    {
+                        return;
+                    }
+                }
+                else
+                {
+                    filePath = await DownloadAsync(zipUri);
+                }
+
                 UnpackPackage(filePath, options.ScriptPath);
 
                 string bundlePath = Path.Combine(options.ScriptPath, "worker-bundle");

src/WebJobs.Script.WebHost/Models/HostAssignmentContext.cs

@@ -24,26 +24,31 @@ public class HostAssignmentContext
         [JsonProperty("MSISpecializationPayload")]
         public MSIContext MSIContext { get; set; }
 
-        public string ZipUrl
+        public RunFromPackageContext GetRunFromPkgContext()
         {
-            get
+            if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteRunFromPackage))
             {
-                if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteRunFromPackage))
-                {
-                    return Environment[EnvironmentSettingNames.AzureWebsiteRunFromPackage];
-                }
-                else if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteAltZipDeployment))
-                {
-                    return Environment[EnvironmentSettingNames.AzureWebsiteAltZipDeployment];
-                }
-                else if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteZipDeployment))
-                {
-                    return Environment[EnvironmentSettingNames.AzureWebsiteZipDeployment];
-                }
-                else
-                {
-                    return string.Empty;
-                }
+                return new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage,
+                    Environment[EnvironmentSettingNames.AzureWebsiteRunFromPackage]);
+            }
+            else if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteAltZipDeployment))
+            {
+                return new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteAltZipDeployment,
+                    Environment[EnvironmentSettingNames.AzureWebsiteAltZipDeployment]);
+            }
+            else if (Environment.ContainsKey(EnvironmentSettingNames.AzureWebsiteZipDeployment))
+            {
+                return new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteZipDeployment,
+                    Environment[EnvironmentSettingNames.AzureWebsiteZipDeployment]);
+            }
+            else if (Environment.ContainsKey(EnvironmentSettingNames.ScmRunFromPackage))
+            {
+                return new RunFromPackageContext(EnvironmentSettingNames.ScmRunFromPackage,
+                    Environment[EnvironmentSettingNames.ScmRunFromPackage]);
+            }
+            else
+            {
+                return new RunFromPackageContext(string.Empty, string.Empty);
             }
         }
 

src/WebJobs.Script.WebHost/Models/RunFromPackageContext.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost.Models
+{
+    public class RunFromPackageContext
+    {
+        public RunFromPackageContext(string envVarName, string url)
+        {
+            EnvironmentVariableName = envVarName;
+            Url = url;
+        }
+
+        public string EnvironmentVariableName { get; set; }
+
+        public string Url { get; set; }
+
+        public bool IsScmRunFromPackage()
+        {
+            return string.Equals(EnvironmentVariableName, EnvironmentSettingNames.ScmRunFromPackage,
+                        StringComparison.OrdinalIgnoreCase);
+        }
+
+        public async Task<bool> BlobExistsAsync(ILogger logger)
+        {
+            bool exists = false;
+            await Utility.InvokeWithRetriesAsync(async () =>
+            {
+                try
+                {
+                    CloudBlockBlob blob = new CloudBlockBlob(new Uri(Url));
+                    exists = await blob.ExistsAsync();
+                }
+                catch (Exception e)
+                {
+                    logger.LogError(e, $"Failed to check if zip url blob exists");
+                    throw;
+                }
+            }, maxRetries: 2, retryInterval: TimeSpan.FromSeconds(0.3));
+
+            return exists;
+        }
+    }
+}

src/WebJobs.Script/Environment/EnvironmentExtensions.cs

@@ -70,13 +70,19 @@ public static bool IsZipDeployment(this IEnvironment environment)
             // Run From Package app setting exists
             return IsValidZipSetting(environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment)) ||
                 IsValidZipSetting(environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteAltZipDeployment)) ||
-                IsValidZipSetting(environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteRunFromPackage));
+                IsValidZipSetting(environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteRunFromPackage)) ||
+                IsValidZipUrl(environment.GetEnvironmentVariable(EnvironmentSettingNames.ScmRunFromPackage));
         }
 
-        private static bool IsValidZipSetting(string appSetting)
+        public static bool IsValidZipSetting(string appSetting)
         {
             // valid values are 1 or an absolute URI
-            return string.Equals(appSetting, "1") || Uri.TryCreate(appSetting, UriKind.Absolute, out Uri result);
+            return string.Equals(appSetting, "1") || IsValidZipUrl(appSetting);
+        }
+
+        public static bool IsValidZipUrl(string appSetting)
+        {
+            return Uri.TryCreate(appSetting, UriKind.Absolute, out Uri result);
         }
 
         public static bool IsAppServiceWindowsEnvironment(this IEnvironment environment)

src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

@@ -56,6 +56,9 @@ public static class EnvironmentSettingNames
         public const string AzureWebsiteRunFromPackage = "WEBSITE_RUN_FROM_PACKAGE";
         public const string RegionName = "REGION_NAME";
 
+        // handling server side builds
+        public const string ScmRunFromPackage = "SCM_RUN_FROM_PACKAGE";
+
         public const string LinuxAzureAppServiceStorage = "WEBSITES_ENABLE_APP_SERVICE_STORAGE";
         public const string CoreToolsEnvironment = "FUNCTIONS_CORETOOLS_ENVIRONMENT";
         public const string RunningInContainer = "DOTNET_RUNNING_IN_CONTAINER";

test/WebJobs.Script.Tests.Integration/Host/StandbyManager/StandbyManagerE2ETests_Linux.cs

@@ -139,10 +139,12 @@ private async Task Assign(string encryptionKey)
             var contentRoot = Path.Combine(Path.GetTempPath(), @"FunctionsTest");
             var sourcePath = Path.Combine(Directory.GetCurrentDirectory(), @"TestScripts\Node\HttpTrigger");
             var zipFilePath = Path.Combine(contentRoot, "content.zip");
-            await CreateContentZip(contentRoot, zipFilePath, @"TestScripts\Node\HttpTrigger");
+            await TestHelpers.CreateContentZip(contentRoot, zipFilePath, @"TestScripts\Node\HttpTrigger");
 
             // upload the blob and get a SAS uri
-            var sasUri = await CreateBlobSas(zipFilePath, "azure-functions-test", "appcontents.zip");
+            var configuration = _httpServer.Host.Services.GetService<IConfiguration>();
+            string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
+            var sasUri = await TestHelpers.CreateBlobSas(connectionString, zipFilePath, "azure-functions-test", "appcontents.zip");
 
             // Now specialize the host by invoking assign
             var secretManager = _httpServer.Host.Services.GetService<ISecretManagerProvider>().Current;
@@ -166,45 +168,5 @@ private async Task Assign(string encryptionKey)
             var response = await _httpClient.SendAsync(request);
             Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
         }
-
-        private static async Task CreateContentZip(string contentRoot, string zipPath, params string[] copyDirs)
-        {
-            var contentTemp = Path.Combine(contentRoot, @"ZipContent");
-            await FileUtility.DeleteDirectoryAsync(contentTemp, true);
-
-            foreach (var sourceDir in copyDirs)
-            {
-                var directoryName = Path.GetFileName(sourceDir);
-                var targetPath = Path.Combine(contentTemp, directoryName);
-                FileUtility.EnsureDirectoryExists(targetPath);
-                var sourcePath = Path.Combine(Directory.GetCurrentDirectory(), sourceDir);
-                FileUtility.CopyDirectory(sourcePath, targetPath);
-            }
-
-            FileUtility.DeleteFileSafe(zipPath);
-            ZipFile.CreateFromDirectory(contentTemp, zipPath);
-        }
-
-        private async Task<Uri> CreateBlobSas(string filePath, string blobContainer, string blobName)
-        {
-            var configuration = _httpServer.Host.Services.GetService<IConfiguration>();
-            string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
-            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
-            var blobClient = storageAccount.CreateCloudBlobClient();
-            var container = blobClient.GetContainerReference(blobContainer);
-            await container.CreateIfNotExistsAsync();
-            var blob = container.GetBlockBlobReference(blobName);
-            await blob.UploadFromFileAsync(filePath);
-            var policy = new SharedAccessBlobPolicy
-            {
-                SharedAccessStartTime = DateTime.UtcNow,
-                SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1),
-                Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
-            };
-            var sas = blob.GetSharedAccessSignature(policy);
-            var sasUri = new Uri(blob.Uri, sas);
-
-            return sasUri;
-        }
     }
 }