Stop inlining large TestData (#5647) (#6292)

Author: mathewc

src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs

@@ -125,7 +125,18 @@ private static async Task<string> GetTestData(string testDataPath, ScriptHostCon
                 await FileUtility.WriteAsync(testDataPath, string.Empty);
             }
 
-            return await FileUtility.ReadAsync(testDataPath);
+            string data = await FileUtility.ReadAsync(testDataPath);
+
+            // We avoid inlining test data beyond the specified length. We do this to
+            // provide back compat for the majority of cases. In all other cases, the caller
+            // is expected to follow TestDataHref to retrieve the data.
+            bool testDataCapEnabled = (Environment.GetEnvironmentVariable(EnvironmentSettingNames.TestDataCapEnabled) ?? "1") == "1";
+            if (data.Length > ScriptConstants.MaxTestDataInlineStringLength && testDataCapEnabled)
+            {
+                return null;
+            }
+
+            return data;
         }
 
         internal static Uri GetFunctionInvokeUrlTemplate(string baseUrl, FunctionMetadata functionMetadata, string routePrefix)

src/WebJobs.Script/EnvironmentSettingNames.cs

@@ -30,6 +30,7 @@ public static class EnvironmentSettingNames
         public const string SkipSslValidation = "SCM_SKIP_SSL_VALIDATION";
         public const string CoreToolsEnvironment = "FUNCTIONS_CORETOOLS_ENVIRONMENT";
         public const string AzureWebsiteArmCacheEnabled = "WEBSITE_FUNCTIONS_ARMCACHE_ENABLED";
+        public const string TestDataCapEnabled = "WEBSITE_FUNCTIONS_TESTDATA_CAP_ENABLED";
 
         /// <summary>
         /// Environment variable dynamically set by the platform when it is safe to

src/WebJobs.Script/ScriptConstants.cs

@@ -109,5 +109,6 @@ public static class ScriptConstants
         /// for settriggers calls. If we raise that limit there, we should raise here as well.
         /// </summary>
         public const int MaxTriggersStringLength = 204800;
+        public const int MaxTestDataInlineStringLength = 4 * 1024;
     }
 }

test/WebJobs.Script.Tests/Management/WebFunctionsManagerTests.cs

@@ -35,37 +35,63 @@ public WebFunctionsManagerTests()
 
             _hostConfig = new ScriptHostConfiguration
             {
-                RootScriptPath = @"x:\root\site\wwwroot",
+                RootScriptPath = @"x:\root",
                 IsSelfHost = false,
                 RootLogPath = @"x:\root\LogFiles\Application\Functions",
-                TestDataPath = @"x:\root\data\functions\sampledata"
+                TestDataPath = @"x:\root\data"
             };
             _hostConfig.HostConfig.HostId = "testhostid123";
 
             HostNameProvider.Reset();
         }
 
-        [Fact]
-        public async Task ReadFunctionsMetadataSucceeds()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task ReadFunctionsMetadataSucceeds(bool testDataCapEnabled)
         {
-            // Setup
-            var fileSystem = CreateFileSystem(_hostConfig);
-            var loggerProvider = new TestLoggerProvider();
-            var loggerFactory = new LoggerFactory();
-            loggerFactory.AddProvider(loggerProvider);
-
-            var contentBuilder = new StringBuilder();
-            var httpClient = CreateHttpClient(contentBuilder);
-            var webManager = new WebFunctionsManager(_hostConfig, loggerFactory);
-
-            FileUtility.Instance = fileSystem;
-            var functions = await webManager.GetFunctionsMetadata();
-            var jsFunctions = functions.Where(funcMetadata => funcMetadata.Language == ScriptType.Javascript.ToString()).ToList();
-            var pytonFunctions = functions.Where(funcMetadata => funcMetadata.Language == ScriptType.Python.ToString()).ToList();
-
-            Assert.Equal(3, functions.Count());
-            Assert.Equal(2, jsFunctions.Count());
-            Assert.Equal(1, pytonFunctions.Count());
+            var env = new Dictionary<string, string>();
+            if (!testDataCapEnabled)
+            {
+                env.Add(EnvironmentSettingNames.TestDataCapEnabled, "0");
+            }
+            var envScope = new TestScopedEnvironmentVariable(env);
+            using (envScope)
+            {
+                // Setup
+                var fileSystem = CreateFileSystem(_hostConfig);
+                var loggerProvider = new TestLoggerProvider();
+                var loggerFactory = new LoggerFactory();
+                loggerFactory.AddProvider(loggerProvider);
+
+                var contentBuilder = new StringBuilder();
+                var httpClient = CreateHttpClient(contentBuilder);
+                var webManager = new WebFunctionsManager(_hostConfig, loggerFactory);
+
+                FileUtility.Instance = fileSystem;
+                var metadata = (await webManager.GetFunctionsMetadata()).ToArray();
+
+                Assert.Equal(3, metadata.Count());
+                Assert.Equal(2, metadata.Count(p => p.Language == ScriptType.Javascript.ToString()));
+                Assert.Equal(1, metadata.Count(p => p.Language == ScriptType.Python.ToString()));
+
+                Assert.Equal("https://localhost/api/vfs/data/function1.dat", metadata[0].TestDataHref.AbsoluteUri);
+                Assert.Equal("https://localhost/api/vfs/data/function2.dat", metadata[1].TestDataHref.AbsoluteUri);
+                Assert.Equal("https://localhost/api/vfs/data/function3.dat", metadata[2].TestDataHref.AbsoluteUri);
+
+                Assert.Equal("Test Data 1", metadata[0].TestData);
+                Assert.Equal("Test Data 2", metadata[1].TestData);
+
+                if (testDataCapEnabled)
+                {
+                    // TestData size is capped by default
+                    Assert.Null(metadata[2].TestData);
+                }
+                else
+                {
+                    Assert.Equal(ScriptConstants.MaxTestDataInlineStringLength + 1, metadata[2].TestData.Length);
+                }
+            }
         }
 
         [Theory]
@@ -236,6 +262,10 @@ private static IFileSystem CreateFileSystem(ScriptHostConfiguration hostConfig)
   ]
 }";
 
+            string testData1 = "Test Data 1";
+            string testData2 = "Test Data 2";
+            string testData3 = TestHelpers.NewRandomString(ScriptConstants.MaxTestDataInlineStringLength + 1);
+
             fileBase.Setup(f => f.Exists(Path.Combine(rootScriptPath, @"function1\function.json"))).Returns(true);
             fileBase.Setup(f => f.Exists(Path.Combine(rootScriptPath, @"function1\main.py"))).Returns(true);
             fileBase.Setup(f => f.ReadAllText(Path.Combine(rootScriptPath, @"function1\function.json"))).Returns(function1);
@@ -245,7 +275,7 @@ private static IFileSystem CreateFileSystem(ScriptHostConfiguration hostConfig)
             });
             fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function1.dat"), It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>())).Returns(() =>
             {
-                return new MemoryStream(Encoding.UTF8.GetBytes(function1));
+                return new MemoryStream(Encoding.UTF8.GetBytes(testData1));
             });
 
             fileBase.Setup(f => f.Exists(Path.Combine(rootScriptPath, @"function2\function.json"))).Returns(true);
@@ -257,7 +287,7 @@ private static IFileSystem CreateFileSystem(ScriptHostConfiguration hostConfig)
             });
             fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function2.dat"), It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>())).Returns(() =>
             {
-                return new MemoryStream(Encoding.UTF8.GetBytes(function1));
+                return new MemoryStream(Encoding.UTF8.GetBytes(testData2));
             });
 
             fileBase.Setup(f => f.Exists(Path.Combine(rootScriptPath, @"function3\function.json"))).Returns(true);
@@ -269,7 +299,7 @@ private static IFileSystem CreateFileSystem(ScriptHostConfiguration hostConfig)
             });
             fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function3.dat"), It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>())).Returns(() =>
             {
-                return new MemoryStream(Encoding.UTF8.GetBytes(function1));
+                return new MemoryStream(Encoding.UTF8.GetBytes(testData3));
             });
 
             return fileSystem.Object;