Skip to content

Commit 8e8c1ce

Browse files
authored
Exclude test data from functions endpoint response (#10943)
* Exclude test data from functions endpoint response. * Rename feature flag to be more generic (for READ and WRITE). Update write code path to check feature flag. * reset FileUtility.Instance after test * fix Property_ValidateAccess test * Adding a test for the write code path of the test data file. * PR feedbak updates. - Switched from `IOptions<FunctionsHostingConfigOptions>` to `IOptionsMonitor<FunctionsHostingConfigOptions>` - renamed feature flag.
1 parent ca8f59d commit 8e8c1ce

File tree

9 files changed

+171
-38
lines changed

9 files changed

+171
-38
lines changed

release_notes.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
- Fixing invalid DateTimes in status blobs when invoking via portal (#10916)
1414
- Bug fix for platform release channel bundles resolution casing issue and additional logging (#10921)
1515
- Adding support for faas.invoke_duration metric and other spec related updates (#10929)
16+
- Added the option to exclude test data from the `/functions` endpoint API response (#10943)
1617
- Increased the GC allocation budget value to improve cold start (#10953)
1718
- Fixed bug that could result in "Binding names must be unique" error (#10938)
1819
- Fix race condition that leads the host to initialize placeholder (warmup) function in Linux environments (#10848)

src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ public static class FunctionMetadataExtensions
1919
/// </summary>
2020
/// <param name="functionMetadata">FunctionMetadata to be mapped.</param>
2121
/// <param name="hostOptions">The host options.</param>
22+
/// <param name="excludeTestData">If true, the returned <see cref="FunctionMetadataResponse"/> will not populate the <see cref="FunctionMetadataResponse.TestData"/> property.</param>
2223
/// <returns>Promise of a FunctionMetadataResponse.</returns>
23-
public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl)
24+
public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl, bool excludeTestData)
2425
{
2526
string functionPath = GetFunctionPathOrNull(hostOptions.RootScriptPath, functionMetadata.Name);
2627
string functionMetadataFilePath = GetMetadataPathOrNull(functionPath);
@@ -54,7 +55,7 @@ public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(th
5455
response.ConfigHref = VirtualFileSystem.FilePathToVfsUri(functionMetadataFilePath, baseUrl, hostOptions);
5556
}
5657

57-
if (!string.IsNullOrEmpty(hostOptions.TestDataPath))
58+
if (!excludeTestData && !string.IsNullOrEmpty(hostOptions.TestDataPath))
5859
{
5960
var testDataFilePath = functionMetadata.GetTestDataFilePath(hostOptions);
6061
response.TestDataHref = VirtualFileSystem.FilePathToVfsUri(testDataFilePath, baseUrl, hostOptions);
@@ -194,7 +195,7 @@ private static JObject GetFunctionConfigFromMetadata(FunctionMetadata metadata)
194195

195196
private static async Task<string> GetTestData(string testDataPath, ScriptJobHostOptions config)
196197
{
197-
if (!File.Exists(testDataPath))
198+
if (!FileUtility.FileExists(testDataPath))
198199
{
199200
FileUtility.EnsureDirectoryExists(Path.GetDirectoryName(testDataPath));
200201
await FileUtility.WriteAsync(testDataPath, string.Empty);

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ private async Task<SyncTriggersPayload> GetSyncTriggersPayload()
339339

340340
// Add all listable functions details to the payload
341341
var listableFunctions = _functionMetadataManager.GetFunctionMetadata().Where(m => !m.IsCodeless());
342-
var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(listableFunctions, hostOptions, _hostNameProvider);
342+
var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(listableFunctions, hostOptions, _hostNameProvider, excludeTestData: _hostingConfigOptions.Value.IsTestDataSuppressionEnabled);
343343
result.Add("functions", new JArray(functionDetails.Select(p => JObject.FromObject(p))));
344344

345345
// TEMP: refactor this code to properly add extensions in all scenario(#7394)

src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,12 @@
77
using System.Linq;
88
using System.Net.Http;
99
using System.Threading.Tasks;
10-
using Azure;
11-
using Azure.Core;
1210
using Microsoft.AspNetCore.Http;
11+
using Microsoft.Azure.WebJobs.Script.Config;
1312
using Microsoft.Azure.WebJobs.Script.Description;
1413
using Microsoft.Azure.WebJobs.Script.Management.Models;
1514
using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
1615
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
17-
using Microsoft.Extensions.DependencyInjection;
1816
using Microsoft.Extensions.Logging;
1917
using Microsoft.Extensions.Options;
2018
using Newtonsoft.Json;
@@ -33,9 +31,10 @@ public class WebFunctionsManager : IWebFunctionsManager
3331
private readonly IFunctionMetadataManager _functionMetadataManager;
3432
private readonly IHostFunctionMetadataProvider _hostFunctionMetadataProvider;
3533
private readonly IOptionsMonitor<LanguageWorkerOptions> _languageWorkerOptions;
34+
private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
3635

3736
public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IHostFunctionMetadataProvider hostFunctionMetadataProvider,
38-
IOptionsMonitor<LanguageWorkerOptions> languageWorkerOptions)
37+
IOptionsMonitor<LanguageWorkerOptions> languageWorkerOptions, IOptionsMonitor<FunctionsHostingConfigOptions> hostingConfigOptions)
3938
{
4039
_applicationHostOptions = applicationHostOptions;
4140
_logger = loggerFactory?.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
@@ -46,21 +45,22 @@ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applica
4645
_functionMetadataManager = functionMetadataManager;
4746
_hostFunctionMetadataProvider = hostFunctionMetadataProvider;
4847
_languageWorkerOptions = languageWorkerOptions;
48+
_hostingConfigOptions = hostingConfigOptions;
4949
}
5050

5151
public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies)
5252
{
5353
var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
5454
var functionsMetadata = GetFunctionsMetadata(includeProxies, forceRefresh: false);
5555

56-
return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider);
56+
return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider, excludeTestData: _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled);
5757
}
5858

59-
internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider)
59+
internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider, bool excludeTestData)
6060
{
6161
string baseUrl = GetBaseUrl(hostNameProvider);
6262
string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
63-
var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));
63+
var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData));
6464

6565
return await tasks.WhenAll();
6666
}
@@ -148,7 +148,7 @@ await functionMetadata
148148
configChanged = true;
149149
}
150150

151-
if (functionMetadata.TestData != null)
151+
if (functionMetadata.TestData != null && !_hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled)
152152
{
153153
await FileUtility.WriteAsync(dataFilePath, functionMetadata.TestData);
154154
}
@@ -161,7 +161,7 @@ await functionMetadata
161161
FunctionMetadataResponse functionMetadataResult = null;
162162
if (metadata != null)
163163
{
164-
functionMetadataResult = await GetFunctionMetadataResponseAsync(metadata, hostOptions, request);
164+
functionMetadataResult = await GetFunctionMetadataResponseAsync(metadata, hostOptions, request, _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled);
165165
success = true;
166166
}
167167

@@ -197,7 +197,7 @@ await functionMetadata
197197

198198
if (functionMetadata != null)
199199
{
200-
var functionMetadataResponse = await GetFunctionMetadataResponseAsync(functionMetadata, hostOptions, request);
200+
var functionMetadataResponse = await GetFunctionMetadataResponseAsync(functionMetadata, hostOptions, request, _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled);
201201
return (true, functionMetadataResponse);
202202
}
203203
else
@@ -245,11 +245,11 @@ private void DeleteFunctionArtifacts(FunctionMetadataResponse function)
245245
}
246246
}
247247

248-
private static async Task<FunctionMetadataResponse> GetFunctionMetadataResponseAsync(FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, HttpRequest request)
248+
private async Task<FunctionMetadataResponse> GetFunctionMetadataResponseAsync(FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, HttpRequest request, bool excludeTestData)
249249
{
250250
string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
251251
var baseUrl = $"{request.Scheme}://{request.Host}";
252-
return await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl);
252+
return await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData);
253253
}
254254

255255
// TODO : Due to lifetime scoping issues (this service lifetime is longer than the lifetime

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,22 @@ internal bool EnableOrderedInvocationMessages
147147
}
148148
}
149149

150+
/// <summary>
151+
/// Gets or sets a value indicating whether to ignore the TestData property during read and write operations of functions metadata.
152+
/// </summary>
153+
internal bool IsTestDataSuppressionEnabled
154+
{
155+
get
156+
{
157+
return GetFeatureAsBooleanOrDefault(ScriptConstants.FeatureFlagEnableTestDataSuppression, false);
158+
}
159+
160+
set
161+
{
162+
_features[ScriptConstants.FeatureFlagEnableTestDataSuppression] = value ? "1" : "0";
163+
}
164+
}
165+
150166
/// <summary>
151167
/// Gets the highest version of extension bundle v3 supported.
152168
/// </summary>

src/WebJobs.Script/ScriptConstants.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ public static class ScriptConstants
141141
public const string FeatureFlagEnableResponseCompression = "EnableResponseCompression";
142142
public const string FeatureFlagDisableOrderedInvocationMessages = "DisableOrderedInvocationMessages";
143143
public const string FeatureFlagEnableAzureMonitorTimeIsoFormat = "EnableAzureMonitorTimeIsoFormat";
144+
public const string FeatureFlagEnableTestDataSuppression = "EnableTestDataSuppression";
144145
public const string HostingConfigDisableLinuxAppServiceDetailedExecutionEvents = "DisableLinuxExecutionDetails";
145146
public const string HostingConfigDisableLinuxAppServiceExecutionEventLogBackoff = "DisableLinuxLogBackoff";
146147
public const string FeatureFlagEnableLegacyDurableVersionCheck = "EnableLegacyDurableVersionCheck";

test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ public static IEnumerable<object[]> PropertyValues
7474
yield return [ nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=True", true ];
7575
yield return [ nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=1", true ];
7676
yield return [ nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=0", false ];
77+
78+
yield return [nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true];
79+
7780
#pragma warning restore SA1011 // Closing square brackets should be spaced correctly
7881
#pragma warning restore SA1010 // Opening square brackets should be spaced correctly
7982
}

test/WebJobs.Script.Tests/Extensions/FunctionMetadataExtensionsTests.cs

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

44
using System.IO;
5+
using System.IO.Abstractions;
6+
using System.Text;
57
using System.Threading.Tasks;
68
using Microsoft.Azure.WebJobs.Script.Description;
79
using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
10+
using Moq;
811
using Newtonsoft.Json.Linq;
912
using Xunit;
1013

@@ -168,7 +171,7 @@ public async Task ToFunctionMetadataResponse_WithoutFiles_ReturnsExpected()
168171
};
169172

170173
AddSampleBindings(functionMetadata);
171-
var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null);
174+
var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null, excludeTestData: false);
172175

173176
Assert.Null(result.ScriptRootPathHref);
174177
Assert.Null(result.ConfigHref);
@@ -178,6 +181,66 @@ public async Task ToFunctionMetadataResponse_WithoutFiles_ReturnsExpected()
178181
Assert.Equal("httpTrigger", binding[0]["type"].Value<string>());
179182
}
180183

184+
[Theory]
185+
[InlineData(true)]
186+
[InlineData(false)]
187+
public async Task ToFunctionMetadataResponse_TestData_Exclusion(bool shouldExcludeTestDataFromApiResponse)
188+
{
189+
var functionName = "TestFunction1";
190+
var functionMetadata = new FunctionMetadata
191+
{
192+
Name = functionName
193+
};
194+
var testDataFileName = $"{functionName}.dat";
195+
var testDataFilePath = Path.Combine(_testRootScriptPath, testDataFileName);
196+
var options = new ScriptJobHostOptions
197+
{
198+
RootScriptPath = _testRootScriptPath,
199+
TestDataPath = testDataFilePath
200+
};
201+
202+
IFileSystem fileSystem = FileUtility.Instance;
203+
204+
try
205+
{
206+
var testDataContent = @"
207+
{
208+
""method"": ""POST"",
209+
""headers"": [],
210+
""body"": ""foo""
211+
}";
212+
213+
var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(testDataContent))
214+
{
215+
Position = 0
216+
};
217+
// Mock the file system
218+
var mockFileSystem = new Mock<IFileSystem>();
219+
var mockFile = new Mock<FileBase>();
220+
mockFileSystem.Setup(f => f.Directory.Exists(It.IsAny<string>())).Returns(true);
221+
mockFile.Setup(f => f.Exists(It.Is<string>(path => path.EndsWith(Path.Combine(functionName, "function.json"))))).Returns(true);
222+
mockFile.Setup(f => f.Exists(It.Is<string>(path => path.EndsWith(testDataFileName)))).Returns(true);
223+
mockFile.Setup(f => f.Open(It.Is<string>(path => path.EndsWith(testDataFileName)), It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>()))
224+
.Returns(memoryStream);
225+
mockFileSystem.Setup(fs => fs.File).Returns(mockFile.Object);
226+
FileUtility.Instance = mockFileSystem.Object;
227+
228+
AddSampleBindings(functionMetadata);
229+
var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null, excludeTestData: shouldExcludeTestDataFromApiResponse);
230+
231+
Assert.Equal(functionName, result.Name);
232+
Assert.Equal(shouldExcludeTestDataFromApiResponse, result.TestData == null);
233+
if (!shouldExcludeTestDataFromApiResponse)
234+
{
235+
Assert.True(result.TestData.Trim().Length > 0);
236+
}
237+
}
238+
finally
239+
{
240+
FileUtility.Instance = fileSystem;
241+
}
242+
}
243+
181244
[Fact]
182245
public void GetFunctionInvokeUrlTemplate_ReturnsExpectedResult()
183246
{

0 commit comments

Comments
 (0)