Skip to content

Commit f8bd5cb

Browse files
NJLangleyNJLangley
committed
Basic code infrastructure for running integration tests using the Azure Functions Emulator, or against an actual Azure Functions instance based on UseFunctionsEmulator option in the .runSettings file
1 parent 2fd14a3 commit f8bd5cb

File tree

4 files changed

+216
-33
lines changed

4 files changed

+216
-33
lines changed

‎Azure-DevTestLab/Environments/sqlcollaborative_AzureDataPipelineTools/azuredeploy.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -590,7 +590,7 @@
590590
concat('<?xml version=\"1.0\" encoding=\"utf-8\"?>',
591591
'<RunSettings>',
592592
' <TestRunParameters>',
593-
' <Parameter name=\"UseFunctionsEmulator\" value=\"true\" />',
593+
' <Parameter name=\"UseFunctionsEmulator\" value=\"false\" />',
594594
' <Parameter name=\"FunctionsAppName\" value=\"', variables('functionsAppName'), '\" />',
595595
' <Parameter name=\"FunctionsAppUrl\" value=\"', reference(resourceId('Microsoft.Web/sites', variables('functionsAppName')), variables('functionsAppApiVersion'), 'full').properties.hostNames[0], '\" />',
596596
' <Parameter name=\"StorageAccountName\" value=\"', variables('adlsStorageAccountName'), '\" />',

‎DataPipelineTools.Functions.Tests/DataLake/DataLakeFunctions/Integration/DataLakeGetItemsIntegrationTests.cs

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
using System;
2+
using System.Net;
3+
using System.Net.Http;
4+
using System.Threading.Tasks;
5+
using System.Web;
16
using DataPipelineTools.Tests.Common;
27
using Microsoft.Extensions.Logging;
38
using NUnit.Framework;
@@ -8,21 +13,43 @@ namespace DataPipelineTools.Functions.Tests.DataLake.DataLakeFunctions.Integrati
813
[Category(nameof(TestType.IntegrationTest))]
914
public class DataLakeGetItemsIntegrationTests: IntegrationTestBase
1015
{
11-
16+
protected string FunctionUri => $"{FunctionsAppUrl}/api/DataLakeGetItems";
1217

1318
[SetUp]
1419
public void Setup()
1520
{
16-
Logger.LogInformation($"Running tests in { (IsRunningOnCIServer ? "CI" : "local") } environment");
21+
Logger.LogInformation($"Running tests in { (IsRunningOnCIServer ? "CI" : "local") } environment using Functions App '{FunctionsAppUrl}'");
1722
Logger.LogInformation($"TestContext.Parameters.Count: { TestContext.Parameters.Count }");
1823
}
1924

20-
//[Test]
21-
//public void Test_RunSettingsLoadedOk()
22-
//{
23-
// Assert.
25+
[Test]
26+
public async Task Test_FunctionIsRunnable()
27+
{
28+
using (var client = new HttpClient())
29+
{
30+
var queryParams = HttpUtility.ParseQueryString(string.Empty);
31+
queryParams["AccountUri"] = this.StorageAccountName;
32+
queryParams["container"] = this.StorageContainerName;
33+
34+
if (!IsEmulatorRunning)
35+
queryParams["code"] = this.FunctionsAppKey;
36+
37+
var urlBuilder = new UriBuilder(FunctionUri)
38+
{
39+
Query = queryParams.ToString() ?? string.Empty
40+
};
41+
var queryUrl = urlBuilder.ToString();
2442

25-
// //Assert.Fail("Integration tests not implemented yet.");
26-
//}
43+
if (!IsRunningOnCIServer)
44+
Logger.LogInformation($"Query URL: {queryUrl}");
45+
46+
var result = await client.GetAsync(queryUrl);
47+
var content = result.Content.ReadAsStringAsync().Result;
48+
49+
Logger.LogInformation($"Content: {content}");
50+
51+
Assert.AreEqual(HttpStatusCode.OK, result.StatusCode);
52+
}
53+
}
2754
}
2855
}

‎DataPipelineTools.Functions.Tests/IntegrationTestBase.cs

Lines changed: 177 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
using System;
2-
using System.Runtime.InteropServices.ComTypes;
2+
using System.Diagnostics;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Reflection;
6+
using System.Threading;
37
using Azure.Identity;
48
using Azure.Security.KeyVault.Secrets;
59
using DataPipelineTools.Tests.Common;
6-
using Microsoft.Extensions.Logging;
10+
using Microsoft.VisualBasic.FileIO;
711
using NUnit.Framework;
12+
using SearchOption = System.IO.SearchOption;
813

914
namespace DataPipelineTools.Functions.Tests
1015
{
1116
/// <summary>
12-
/// Base class for functions tests. Exposes the
17+
/// Base class for functions integration tests. Exposes the run settings as properties, along with secrets from either the .runsettings file directly, or
18+
/// Azure Key Vault as specified by the .runsettings file.
1319
/// </summary>
1420
public abstract class IntegrationTestBase : TestBase
1521
{
16-
public IntegrationTestBase()
22+
protected IntegrationTestBase()
1723
{
1824
if (TestContext.Parameters.Count == 0)
1925
throw new ArgumentException("No setting file is configured for the integration tests.");
@@ -29,22 +35,80 @@ protected bool UseFunctionsEmulator
2935
}
3036
}
3137

32-
protected string FunctionsAppName => TestContext.Parameters["FunctionsAppName"];
33-
protected string FunctionsAppUrl => TestContext.Parameters["FunctionsAppUrl"];
38+
protected string FunctionsAppName => UseFunctionsEmulator ? "localhost" : TestContext.Parameters["FunctionsAppName"];
39+
protected string FunctionsAppUrl => UseFunctionsEmulator ? "http://localhost:7071" : $"https://{TestContext.Parameters["FunctionsAppUrl"]}";
3440
protected string StorageAccountName => TestContext.Parameters["StorageAccountName"];
3541
protected string StorageContainerName => TestContext.Parameters["StorageContainerName"];
3642
protected string KeyVaultName => TestContext.Parameters["KeyVaultName"];
3743
protected string ServicePrincipalName => TestContext.Parameters["ServicePrincipalName"];
3844
protected string ApplicationInsightsName => TestContext.Parameters["ApplicationInsightsName"];
39-
40-
41-
42-
43-
protected string FunctionsAppKey => TestContext.Parameters["FunctionsAppKey"] ?? GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretFunctionsAppKey"]);
44-
protected string ServicePrincipalSecretKey => TestContext.Parameters["ServicePrincipalSecretKey"] ?? GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretServicePrincipalSecretKey"]);
45-
protected string StorageContainerSasToken => TestContext.Parameters["StorageContainerSasToken"] ?? GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretStorageContainerSasToken"]);
46-
protected string StorageAccountAccessKey => TestContext.Parameters["StorageAccountAccessKey"] ?? GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretStorageAccountAccessKey"]);
47-
protected string ApplicationInsightsKey => TestContext.Parameters["ApplicationInsightsKey"] ?? GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretApplicationInsightsKey"]);
45+
46+
47+
// The properties that we get from Azure Key Vault are cached for reuse
48+
private string _functionsAppKey;
49+
protected string FunctionsAppKey
50+
{
51+
get
52+
{
53+
if (_functionsAppKey == null)
54+
_functionsAppKey = TestContext.Parameters["FunctionsAppKey"] ??
55+
GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretFunctionsAppKey"]);
56+
57+
return _functionsAppKey;
58+
}
59+
}
60+
61+
private string _servicePrincipalSecretKey;
62+
protected string ServicePrincipalSecretKey
63+
{
64+
get
65+
{
66+
if (_servicePrincipalSecretKey == null)
67+
_servicePrincipalSecretKey = TestContext.Parameters["ServicePrincipalSecretKey"] ??
68+
GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretServicePrincipalSecretKey"]);
69+
70+
return _servicePrincipalSecretKey;
71+
}
72+
}
73+
74+
private string _storageContainerSasToken;
75+
protected string StorageContainerSasToken
76+
{
77+
get
78+
{
79+
if (_storageContainerSasToken == null)
80+
_storageContainerSasToken = TestContext.Parameters["StorageContainerSasToken"] ??
81+
GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretStorageContainerSasToken"]);
82+
83+
return _storageContainerSasToken;
84+
}
85+
}
86+
87+
private string _storageAccountAccessKey;
88+
protected string StorageAccountAccessKey
89+
{
90+
get
91+
{
92+
if (_storageAccountAccessKey == null)
93+
_storageAccountAccessKey = TestContext.Parameters["StorageAccountAccessKey"] ??
94+
GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretStorageAccountAccessKey"]);
95+
96+
return _storageAccountAccessKey;
97+
}
98+
}
99+
100+
private string _applicationInsightsKey;
101+
protected string ApplicationInsightsKey
102+
{
103+
get
104+
{
105+
if (_applicationInsightsKey == null)
106+
_applicationInsightsKey = TestContext.Parameters["ApplicationInsightsKey"] ??
107+
GetKeyVaultSecretValue(TestContext.Parameters["KeyVaultSecretApplicationInsightsKey"]);
108+
109+
return _applicationInsightsKey;
110+
}
111+
}
48112

49113

50114
[Test]
@@ -59,12 +123,13 @@ public void Test_RunSettingsLoadedOk()
59123
Assert.IsNotNull(ServicePrincipalName);
60124
Assert.IsNotNull(ApplicationInsightsName);
61125
Assert.IsNotNull(FunctionsAppKey);
126+
62127
Assert.IsNotNull(ServicePrincipalSecretKey);
63128
Assert.IsNotNull(StorageContainerSasToken);
64129
Assert.IsNotNull(StorageAccountAccessKey);
65130
Assert.IsNotNull(ApplicationInsightsKey);
66131

67-
// Check that the StorageContainerSasToken got unquoted correctly
132+
// Check that the StorageContainerSasToken got unquoted correctly from the .runsettings XML file.
68133
Assert.IsFalse(StorageContainerSasToken.Contains("&amp;"));
69134
}
70135

@@ -84,12 +149,7 @@ protected bool IsRunningOnCIServer
84149
return false;
85150
}
86151
}
87-
88-
protected void StartLocalFunctionsInstance()
89-
{
90-
throw new NotImplementedException();
91-
}
92-
152+
93153
protected string GetKeyVaultSecretValue(string secretName)
94154
{
95155
/* For some reason the DefaultAzureCredential (SharedTokenCacheCredential / VisualStudioCredential) returns a 403 trying to access the key vault, even when access policies are configured correctly
@@ -111,7 +171,101 @@ protected string GetKeyVaultSecretValue(string secretName)
111171
return result?.Value?.Value;
112172
}
113173

114-
115174

175+
176+
177+
178+
179+
180+
181+
#region Azure Functions Local Host
182+
// We use one time setup and teardown to generate a single instance of the emulator across all classes that implement this base class
183+
184+
private static object _functionsProcessLock = new object();
185+
186+
[OneTimeSetUp]
187+
public void StartFunctionsEmulator()
188+
{
189+
lock (_functionsProcessLock)
190+
{
191+
if (UseFunctionsEmulator)
192+
{
193+
if (InstanceCount == 0)
194+
StartFunctionsEmulatorInternal();
195+
196+
InstanceCount++;
197+
}
198+
}
199+
}
200+
201+
[OneTimeTearDown]
202+
public void StopFunctionsEmulator()
203+
{
204+
lock (_functionsProcessLock)
205+
{
206+
if (UseFunctionsEmulator)
207+
{
208+
InstanceCount--;
209+
210+
if (InstanceCount == 0)
211+
StopFunctionsEmulatorInternal();
212+
}
213+
}
214+
}
215+
216+
217+
protected static bool IsEmulatorRunning => LocalFunctionsHostProcess != null;
218+
private static Process LocalFunctionsHostProcess { get; set; }
219+
private static int InstanceCount { get; set; }
220+
221+
222+
223+
private void StartFunctionsEmulatorInternal()
224+
{
225+
if (IsEmulatorRunning)
226+
return;
227+
228+
string appData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
229+
string toolsPath = Path.Join(appData, "AzureFunctionsTools", "Releases");
230+
231+
var toolsVersions = Directory.GetFiles(toolsPath, "func.exe", SearchOption.AllDirectories);
232+
var latestToolsVersion = toolsVersions.OrderBy(x => x).FirstOrDefault();
233+
234+
if (latestToolsVersion == null)
235+
throw new FileNotFoundException("The Azure Functions Core tools are not installed. Run the functions app locally to install the tools.");
236+
237+
const string args = "host start";
238+
string binDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
239+
240+
binDir = binDir.Replace("DataPipelineTools.Functions.Tests", "DataPipelineTools.Functions");
241+
242+
binDir = @"C:\Users\Niall\src\sqlcollaborative\AzureDataPipelineTools\DataPipelineTools.Functions\bin\Debug";
243+
244+
ProcessStartInfo hostProcess = new ProcessStartInfo
245+
{
246+
FileName = latestToolsVersion,
247+
Arguments = args,
248+
WorkingDirectory = binDir,
249+
CreateNoWindow = false,
250+
WindowStyle = ProcessWindowStyle.Normal
251+
};
252+
253+
LocalFunctionsHostProcess = Process.Start(hostProcess);
254+
255+
// Sleep for 5 seconds to allow the emulated functions app to start
256+
Thread.Sleep(5000);
257+
}
258+
259+
private void StopFunctionsEmulatorInternal()
260+
{
261+
if (!IsEmulatorRunning)
262+
return;
263+
264+
LocalFunctionsHostProcess.Kill();
265+
LocalFunctionsHostProcess.WaitForExit();
266+
LocalFunctionsHostProcess.Dispose();
267+
}
268+
269+
#endregion Azure Functions Local Host
116270
}
117271
}

‎DataPipelineTools/DataLake/DataLakeClientFactory.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ public DataLakeFileSystemClient GetDataLakeClient(DataLakeConfig dataLakeConfig)
1919
// This works as long as the account accessing (managed identity or visual studio user) has both of the following IAM permissions on the storage account:
2020
// - Reader
2121
// - Storage Blob Data Reader
22-
var credential = new DefaultAzureCredential();
22+
//
23+
// Note: The SharedTokenCacheCredential type is excluded as it seems to give auth errors
24+
var credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ExcludeSharedTokenCacheCredential = true });
2325
_logger.LogInformation($"Using credential Type: {credential.GetType().Name}");
2426

2527
var client = new DataLakeFileSystemClient(new Uri(dataLakeConfig.BaseUrl), credential);

0 commit comments

Comments
 (0)