Skip to content

Commit 4d313cf

Browse files
author
MHD\nialan
committed
Refeactoring to use dependency injection for logging and service factories
1 parent 374edfd commit 4d313cf

File tree

9 files changed

+219
-172
lines changed

9 files changed

+219
-172
lines changed

‎ADF.Functions/ADF.Functions.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
<PackageReference Include="Azure.Identity" Version="1.3.0" />
2222
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.6.0" />
2323
<PackageReference Include="Flurl" Version="3.0.1" />
24+
<PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.1.0" />
25+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.13" />
2426
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
2527
<PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.8" />
2628
</ItemGroup>

‎ADF.Functions/DataLakeConfigFactory.cs

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using Azure.Datafactory.Extensions.DataLake.Model;
2+
using Microsoft.AspNetCore.Http;
3+
using Microsoft.Extensions.Logging;
4+
using Microsoft.Extensions.Primitives;
5+
using Newtonsoft.Json;
6+
using System;
7+
using System.Collections.Generic;
8+
using System.IO;
9+
using System.Linq;
10+
using System.Text;
11+
12+
namespace Azure.Datafactory.Extensions.Functions
13+
{
14+
public class DataLakeConfigFactory
15+
{
16+
private const string AccountUriParam = "accountUri";
17+
private const string ContainerParam = "container";
18+
private const string PathParam = "path";
19+
private const string DirectoryParam = "directory";
20+
private const string IgnoreDirectoryCaseParam = "ignoreDirectoryCase";
21+
private const string RecursiveParam = "recursive";
22+
private const string OrderByColumnParam = "orderBy";
23+
private const string OrderByDescendingParam = "orderByDesc";
24+
private const string LimitParam = "limit";
25+
26+
private readonly ILogger _logger;
27+
public DataLakeConfigFactory(ILogger logger)
28+
{
29+
_logger = logger;
30+
}
31+
32+
public DataLakeConfig GetDataLakeConfig (HttpRequest req)
33+
{
34+
var config = new DataLakeConfig();
35+
//_logger.LogInformation($"req.GetQueryParameterDictionary(): {JsonConvert.SerializeObject(req.GetQueryParameterDictionary(), Formatting.Indented)}");
36+
37+
var data = GetRequestData(req);
38+
config.AccountUri = req.Query["accountUri"] != StringValues.Empty ? (string)req.Query["accountUri"] : data?.accountUri;
39+
config.Container = req.Query["container"] != StringValues.Empty ? (string)req.Query["container"] : data?.container;
40+
41+
return config;
42+
}
43+
44+
public DataLakeCheckPathCaseConfig GetCheckPathCaseConfig (HttpRequest req)
45+
{
46+
var config = new DataLakeCheckPathCaseConfig();
47+
48+
var data = GetRequestData(req);
49+
config.Path = req.Query[PathParam] != StringValues.Empty ? (string)req.Query[PathParam] : data?.directory;
50+
51+
return config;
52+
}
53+
54+
public DataLakeGetItemsConfig GetItemsConfig (HttpRequest req)
55+
{
56+
var config = new DataLakeGetItemsConfig();
57+
58+
var data = GetRequestData(req);
59+
60+
bool recursive;
61+
bool orderByDesc;
62+
bool ignoreDirectoryCase = true;
63+
int limit = 0;
64+
bool.TryParse(req.Query[RecursiveParam] != StringValues.Empty ? (string)req.Query[RecursiveParam] : data?.recursive, out recursive);
65+
bool.TryParse(req.Query[OrderByDescendingParam] != StringValues.Empty ? (string)req.Query[OrderByDescendingParam] : data?.orderByDesc, out orderByDesc);
66+
bool.TryParse(req.Query[IgnoreDirectoryCaseParam] != StringValues.Empty ? (string)req.Query[IgnoreDirectoryCaseParam] : data?.ignoreDirectoryCase, out ignoreDirectoryCase);
67+
int.TryParse(req.Query[LimitParam] != StringValues.Empty ? (string)req.Query[LimitParam] : data?.orderByDesc, out limit);
68+
69+
config.Directory = req.Query[DirectoryParam] != StringValues.Empty ? (string)req.Query[DirectoryParam] : data?.directory;
70+
config.IgnoreDirectoryCase = ignoreDirectoryCase;
71+
config.Recursive = recursive;
72+
config.OrderByColumn = req.Query[OrderByColumnParam] != StringValues.Empty ? (string)req.Query[OrderByColumnParam] : data?.orderBy;
73+
config.OrderByDescending = orderByDesc;
74+
config.Limit = limit;
75+
76+
config.Filters = ParseFilters(req);
77+
78+
return config;
79+
}
80+
81+
private IEnumerable<Filter<DataLakeFile>> ParseFilters(HttpRequest req)
82+
{
83+
var filters = req.Query.Keys
84+
.Where(k => k.StartsWith("filter[") && k.EndsWith("]"))
85+
.SelectMany(k => req.Query[k].Select(v => Filter<DataLakeFile>.ParseFilter(k, v, _logger)))
86+
.Where(f => f != null);
87+
88+
return filters;
89+
}
90+
91+
private dynamic GetRequestData(HttpRequest req)
92+
{
93+
var task = new StreamReader(req.Body).ReadToEndAsync();
94+
//task.Wait(250);
95+
return JsonConvert.DeserializeObject(task.Result);
96+
}
97+
}
98+
}

‎ADF.Functions/DataLakeHelpers.cs renamed to ‎ADF.Functions/DataLakeFunctions.cs

Lines changed: 80 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -21,110 +21,130 @@
2121

2222
namespace Azure.Datafactory.Extensions.Functions
2323
{
24-
public static partial class DataLakeHelpers
24+
public partial class DataLakeFunctions
2525
{
26+
private readonly ILogger<DataLakeFunctions> _logger;
27+
private readonly DataLakeConfigFactory _configFactory;
28+
public DataLakeFunctions(ILogger<DataLakeFunctions> logger, DataLakeConfigFactory configFactory)
29+
{
30+
_logger = logger;
31+
_configFactory = configFactory;
32+
}
33+
34+
35+
2636
[FunctionName("DataLakeGetItems")]
27-
public static async Task<IActionResult> DataLakeGetItems(
28-
[HttpTrigger(AuthorizationLevel.Function, "get" /*, "post"*/, Route = null)] HttpRequest req,
29-
ILogger log)
37+
public async Task<IActionResult> DataLakeGetItems(
38+
[HttpTrigger(AuthorizationLevel.Function, "get" /*, "post"*/, Route = null)] HttpRequest req)
3039
{
3140
req.GetQueryParameterDictionary();
3241

3342
var userAgentKey = req.Headers.Keys.FirstOrDefault(k => k.ToLower() == "user-agent" || k.ToLower() == "useragent");
34-
log.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
43+
_logger.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
3544

3645
try
3746
{
38-
var settings = DataLakeGetItemsConfig.ParseFromRequestBody(req, log);
39-
if (string.IsNullOrWhiteSpace(settings.AccountUri))
40-
throw new ArgumentException($"Account Uri '{settings.AccountUri}' not found. Check the URI is correct.");
47+
var dataLakeConfig = _configFactory.GetDataLakeConfig(req);
48+
var getItemsConfig = _configFactory.GetItemsConfig(req);
49+
50+
if (string.IsNullOrWhiteSpace(dataLakeConfig.AccountUri))
51+
throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
4152

42-
var client = DataLakeClientFactory.GetDataLakeClient(settings, log);
43-
return await GetItemsAsync(client, settings, log);
53+
var clientFactory = new DataLakeClientFactory(_logger);
54+
var client = clientFactory.GetDataLakeClient(dataLakeConfig);
55+
return await GetItemsAsync(client, dataLakeConfig, getItemsConfig, _logger);
4456
}
4557
catch (ArgumentException ex)
4658
{
47-
log.LogError(ex.Message);
59+
_logger.LogError(ex.Message);
4860
return new BadRequestObjectResult(ex.Message);
4961
}
5062
catch (Exception ex)
5163
{
52-
log.LogError(ex.ToString());
64+
_logger.LogError(ex.ToString());
5365
return new BadRequestObjectResult("An error occurred, see the Azure Function logs for more details");
5466
}
5567
}
5668

5769

5870

5971
[FunctionName("DataLakeCheckPathCase")]
60-
public static async Task<IActionResult> DataLakeCheckPathCase(
61-
[HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req,
62-
ILogger log)
72+
public async Task<IActionResult> DataLakeCheckPathCase(
73+
[HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req)
6374
{
6475
var userAgentKey = req.Headers.Keys.FirstOrDefault(k => k.ToLower() == "user-agent" || k.ToLower() == "useragent");
65-
log.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
76+
_logger.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
6677

6778
try
6879
{
69-
var settings = DataLakeCheckPathCaseConfig.ParseFromRequestBody(req, log);
70-
if (string.IsNullOrWhiteSpace(settings.AccountUri))
71-
throw new ArgumentException($"Account Uri '{settings.AccountUri}' not found. Check the URI is correct.");
80+
var dataLakeConfig = _configFactory.GetDataLakeConfig(req);
81+
var getItemsConfig = _configFactory.GetCheckPathCaseConfig(req);
82+
83+
if (string.IsNullOrWhiteSpace(dataLakeConfig.AccountUri))
84+
throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
7285

73-
var client = DataLakeClientFactory.GetDataLakeClient(settings, log);
86+
var clientFactory = new DataLakeClientFactory(_logger);
87+
var client = clientFactory.GetDataLakeClient(dataLakeConfig);
7488

75-
var paramsJsonFragment = GetParamsJsonFragment(settings);
76-
var validatedPath = await CheckPathAsync(client, settings.Path, true, log);
89+
var paramsJsonFragment = GetParamsJsonFragment(dataLakeConfig, getItemsConfig);
90+
var validatedPath = await CheckPathAsync(client, getItemsConfig.Path, true, _logger);
7791

7892
// If multiple files match, the function will throw and the catch block will return a BadRequestObjectResult
7993
// If the path could not be found as a directory, try for a file...
80-
validatedPath = validatedPath ?? await CheckPathAsync(client, settings.Path, false, log);
94+
validatedPath = validatedPath ?? await CheckPathAsync(client, getItemsConfig.Path, false, _logger);
8195

8296
var resultJson = "{" +
8397
$"{paramsJsonFragment}, \"validatedPath\":\"{validatedPath}\" " +
8498
"}";
8599

86100
return validatedPath != null ?
87-
(IActionResult) new OkObjectResult(JObject.Parse(resultJson)) :
88-
(IActionResult) new NotFoundObjectResult(JObject.Parse(resultJson));
101+
(IActionResult)new OkObjectResult(JObject.Parse(resultJson)) :
102+
(IActionResult)new NotFoundObjectResult(JObject.Parse(resultJson));
89103
}
90104
catch (ArgumentException ex)
91105
{
92-
log.LogError(ex.Message);
106+
_logger.LogError(ex.Message);
93107
return new BadRequestObjectResult(ex.Message);
94108
}
95109
catch (Exception ex)
96110
{
97-
log.LogError(ex.ToString());
111+
_logger.LogError(ex.ToString());
98112
return new BadRequestObjectResult("An error occurred, see the Azure Function logs for more details");
99113
}
100114
}
101115

102116

103117

104-
private static string GetParamsJsonFragment(DataLakeConfig settings)
105-
{
106-
return $"\"debugInfo\": {AssemblyHelpers.GetAssemblyVersionInfoJson()}," +
107-
$"\"parameters\": {JsonConvert.SerializeObject(settings, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore })}";
108-
}
109118

110119

111-
//private static DataLakeFileSystemClient GetDataLakeClient(DataLakeConfig settings, ILogger log)
112-
//{
113-
// // This works as long as the account accessing (managed identity or visual studio user) has both of the following IAM permissions on the storage account:
114-
// // - Reader
115-
// // - Storage Blob Data Reader
116-
// var credential = new DefaultAzureCredential();
117-
// log.LogInformation($"Using credential Type: {credential.GetType().Name}");
118120

119-
// var client = new DataLakeFileSystemClient(new Uri(settings.BaseUrl), credential);
120-
// if (!client.Exists())
121-
// return null;
122121

123-
// return client;
124-
//}
122+
123+
124+
125+
126+
127+
128+
129+
130+
131+
132+
133+
134+
135+
136+
137+
private string GetParamsJsonFragment(DataLakeConfig dataLakeConfig, object parameters)
138+
{
139+
return $"\"debugInfo\": {AssemblyHelpers.GetAssemblyVersionInfoJson()}," +
140+
$"\"storageContainerUrl\": {dataLakeConfig.BaseUrl}," +
141+
parameters == null ?
142+
string.Empty :
143+
$"\"parameters\": {JsonConvert.SerializeObject(parameters, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore })}";
144+
}
125145

126146

127-
private async static Task<string> CheckPathAsync(DataLakeFileSystemClient client, string path, bool isDirectory, ILogger log)
147+
private async Task<string> CheckPathAsync(DataLakeFileSystemClient client, string path, bool isDirectory, ILogger log)
128148
{
129149
if (path == null || path.Trim() == "/")
130150
return null;
@@ -175,7 +195,7 @@ private async static Task<string> CheckPathAsync(DataLakeFileSystemClient client
175195
return files.FirstOrDefault();
176196
}
177197

178-
private static IList<string> MatchPathItemsCaseInsensitive(DataLakeFileSystemClient client, string basePath, string searchItem, bool isDirectory, ILogger log)
198+
private IList<string> MatchPathItemsCaseInsensitive(DataLakeFileSystemClient client, string basePath, string searchItem, bool isDirectory, ILogger log)
179199
{
180200
var paths = client.GetPaths(basePath).ToList();
181201
return paths.Where(p => p.IsDirectory == isDirectory && Path.GetFileName(p.Name).Equals(searchItem, StringComparison.CurrentCultureIgnoreCase))
@@ -185,35 +205,35 @@ private static IList<string> MatchPathItemsCaseInsensitive(DataLakeFileSystemCli
185205
}
186206

187207

188-
private static async Task<IActionResult> GetItemsAsync(DataLakeFileSystemClient client, DataLakeGetItemsConfig settings, ILogger log)
208+
private async Task<IActionResult> GetItemsAsync(DataLakeFileSystemClient client, DataLakeConfig dataLakeConfig, DataLakeGetItemsConfig getItemsConfig, ILogger log)
189209
{
190-
var directory = settings.IgnoreDirectoryCase ?
191-
await CheckPathAsync(client, settings.Directory, true, log) :
192-
settings.Directory;
210+
var directory = getItemsConfig.IgnoreDirectoryCase ?
211+
await CheckPathAsync(client, getItemsConfig.Directory, true, log) :
212+
getItemsConfig.Directory;
193213

194-
var paramsJsonFragment = GetParamsJsonFragment(settings);
214+
var paramsJsonFragment = GetParamsJsonFragment(dataLakeConfig, getItemsConfig);
195215

196216
if (!client.GetDirectoryClient(directory).Exists())
197217
return new BadRequestObjectResult(JObject.Parse($"{{ {paramsJsonFragment}, \"error\": \"Directory '{directory} could not be found'\" }}"));
198218

199219
var paths = client
200-
.GetPaths(path: directory ?? string.Empty, recursive: settings.Recursive)
220+
.GetPaths(path: directory ?? string.Empty, recursive: getItemsConfig.Recursive)
201221
.Select(p => new DataLakeFile
202222
{
203223
Name = Path.GetFileName(p.Name),
204224
Directory = p.IsDirectory.GetValueOrDefault(false) ?
205225
p.Name :
206226
Path.GetDirectoryName(p.Name).Replace(Path.DirectorySeparatorChar, '/'),
207227
FullPath = p.Name,
208-
Url = Url.Combine(settings.BaseUrl, p.Name),
228+
Url = Url.Combine(dataLakeConfig.BaseUrl, p.Name),
209229
IsDirectory = p.IsDirectory.GetValueOrDefault(false),
210230
ContentLength = p.ContentLength.GetValueOrDefault(0),
211231
LastModified = p.LastModified.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")
212232
})
213233
.ToList();
214234

215235
// 1: Filter the results using dynamic LINQ
216-
foreach (var filter in settings.Filters.Where(f => f.IsValid))
236+
foreach (var filter in getItemsConfig.Filters.Where(f => f.IsValid))
217237
{
218238
var dynamicLinqQuery = filter.GetDynamicLinqString();
219239
string dynamicLinqQueryValue = filter.GetDynamicLinqValue();
@@ -222,29 +242,29 @@ await CheckPathAsync(client, settings.Directory, true, log) :
222242
}
223243

224244
// 2: Sort the results
225-
if (!string.IsNullOrWhiteSpace(settings.OrderByColumn))
245+
if (!string.IsNullOrWhiteSpace(getItemsConfig.OrderByColumn))
226246
{
227247
paths = paths.AsQueryable()
228-
.OrderBy(settings.OrderByColumn + (settings.OrderByDescending ? " descending" : string.Empty))
248+
.OrderBy(getItemsConfig.OrderByColumn + (getItemsConfig.OrderByDescending ? " descending" : string.Empty))
229249
.ToList();
230250
}
231251

232252
// 3: Do a top N if required
233-
if (settings.Limit > 0 && settings.Limit < paths.Count)
234-
paths = paths.Take(settings.Limit).ToList();
253+
if (getItemsConfig.Limit > 0 && getItemsConfig.Limit < paths.Count)
254+
paths = paths.Take(getItemsConfig.Limit).ToList();
235255

236256

237257

238258
// Output the results
239259
var versionAttribute = Attribute.GetCustomAttribute(Assembly.GetExecutingAssembly(), typeof(AssemblyInformationalVersionAttribute)) as AssemblyInformationalVersionAttribute;
240260

241-
var IsEveryFilterValid = settings.Filters.All(f => f.IsValid);
261+
var IsEveryFilterValid = getItemsConfig.Filters.All(f => f.IsValid);
242262
var filesListJson = IsEveryFilterValid ?
243263
$"\"fileCount\": {paths.Count}," +
244264
$"\"files\": {JsonConvert.SerializeObject(paths, Formatting.Indented)}" :
245265
string.Empty;
246266

247-
var resultJson = $"{{ {paramsJsonFragment}, {(settings.IgnoreDirectoryCase && directory != settings.Directory ? $"\"correctedFilePath\": \"{directory}\"," : string.Empty)} {filesListJson} }}";
267+
var resultJson = $"{{ {paramsJsonFragment}, {(getItemsConfig.IgnoreDirectoryCase && directory != getItemsConfig.Directory ? $"\"correctedFilePath\": \"{directory}\"," : string.Empty)} {filesListJson} }}";
248268

249269
return IsEveryFilterValid ?
250270
(IActionResult)new OkObjectResult(JObject.Parse(resultJson)) :

‎ADF.Functions/Startup.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using Microsoft.Azure.Functions.Extensions.DependencyInjection;
2+
using Microsoft.Extensions.DependencyInjection;
3+
4+
[assembly: FunctionsStartup(typeof(Azure.Datafactory.Extensions.Functions.Startup))]
5+
namespace Azure.Datafactory.Extensions.Functions
6+
{
7+
public class Startup : FunctionsStartup
8+
{
9+
public override void Configure(IFunctionsHostBuilder builder)
10+
{
11+
// ** Registers the ILogger instance **
12+
builder.Services.AddLogging();
13+
14+
builder.Services.AddSingleton(typeof(DataLakeConfigFactory));
15+
16+
// Registers the application settings' class.
17+
//...
18+
19+
//...omitted for brevity
20+
}
21+
}
22+
}

0 commit comments

Comments
 (0)