Refactored main logic into seperate libary project to more easily unit test

MHD\nialan · MHD\nialan · commit 374edfde90b0 · 2021-03-10T14:17:36.000Z
diff --git a/ADF.Functions/ADF.Functions.csproj b/ADF.Functions/ADF.Functions.csproj
@@ -18,11 +18,11 @@
     <RootNamespace>Azure.Datafactory.Extensions.Functions</RootNamespace>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.2.2" />
-    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.4.0" />
-    <PackageReference Include="Flurl" Version="2.8.2" />
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.3" />
-    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.2" />
+    <PackageReference Include="Azure.Identity" Version="1.3.0" />
+    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.6.0" />
+    <PackageReference Include="Flurl" Version="3.0.1" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
+    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.8" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">
@@ -37,4 +37,7 @@
       <PackagePath></PackagePath>
     </None>
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Azure.DataFactory.Extensions\Azure.DataFactory.Extensions.csproj" />
+  </ItemGroup>
 </Project>
diff --git a/ADF.Functions/DataLakeHelpers.cs b/ADF.Functions/DataLakeHelpers.cs
@@ -16,6 +16,8 @@
 using System.Reflection;
 using System.Linq.Dynamic.Core;
 using Azure.Storage.Files.DataLake.Models;
+using Azure.Datafactory.Extensions.DataLake;
+using Azure.Datafactory.Extensions.DataLake.Model;
 
 namespace Azure.Datafactory.Extensions.Functions
 {
@@ -26,6 +28,8 @@ public static async Task<IActionResult> DataLakeGetItems(
             [HttpTrigger(AuthorizationLevel.Function, "get" /*, "post"*/, Route = null)] HttpRequest req,
             ILogger log)
         {
+            req.GetQueryParameterDictionary();
+
             var userAgentKey = req.Headers.Keys.FirstOrDefault(k => k.ToLower() == "user-agent" || k.ToLower() == "useragent");
             log.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
 
diff --git a/Azure.DataFactory.Extensions/AssemblyHelpers.cs b/Azure.DataFactory.Extensions/AssemblyHelpers.cs
@@ -11,7 +11,7 @@ public static string GetAssemblyVersionInfoJson()
             return GetAssemblyVersionInfoJson(callingAssembly);
         }
 
-            public static string GetAssemblyVersionInfoJson(System.Reflection.Assembly assembly)
+        public static string GetAssemblyVersionInfoJson(Assembly assembly)
         {
             string buildDate = assembly.GetCustomAttributes().OfType<AssemblyMetadataAttribute>().FirstOrDefault(a => a.Key == "BuildDate")?.Value;
             string informationalVersion = assembly.GetCustomAttributes().OfType<AssemblyInformationalVersionAttribute>().FirstOrDefault()?.InformationalVersion;
diff --git a/Azure.DataFactory.Extensions/Azure.DataFactory.Extensions.csproj b/Azure.DataFactory.Extensions/Azure.DataFactory.Extensions.csproj
@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <OutDir>bin\$(Configuration)</OutDir>
+    <Version>1.0.0</Version>
+    <Company />
+    <Authors>Niall Langley</Authors>
+    <Product>azure.datafactory.extensions</Product>
+    <PackageId>Azure.Datafactory.Extensions</PackageId>
+    <PackageLicenseFile>LICENSE</PackageLicenseFile>
+    <PackageProjectUrl>https://github.com/NJLangley/azure.datafactory.extensions</PackageProjectUrl>
+    <AssemblyName>Azure.Datafactory.Extensions</AssemblyName>
+    <RootNamespace>Azure.Datafactory.Extensions</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Identity" Version="1.3.0" />
+    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.6.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.13" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.8" />
+  </ItemGroup>
+
+</Project>
diff --git a/Azure.DataFactory.Extensions/DataLake/DataLakeClientFactory.cs b/Azure.DataFactory.Extensions/DataLake/DataLakeClientFactory.cs
@@ -1,9 +1,8 @@
-﻿using Azure.Identity;
+﻿using Azure.Datafactory.Extensions.DataLake.Model;
+using Azure.Identity;
 using Azure.Storage.Files.DataLake;
 using Microsoft.Extensions.Logging;
 using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace Azure.Datafactory.Extensions.Functions
 {
diff --git a/Azure.DataFactory.Extensions/DataLake/Model/DataLakeCheckPathCaseConfig.cs b/Azure.DataFactory.Extensions/DataLake/Model/DataLakeCheckPathCaseConfig.cs
@@ -2,7 +2,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 
-namespace Azure.Datafactory.Extensions.Functions
+namespace Azure.Datafactory.Extensions.DataLake.Model
 {
     public class DataLakeCheckPathCaseConfig : DataLakeConfig
     {
diff --git a/Azure.DataFactory.Extensions/DataLake/Model/DataLakeConfig.cs b/Azure.DataFactory.Extensions/DataLake/Model/DataLakeConfig.cs
@@ -1,11 +1,11 @@
 ﻿using Microsoft.AspNetCore.Http;
-using Microsoft.Azure.WebJobs.Extensions.Http;
+//using Microsoft.Azure.WebJobs.Extensions.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 using Newtonsoft.Json;
 using System.IO;
 
-namespace Azure.Datafactory.Extensions.Functions
+namespace Azure.Datafactory.Extensions.DataLake.Model
 {
     public class DataLakeConfig
     {
@@ -23,7 +23,7 @@ public class DataLakeConfig
         protected DataLakeConfig(HttpRequest req, ILogger logger)
         {
             Logger = logger;
-            Logger?.LogInformation($"req.GetQueryParameterDictionary(): {JsonConvert.SerializeObject(req.GetQueryParameterDictionary(), Formatting.Indented)}");
+            //Logger?.LogInformation($"req.GetQueryParameterDictionary(): {JsonConvert.SerializeObject(req.GetQueryParameterDictionary(), Formatting.Indented)}");
 
             Data = GetRequestData(req);
 
@@ -39,7 +39,7 @@ public static DataLakeConfig ParseFromRequestBody(HttpRequest req, ILogger logge
         private static dynamic GetRequestData(HttpRequest req)
         {
             var task = new StreamReader(req.Body).ReadToEndAsync();
-            task.Wait(250);
+            //task.Wait(250);
             return JsonConvert.DeserializeObject(task.Result);
         }
     }
diff --git a/Azure.DataFactory.Extensions/DataLake/Model/DataLakeFile.cs b/Azure.DataFactory.Extensions/DataLake/Model/DataLakeFile.cs
@@ -1,4 +1,4 @@
-﻿namespace Azure.Datafactory.Extensions.Functions
+﻿namespace Azure.Datafactory.Extensions.DataLake.Model
 {
     public struct DataLakeFile
     {
diff --git a/Azure.DataFactory.Extensions/DataLake/Model/DataLakeGetItemsConfig.cs b/Azure.DataFactory.Extensions/DataLake/Model/DataLakeGetItemsConfig.cs
@@ -1,10 +1,11 @@
-﻿using Microsoft.AspNetCore.Http;
+﻿using Azure.Datafactory.Extensions.Functions;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace Azure.Datafactory.Extensions.Functions
+namespace Azure.Datafactory.Extensions.DataLake.Model
 {
 
     public class DataLakeGetItemsConfig : DataLakeConfig
@@ -15,7 +16,7 @@ public class DataLakeGetItemsConfig : DataLakeConfig
         public string OrderByColumn { get; set; }
         public bool OrderByDescending { get; set; }
         public int Limit { get; set; }
-        public IEnumerable<Filter> Filters { get; set; }
+        public IEnumerable<Filter<DataLakeFile>> Filters { get; set; }
 
         private const string DirectoryParam = "directory";
         private const string IgnoreDirectoryCaseParam = "ignoreDirectoryCase";
@@ -47,11 +48,11 @@ private DataLakeGetItemsConfig(HttpRequest req, ILogger logger) :
             Filters = ParseFilters(req);
         }
 
-        private IEnumerable<Filter> ParseFilters(HttpRequest req)
+        private IEnumerable<Filter<DataLakeFile>> ParseFilters(HttpRequest req)
         {
             var filters = req.Query.Keys
                             .Where(k => k.StartsWith("filter[") && k.EndsWith("]"))
-                            .SelectMany(k => req.Query[k].Select(v => Filter.ParseFilter(k, v, Logger)))
+                            .SelectMany(k => req.Query[k].Select(v => Filter<DataLakeFile>.ParseFilter(k, v, Logger)))
                             .Where(f => f != null);
 
             return filters;
diff --git a/Azure.DataFactory.Extensions/DynamicLinqUtils.cs b/Azure.DataFactory.Extensions/DynamicLinqUtils.cs
diff --git a/Azure.DataFactory.Extensions/Filter.cs b/Azure.DataFactory.Extensions/Filter.cs
@@ -7,7 +7,7 @@
 
 namespace Azure.Datafactory.Extensions.Functions
 {
-    public class Filter
+    public class Filter<T>
     {
         public string PropertyName { get; set; }
         public string Operator { get; set; }
@@ -19,10 +19,10 @@ public class Filter
 
 
 
-        private static IEnumerable<PropertyInfo> DataLakeFileProperties { get; } = typeof(DataLakeFile).GetProperties();
-        private static String FilterableProperties { get; } = string.Join(", ", typeof(DataLakeFile).GetProperties().Select(p => p.Name).OrderBy(p => p));
+        private static IEnumerable<PropertyInfo> DataLakeFileProperties { get; } = typeof(T).GetProperties();
+        private static String FilterableProperties { get; } = string.Join(", ", typeof(T).GetProperties().Select(p => p.Name).OrderBy(p => p));
 
-        public static Filter ParseFilter(string columnName, string filter, ILogger log)
+        public static Filter<T> ParseFilter(string columnName, string filter, ILogger log)
         {
             // Clean up the column name by removing the filter[...] parts
             var columnNameClean = columnName[7..^1];
@@ -35,7 +35,7 @@ public static Filter ParseFilter(string columnName, string filter, ILogger log)
             {
                 var error = $"The filter column '{columnNameClean}' does not exist. Filter columns must be one of the following: {FilterableProperties}.";
                 log?.LogWarning(error);
-                return new Filter
+                return new Filter<T>
                 {
                     PropertyName = columnNameClean,
                     ErrorMessage = error
@@ -49,7 +49,7 @@ public static Filter ParseFilter(string columnName, string filter, ILogger log)
             {
                 var error = $"The filter string '{filter}' for column '{columnNameClean}' is not valid. It should match the format '{operatorRegex}'";
                 log?.LogWarning(error);
-                return new Filter
+                return new Filter<T>
                 {
                     PropertyName = columnNameClean,
                     ErrorMessage = error
@@ -92,7 +92,7 @@ public static Filter ParseFilter(string columnName, string filter, ILogger log)
             if (!isValueParseable)
                 log?.LogWarning(parseError);
 
-            return new Filter
+            return new Filter<T>
             {
                 PropertyName = columnNameClean,
                 Operator = op,
diff --git a/Azure.DataFactory.Extensions/Properties/launchSettings.json b/Azure.DataFactory.Extensions/Properties/launchSettings.json
@@ -0,0 +1,8 @@
+{
+  "profiles": {
+    "Azure.DataFactory.Extensions": {
+      "commandName": "Project",
+      "workingDirectory": "bin/Debug"
+    }
+  }
+}
diff --git a/azure.datafactory.extensions.sln b/azure.datafactory.extensions.sln
@@ -3,7 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.30225.117
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ADF.Functions", "ADF.Functions\ADF.Functions.csproj", "{12DD4B77-8B6C-4F4F-B0D2-58E60D2735BF}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ADF.Functions", "ADF.Functions\ADF.Functions.csproj", "{12DD4B77-8B6C-4F4F-B0D2-58E60D2735BF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.DataFactory.Extensions", "Azure.DataFactory.Extensions\Azure.DataFactory.Extensions.csproj", "{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -15,6 +17,10 @@ Global
 		{12DD4B77-8B6C-4F4F-B0D2-58E60D2735BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{12DD4B77-8B6C-4F4F-B0D2-58E60D2735BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{12DD4B77-8B6C-4F4F-B0D2-58E60D2735BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ public static string GetAssemblyVersionInfoJson()`
`11`	`11`	`return GetAssemblyVersionInfoJson(callingAssembly);`
`12`	`12`	`}`
`13`	`13`
`14`		`- public static string GetAssemblyVersionInfoJson(System.Reflection.Assembly assembly)`
	`14`	`+ public static string GetAssemblyVersionInfoJson(Assembly assembly)`
`15`	`15`	`{`
`16`	`16`	`string buildDate = assembly.GetCustomAttributes().OfType<AssemblyMetadataAttribute>().FirstOrDefault(a => a.Key == "BuildDate")?.Value;`
`17`	`17`	`string informationalVersion = assembly.GetCustomAttributes().OfType<AssemblyInformationalVersionAttribute>().FirstOrDefault()?.InformationalVersion;`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using Microsoft.Extensions.Logging;`
`3`	`3`	`using Microsoft.Extensions.Primitives;`
`4`	`4`
`5`		`-namespace Azure.Datafactory.Extensions.Functions`
	`5`	`+namespace Azure.Datafactory.Extensions.DataLake.Model`
`6`	`6`	`{`
`7`	`7`	`public class DataLakeCheckPathCaseConfig : DataLakeConfig`
`8`	`8`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`using Microsoft.AspNetCore.Http;`
`2`		`-using Microsoft.Azure.WebJobs.Extensions.Http;`
	`2`	`+//using Microsoft.Azure.WebJobs.Extensions.Http;`
`3`	`3`	`using Microsoft.Extensions.Logging;`
`4`	`4`	`using Microsoft.Extensions.Primitives;`
`5`	`5`	`using Newtonsoft.Json;`
`6`	`6`	`using System.IO;`
`7`	`7`
`8`		`-namespace Azure.Datafactory.Extensions.Functions`
	`8`	`+namespace Azure.Datafactory.Extensions.DataLake.Model`
`9`	`9`	`{`
`10`	`10`	`public class DataLakeConfig`
`11`	`11`	`{`
`@@ -23,7 +23,7 @@ public class DataLakeConfig`
`23`	`23`	`protected DataLakeConfig(HttpRequest req, ILogger logger)`
`24`	`24`	`{`
`25`	`25`	`Logger = logger;`
`26`		`- Logger?.LogInformation($"req.GetQueryParameterDictionary(): {JsonConvert.SerializeObject(req.GetQueryParameterDictionary(), Formatting.Indented)}");`
	`26`	`+ //Logger?.LogInformation($"req.GetQueryParameterDictionary(): {JsonConvert.SerializeObject(req.GetQueryParameterDictionary(), Formatting.Indented)}");`
`27`	`27`
`28`	`28`	`Data = GetRequestData(req);`
`29`	`29`
`@@ -39,7 +39,7 @@ public static DataLakeConfig ParseFromRequestBody(HttpRequest req, ILogger logge`
`39`	`39`	`private static dynamic GetRequestData(HttpRequest req)`
`40`	`40`	`{`
`41`	`41`	`var task = new StreamReader(req.Body).ReadToEndAsync();`
`42`		`- task.Wait(250);`
	`42`	`+ //task.Wait(250);`
`43`	`43`	`return JsonConvert.DeserializeObject(task.Result);`
`44`	`44`	`}`
`45`	`45`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-namespace Azure.Datafactory.Extensions.Functions`
	`1`	`+namespace Azure.Datafactory.Extensions.DataLake.Model`
`2`	`2`	`{`
`3`	`3`	`public struct DataLakeFile`
`4`	`4`	`{`