Started adding mocks for testing data lake services, refactored controllers to be services

Author: MHD\nialan

Azure.DataPipelineTools.sln

@@ -12,6 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		LICENSE = LICENSE
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataPipelineTools.Tests", "DataPipelineTools.Tests\DataPipelineTools.Tests.csproj", "{A2C01394-16F9-4783-9629-2857C400D6E6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -26,6 +28,10 @@ Global
 		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{84B7E7A9-BDE5-4602-B99C-853CC1159CE5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A2C01394-16F9-4783-9629-2857C400D6E6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A2C01394-16F9-4783-9629-2857C400D6E6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A2C01394-16F9-4783-9629-2857C400D6E6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A2C01394-16F9-4783-9629-2857C400D6E6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

DataPipelineTools.Functions/Common/FunctionsBase.cs

@@ -14,7 +14,7 @@ public FunctionsBase(ILogger<FunctionsBase> logger)
             _logger = logger;
         }
 
-        protected JObject GetBaseResponse(DataLakeConfig dataLakeConfig, object parameters)
+        protected JObject GetTemplateResponse(DataLakeConfig dataLakeConfig, object parameters)
         {
             var assemblyInfo = AssemblyHelpers.GetAssemblyVersionInfoJson();
 

DataPipelineTools.Functions/Common/Startup.cs

@@ -16,7 +16,7 @@ public override void Configure(IFunctionsHostBuilder builder)
 
             builder.Services.AddTransient(typeof(DataLakeConfigFactory));
             builder.Services.AddTransient<IDataLakeClientFactory, DataLakeClientFactory>();
-            builder.Services.AddTransient(typeof(DataLakeControllerFactory));
+            builder.Services.AddTransient(typeof(DataLakeServiceFactory));
         }
     }
 }

DataPipelineTools.Functions/DataLake/DataLakeConfigFactory.cs

@@ -77,11 +77,11 @@ public DataLakeGetItemsConfig GetItemsConfig (HttpRequest req)
             return config;
         }
 
-        private IEnumerable<Filter<DataLakeFile>> ParseFilters(HttpRequest req)
+        private IEnumerable<Filter<DataLakeItem>> ParseFilters(HttpRequest req)
         {
             var filters = req.Query.Keys
                             .Where(k => k.StartsWith("filter[") && k.EndsWith("]"))
-                            .SelectMany(k => req.Query[k].Select(v => Filter<DataLakeFile>.ParseFilter(k, v, _logger)))
+                            .SelectMany(k => req.Query[k].Select(v => Filter<DataLakeItem>.ParseFilter(k, v, _logger)))
                             .Where(f => f != null);
 
             return filters;

DataPipelineTools.Functions/DataLake/DataLakeFunctions.cs

@@ -15,13 +15,13 @@ public partial class DataLakeFunctions: FunctionsBase
     {
         private readonly DataLakeConfigFactory _configFactory;
         private readonly IDataLakeClientFactory _clientFactory;
-        private readonly DataLakeControllerFactory _controllerFactory;
-        public DataLakeFunctions(ILogger<DataLakeFunctions> logger, DataLakeConfigFactory configFactory, IDataLakeClientFactory clientFactory, DataLakeControllerFactory controllerFactory):
+        private readonly DataLakeServiceFactory _serviceFactory;
+        public DataLakeFunctions(ILogger<DataLakeFunctions> logger, DataLakeConfigFactory configFactory, IDataLakeClientFactory clientFactory, DataLakeServiceFactory serviceFactory):
             base(logger)
         {
             _configFactory = configFactory;
             _clientFactory = clientFactory;
-            _controllerFactory = controllerFactory;
+            _serviceFactory = serviceFactory;
         }
 
 
@@ -45,9 +45,9 @@ public async Task<IActionResult> DataLakeGetItems(
                     throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
 
                 var client = _clientFactory.GetDataLakeClient(dataLakeConfig);
-                var controller = _controllerFactory.CreateDataLakeController(client);
+                var controller = _serviceFactory.CreateDataLakeService(client);
 
-                var responseJson = GetBaseResponse(dataLakeConfig, getItemsConfig);
+                var responseJson = GetTemplateResponse(dataLakeConfig, getItemsConfig);
                 var items = await controller.GetItemsAsync(dataLakeConfig, getItemsConfig);
                 foreach (var item in items)
                     responseJson.Add(item.Key, item.Value);
@@ -84,15 +84,15 @@ public async Task<IActionResult> DataLakeCheckPathCase(
                     throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
 
                 var client = _clientFactory.GetDataLakeClient(dataLakeConfig);
-                var controller = _controllerFactory.CreateDataLakeController(client);
+                var controller = _serviceFactory.CreateDataLakeService(client);
 
                 var validatedPath = await controller.CheckPathAsync(getItemsConfig.Path, true);
 
                 // If multiple files match, the function will throw and the catch block will return a BadRequestObjectResult
                 // If the path could not be found as a directory, try for a file...
                 validatedPath ??= await controller.CheckPathAsync(getItemsConfig.Path, false);
 
-                var responseJson = GetBaseResponse(dataLakeConfig, getItemsConfig);
+                var responseJson = GetTemplateResponse(dataLakeConfig, getItemsConfig);
                 responseJson.Add("validatedPath", validatedPath);
 
                 return validatedPath != null ?

DataPipelineTools.Tests/DataLake/DataLakeServiceTests.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Core.Extensions;
+using Azure.Storage.Files.DataLake;
+using Azure.Storage.Files.DataLake.Models;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NUnit.Framework;
+using SqlCollaborative.Azure.DataPipelineTools.DataLake;
+using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
+
+namespace DataPipelineTools.Tests.DataLake
+{
+    [TestFixture]
+    public class DataLakeServiceTests: TestBase
+    {
+        private const string AccountUri = "mydatalake";
+        private const string ContainerName = "mycontainer";
+
+        private readonly IEnumerable<PathItem> TestData;
+        private readonly Mock<DataLakeFileSystemClient> mockFileSystemClient;
+
+        //private readonly Mock<DataLakeDirectoryClient> mockDirectoryClient;
+        //private readonly Mock<DataLakeFileClient> mockFileClient;
+
+        private readonly Mock<ILogger<DataLakeServiceFactory>> mockLogger;
+       
+        private readonly DataLakeService Sut;
+
+        public DataLakeServiceTests()
+        {
+            // Get test data to mock the file system
+            TestData = GetTestData();
+
+            // Mock the logger to test where it is called
+            mockLogger = new Mock<ILogger<DataLakeServiceFactory>>();
+
+            // Mock the file system client
+            mockFileSystemClient = BuildMockDataLakeFileSystemClient();
+
+            // Use the factory to inject the mock logger to get the mock client...
+            var factory = new DataLakeServiceFactory(mockLogger.Object);
+            Sut = factory.CreateDataLakeService(mockFileSystemClient.Object);
+        }
+
+        private Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
+        {
+            var mockFileSystemClient = new Mock<DataLakeFileSystemClient>();
+            mockFileSystemClient.SetupGet(x => x.Name).Returns(ContainerName);
+            mockFileSystemClient.SetupGet(x => x.AccountName).Returns(AccountUri);
+
+            mockFileSystemClient
+                .Setup(x => x.GetDirectoryClient(It.IsAny<String>()))
+                .Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeDirectoryClient>);
+            mockFileSystemClient
+                .Setup(x => x.GetFileClient(It.IsAny<String>()))
+                .Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeFileClient>);
+
+            mockFileSystemClient
+                .Setup(x => x.GetPaths(It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                .Returns((string p, bool r) =>
+                {
+                    var items = TestData
+                        // Include all files starting with the test path
+                        .Where(x => x.Name.StartsWith(p))
+                        // Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains 
+                        // directory separator to exclude sub dirs
+                        .Where(x => r || !x.Name.Substring(p.Length).Contains('/'))
+                        .ToList()
+                        .AsReadOnly();
+
+                    var page = Page<PathItem>.FromValues(items, null, Mock.Of<Response>());
+                    return Pageable<PathItem>.FromPages(new[] { page });
+                });
+
+            return mockFileSystemClient;
+        }
+
+        private T BuildMockDataLakeDirectoryClient<T>(string directoryName) where T: DataLakePathClient
+        {
+            var mockDirectoryClient = new Mock<T>();
+            mockDirectoryClient.SetupGet(x => x.FileSystemName).Returns(ContainerName);
+            mockDirectoryClient.SetupGet(x => x.AccountName).Returns(AccountUri);
+            mockDirectoryClient.SetupGet(x => x.Name).Returns(directoryName);
+
+            var directoryNameExists = TestData.Any(i => i.Name == directoryName);
+            mockDirectoryClient
+                .Setup(x => x.ExistsAsync(It.IsAny<CancellationToken>()))
+                .ReturnsAsync(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
+
+            mockDirectoryClient
+                .Setup(x => x.Exists(It.IsAny<CancellationToken>()))
+                .Returns(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
+
+            return mockDirectoryClient.Object;
+        }
+
+        private IEnumerable<PathItem> GetTestData()
+        {
+            return GetTestData(",", properties =>
+            {
+                return DataLakeModelFactory.PathItem(
+                    properties[nameof(PathItem.Name)],
+                    Convert.ToBoolean(properties[nameof(PathItem.IsDirectory)]),
+                    Convert.ToDateTime(properties[nameof(PathItem.LastModified)]),
+                    ETag.All,
+                    Convert.ToInt32(properties[nameof(PathItem.ContentLength)]), 
+                    null,
+                    null,
+                    null
+                    );
+            }).ToArray();
+        }
+
+//private IEnumerable<DataLakeItem> GetTestData()
+//        { 
+//            return GetTestData(",", properties =>
+//            {
+//                return new DataLakeItem()
+//                {
+//                    Directory = properties[nameof(DataLakeItem.Directory)],
+//                    Name = properties[nameof(DataLakeItem.Name)],
+//                    IsDirectory = Convert.ToBoolean(properties[nameof(DataLakeItem.IsDirectory)]),
+//                    ContentLength = Convert.ToInt32(properties[nameof(DataLakeItem.ContentLength)]),
+//                    LastModified = Convert.ToDateTime(properties[nameof(DataLakeItem.LastModified)])
+//                };
+//            }).ToArray();
+//        }
+
+        [SetUp]
+        public void Setup()
+        {
+
+        }
+
+        [Test]
+        public void CheckPathAsync_ShouldReturn_()
+        {
+            
+        }
+
+        
+    }
+}

DataPipelineTools.Tests/DataLake/DataLakeServiceTests_Data_DataLakeItem.csv

@@ -0,0 +1,11 @@
+Name,IsDirectory,ContentLength,LastModified
+raw,TRUE,0,01/01/2021 12:00
+raw/database,TRUE,0,01/01/2021 13:00
+raw/database/jan/extract_1.csv,FALSE,10,02/01/2021 13:00
+raw/database/feb/extract_2.csv,FALSE,20,03/01/2021 13:00
+raw/api,TRUE,0,01/01/2021 14:00
+raw/api/jan/delta_extract_1.json,FALSE,10,01/01/2021 14:00
+raw/api/jan/delta_extract_2.json,FALSE,20,02/01/2021 14:00
+raw/api/feb/delta_extract_3.json,FALSE,30,03/01/2021 14:00
+raw/API,TRUE,0,01/01/2021 15:00
+raw/API/jan/delta_extract_1.json,FALSE,10,01/01/2021 15:00

DataPipelineTools.Tests/DataLake/Model/DataLakeConfigTests.cs

@@ -0,0 +1,31 @@
+using NUnit.Framework;
+using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
+
+namespace DataPipelineTools.Tests.DataLake.Model
+{
+    [TestFixture]
+    public class DataLakeConfigTests
+    {
+        private const string AccountUri = "mydatalake";
+        private const string ContainerName = "mycontainer";
+        [SetUp]
+        public void Setup()
+        {
+        }
+
+        [Test]
+        public void BaseUrl_ShouldContain_AccountUri()
+        {
+            var config = new DataLakeConfig { AccountUri = AccountUri, Container = ContainerName };
+
+            Assert.IsTrue(config.BaseUrl.StartsWith($"https://{AccountUri}"));
+        }
+
+        public void BaseUrl_ShouldContain_ContainerName()
+        {
+            var config = new DataLakeConfig { AccountUri = AccountUri, Container = ContainerName };
+
+            Assert.IsTrue(config.BaseUrl.EndsWith(ContainerName));
+        }
+    }
+}

DataPipelineTools.Tests/DataPipelineTools.Tests.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="NUnit" Version="3.12.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataPipelineTools\DataPipelineTools.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="DataLake\DataLakeServiceTests_Data_DataLakeItem - Copy.csv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="DataLake\DataLakeServiceTests_Data_DataLakeItem.csv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

DataPipelineTools.Tests/TestBase.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Text;
+using DataPipelineTools.Tests.DataLake;
+using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
+
+namespace DataPipelineTools.Tests
+{
+    public abstract class TestBase
+    {
+        //protected abstract T ParseCsv(string csvLine);
+
+        protected IEnumerable<T> GetTestData<T>(string delimiter, Func<Dictionary<string, string>, T> conversionFunc)
+        {
+            var thisAssembly = Assembly.GetExecutingAssembly();
+            var assemblyPath = thisAssembly.Location;
+            var assemblyName = thisAssembly.GetName().Name;
+            var nameSpace = GetType().Namespace;
+
+            var testDataRelativePath = nameSpace.Replace(assemblyName, "").Replace(".", "\\").TrimStart('\\');
+            var testDataPath = Path.Combine(Path.GetDirectoryName(assemblyPath), testDataRelativePath, $"{GetType().Name}_Data_{typeof(T).Name}.csv");
+
+            using (var fs = new FileStream(testDataPath, FileMode.Open, FileAccess.Read, FileShare.Read))
+            {
+                using (StreamReader sr = new StreamReader(fs))
+                {
+                    // The first line is the headers, so grab the values as keys
+                    var keys = sr.ReadLine().Split(delimiter);
+
+                    // Build a dictionary of the properties, as pass it to the parser function
+                    while (!sr.EndOfStream)
+                    {
+                        var dict = new Dictionary<string, string>();
+                        var values = sr.ReadLine().Split(delimiter);
+                        for (int i = 0; i < keys.Length && i < values.Length; i++)
+                        {
+                            dict.Add(keys[i], values[i]);
+                        }
+
+                        yield return conversionFunc(dict);
+                    }
+                }
+            }
+        }
+    }
+}