Skip to content

Commit f45e302

Browse files
author
MHD\nialan
committed
Refactored code for creating DataLakeFileSystemClient mocks into a base class for reuse
1 parent 33170b0 commit f45e302

File tree

3 files changed

+115
-110
lines changed

3 files changed

+115
-110
lines changed

‎DataPipelineTools.Tests/DataLake/DataLakeServiceTests.cs

Lines changed: 5 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,121 +1,21 @@
1-
using System;
2-
using System.Collections.Generic;
3-
using System.IO;
4-
using System.Linq;
5-
using System.Reflection;
6-
using System.Threading;
7-
using System.Threading.Tasks;
8-
using Azure;
9-
using Azure.Core.Extensions;
10-
using Azure.Storage.Files.DataLake;
11-
using Azure.Storage.Files.DataLake.Models;
12-
using Microsoft.Extensions.Logging;
13-
using Moq;
14-
using NUnit.Framework;
1+
using NUnit.Framework;
152
using SqlCollaborative.Azure.DataPipelineTools.DataLake;
16-
using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
173

184
namespace DataPipelineTools.Tests.DataLake
195
{
206
[TestFixture]
21-
public class DataLakeServiceTests: TestBase
7+
public class DataLakeServiceTests: DataLakeTestBase
228
{
23-
private const string AccountUri = "mydatalake";
24-
private const string ContainerName = "mycontainer";
259

26-
private readonly IEnumerable<PathItem> TestData;
27-
28-
private readonly Mock<DataLakeFileSystemClient> mockFileSystemClient;
29-
private readonly Mock<ILogger<DataLakeServiceFactory>> mockLogger;
30-
31-
private readonly DataLakeService Sut;
10+
protected readonly DataLakeService Sut;
3211

3312
public DataLakeServiceTests()
3413
{
35-
// Get test data to mock the file system
36-
TestData = GetTestData();
37-
38-
// Mock the logger to test where it is called
39-
mockLogger = new Mock<ILogger<DataLakeServiceFactory>>();
40-
41-
// Mock the file system client
42-
mockFileSystemClient = BuildMockDataLakeFileSystemClient();
43-
4414
// Use the factory to inject the mock logger to get the mock client...
45-
var factory = new DataLakeServiceFactory(mockLogger.Object);
46-
Sut = factory.CreateDataLakeService(mockFileSystemClient.Object);
47-
}
48-
49-
private Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
50-
{
51-
var mockFileSystemClient = new Mock<DataLakeFileSystemClient>();
52-
mockFileSystemClient.SetupGet(x => x.Name).Returns(ContainerName);
53-
mockFileSystemClient.SetupGet(x => x.AccountName).Returns(AccountUri);
54-
55-
mockFileSystemClient
56-
.Setup(x => x.GetDirectoryClient(It.IsAny<String>()))
57-
.Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeDirectoryClient>);
58-
mockFileSystemClient
59-
.Setup(x => x.GetFileClient(It.IsAny<String>()))
60-
.Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeFileClient>);
61-
62-
mockFileSystemClient
63-
.Setup(x => x.GetPaths(It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
64-
.Returns((string path, bool recursive, bool userPrinciaplName, CancellationToken token) =>
65-
{
66-
var items = TestData
67-
// Include all files starting with the test path
68-
.Where(x => x.Name.StartsWith(path ?? string.Empty) && path != null)
69-
// Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains
70-
// directory separator to exclude sub dirs
71-
.Where(x => recursive || !x.Name.Substring(path.Length).Contains('/'))
72-
.ToList()
73-
.AsReadOnly();
74-
75-
var page = Page<PathItem>.FromValues(items, null, Mock.Of<Response>());
76-
return Pageable<PathItem>.FromPages(new[] { page });
77-
});
78-
79-
return mockFileSystemClient;
15+
var factory = new DataLakeServiceFactory(MockLogger.Object);
16+
Sut = factory.CreateDataLakeService(MockFileSystemClient.Object);
8017
}
8118

82-
private T BuildMockDataLakeDirectoryClient<T>(string directoryName) where T: DataLakePathClient
83-
{
84-
var mockDirectoryClient = new Mock<T>();
85-
mockDirectoryClient.SetupGet(x => x.FileSystemName).Returns(ContainerName);
86-
mockDirectoryClient.SetupGet(x => x.AccountName).Returns(AccountUri);
87-
mockDirectoryClient.SetupGet(x => x.Name).Returns(directoryName);
88-
89-
var directoryNameExists = TestData.Any(i => i.Name == directoryName);
90-
mockDirectoryClient
91-
.Setup(x => x.ExistsAsync(It.IsAny<CancellationToken>()))
92-
.ReturnsAsync(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
93-
94-
mockDirectoryClient
95-
.Setup(x => x.Exists(It.IsAny<CancellationToken>()))
96-
.Returns(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
97-
98-
return mockDirectoryClient.Object;
99-
}
100-
101-
private IEnumerable<PathItem> GetTestData()
102-
{
103-
return GetTestData(",", properties =>
104-
{
105-
return DataLakeModelFactory.PathItem(
106-
properties[nameof(PathItem.Name)],
107-
Convert.ToBoolean(properties[nameof(PathItem.IsDirectory)]),
108-
Convert.ToDateTime(properties[nameof(PathItem.LastModified)]),
109-
ETag.All,
110-
Convert.ToInt32(properties[nameof(PathItem.ContentLength)]),
111-
null,
112-
null,
113-
null
114-
);
115-
}).ToArray();
116-
}
117-
118-
11919
[SetUp]
12020
public void Setup()
12121
{

‎DataPipelineTools.Tests/DataLake/DataLakeTestBase.cs

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Threading;
5+
using Azure;
6+
using Azure.Storage.Files.DataLake;
7+
using Azure.Storage.Files.DataLake.Models;
8+
using Microsoft.Extensions.Logging;
9+
using Moq;
10+
using SqlCollaborative.Azure.DataPipelineTools.DataLake;
11+
12+
namespace DataPipelineTools.Tests.DataLake
13+
{
14+
/// <summary>
15+
/// Base class for tests that need to mock the DataLakeFileSystemClient classes
16+
/// </summary>
17+
public class DataLakeTestBase : TestBase
18+
{
19+
protected const string AccountUri = "mydatalake";
20+
protected const string ContainerName = "mycontainer";
21+
22+
protected readonly IEnumerable<PathItem> TestData;
23+
24+
protected readonly Mock<DataLakeFileSystemClient> MockFileSystemClient;
25+
protected readonly Mock<ILogger<DataLakeServiceFactory>> MockLogger;
26+
27+
28+
29+
public DataLakeTestBase()
30+
{
31+
// Get test data to mock the file system
32+
TestData = GetTestData();
33+
34+
// Mock the logger to test where it is called
35+
MockLogger = new Mock<ILogger<DataLakeServiceFactory>>();
36+
37+
// Mock the file system client
38+
MockFileSystemClient = BuildMockDataLakeFileSystemClient();
39+
}
40+
41+
protected Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
42+
{
43+
var mockFileSystemClient = new Mock<DataLakeFileSystemClient>();
44+
mockFileSystemClient.SetupGet(x => x.Name).Returns(ContainerName);
45+
mockFileSystemClient.SetupGet(x => x.AccountName).Returns(AccountUri);
46+
47+
mockFileSystemClient
48+
.Setup(x => x.GetDirectoryClient(It.IsAny<String>()))
49+
.Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeDirectoryClient>);
50+
mockFileSystemClient
51+
.Setup(x => x.GetFileClient(It.IsAny<String>()))
52+
.Returns<string>(BuildMockDataLakeDirectoryClient<DataLakeFileClient>);
53+
54+
mockFileSystemClient
55+
.Setup(x => x.GetPaths(It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
56+
.Returns((string path, bool recursive, bool userPrinciaplName, CancellationToken token) =>
57+
{
58+
var items = TestData
59+
// Include all files starting with the test path
60+
.Where(x => x.Name.StartsWith(path ?? string.Empty) && path != null)
61+
// Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains
62+
// directory separator to exclude sub dirs
63+
.Where(x => recursive || !x.Name.Substring(path.Length).Contains('/'))
64+
.ToList()
65+
.AsReadOnly();
66+
67+
var page = Page<PathItem>.FromValues(items, null, Mock.Of<Response>());
68+
return Pageable<PathItem>.FromPages(new[] { page });
69+
});
70+
71+
return mockFileSystemClient;
72+
}
73+
74+
protected T BuildMockDataLakeDirectoryClient<T>(string directoryName) where T : DataLakePathClient
75+
{
76+
var mockDirectoryClient = new Mock<T>();
77+
mockDirectoryClient.SetupGet(x => x.FileSystemName).Returns(ContainerName);
78+
mockDirectoryClient.SetupGet(x => x.AccountName).Returns(AccountUri);
79+
mockDirectoryClient.SetupGet(x => x.Name).Returns(directoryName);
80+
81+
var directoryNameExists = TestData.Any(i => i.Name == directoryName);
82+
mockDirectoryClient
83+
.Setup(x => x.ExistsAsync(It.IsAny<CancellationToken>()))
84+
.ReturnsAsync(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
85+
86+
mockDirectoryClient
87+
.Setup(x => x.Exists(It.IsAny<CancellationToken>()))
88+
.Returns(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
89+
90+
return mockDirectoryClient.Object;
91+
}
92+
93+
protected IEnumerable<PathItem> GetTestData()
94+
{
95+
return GetTestData(",", properties =>
96+
{
97+
return DataLakeModelFactory.PathItem(
98+
properties[nameof(PathItem.Name)],
99+
Convert.ToBoolean(properties[nameof(PathItem.IsDirectory)]),
100+
Convert.ToDateTime(properties[nameof(PathItem.LastModified)]),
101+
ETag.All,
102+
Convert.ToInt32(properties[nameof(PathItem.ContentLength)]),
103+
null,
104+
null,
105+
null
106+
);
107+
}).ToArray();
108+
}
109+
}
110+
}

‎DataPipelineTools.Tests/TestBase.cs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,11 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Reflection;
5-
using System.Text;
6-
using DataPipelineTools.Tests.DataLake;
7-
using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
85

96
namespace DataPipelineTools.Tests
107
{
118
public abstract class TestBase
129
{
13-
//protected abstract T ParseCsv(string csvLine);
14-
1510
protected IEnumerable<T> GetTestData<T>(string delimiter, Func<Dictionary<string, string>, T> conversionFunc)
1611
{
1712
var thisAssembly = Assembly.GetExecutingAssembly();

0 commit comments

Comments
 (0)