Skip to content

Commit b044bce

Browse files
author
Niall Langley
committed
Fixed problems with unit test mock data structure
1 parent f45e302 commit b044bce

File tree

7 files changed

+143
-44
lines changed

7 files changed

+143
-44
lines changed

‎DataPipelineTools.Functions/DataLake/DataLakeFunctions.cs

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,10 @@ public async Task<IActionResult> DataLakeGetItems(
4242
var getItemsConfig = _configFactory.GetItemsConfig(req);
4343

4444
if (string.IsNullOrWhiteSpace(dataLakeConfig.AccountUri))
45-
throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
45+
throw new ArgumentException($"Parameter 'accountUri' with value '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
46+
47+
if (getItemsConfig.Directory == null)
48+
throw new ArgumentException($"Parameter 'directory' is required.");
4649

4750
var client = _clientFactory.GetDataLakeClient(dataLakeConfig);
4851
var controller = _serviceFactory.CreateDataLakeService(client);
@@ -81,16 +84,19 @@ public async Task<IActionResult> DataLakeCheckPathCase(
8184
var getItemsConfig = _configFactory.GetCheckPathCaseConfig(req);
8285

8386
if (string.IsNullOrWhiteSpace(dataLakeConfig.AccountUri))
84-
throw new ArgumentException($"Account Uri '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
87+
throw new ArgumentException($"Parameter 'accountUri' with value '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
88+
89+
if (getItemsConfig.Path == null)
90+
throw new ArgumentException($"Parameter 'path' is required.");
8591

8692
var client = _clientFactory.GetDataLakeClient(dataLakeConfig);
87-
var controller = _serviceFactory.CreateDataLakeService(client);
93+
var dataLakeService = _serviceFactory.CreateDataLakeService(client);
8894

89-
var validatedPath = await controller.CheckPathAsync(getItemsConfig.Path, true);
95+
var validatedPath = await dataLakeService.CheckPathAsync(getItemsConfig.Path, true);
9096

9197
// If multiple files match, the function will throw and the catch block will return a BadRequestObjectResult
9298
// If the path could not be found as a directory, try for a file...
93-
validatedPath ??= await controller.CheckPathAsync(getItemsConfig.Path, false);
99+
validatedPath ??= await dataLakeService.CheckPathAsync(getItemsConfig.Path, false);
94100

95101
var responseJson = GetTemplateResponse(dataLakeConfig, getItemsConfig);
96102
responseJson.Add("validatedPath", validatedPath);

‎DataPipelineTools.Functions/host.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
}
99
},
1010
"logLevel": {
11+
"default": "Error" ,
1112
"SqlCollaborative.Azure.DataPipelineTools.Functions.DataLakeFunctions": "Information",
1213
"SqlCollaborative.Azure.DataPipelineTools.Functions.FunctionsBase": "Information",
1314
"SqlCollaborative.Azure.DataPipelineTools.Functions.DataLakeConfigFactory": "Information",

‎DataPipelineTools.Tests/DataLake/DataLakeServiceTests.cs

Lines changed: 69 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,50 +23,103 @@ public void Setup()
2323
}
2424

2525
[Test]
26-
public void CheckPathAsync_ShouldReturnThePath_WhenTheDirectoryPathExists()
26+
public void CheckPathAsync_Given_ValidDirectoryPath_Should_ReturnDirectoryPath()
2727
{
2828
var testPath = "raw/database";
29-
var correctPath = Sut.CheckPathAsync(testPath, true).Result;
29+
var resultPath = Sut.CheckPathAsync(testPath, true).Result;
3030

31-
Assert.That(testPath, Is.EqualTo(correctPath));
31+
Assert.That(testPath, Is.EqualTo(resultPath));
3232
}
33-
33+
34+
[Test]
35+
public void CheckPathAsync_Given_DirectoryPathWithIncorrectCase_Should_ReturnCorrectedDirectoryPath()
36+
{
37+
var testPath = "raw/DATABASE";
38+
var resultPath = Sut.CheckPathAsync(testPath, true).Result;
39+
40+
Assert.That("raw/database", Is.EqualTo(resultPath));
41+
}
42+
3443
[Test]
35-
public void CheckPathAsync_ShouldReturnThePath_WhenTheFilePathExists()
44+
public void CheckPathAsync_Given_ValidFilePath_Should_ReturnValidPath()
3645
{
3746
var testPath = "raw/database/jan/extract_1.csv";
38-
var correctPath = Sut.CheckPathAsync(testPath, false).Result;
47+
var resultPath = Sut.CheckPathAsync(testPath, false).Result;
3948

40-
Assert.That(testPath, Is.EqualTo(correctPath));
49+
Assert.That(testPath, Is.EqualTo(resultPath));
4150
}
4251

4352
[Test]
44-
public void CheckPathAsync_ShouldReturnNull_WhenTheIsDirectoryIsIncorrectlyFalse()
53+
public void CheckPathAsync_Given_FilePathWithIncorrectCase_Should_ReturnCorrectedFilePath()
54+
{
55+
var testPath = "raw/DATABASE/jan/extract_1.csv";
56+
var resultPath = Sut.CheckPathAsync(testPath, false).Result;
57+
58+
Assert.That("raw/database/jan/extract_1.csv", Is.EqualTo(resultPath));
59+
}
60+
61+
[Test]
62+
public void CheckPathAsync_Given_ValidDirectoryPath_And_IsDirectoryFlagIsFalse_Should_ReturnNull()
4563
{
4664
var testPath = "raw/database";
47-
var correctPath = Sut.CheckPathAsync(testPath, false).Result;
65+
var resultPath = Sut.CheckPathAsync(testPath, false).Result;
4866

49-
Assert.That(testPath, Is.EqualTo(correctPath));
67+
Assert.That(testPath, Is.EqualTo(resultPath));
5068
}
5169

5270
[Test]
53-
public void CheckPathAsync_ShouldReturnNull_WhenTheIsDirectoryIsIncorrectlyTrue()
71+
public void CheckPathAsync_Given_ValidFilePath_And_IsDirectoryFlagIsTrue_Should_ReturnNull()
5472
{
5573
var testPath = "raw/database/jan/extract_1.csv";
56-
var correctPath = Sut.CheckPathAsync(testPath, true).Result;
74+
var resultPath = Sut.CheckPathAsync(testPath, true).Result;
5775

58-
Assert.That(testPath, Is.EqualTo(correctPath));
76+
Assert.That(testPath, Is.EqualTo(resultPath));
5977
}
6078

6179
[Test]
62-
public void CheckPathAsync_ShouldReturnNull_WhenPathDoesNotExist()
80+
public void CheckPathAsync_Given_InvalidDirectoryPath_Should_ReturnNull()
6381
{
6482
var testPath = "some/invalid/path";
65-
var correctPath = Sut.CheckPathAsync(testPath, true).Result;
83+
var resultPath = Sut.CheckPathAsync(testPath, true).Result;
6684

67-
Assert.That(null, Is.EqualTo(correctPath));
85+
Assert.That(null, Is.EqualTo(resultPath));
6886
}
6987

88+
[Test]
89+
public void CheckPathAsync_Given_InvalidFilePath_Should_ReturnNull()
90+
{
91+
var testPath = "some/invalid/path.csv";
92+
var resultPath = Sut.CheckPathAsync(testPath, false).Result;
93+
94+
Assert.That(null, Is.EqualTo(resultPath));
95+
}
96+
97+
[Test]
98+
public void CheckPathAsync_Given_DirectoryPathWithIncorrectCase_When_MatchesMultiplePaths_Should_Throw()
99+
{
100+
var testPath = "raw/aPi";
101+
102+
Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, true));
103+
}
104+
105+
[Test]
106+
public void CheckPathAsync_Given_FilePathWithIncorrectCase_When_MatchesMultiplePaths_Should_Throw()
107+
{
108+
var testPath = "raw/aPi/jan/delta_extract_1.json";
109+
110+
Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, true));
111+
}
112+
113+
114+
115+
//[Test]
116+
//public void CheckPathAsync_Given__Should_()
117+
//{
118+
// var testPath = "some/invalid/path";
119+
// var resultPath = Sut.CheckPathAsync(testPath, true).Result;
120+
121+
// Assert.That(null, Is.EqualTo(resultPath));
122+
//}
70123

71124
}
72125
}

‎DataPipelineTools.Tests/DataLake/DataLakeServiceTests_Data_PathItem.csv

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,7 @@
11
Name,IsDirectory,ContentLength,LastModified
2-
raw,TRUE,0,01/01/2021 12:00
3-
raw/database,TRUE,0,01/01/2021 13:00
42
raw/database/jan/extract_1.csv,FALSE,10,02/01/2021 13:00
53
raw/database/feb/extract_2.csv,FALSE,20,03/01/2021 13:00
6-
raw/api,TRUE,0,01/01/2021 14:00
74
raw/api/jan/delta_extract_1.json,FALSE,10,01/01/2021 14:00
85
raw/api/jan/delta_extract_2.json,FALSE,20,02/01/2021 14:00
96
raw/api/feb/delta_extract_3.json,FALSE,30,03/01/2021 14:00
10-
raw/API,TRUE,0,01/01/2021 15:00
117
raw/API/jan/delta_extract_1.json,FALSE,10,01/01/2021 15:00

‎DataPipelineTools.Tests/DataLake/DataLakeTestBase.cs

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.IO;
34
using System.Linq;
45
using System.Threading;
56
using Azure;
@@ -55,12 +56,13 @@ protected Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
5556
.Setup(x => x.GetPaths(It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
5657
.Returns((string path, bool recursive, bool userPrinciaplName, CancellationToken token) =>
5758
{
59+
var pathLength = 1 + (path?.Length ?? 0);
5860
var items = TestData
59-
// Include all files starting with the test path
60-
.Where(x => x.Name.StartsWith(path ?? string.Empty) && path != null)
61+
// Include all files starting with the test path, or root paths if the test path is null
62+
.Where(x => x.Name.StartsWith(path ?? string.Empty))// || (path == null && !x.Name.Contains('/')))
6163
// Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains
6264
// directory separator to exclude sub dirs
63-
.Where(x => recursive || !x.Name.Substring(path.Length).Contains('/'))
65+
.Where(x => recursive || !x.Name.Substring(pathLength > x.Name.Length ? x.Name.Length : pathLength).Contains('/'))
6466
.ToList()
6567
.AsReadOnly();
6668

@@ -71,28 +73,28 @@ protected Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
7173
return mockFileSystemClient;
7274
}
7375

74-
protected T BuildMockDataLakeDirectoryClient<T>(string directoryName) where T : DataLakePathClient
76+
protected T BuildMockDataLakeDirectoryClient<T>(string path) where T : DataLakePathClient
7577
{
76-
var mockDirectoryClient = new Mock<T>();
77-
mockDirectoryClient.SetupGet(x => x.FileSystemName).Returns(ContainerName);
78-
mockDirectoryClient.SetupGet(x => x.AccountName).Returns(AccountUri);
79-
mockDirectoryClient.SetupGet(x => x.Name).Returns(directoryName);
78+
var mockDataLakePathClient = new Mock<T>();
79+
mockDataLakePathClient.SetupGet(x => x.FileSystemName).Returns(ContainerName);
80+
mockDataLakePathClient.SetupGet(x => x.AccountName).Returns(AccountUri);
81+
mockDataLakePathClient.SetupGet(x => x.Name).Returns(path);
8082

81-
var directoryNameExists = TestData.Any(i => i.Name == directoryName);
82-
mockDirectoryClient
83+
var pathExists = TestData.Any(i => i.Name == path);
84+
mockDataLakePathClient
8385
.Setup(x => x.ExistsAsync(It.IsAny<CancellationToken>()))
84-
.ReturnsAsync(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
86+
.ReturnsAsync(() => Response.FromValue(pathExists, new Mock<Response>().Object));
8587

86-
mockDirectoryClient
88+
mockDataLakePathClient
8789
.Setup(x => x.Exists(It.IsAny<CancellationToken>()))
88-
.Returns(() => Response.FromValue(directoryNameExists, new Mock<Response>().Object));
90+
.Returns(() => Response.FromValue(pathExists, new Mock<Response>().Object));
8991

90-
return mockDirectoryClient.Object;
92+
return mockDataLakePathClient.Object;
9193
}
9294

9395
protected IEnumerable<PathItem> GetTestData()
9496
{
95-
return GetTestData(",", properties =>
97+
var files = GetTestData(",", properties =>
9698
{
9799
return DataLakeModelFactory.PathItem(
98100
properties[nameof(PathItem.Name)],
@@ -104,7 +106,46 @@ protected IEnumerable<PathItem> GetTestData()
104106
null,
105107
null
106108
);
107-
}).ToArray();
109+
}).ToList();
110+
111+
// Build the directories by expanding the paths in the file to add all required levels of the directory structure
112+
var directories = files.SelectMany(f => ExpandAllParentPaths(f.Name))
113+
.Distinct()
114+
.Select(d =>
115+
{
116+
var lastModified = files.Where(f => f.Name.StartsWith(d))
117+
.Select(f => f.LastModified)
118+
.Min();
119+
120+
return DataLakeModelFactory.PathItem(
121+
d,
122+
true,
123+
lastModified,
124+
ETag.All,
125+
0,
126+
null,
127+
null,
128+
null
129+
);
130+
})
131+
// Remove dupes if the directory was listed in the source file as an empty dir
132+
.Where(d => !files.Exists(f => f.Name == d.Name))
133+
.ToList();
134+
135+
files.AddRange(directories);
136+
return files.OrderBy(x => x.Name).ToList();
137+
}
138+
139+
private IEnumerable<string> ExpandAllParentPaths(string path)
140+
{
141+
if (string.IsNullOrWhiteSpace(path))
142+
return new string[0];
143+
144+
// Directory separators are changed when using Path.GetDirectoryName(...)
145+
var parent = Path.GetDirectoryName(path).Replace('\\', '/');
146+
var grandParents = ExpandAllParentPaths(parent);
147+
148+
return grandParents.Append(parent).Where(s => !string.IsNullOrWhiteSpace(s));
108149
}
109150
}
110151
}

‎DataPipelineTools.Tests/DataPipelineTools.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
</PropertyGroup>
88

99
<ItemGroup>
10+
<PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.13" />
1011
<PackageReference Include="Moq" Version="4.16.1" />
1112
<PackageReference Include="NUnit" Version="3.12.0" />
1213
<PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />

‎DataPipelineTools/DataLake/DataLakeService.cs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,10 @@ internal DataLakeService (ILogger logger, DataLakeFileSystemClient client)
2727

2828
public async Task<string> CheckPathAsync(string path, bool isDirectory)
2929
{
30-
if (path == null || path.Trim() == "/")
31-
return null;
30+
// If the string is null, empty, whitespace or a single "/" return an empty string so that the client can check a directory for
31+
// files or concatenate a directory name with a filename etc without an error getting returned
32+
if (string.IsNullOrWhiteSpace(path) || path.Trim() == "/")
33+
return string.Empty;
3234

3335
// Check if the path exists with the casing as is...
3436
var pathExists = isDirectory ?
@@ -45,7 +47,6 @@ public async Task<string> CheckPathAsync(string path, bool isDirectory)
4547

4648
// If the directory does not exist, we find it
4749
string validDirectory = null;
48-
var tr = _client.GetDirectoryClient(path).ExistsAsync().Result;
4950
if (!await _client.GetDirectoryClient(path).ExistsAsync())
5051
{
5152
var directoryParts = directoryPath.Split('/');

0 commit comments

Comments
 (0)