Skip to content

Commit d99991f

Browse files
jaschrep-msftjaschrep-msft
authored
DataMovement ShareDirectoryStorageResourceContainer implementation (Azure#38892)
* resource container listing implementation PathScanner for share directories. Test PathScanner. * tests * getstorageresourcereference * fix net462 issue
1 parent faae52e commit d99991f

8 files changed

+415
-29
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,8 @@
55
<IsPackable>false</IsPackable>
66
</PropertyGroup>
77
<ItemGroup>
8-
<Compile Include="$(AzureStorageSharedTestSources)\**\*.cs" LinkBase="Shared" />
9-
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteFixture.cs" />
10-
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteNUnitFixture.cs" />
11-
<Compile Remove="$(AzureStorageSharedTestSources)\StorageTestBase.SasVersion.cs" />
12-
<Compile Remove="$(AzureStorageSharedTestSources)\ClientSideEncryptionTestExtensions.cs" />
13-
<Compile Remove="$(AzureStorageSharedTestSources)\RepeatingStream.cs" />
14-
<Compile Remove="$(AzureStorageSharedTestSources)\TransferValidationTestBase.cs" />
15-
<Compile Remove="$(AzureStorageSharedTestSources)\Sas\*.cs" />
168
<None Include="$(AzureStorageSharedTestSources)\*.xml">
179
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1810
</None>
1911
</ItemGroup>
20-
<ItemGroup>
21-
<Compile Include="$(AzureCoreSharedSources)ArrayBufferWriter.cs" LinkBase="Shared\Core" />
22-
<Compile Include="$(AzureCoreSharedSources)NoBodyResponseOfT.cs" LinkBase="Shared\Core" />
23-
<Compile Include="$(AzureCoreSharedSources)TaskExtensions.cs" LinkBase="Shared\Core" />
24-
</ItemGroup>
25-
<ItemGroup>
26-
<Compile Include="$(AzureStorageSharedSources)Constants.cs" LinkBase="Shared" />
27-
<Compile Include="$(AzureStorageSharedSources)Errors.cs" LinkBase="Shared" />
28-
<Compile Include="$(AzureStorageSharedSources)SasExtensions.cs" LinkBase="Shared" />
29-
<Compile Include="$(AzureStorageSharedSources)StorageConnectionString.cs" LinkBase="Shared" />
30-
<Compile Include="$(AzureStorageSharedSources)SharedAccessSignatureCredentials.cs" LinkBase="Shared" />
31-
<Compile Include="$(AzureStorageSharedSources)UriExtensions.cs" LinkBase="Shared" />
32-
<Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
33-
<Compile Include="$(AzureStorageSharedSources)StorageExceptionExtensions.cs" LinkBase="Shared" />
34-
</ItemGroup>
3512
</Project>

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/AssemblyInfo.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Runtime.CompilerServices;
5+
[assembly: InternalsVisibleTo("Azure.Storage.DataMovement.Files.Shares.Tests, PublicKey=" +
6+
"0024000004800000940000000602000000240000525341310004000001000100d15ddcb2968829" +
7+
"5338af4b7686603fe614abd555e09efba8fb88ee09e1f7b1ccaeed2e8f823fa9eef3fdd60217fc" +
8+
"012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265" +
9+
"e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593d" +
10+
"aa7b11b4")]
11+
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/PathScanner.cs

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections.Generic;
5+
using System.Runtime.CompilerServices;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using Azure.Core;
9+
using Azure.Storage.Files.Shares;
10+
using Azure.Storage.Files.Shares.Models;
11+
12+
namespace Azure.Storage.DataMovement.Files.Shares
13+
{
14+
internal class PathScanner
15+
{
16+
public virtual async IAsyncEnumerable<ShareFileClient> ScanFilesAsync(
17+
ShareDirectoryClient directory,
18+
[EnumeratorCancellation] CancellationToken cancellationToken)
19+
{
20+
await foreach ((ShareDirectoryClient _, ShareFileClient file) in
21+
ScanAsync(directory, cancellationToken).ConfigureAwait(false))
22+
{
23+
if (file != default)
24+
{
25+
yield return file;
26+
}
27+
}
28+
}
29+
30+
public virtual async IAsyncEnumerable<(ShareDirectoryClient Dir, ShareFileClient File)> ScanAsync(
31+
ShareDirectoryClient directory,
32+
[EnumeratorCancellation] CancellationToken cancellationToken)
33+
{
34+
Argument.AssertNotNull(directory, nameof(directory));
35+
36+
Queue<ShareDirectoryClient> toScan = new();
37+
toScan.Enqueue(directory);
38+
39+
while (toScan.Count > 0)
40+
{
41+
ShareDirectoryClient current = toScan.Dequeue();
42+
await foreach (ShareFileItem item in current.GetFilesAndDirectoriesAsync(
43+
cancellationToken: cancellationToken).ConfigureAwait(false))
44+
{
45+
if (item.IsDirectory)
46+
{
47+
ShareDirectoryClient subdir = current.GetSubdirectoryClient(item.Name);
48+
toScan.Enqueue(subdir);
49+
yield return (Dir: subdir, File: null);
50+
}
51+
else
52+
{
53+
yield return (Dir: null, File: current.GetFileClient(item.Name));
54+
}
55+
}
56+
}
57+
}
58+
}
59+
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,18 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Runtime.CompilerServices;
68
using System.Threading;
9+
using System.Threading.Tasks;
710
using Azure.Storage.Files.Shares;
811

912
namespace Azure.Storage.DataMovement.Files.Shares
1013
{
1114
internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer
1215
{
13-
internal readonly ShareFileStorageResourceOptions _options;
16+
internal ShareFileStorageResourceOptions ResourceOptions { get; set; }
17+
internal PathScanner PathScanner { get; set; }
1418

1519
internal ShareDirectoryClient ShareDirectoryClient { get; }
1620

@@ -19,17 +23,44 @@ internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer
1923
internal ShareDirectoryStorageResourceContainer(ShareDirectoryClient shareDirectoryClient, ShareFileStorageResourceOptions options)
2024
{
2125
ShareDirectoryClient = shareDirectoryClient;
22-
_options = options;
26+
ResourceOptions = options;
2327
}
2428

2529
protected override StorageResourceItem GetStorageResourceReference(string path)
2630
{
27-
throw new NotImplementedException();
31+
List<string> pathSegments = path.Split('/').Where(s => !string.IsNullOrEmpty(s)).ToList();
32+
ShareDirectoryClient dir = ShareDirectoryClient;
33+
foreach (string pathSegment in pathSegments.Take(pathSegments.Count - 1))
34+
{
35+
dir = dir.GetSubdirectoryClient(pathSegment);
36+
}
37+
ShareFileClient file = dir.GetFileClient(pathSegments.Last());
38+
return new ShareFileStorageResourceItem(file, ResourceOptions);
2839
}
2940

30-
protected override IAsyncEnumerable<StorageResource> GetStorageResourcesAsync(CancellationToken cancellationToken = default)
41+
protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAsync(
42+
[EnumeratorCancellation] CancellationToken cancellationToken = default)
3143
{
32-
throw new NotImplementedException();
44+
await foreach (ShareFileClient client in PathScanner.ScanFilesAsync(
45+
ShareDirectoryClient, cancellationToken).ConfigureAwait(false))
46+
{
47+
yield return new ShareFileStorageResourceItem(client, ResourceOptions);
48+
}
3349
}
50+
51+
#region Protected Hooks
52+
// Internal func to access protected member for testing.
53+
internal async IAsyncEnumerable<StorageResource> GetStorageResourcesInternal(
54+
[EnumeratorCancellation] CancellationToken cancellationToken = default)
55+
{
56+
await foreach (StorageResource resource in GetStorageResourcesAsync(cancellationToken).ConfigureAwait(false))
57+
{
58+
yield return resource;
59+
}
60+
}
61+
62+
internal StorageResourceItem GetStorageResourceReferenceInternal(string path)
63+
=> GetStorageResourceReference(path);
64+
#endregion
3465
}
3566
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
33
<TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks>
44
<AssemblyTitle>Microsoft Azure.Storage.DataMovement.Files.Shareas client library tests</AssemblyTitle>
@@ -10,10 +10,14 @@
1010
<Aliases>DMBlobs</Aliases>
1111
</ProjectReference>
1212
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.DataMovement\src\Azure.Storage.DataMovement.csproj" />
13+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.DataMovement.Files.Shares\src\Azure.Storage.DataMovement.Files.Shares.csproj" />
1314
</ItemGroup>
1415
<ItemGroup>
1516
</ItemGroup>
1617
<ItemGroup>
18+
<Compile Include="$(AzureStorageSharedTestSources)AsyncPageableExtensions.cs" LinkBase="Shared\Test" />
19+
<Compile Include="$(AzureStorageSharedTestSources)RandomExtensions.cs" LinkBase="Shared\Test" />
20+
<Compile Include="$(AzureStorageSharedTestSources)Tree.cs" LinkBase="Shared\Test" />
1721
</ItemGroup>
1822
<ItemGroup>
1923
<None Include="$(AzureStorageSharedTestSources)\*.xml">

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientMockingExtensions.cs

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Threading;
9+
using Azure.Storage.Files.Shares;
10+
using Azure.Storage.Files.Shares.Models;
11+
using Moq;
12+
using Azure.Storage.Tests;
13+
14+
namespace Azure.Storage.DataMovement.Files.Shares.Tests
15+
{
16+
internal static class ClientMockingExtensions
17+
{
18+
public static Mock<ShareFileClient> WithUri(
19+
this Mock<ShareFileClient> mock,
20+
Uri uri)
21+
{
22+
mock.Setup(m => m.Uri).Returns(uri);
23+
ShareUriBuilder builder = new(uri);
24+
mock.Setup(m => m.AccountName).Returns(builder.AccountName);
25+
mock.Setup(m => m.ShareName).Returns(builder.ShareName);
26+
mock.Setup(m => m.Path).Returns(builder.DirectoryOrFilePath);
27+
mock.Setup(m => m.Name).Returns(builder.DirectoryOrFilePath.Split('/').Last());
28+
return mock;
29+
}
30+
31+
public static Mock<ShareDirectoryClient> WithUri(
32+
this Mock<ShareDirectoryClient> mock,
33+
Uri uri)
34+
{
35+
mock.Setup(m => m.Uri).Returns(uri);
36+
ShareUriBuilder builder = new(uri);
37+
mock.Setup(m => m.AccountName).Returns(builder.AccountName);
38+
mock.Setup(m => m.ShareName).Returns(builder.ShareName);
39+
mock.Setup(m => m.Path).Returns(builder.DirectoryOrFilePath);
40+
mock.Setup(m => m.Name).Returns(builder.DirectoryOrFilePath.Split('/').Last());
41+
return mock;
42+
}
43+
44+
/// <summary>
45+
/// </summary>
46+
/// <param name="mock"></param>
47+
/// <param name="fileStructure">
48+
/// Tree of strings signifying file structure to mock.
49+
/// When provided, calls to <see cref="ShareDirectoryClient.GetSubdirectoryClient(string)"/>
50+
/// and <see cref="ShareDirectoryClient.GetFileClient(string)"/> will only work according to
51+
/// the tree. Otherwise, all invocations will work.
52+
/// </param>
53+
public static Mock<ShareDirectoryClient> WithDirectoryStructure(
54+
this Mock<ShareDirectoryClient> mock,
55+
Tree<(string Name, bool IsDirectory)> directoryStructure = default)
56+
{
57+
mock.Setup(m => m.GetFileClient(It.IsAny<string>()))
58+
.Returns<string>(name =>
59+
{
60+
_ = directoryStructure.FirstOrDefault(d => !d.Value.IsDirectory && d.Value.Name == name)
61+
?? throw new FileNotFoundException("File not in test directory structure.");
62+
UriBuilder uriBuilder = new UriBuilder(mock.Object.Uri);
63+
uriBuilder.Path = Path.Combine(uriBuilder.Path, name);
64+
Mock<ShareFileClient> file = new();
65+
return file.WithUri(uriBuilder.Uri).Object;
66+
});
67+
mock.Setup(m => m.GetSubdirectoryClient(It.IsAny<string>()))
68+
.Returns<string>(name =>
69+
{
70+
var subDirStructure = directoryStructure.FirstOrDefault(d => d.Value.IsDirectory && d.Value.Name == name)
71+
?? throw new DirectoryNotFoundException("Directory not in test directory structure.");
72+
UriBuilder uriBuilder = new UriBuilder(mock.Object.Uri);
73+
uriBuilder.Path = Path.Combine(uriBuilder.Path, name);
74+
Mock<ShareDirectoryClient> directory = new();
75+
return directory.WithUri(uriBuilder.Uri).WithDirectoryStructure(subDirStructure).Object;
76+
});
77+
mock.Setup(m => m.GetFilesAndDirectoriesAsync(
78+
It.IsAny<ShareDirectoryGetFilesAndDirectoriesOptions>(),
79+
It.IsAny<CancellationToken>())
80+
).Returns<ShareDirectoryGetFilesAndDirectoriesOptions, CancellationToken>(
81+
(options, cancellationToken) => directoryStructure
82+
.Select(elem => FilesModelFactory.ShareFileItem(
83+
isDirectory: elem.Value.IsDirectory, name: elem.Value.Name))
84+
.AsAsyncPageable()
85+
);
86+
87+
return mock;
88+
}
89+
90+
public static Tree<(string Name, bool IsDirectory)> AsTree(this IEnumerable<(string Path, bool IsDirectory)> files, string rootDirName = "")
91+
{
92+
Tree<(string Name, bool IsDirectory)> tree = new()
93+
{
94+
Value = (rootDirName, true)
95+
};
96+
foreach ((string path, bool isDirectory) in files)
97+
{
98+
Tree<(string Name, bool IsDirectory)> curr = tree;
99+
string[] pathSegments = path.Split('/');
100+
foreach (string pathSegment in pathSegments.Take(pathSegments.Length - 1))
101+
{
102+
Tree<(string Name, bool IsDirectory)> next = curr.FirstOrDefault(
103+
t => t.Value.Name == pathSegment && t.Value.IsDirectory == true)
104+
?? new() { Value = (Name: pathSegment, IsDirectory: true) };
105+
curr.Add(next);
106+
curr = next;
107+
}
108+
curr.Add(new() { Value = (Name: pathSegments.Last(), IsDirectory: isDirectory) });
109+
}
110+
return tree;
111+
}
112+
}
113+
}

0 commit comments

Comments
 (0)