Skip to content

Commit 99a885b

Browse files
jaschrep-msftjaschrep-msft
authored
DataMovement Share File Rehydration (Azure#39990)
* initial impl * first test and bugfixes * tests
1 parent a3ecd60 commit 99a885b

File tree

6 files changed

+229
-18
lines changed

6 files changed

+229
-18
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
<Compile Include="$(AzureStorageDataMovementSharedSources)DataMovementConstants.cs" LinkBase="Shared\DataMovement" />
4141
<Compile Include="$(AzureStorageDataMovementSharedSources)Errors.DataMovement.cs" LinkBase="Shared\DataMovement" />
4242
<Compile Include="$(AzureStorageDataMovementSharedSources)CheckpointerExtensions.cs" LinkBase="Shared\DataMovement" />
43+
<Compile Include="$(AzureStorageDataMovementSharedSources)StorageResourceCheckpointDataInternal.cs" LinkBase="Shared\DataMovement" />
4344
<Compile Include="$(AzureStorageDataMovementSharedSources)StorageResourceItemInternal.cs" LinkBase="Shared\DataMovement" />
4445
<Compile Include="$(AzureStorageDataMovementSharedSources)StorageResourceContainerInternal.cs" LinkBase="Shared\DataMovement" />
4546
</ItemGroup>

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
namespace Azure.Storage.DataMovement.Files.Shares
1212
{
13-
internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointData
13+
internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointDataInternal
1414
{
1515
private const char HeaderDelimiter = Constants.CommaChar;
1616

@@ -67,8 +67,6 @@ public ShareFileDestinationCheckpointData(
6767
_filePermissionKeyBytes = SmbProperties?.FilePermissionKey != default ? Encoding.UTF8.GetBytes(SmbProperties.FilePermissionKey) : Array.Empty<byte>();
6868
}
6969

70-
internal void SerializeInternal(Stream stream) => Serialize(stream);
71-
7270
protected override void Serialize(Stream stream)
7371
{
7472
Argument.AssertNotNull(stream, nameof(stream));
@@ -204,8 +202,8 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
204202
ShareFileHttpHeaders contentHeaders = new()
205203
{
206204
ContentType = contentType,
207-
ContentEncoding = contentEncoding.Split(HeaderDelimiter),
208-
ContentLanguage = contentLanguage.Split(HeaderDelimiter),
205+
ContentEncoding = contentEncoding?.Split(HeaderDelimiter),
206+
ContentLanguage = contentLanguage?.Split(HeaderDelimiter),
209207
ContentDisposition = contentDisposition,
210208
CacheControl = cacheControl,
211209
};

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55

66
namespace Azure.Storage.DataMovement.Files.Shares
77
{
8-
internal class ShareFileSourceCheckpointData : StorageResourceCheckpointData
8+
internal class ShareFileSourceCheckpointData : StorageResourceCheckpointDataInternal
99
{
1010
public override int Length => 0;
1111

12-
internal void SerializeInternal(Stream stream) => Serialize(stream);
13-
1412
protected override void Serialize(Stream stream)
1513
{
1614
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33

44
using System;
5+
using System.IO;
56
using System.Threading;
67
using System.Threading.Tasks;
78
using Azure.Core;
@@ -222,19 +223,35 @@ public ShareFilesStorageResourceProvider(GetAzureSasCredential getAzureSasCreden
222223

223224
#region Abstract Class Implementation
224225
/// <inheritdoc/>
225-
protected override async Task<StorageResource> FromSourceAsync(DataTransferProperties properties, CancellationToken cancellationToken)
226-
=> await FromTransferPropertiesAsync(properties, getSource: true, cancellationToken).ConfigureAwait(false);
226+
protected override Task<StorageResource> FromSourceAsync(DataTransferProperties properties, CancellationToken cancellationToken)
227+
{
228+
// Source share file data currently empty, so no specific properties to grab
227229

228-
/// <inheritdoc/>
229-
protected override async Task<StorageResource> FromDestinationAsync(DataTransferProperties properties, CancellationToken cancellationToken)
230-
=> await FromTransferPropertiesAsync(properties, getSource: false, cancellationToken).ConfigureAwait(false);
230+
return Task.FromResult(properties.IsContainer
231+
? FromDirectory(properties.SourceUri.AbsoluteUri)
232+
: FromFile(properties.SourceUri.AbsoluteUri));
233+
}
231234

232-
private Task<StorageResource> FromTransferPropertiesAsync(
233-
DataTransferProperties properties,
234-
bool getSource,
235-
CancellationToken cancellationToken)
235+
/// <inheritdoc/>
236+
protected override Task<StorageResource> FromDestinationAsync(DataTransferProperties properties, CancellationToken cancellationToken)
236237
{
237-
throw new NotImplementedException();
238+
ShareFileDestinationCheckpointData checkpointData;
239+
using (MemoryStream stream = new(properties.DestinationCheckpointData))
240+
{
241+
checkpointData = ShareFileDestinationCheckpointData.Deserialize(stream);
242+
}
243+
244+
ShareFileStorageResourceOptions options = new()
245+
{
246+
SmbProperties = checkpointData.SmbProperties,
247+
HttpHeaders = checkpointData.ContentHeaders,
248+
DirectoryMetadata = checkpointData.DirectoryMetadata,
249+
FileMetadata = checkpointData.FileMetadata,
250+
};
251+
252+
return Task.FromResult(properties.IsContainer
253+
? FromDirectory(properties.DestinationUri.AbsoluteUri, options)
254+
: FromFile(properties.DestinationUri.AbsoluteUri, options));
238255
}
239256

240257
/// <summary>

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/RehydrateShareResourceTests.cs

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Text;
9+
using System.Threading.Tasks;
10+
using Azure.Storage.Files.Shares.Models;
11+
using Azure.Storage.Test;
12+
using Azure.Storage.Tests;
13+
using Moq;
14+
using NUnit.Framework;
15+
16+
namespace Azure.Storage.DataMovement.Files.Shares.Tests
17+
{
18+
public class RehydrateShareResourceTests
19+
{
20+
public const string ShareProviderId = "share";
21+
22+
private static byte[] GetBytes(StorageResourceCheckpointDataInternal checkpointData)
23+
{
24+
using MemoryStream stream = new();
25+
checkpointData.SerializeInternal(stream);
26+
return stream.ToArray();
27+
}
28+
29+
private static Mock<DataTransferProperties> GetProperties(
30+
string transferId,
31+
string sourcePath,
32+
string destinationPath,
33+
string sourceProviderId,
34+
string destinationProviderId,
35+
bool isContainer,
36+
ShareFileSourceCheckpointData sourceCheckpointData,
37+
ShareFileDestinationCheckpointData destinationCheckpointData)
38+
{
39+
var mock = new Mock<DataTransferProperties>(MockBehavior.Strict);
40+
mock.Setup(p => p.TransferId).Returns(transferId);
41+
mock.Setup(p => p.SourceUri).Returns(new Uri(sourcePath));
42+
mock.Setup(p => p.DestinationUri).Returns(new Uri(destinationPath));
43+
mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId);
44+
mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId);
45+
mock.Setup(p => p.SourceCheckpointData).Returns(GetBytes(sourceCheckpointData));
46+
mock.Setup(p => p.DestinationCheckpointData).Returns(GetBytes(destinationCheckpointData));
47+
mock.Setup(p => p.IsContainer).Returns(isContainer);
48+
return mock;
49+
}
50+
51+
[Test]
52+
public async Task RehydrateFile(
53+
[Values(true, false)] bool isSource)
54+
{
55+
string transferId = Guid.NewGuid().ToString();
56+
string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1/file1";
57+
string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2/file2";
58+
string originalPath = isSource ? sourcePath : destinationPath;
59+
60+
DataTransferProperties transferProperties = GetProperties(
61+
transferId,
62+
sourcePath,
63+
destinationPath,
64+
ShareProviderId,
65+
ShareProviderId,
66+
isContainer: false,
67+
new ShareFileSourceCheckpointData(),
68+
new ShareFileDestinationCheckpointData(null, null, null, null)).Object;
69+
70+
StorageResource storageResource = isSource
71+
? await new ShareFilesStorageResourceProvider().FromSourceInternalHookAsync(transferProperties)
72+
: await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties);
73+
74+
Assert.That(originalPath, Is.EqualTo(storageResource.Uri.AbsoluteUri));
75+
Assert.That(storageResource, Is.TypeOf<ShareFileStorageResource>());
76+
}
77+
78+
[Test]
79+
public async Task RehydrateFile_DestinationOptions()
80+
{
81+
string transferId = Guid.NewGuid().ToString();
82+
string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1/file1";
83+
string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2/file2";
84+
85+
Random r = new();
86+
ShareFileDestinationCheckpointData originalDestinationData = new(
87+
new ShareFileHttpHeaders
88+
{
89+
ContentType = "text/plain",
90+
ContentEncoding = new string[] { "gzip" },
91+
ContentLanguage = new string[] { "en-US" },
92+
ContentDisposition = "inline",
93+
CacheControl = "no-cache",
94+
},
95+
new Dictionary<string, string>
96+
{
97+
{ r.NextString(8), r.NextString(8) }
98+
},
99+
new Dictionary<string, string>
100+
{
101+
{ r.NextString(8), r.NextString(8) }
102+
},
103+
new FileSmbProperties
104+
{
105+
FileAttributes = NtfsFileAttributes.Archive,
106+
FilePermissionKey = r.NextString(8),
107+
FileLastWrittenOn = DateTimeOffset.Now,
108+
FileChangedOn = DateTimeOffset.Now,
109+
FileCreatedOn = DateTimeOffset.Now,
110+
});
111+
DataTransferProperties transferProperties = GetProperties(
112+
transferId,
113+
sourcePath,
114+
destinationPath,
115+
ShareProviderId,
116+
ShareProviderId,
117+
isContainer: false,
118+
new ShareFileSourceCheckpointData(),
119+
originalDestinationData).Object;
120+
121+
ShareFileStorageResource storageResource = (ShareFileStorageResource)
122+
await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties);
123+
124+
Assert.That(destinationPath, Is.EqualTo(storageResource.Uri.AbsoluteUri));
125+
Assert.That(storageResource, Is.TypeOf<ShareFileStorageResource>());
126+
Assert.That(storageResource._options.HttpHeaders.ContentType, Is.EqualTo(originalDestinationData.ContentHeaders.ContentType));
127+
Assert.That(storageResource._options.HttpHeaders.ContentEncoding, Is.EqualTo(originalDestinationData.ContentHeaders.ContentEncoding));
128+
Assert.That(storageResource._options.HttpHeaders.ContentLanguage, Is.EqualTo(originalDestinationData.ContentHeaders.ContentLanguage));
129+
Assert.That(storageResource._options.HttpHeaders.ContentDisposition, Is.EqualTo(originalDestinationData.ContentHeaders.ContentDisposition));
130+
Assert.That(storageResource._options.HttpHeaders.CacheControl, Is.EqualTo(originalDestinationData.ContentHeaders.CacheControl));
131+
Assert.That(storageResource._options.FileMetadata, Is.EqualTo(originalDestinationData.FileMetadata));
132+
Assert.That(storageResource._options.DirectoryMetadata, Is.EqualTo(originalDestinationData.DirectoryMetadata));
133+
Assert.That(storageResource._options.SmbProperties.FileAttributes, Is.EqualTo(originalDestinationData.SmbProperties.FileAttributes));
134+
Assert.That(storageResource._options.SmbProperties.FilePermissionKey, Is.EqualTo(originalDestinationData.SmbProperties.FilePermissionKey));
135+
Assert.That(storageResource._options.SmbProperties.FileCreatedOn, Is.EqualTo(originalDestinationData.SmbProperties.FileCreatedOn));
136+
Assert.That(storageResource._options.SmbProperties.FileLastWrittenOn, Is.EqualTo(originalDestinationData.SmbProperties.FileLastWrittenOn));
137+
Assert.That(storageResource._options.SmbProperties.FileChangedOn, Is.EqualTo(originalDestinationData.SmbProperties.FileChangedOn));
138+
}
139+
140+
[Test]
141+
public async Task RehydrateDirectory(
142+
[Values(true, false)] bool isSource)
143+
{
144+
string transferId = Guid.NewGuid().ToString();
145+
List<string> sourcePaths = new List<string>();
146+
string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1";
147+
List<string> destinationPaths = new List<string>();
148+
string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2";
149+
string originalPath = isSource ? sourcePath : destinationPath;
150+
int jobPartCount = 10;
151+
for (int i = 0; i < jobPartCount; i++)
152+
{
153+
string childPath = DataProvider.GetNewString(5);
154+
sourcePaths.Add(string.Join("/", sourcePath, childPath));
155+
destinationPaths.Add(string.Join("/", destinationPath, childPath));
156+
}
157+
158+
DataTransferProperties transferProperties = GetProperties(
159+
transferId,
160+
sourcePath,
161+
destinationPath,
162+
ShareProviderId,
163+
ShareProviderId,
164+
isContainer: true,
165+
new ShareFileSourceCheckpointData(),
166+
new ShareFileDestinationCheckpointData(null, null, null, null)).Object;
167+
168+
StorageResource storageResource = isSource
169+
? await new ShareFilesStorageResourceProvider().FromSourceInternalHookAsync(transferProperties)
170+
: await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties);
171+
172+
Assert.That(originalPath, Is.EqualTo(storageResource.Uri.AbsoluteUri));
173+
Assert.That(storageResource, Is.TypeOf<ShareDirectoryStorageResourceContainer>());
174+
}
175+
}
176+
}

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCheckpointDataInternal.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
9+
namespace Azure.Storage.DataMovement
10+
{
11+
/// <summary>
12+
/// Middle class between the public type and implementation class.
13+
/// Gives internal hook methods to protected methods of
14+
/// <see cref="StorageResourceCheckpointData"/>, allowing for internal
15+
/// package use as well as testing access.
16+
/// </summary>
17+
internal abstract class StorageResourceCheckpointDataInternal : StorageResourceCheckpointData
18+
{
19+
internal void SerializeInternal(Stream stream) => Serialize(stream);
20+
}
21+
}

0 commit comments

Comments
 (0)