Skip to content

Commit 7963167

Browse files
nickliu-msftnickliu-msft
authored
[Storage][DataMovement] Checkpointing for NFS over REST (Azure#50225)
* Introduced new ShareProtocol enum to replace existing * Initial Dest and Source Checkpointing * Moved ShareProtocol serialization/deserialization location to before variable length + added index constants * Added and modified some tests
1 parent f59550a commit 7963167

20 files changed

+683
-130
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,15 @@ public ShareFileStorageResourceOptions() { }
4040
public System.DateTimeOffset? FileLastWrittenOn { get { throw null; } set { } }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
43-
public Azure.Storage.Files.Shares.Models.ShareProtocols ShareProtocol { get { throw null; } set { } }
43+
public Azure.Storage.DataMovement.Files.Shares.ShareProtocol ShareProtocol { get { throw null; } set { } }
4444
public bool SkipProtocolValidation { get { throw null; } set { } }
4545
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4646
}
47+
public enum ShareProtocol : byte
48+
{
49+
Smb = (byte)1,
50+
Nfs = (byte)2,
51+
}
4752
}
4853
namespace Azure.Storage.Files.Shares
4954
{

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net8.0.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,15 @@ public ShareFileStorageResourceOptions() { }
4040
public System.DateTimeOffset? FileLastWrittenOn { get { throw null; } set { } }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
43-
public Azure.Storage.Files.Shares.Models.ShareProtocols ShareProtocol { get { throw null; } set { } }
43+
public Azure.Storage.DataMovement.Files.Shares.ShareProtocol ShareProtocol { get { throw null; } set { } }
4444
public bool SkipProtocolValidation { get { throw null; } set { } }
4545
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4646
}
47+
public enum ShareProtocol : byte
48+
{
49+
Smb = (byte)1,
50+
Nfs = (byte)2,
51+
}
4752
}
4853
namespace Azure.Storage.Files.Shares
4954
{

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,15 @@ public ShareFileStorageResourceOptions() { }
4040
public System.DateTimeOffset? FileLastWrittenOn { get { throw null; } set { } }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
43-
public Azure.Storage.Files.Shares.Models.ShareProtocols ShareProtocol { get { throw null; } set { } }
43+
public Azure.Storage.DataMovement.Files.Shares.ShareProtocol ShareProtocol { get { throw null; } set { } }
4444
public bool SkipProtocolValidation { get { throw null; } set { } }
4545
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4646
}
47+
public enum ShareProtocol : byte
48+
{
49+
Smb = (byte)1,
50+
Nfs = (byte)2,
51+
}
4752
}
4853
namespace Azure.Storage.Files.Shares
4954
{

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,32 @@ internal class DataMovementShareConstants
1414

1515
internal class SourceCheckpointDetails
1616
{
17-
internal const int DataSize = 0;
17+
// Prior to Source Files Schema Version 1, the SourceCheckpointDetails was empty and Version was not present.
18+
internal const int SchemaVersion_1 = 1;
19+
internal const int SchemaVersion = SchemaVersion_1;
20+
21+
internal const int VersionEncodedSize = IntSizeInBytes;
22+
internal const int ShareProtocolEncodedSize = OneByte;
23+
24+
internal const int VersionIndex = 0;
25+
internal const int ShareProtocolIndex = VersionIndex + VersionEncodedSize;
26+
internal const int DataSize = ShareProtocolIndex + ShareProtocolEncodedSize;
1827
}
1928

2029
internal class DestinationCheckpointDetails
2130
{
31+
// Destination Files Schema Versions 1 and 2 were the beta version of the schema and do not need to be serialized and deserialized backwards compatible.
32+
// Only Destination Files Schema Versions 3 and beyond need to be backwards compatible.
2233
internal const int SchemaVersion_3 = 3;
23-
internal const int SchemaVersion = SchemaVersion_3;
34+
internal const int SchemaVersion_4 = 4;
35+
internal const int SchemaVersion = SchemaVersion_4;
36+
internal const int MinValidSchemaVersion = SchemaVersion_3;
37+
internal const int MaxValidSchemaVersion = SchemaVersion_4;
2438

2539
internal const int VersionEncodedSize = IntSizeInBytes;
2640
internal const int PreserveEncodedSize = OneByte;
2741
internal const int OffsetLengthEncodedSize = IntSizeInBytes;
42+
internal const int ShareProtocolEncodedSize = OneByte;
2843

2944
internal const int VersionIndex = 0;
3045

@@ -74,7 +89,9 @@ internal class DestinationCheckpointDetails
7489
internal const int DirectoryMetadataOffsetIndex = PreserveDirectoryMetadataIndex + PreserveEncodedSize;
7590
internal const int DirectoryMetadataLengthIndex = DirectoryMetadataOffsetIndex + OffsetLengthEncodedSize;
7691

77-
internal const int VariableLengthStartIndex = DirectoryMetadataLengthIndex + OffsetLengthEncodedSize;
92+
internal const int ShareProtocolIndex = DirectoryMetadataLengthIndex + OffsetLengthEncodedSize;
93+
94+
internal const int VariableLengthStartIndex = ShareProtocolIndex + ShareProtocolEncodedSize;
7895
}
7996
}
8097
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,9 @@ public static string GetFilePermission(
5959
StorageResourceItemProperties sourceProperties)
6060
{
6161
bool setPermissions = options?.FilePermissions ?? false;
62-
ShareProtocols protocol = options?.ShareProtocol ?? ShareProtocols.Smb;
62+
ShareProtocol protocol = options?.ShareProtocol ?? ShareProtocol.Smb;
6363

64-
if (protocol == ShareProtocols.Smb && setPermissions)
64+
if (protocol == ShareProtocol.Smb && setPermissions)
6565
{
6666
return sourceProperties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FilePermissions, out object permission) == true
6767
? (string)permission
@@ -76,9 +76,9 @@ public static string GetFilePermission(
7676
{
7777
// Only set permissions if Copy transfer and FilePermissions is on.
7878
bool setPermissions = options?.FilePermissions ?? false;
79-
ShareProtocols protocol = options?.ShareProtocol ?? ShareProtocols.Smb;
79+
ShareProtocol protocol = options?.ShareProtocol ?? ShareProtocol.Smb;
8080

81-
if (protocol == ShareProtocols.Smb && setPermissions)
81+
if (protocol == ShareProtocol.Smb && setPermissions)
8282
{
8383
return sourceProperties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FilePermissions, out object permission) == true
8484
? (string)permission
@@ -126,7 +126,7 @@ public static FileSmbProperties GetFileSmbProperties(
126126
{
127127
FileAttributes = options?.ShareProtocol switch
128128
{
129-
ShareProtocols.Nfs => default,
129+
ShareProtocol.Nfs => default,
130130
_ => GetPropertyValue<NtfsFileAttributes>(
131131
options?._isFileAttributesSet ?? false,
132132
options?.FileAttributes,
@@ -135,7 +135,7 @@ public static FileSmbProperties GetFileSmbProperties(
135135
},
136136
FilePermissionKey = options?.ShareProtocol switch
137137
{
138-
ShareProtocols.Nfs => default,
138+
ShareProtocol.Nfs => default,
139139
_ => permissionKeyValue
140140
},
141141
FileCreatedOn = GetPropertyValue<DateTimeOffset>(
@@ -150,7 +150,7 @@ public static FileSmbProperties GetFileSmbProperties(
150150
DataMovementConstants.ResourceProperties.LastWrittenOn),
151151
FileChangedOn = options?.ShareProtocol switch
152152
{
153-
ShareProtocols.Nfs => default,
153+
ShareProtocol.Nfs => default,
154154
_ => GetPropertyValue<DateTimeOffset>(
155155
options?._isFileChangedOnSet ?? false,
156156
options?.FileChangedOn,
@@ -176,7 +176,7 @@ public static FileSmbProperties GetFileSmbProperties(
176176
{
177177
FileAttributes = options?.ShareProtocol switch
178178
{
179-
ShareProtocols.Nfs => default,
179+
ShareProtocol.Nfs => default,
180180
_ => GetPropertyValue<NtfsFileAttributes>(
181181
options?._isFileAttributesSet ?? false,
182182
options?.FileAttributes,
@@ -185,7 +185,7 @@ public static FileSmbProperties GetFileSmbProperties(
185185
},
186186
FilePermissionKey = options?.ShareProtocol switch
187187
{
188-
ShareProtocols.Nfs => default,
188+
ShareProtocol.Nfs => default,
189189
_ => permissionKeyValue
190190
},
191191
FileCreatedOn = GetPropertyValue<DateTimeOffset>(
@@ -200,7 +200,7 @@ public static FileSmbProperties GetFileSmbProperties(
200200
DataMovementConstants.ResourceProperties.LastWrittenOn),
201201
FileChangedOn = options?.ShareProtocol switch
202202
{
203-
ShareProtocols.Nfs => default,
203+
ShareProtocol.Nfs => default,
204204
_ => GetPropertyValue<DateTimeOffset>(
205205
options?._isFileChangedOnSet ?? false,
206206
options?.FileChangedOn,
@@ -232,9 +232,9 @@ public static FilePosixProperties GetFilePosixProperties(
232232
{
233233
// Only set NFS permissions if Copy transfer and FilePermissions is on.
234234
bool setPermissions = options?.FilePermissions ?? false;
235-
ShareProtocols protocol = options?.ShareProtocol ?? ShareProtocols.Smb;
235+
ShareProtocol protocol = options?.ShareProtocol ?? ShareProtocol.Smb;
236236

237-
if (protocol == ShareProtocols.Nfs)
237+
if (protocol == ShareProtocol.Nfs)
238238
{
239239
NfsFileMode FileMode = default;
240240
string Owner = default;
@@ -274,9 +274,9 @@ public static FilePosixProperties GetFilePosixProperties(
274274

275275
// Only set NFS permissions if Copy transfer and FilePermissions is on.
276276
bool setPermissions = options?.FilePermissions ?? false;
277-
ShareProtocols protocol = options?.ShareProtocol ?? ShareProtocols.Smb;
277+
ShareProtocol protocol = options?.ShareProtocol ?? ShareProtocol.Smb;
278278

279-
if (protocol == ShareProtocols.Nfs && setPermissions)
279+
if (protocol == ShareProtocol.Nfs && setPermissions)
280280
{
281281
FileMode = sourceProperties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FileMode, out object fileMode) == true
282282
? (NfsFileMode)fileMode
@@ -728,8 +728,9 @@ public static async Task ValidateProtocolAsync(
728728
try
729729
{
730730
ShareProperties properties = await parentShareClient.GetPropertiesAsync(cancellationToken).ConfigureAwait(false);
731-
ShareProtocols expectedProtocol = options?.ShareProtocol ?? ShareProtocols.Smb;
732-
ShareProtocols actualProtocol = properties.Protocols ?? ShareProtocols.Smb;
731+
ShareProtocol expectedProtocol = options?.ShareProtocol ?? ShareProtocol.Smb;
732+
ShareProtocols effectiveProtocol = properties.Protocols ?? ShareProtocols.Smb;
733+
ShareProtocol actualProtocol = effectiveProtocol == ShareProtocols.Smb ? ShareProtocol.Smb : ShareProtocol.Nfs;
733734

734735
if (actualProtocol != expectedProtocol)
735736
{

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ ShareDirectoryStorageResourceContainer destinationStorageResourceContainer
6868
= destinationContainer as ShareDirectoryStorageResourceContainer;
6969
ShareFileStorageResourceOptions destinationOptions = destinationStorageResourceContainer.ResourceOptions;
7070
// both source and destination must be SMB
71-
if (((ResourceOptions?.ShareProtocol ?? ShareProtocols.Smb) == ShareProtocols.Smb)
72-
&& ((destinationOptions?.ShareProtocol ?? ShareProtocols.Smb) == ShareProtocols.Smb))
71+
if (((ResourceOptions?.ShareProtocol ?? ShareProtocol.Smb) == ShareProtocol.Smb)
72+
&& ((destinationOptions?.ShareProtocol ?? ShareProtocol.Smb) == ShareProtocol.Smb))
7373
{
7474
traits = ShareFileTraits.Attributes;
7575
if (destinationOptions?.FilePermissions ?? false)
@@ -96,7 +96,7 @@ ShareDirectoryStorageResourceContainer destinationStorageResourceContainer
9696

9797
protected override StorageResourceCheckpointDetails GetSourceCheckpointDetails()
9898
{
99-
return new ShareFileSourceCheckpointDetails();
99+
return new ShareFileSourceCheckpointDetails(shareProtocol: ResourceOptions?.ShareProtocol ?? ShareProtocol.Smb);
100100
}
101101

102102
protected override StorageResourceCheckpointDetails GetDestinationCheckpointDetails()
@@ -124,7 +124,8 @@ protected override StorageResourceCheckpointDetails GetDestinationCheckpointDeta
124124
isFileMetadataSet: ResourceOptions?._isFileMetadataSet ?? false,
125125
fileMetadata: ResourceOptions?.FileMetadata,
126126
isDirectoryMetadataSet: ResourceOptions?._isDirectoryMetadataSet ?? false,
127-
directoryMetadata: ResourceOptions?.DirectoryMetadata)
127+
directoryMetadata: ResourceOptions?.DirectoryMetadata,
128+
shareProtocol: ResourceOptions?.ShareProtocol ?? ShareProtocol.Smb)
128129
{
129130
};
130131
}
@@ -196,8 +197,8 @@ protected override async Task ValidateTransferAsync(
196197
if (sourceResource is ShareDirectoryStorageResourceContainer sourceShareDirectoryResource)
197198
{
198199
// Ensure the transfer is supported (NFS -> NFS and SMB -> SMB)
199-
if ((ResourceOptions?.ShareProtocol ?? ShareProtocols.Smb)
200-
!= (sourceShareDirectoryResource.ResourceOptions?.ShareProtocol ?? ShareProtocols.Smb))
200+
if ((ResourceOptions?.ShareProtocol ?? ShareProtocol.Smb)
201+
!= (sourceShareDirectoryResource.ResourceOptions?.ShareProtocol ?? ShareProtocol.Smb))
201202
{
202203
throw Errors.ShareTransferNotSupported();
203204
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointDetails.cs‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@ internal class ShareFileDestinationCheckpointDetails : StorageResourceCheckpoint
9090
public bool IsDirectoryMetadataSet;
9191
private byte[] _directoryMetadataBytes;
9292

93+
/// <summary>
94+
/// Share Protocol for destination files/directories.
95+
/// </summary>
96+
public ShareProtocol ShareProtocol;
97+
9398
public override int Length => CalculateLength();
9499

95100
public ShareFileDestinationCheckpointDetails(
@@ -115,7 +120,8 @@ public ShareFileDestinationCheckpointDetails(
115120
bool isFileMetadataSet,
116121
Metadata fileMetadata,
117122
bool isDirectoryMetadataSet,
118-
Metadata directoryMetadata)
123+
Metadata directoryMetadata,
124+
ShareProtocol shareProtocol)
119125
{
120126
Version = DataMovementShareConstants.DestinationCheckpointDetails.SchemaVersion;
121127
CacheControl = cacheControl;
@@ -162,6 +168,8 @@ public ShareFileDestinationCheckpointDetails(
162168
DirectoryMetadata = directoryMetadata;
163169
IsDirectoryMetadataSet = isDirectoryMetadataSet;
164170
_directoryMetadataBytes = directoryMetadata != default ? Encoding.UTF8.GetBytes(directoryMetadata.DictionaryToString()) : Array.Empty<byte>();
171+
172+
ShareProtocol = shareProtocol;
165173
}
166174

167175
protected override void Serialize(Stream stream)
@@ -192,6 +200,9 @@ protected override void Serialize(Stream stream)
192200
writer.WritePreservablePropertyOffset(IsFileMetadataSet, _fileMetadataBytes.Length, ref currentVariableLengthIndex);
193201
writer.WritePreservablePropertyOffset(IsDirectoryMetadataSet, _directoryMetadataBytes.Length, ref currentVariableLengthIndex);
194202

203+
// ShareProtocol
204+
writer.Write((byte)ShareProtocol);
205+
195206
// Variable length info
196207
if (IsFileAttributesSet)
197208
{
@@ -254,7 +265,8 @@ internal static ShareFileDestinationCheckpointDetails Deserialize(Stream stream)
254265

255266
// Version
256267
int version = reader.ReadInt32();
257-
if (version != DataMovementShareConstants.DestinationCheckpointDetails.SchemaVersion)
268+
if (version < DataMovementShareConstants.DestinationCheckpointDetails.MinValidSchemaVersion
269+
|| version > DataMovementShareConstants.DestinationCheckpointDetails.MaxValidSchemaVersion)
258270
{
259271
throw Storage.Errors.UnsupportedJobSchemaVersionHeader(version);
260272
}
@@ -277,6 +289,14 @@ internal static ShareFileDestinationCheckpointDetails Deserialize(Stream stream)
277289
(bool isFileMetadataSet, int fileMetadataOffset, int fileMetadataLength) = reader.ReadVariableLengthFieldInfo();
278290
(bool isDirectoryMetadataSet, int directoryMetadataOffset, int directoryMetadataLength) = reader.ReadVariableLengthFieldInfo();
279291

292+
// ShareProtocol
293+
ShareProtocol shareProtocol = ShareProtocol.Smb;
294+
bool shareProtocolSupport = version >= DataMovementShareConstants.DestinationCheckpointDetails.SchemaVersion_4;
295+
if (shareProtocolSupport)
296+
{
297+
shareProtocol = (ShareProtocol)(reader.ReadByte());
298+
}
299+
280300
// NtfsFileAttributes
281301
NtfsFileAttributes? ntfsFileAttributes = null;
282302
if (fileAttributesOffset > 0)
@@ -366,6 +386,7 @@ internal static ShareFileDestinationCheckpointDetails Deserialize(Stream stream)
366386
directoryMetadataString = Encoding.UTF8.GetString(reader.ReadBytes(directoryMetadataLength));
367387
}
368388

389+
// When deserializing, the version of the new CheckpointDetails is always the latest version.
369390
return new(
370391
isContentTypeSet: isContentTypeSet,
371392
contentType: contentType,
@@ -389,7 +410,8 @@ internal static ShareFileDestinationCheckpointDetails Deserialize(Stream stream)
389410
isFileMetadataSet: isFileMetadataSet,
390411
fileMetadata: fileMetadataString?.ToDictionary(nameof(fileMetadataString)),
391412
isDirectoryMetadataSet: isDirectoryMetadataSet,
392-
directoryMetadata: directoryMetadataString?.ToDictionary(nameof(directoryMetadataString)));
413+
directoryMetadata: directoryMetadataString?.ToDictionary(nameof(directoryMetadataString)),
414+
shareProtocol: shareProtocol);
393415
}
394416

395417
private int CalculateLength()

0 commit comments

Comments
 (0)