[Storage][DataMovement] Fix preserving FileChangedOn during transfers (Azure#48009)

jalauzon-msft · web-flow · commit d2200c6f10ee · 2025-01-31T14:53:16.000-08:00
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json b/sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_ce7e1f61c3"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_148b875701"
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs
@@ -36,7 +36,7 @@ public class AppendBlobStartTransferCopyTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "application/octet-stream";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -233,6 +233,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToBlockBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToBlockBlobTests.cs
@@ -36,7 +36,7 @@ public class AppendBlobToBlockBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -229,6 +229,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToPageBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToPageBlobTests.cs
@@ -36,7 +36,7 @@ public class AppendBlobToPageBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -248,6 +248,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobStartTransferCopyTests.cs
@@ -35,7 +35,7 @@ public class BlockBlobStartTransferCopyTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -212,6 +212,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
                 Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToAppendBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToAppendBlobTests.cs
@@ -36,7 +36,7 @@ public class BlockBlobToAppendBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -233,6 +233,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToPageBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToPageBlobTests.cs
@@ -36,7 +36,7 @@ public class BlockBlobToPageBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -234,6 +234,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs
@@ -36,7 +36,7 @@ public class PageBlobStartTransferCopyTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cool;
-        private const string _defaultContentType = "application/octet-stream";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -224,6 +224,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToAppendBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToAppendBlobTests.cs
@@ -36,7 +36,7 @@ public class PageBlobToAppendBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -241,6 +241,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToBlockBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToBlockBlobTests.cs
@@ -36,7 +36,7 @@ public class PageBlobToBlockBlobTests : StartTransferCopyTestBase
         StorageTestEnvironment>
     {
         private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
-        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentType = "image/jpeg";
         private const string _defaultContentLanguage = "en-US";
         private const string _defaultContentDisposition = "inline";
         private const string _defaultCacheControl = "no-cache";
@@ -232,6 +232,7 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
+                Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
                 Assert.IsEmpty(destinationTags.Tags);
diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_91bd99f1ca"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_1510e7b5ae"
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs
@@ -118,6 +118,35 @@ public static FileSmbProperties GetFileSmbProperties(
             };
         }
 
+        public static FileSmbProperties GetFileSmbProperties(
+            this ShareFileStorageResourceOptions options,
+            StorageResourceItemProperties properties)
+        {
+            return new()
+            {
+                FileAttributes = (options?._isFileAttributesSet ?? false)
+                    ? options?.FileAttributes
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FileAttributes, out object fileAttributes) == true
+                        ? (NtfsFileAttributes?)fileAttributes
+                        : default,
+                FileCreatedOn = (options?._isFileCreatedOnSet ?? false)
+                    ? options?.FileCreatedOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.CreationTime, out object fileCreatedOn) == true
+                        ? (DateTimeOffset?)fileCreatedOn
+                        : default,
+                FileLastWrittenOn = (options?._isFileLastWrittenOnSet ?? false)
+                    ? options?.FileLastWrittenOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.LastWrittenOn, out object fileLastWrittenOn) == true
+                        ? (DateTimeOffset?)fileLastWrittenOn
+                        : default,
+                FileChangedOn = (options?._isFileChangedOnSet ?? false)
+                    ? options?.FileChangedOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.ChangedOnTime, out object fileChangedOn) == true
+                        ? (DateTimeOffset?)fileChangedOn
+                        : default,
+            };
+        }
+
         internal static ShareFileUploadRangeOptions ToShareFileUploadRangeOptions(
             this ShareFileStorageResourceOptions options)
             => new()
diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs
@@ -90,13 +90,27 @@ await ShareFileClient.CreateAsync(
                     cancellationToken: cancellationToken).ConfigureAwait(false);
         }
 
-        protected override Task CompleteTransferAsync(
+        protected override async Task CompleteTransferAsync(
             bool overwrite,
             StorageResourceCompleteTransferOptions completeTransferOptions,
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
-            return Task.CompletedTask;
+
+            // Call Set Properties if FileChangedOn is to be preserved or manually set
+            // as it can be changed during a transfer.
+            if (_options?._isFileChangedOnSet == false || _options?.FileChangedOn != null)
+            {
+                StorageResourceItemProperties sourceProperties = completeTransferOptions?.SourceProperties;
+                ShareFileHttpHeaders httpHeaders = _options?.GetShareFileHttpHeaders(sourceProperties?.RawProperties);
+                FileSmbProperties smbProperties = _options?.GetFileSmbProperties(sourceProperties);
+                await ShareFileClient.SetHttpHeadersAsync(new()
+                {
+                    HttpHeaders = httpHeaders,
+                    SmbProperties = smbProperties,
+                },
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
         }
 
         protected override async Task CopyBlockFromUriAsync(
diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs
@@ -7,7 +7,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Azure.Core.TestFramework;
 using Azure.Storage.DataMovement.Tests;
 using BaseShares::Azure.Storage.Files.Shares;
 using BaseShares::Azure.Storage.Files.Shares.Models;
@@ -285,6 +284,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(_defaultContentType, destinationProperties.ContentType);
                 Assert.AreEqual(_defaultFileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(_defaultFileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(_defaultFileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
             }
             else if (transferPropertiesTestType == TransferPropertiesTestType.Preserve)
             {
@@ -298,6 +298,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
 
                 // Check if the permissions are the same. Permission Keys will be different as they are defined by the share service.
                 ShareClient sourceShareClient = sourceClient.GetParentShareClient();
@@ -319,6 +320,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
             }
         }
 
@@ -364,70 +366,5 @@ private ShareFileStorageResourceOptions GetShareFileStorageResourceOptions(Trans
             }
             return options;
         }
-
-        private async Task CopyRemoteObjects_VerifyProperties(
-            ShareClient sourceContainer,
-            ShareClient destinationContainer,
-            TransferPropertiesTestType propertiesType)
-        {
-            // Arrange
-            int size = Constants.KB;
-            string sourcePrefix = "sourceFolder";
-            string destPrefix = "destFolder";
-            await CreateDirectoryInSourceAsync(sourceContainer, sourcePrefix);
-            string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName());
-            string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName1, propertiesType: propertiesType);
-            await CreateShareFileAsync(sourceContainer, size, itemName2, propertiesType: propertiesType);
-
-            string subDirName = string.Join("/", sourcePrefix, "bar");
-            await CreateDirectoryInSourceAsync(sourceContainer, subDirName);
-            string itemName3 = string.Join("/", subDirName, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName3, propertiesType: propertiesType);
-
-            string subDirName2 = string.Join("/", sourcePrefix, "pik");
-            await CreateDirectoryInSourceAsync(sourceContainer, subDirName2);
-            string itemName4 = string.Join("/", subDirName2, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName4, propertiesType: propertiesType);
-
-            await CreateDirectoryInDestinationAsync(destinationContainer, destPrefix);
-
-            // Create storage resource containers
-            StorageResourceContainer sourceResource =
-                GetSourceStorageResourceContainer(sourceContainer, sourcePrefix);
-            StorageResourceContainer destinationResource =
-                GetDestinationStorageResourceContainer(destinationContainer, destPrefix, propertiesType);
-
-            // Create Transfer Manager
-            TransferOptions options = new TransferOptions();
-            TestEventsRaised testEventsRaised = new TestEventsRaised(options);
-            TransferManager transferManager = new TransferManager();
-
-            // Start transfer and await for completion.
-            TransferOperation transfer = await transferManager.StartTransferAsync(
-                sourceResource,
-                destinationResource,
-                options).ConfigureAwait(false);
-
-            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
-            await TestTransferWithTimeout.WaitForCompletionAsync(
-                transfer,
-                testEventsRaised,
-                tokenSource.Token);
-
-            // Verify completion
-            Assert.NotNull(transfer);
-            Assert.IsTrue(transfer.HasCompleted);
-            Assert.AreEqual(TransferState.Completed, transfer.Status.State);
-            await testEventsRaised.AssertContainerCompletedCheck(4);
-
-            // Assert
-            await VerifyResultsAsync(
-                sourceContainer,
-                sourcePrefix,
-                destinationContainer,
-                destPrefix,
-                propertiesType);
-        }
     }
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs

Original file line number	Diff line number	Diff line change
`@@ -2,5 +2,5 @@`
`2`	`2`	`"AssetsRepo": "Azure/azure-sdk-assets",`
`3`	`3`	`"AssetsRepoPrefixPath": "net",`
`4`	`4`	`"TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",`
`5`		`- "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_ce7e1f61c3"`
	`5`	`+ "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_148b875701"`
`6`	`6`	`}`