Checkpointer use testing (Azure#47224)

* fix and test reporting

* testing

* test checpointer use - item transfer

* container transfer test

* use memory checkpointer in tests

* fix test

* fix scope

* simplify container test

* missing assert

* remove tostring used for debugging

Author: jaschrep-msft

sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs

@@ -508,6 +508,7 @@ await TransferFailedEventHandler.RaiseAsync(
                         ClientDiagnostics)
                         .ConfigureAwait(false);
                 }
+                _dataTransfer.TransferStatus.SetFailedItem();
             }
 
             try

sdk/storage/Azure.Storage.DataMovement/tests/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]

sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryTransferCheckpointer.cs

@@ -57,7 +57,7 @@ public JobPartPlanHeader Plan
 
     public Dictionary<string, Job> Jobs { get; set; } = new();
 
-    public Task AddNewJobAsync(string transferId, StorageResource source, StorageResource destination, CancellationToken cancellationToken = default)
+    public virtual Task AddNewJobAsync(string transferId, StorageResource source, StorageResource destination, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.ContainsKey(transferId))
@@ -71,12 +71,12 @@ public Task AddNewJobAsync(string transferId, StorageResource source, StorageRes
             Operation = JobPlanOperation.Upload, // TODO
             Source = source,
             Destination = destination,
-            Status = new()
+            Status = new(DataTransferState.Queued, false, false),
         };
         return Task.CompletedTask;
     }
 
-    public Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHeader header, CancellationToken cancellationToken = default)
+    public virtual Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHeader header, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -90,18 +90,18 @@ public Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHea
         job.Parts.Add(partNumber, new()
         {
             Plan = header,
-            Status = new()
+            Status = new(DataTransferState.Queued, false, false),
         });
         return Task.CompletedTask;
     }
 
-    public Task<int> GetCurrentJobPartCountAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<int> GetCurrentJobPartCountAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.TryGetValue(transferId, out Job job) ? job.Parts.Count : 0);
     }
 
-    public Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -119,7 +119,7 @@ public Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transf
         });
     }
 
-    public Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber, CancellationToken cancellationToken = default)
+    public virtual Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -133,7 +133,7 @@ public Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber
         return Task.FromResult(part.Plan);
     }
 
-    public Task<DataTransferStatus> GetJobStatusAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<DataTransferStatus> GetJobStatusAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -143,19 +143,19 @@ public Task<DataTransferStatus> GetJobStatusAsync(string transferId, Cancellatio
         return Task.FromResult(job.Status.DeepCopy());
     }
 
-    public Task<List<string>> GetStoredTransfersAsync(CancellationToken cancellationToken = default)
+    public virtual Task<List<string>> GetStoredTransfersAsync(CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.Keys.ToList());
     }
 
-    public Task<bool> IsEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<bool> IsEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.TryGetValue(transferId, out Job job) && job.EnumerationComplete);
     }
 
-    public Task SetEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task SetEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job))
@@ -165,7 +165,7 @@ public Task SetEnumerationCompleteAsync(string transferId, CancellationToken can
         return Task.CompletedTask;
     }
 
-    public Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransferStatus status, CancellationToken cancellationToken = default)
+    public virtual Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransferStatus status, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job) && job.Parts.TryGetValue(partNumber, out JobPart part))
@@ -175,7 +175,7 @@ public Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransfe
         return Task.CompletedTask;
     }
 
-    public Task SetJobStatusAsync(string transferId, DataTransferStatus status, CancellationToken cancellationToken = default)
+    public virtual Task SetJobStatusAsync(string transferId, DataTransferStatus status, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job))
@@ -185,7 +185,7 @@ public Task SetJobStatusAsync(string transferId, DataTransferStatus status, Canc
         return Task.CompletedTask;
     }
 
-    public Task<bool> TryRemoveStoredTransferAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<bool> TryRemoveStoredTransferAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.Remove(transferId));