1414
1515namespace Microsoft . WindowsAzure . Commands . Storage . File . Cmdlet
1616{
17+ using global Azure ;
18+ using global Azure . Storage . Files . Shares ;
19+ using global Azure . Storage . Files . Shares . Models ;
20+ using Microsoft . Azure . Storage ;
21+ using Microsoft . Azure . Storage . DataMovement ;
22+ using Microsoft . Azure . Storage . File ;
1723 using Microsoft . WindowsAzure . Commands . Common ;
24+ using Microsoft . WindowsAzure . Commands . Common . Storage . ResourceModel ;
1825 using Microsoft . WindowsAzure . Commands . Storage . Common ;
1926 using Microsoft . WindowsAzure . Commands . Utilities . Common ;
20- using Microsoft . Azure . Storage ;
21- using Microsoft . Azure . Storage . File ;
2227 using System ;
28+ using System . Collections . Generic ;
2329 using System . Globalization ;
2430 using System . IO ;
2531 using System . Management . Automation ;
2632 using System . Net ;
33+ using System . Runtime . InteropServices ;
34+ using System . Security . Cryptography ;
2735 using System . Threading . Tasks ;
2836 using LocalConstants = Microsoft . WindowsAzure . Commands . Storage . File . Constants ;
29- using System . Runtime . InteropServices ;
30- using Microsoft . Azure . Storage . DataMovement ;
31- using Microsoft . WindowsAzure . Commands . Common . CustomAttributes ;
32- using Microsoft . WindowsAzure . Commands . Common . Storage . ResourceModel ;
3337
3438 [ Cmdlet ( "Set" , Azure . Commands . ResourceManager . Common . AzureRMConstants . AzurePrefix + "StorageFileContent" , SupportsShouldProcess = true , DefaultParameterSetName = LocalConstants . ShareNameParameterSetName ) , OutputType ( typeof ( AzureStorageFile ) ) ]
3539 public class SetAzureStorageFileContent : StorageFileDataManagementCmdletBase , IDynamicParameters
@@ -110,11 +114,19 @@ public override void ExecuteCmdlet()
110114 {
111115 throw new FileNotFoundException ( string . Format ( CultureInfo . CurrentCulture , Resources . SourceFileNotFound , this . Source ) ) ;
112116 }
117+ long fileSize = localFile . Length ;
113118
114- // if FIPS policy is enabled, must use native MD5
119+ // if FIPS policy is enabled, must use native MD5 for DMlib.
115120 if ( fipsEnabled )
116121 {
117- CloudStorageAccount . UseV1MD5 = false ;
122+ if ( fileSize < sizeTB )
123+ {
124+ CloudStorageAccount . UseV1MD5 = false ;
125+ }
126+ else // use Track2 SDK
127+ {
128+ WriteWarning ( "The uploaded file won't have Content MD5 hash, since caculate MD5 hash fail, most possiblly caused by FIPS is enabled on this machine." ) ;
129+ }
118130 }
119131
120132 bool isDirectory ;
@@ -133,19 +145,141 @@ public override void ExecuteCmdlet()
133145 cloudFileToBeUploaded . GetFullPath ( ) , cloudFileToBeUploaded . Share . Name ) ,
134146 Resources . PrepareUploadingFile ) ;
135147
136- await DataMovementTransferHelper . DoTransfer ( ( ) =>
137- this . TransferManager . UploadAsync (
138- localFile . FullName ,
139- cloudFileToBeUploaded ,
140- new UploadOptions
148+ if ( fileSize <= sizeTB )
149+ {
150+
151+
152+ await DataMovementTransferHelper . DoTransfer ( ( ) =>
153+ this . TransferManager . UploadAsync (
154+ localFile . FullName ,
155+ cloudFileToBeUploaded ,
156+ new UploadOptions
157+ {
158+ PreserveSMBAttributes = context is null ? false : context . PreserveSMBAttribute . IsPresent
159+ } ,
160+ this . GetTransferContext ( progressRecord , localFile . Length ) ,
161+ this . CmdletCancellationToken ) ,
162+ progressRecord ,
163+ this . OutputStream ) . ConfigureAwait ( false ) ;
164+ }
165+ else // use Track2 SDK
166+ {
167+ //Create File
168+ ShareFileClient fileClient = AzureStorageFile . GetTrack2FileClient ( cloudFileToBeUploaded , Channel . StorageContext ) ;
169+
170+ // confirm overwrite if file exist
171+ if ( ! this . Force . IsPresent &&
172+ fileClient . Exists ( this . CmdletCancellationToken ) &&
173+ ! await this . OutputStream . ConfirmAsync ( string . Format ( CultureInfo . CurrentCulture , Resources . OverwriteConfirmation , Util . ConvertToString ( cloudFileToBeUploaded ) ) ) )
174+ {
175+ return ;
176+ }
177+
178+ await fileClient . CreateAsync ( fileSize , cancellationToken : this . CmdletCancellationToken ) . ConfigureAwait ( false ) ;
179+
180+ //Prepare progress Handler
181+ IProgress < long > progressHandler = new Progress < long > ( ( finishedBytes ) =>
182+ {
183+ if ( progressRecord != null )
184+ {
185+ // Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
186+ progressRecord . PercentComplete = 0 == fileSize ? 100 : ( int ) ( finishedBytes * 100 / fileSize ) ;
187+ progressRecord . StatusDescription = string . Format ( CultureInfo . CurrentCulture , Resources . FileTransmitStatus , progressRecord . PercentComplete ) ;
188+ this . OutputStream . WriteProgress ( progressRecord ) ;
189+ }
190+ } ) ;
191+
192+ long blockSize = 4 * 1024 * 1024 ;
193+ int maxWorkers = 4 ;
194+
195+ List < Task > runningTasks = new List < Task > ( ) ;
196+
197+ IncrementalHash hash = null ;
198+ if ( ! fipsEnabled )
199+ {
200+ hash = IncrementalHash . CreateHash ( HashAlgorithmName . MD5 ) ;
201+ }
202+
203+ using ( FileStream stream = File . OpenRead ( localFile . FullName ) )
204+ {
205+ byte [ ] buffer = null ;
206+ long lastBlockSize = 0 ;
207+ for ( long offset = 0 ; offset < fileSize ; offset += blockSize )
208+ {
209+ long currentBlockSize = offset + blockSize < fileSize ? blockSize : fileSize - offset ;
210+
211+ // Only need to create new buffer when chunk size change
212+ if ( currentBlockSize != lastBlockSize )
213+ {
214+ buffer = new byte [ currentBlockSize ] ;
215+ lastBlockSize = currentBlockSize ;
216+ }
217+ await stream . ReadAsync ( buffer : buffer , offset : 0 , count : ( int ) currentBlockSize ) ;
218+ if ( ! fipsEnabled && hash != null )
219+ {
220+ hash . AppendData ( buffer ) ;
221+ }
222+
223+ Task task = UploadFileRangAsync ( fileClient ,
224+ new HttpRange ( offset , currentBlockSize ) ,
225+ new MemoryStream ( buffer ) ,
226+ progressHandler ) ;
227+ runningTasks . Add ( task ) ;
228+
229+ // Check if any of upload range tasks are still busy
230+ if ( runningTasks . Count >= maxWorkers )
231+ {
232+ await Task . WhenAny ( runningTasks ) . ConfigureAwait ( false ) ;
233+
234+ // Clear any completed blocks from the task list
235+ for ( int i = 0 ; i < runningTasks . Count ; i ++ )
236+ {
237+ Task runningTask = runningTasks [ i ] ;
238+ if ( ! runningTask . IsCompleted )
239+ {
240+ continue ;
241+ }
242+
243+ await runningTask . ConfigureAwait ( false ) ;
244+ runningTasks . RemoveAt ( i ) ;
245+ i -- ;
246+ }
247+ }
248+ }
249+ // Wait for all upload range tasks finished
250+ await Task . WhenAll ( runningTasks ) . ConfigureAwait ( false ) ;
251+ }
252+
253+ // Need set file properties
254+ if ( ( ! fipsEnabled && hash != null ) || ( context != null && context . PreserveSMBAttribute . IsPresent ) )
255+ {
256+ ShareFileHttpHeaders header = null ;
257+ if ( ! fipsEnabled && hash != null )
258+ {
259+ header = new ShareFileHttpHeaders ( ) ;
260+ header . ContentHash = hash . GetHashAndReset ( ) ;
261+ }
262+
263+ FileSmbProperties smbProperties = null ;
264+ if ( context != null && context . PreserveSMBAttribute . IsPresent )
141265 {
142- PreserveSMBAttributes = context is null ? false : context . PreserveSMBAttribute . IsPresent
143- } ,
144- this . GetTransferContext ( progressRecord , localFile . Length ) ,
145- this . CmdletCancellationToken ) ,
146- progressRecord ,
147- this . OutputStream ) . ConfigureAwait ( false ) ;
266+ FileInfo sourceFileInfo = new FileInfo ( localFile . FullName ) ;
267+ smbProperties = new FileSmbProperties ( ) ;
268+ smbProperties . FileCreatedOn = sourceFileInfo . CreationTimeUtc ;
269+ smbProperties . FileLastWrittenOn = sourceFileInfo . LastWriteTimeUtc ;
270+ smbProperties . FileAttributes = Util . LocalAttributesToAzureFileNtfsAttributes ( File . GetAttributes ( localFile . FullName ) ) ;
271+ }
148272
273+ // set file header and attributes to the file
274+ fileClient . SetHttpHeaders ( httpHeaders : header , smbProperties : smbProperties ) ;
275+ }
276+
277+ if ( this . PassThru )
278+ {
279+ // fetch latest file properties for output
280+ cloudFileToBeUploaded . FetchAttributes ( ) ;
281+ }
282+ }
149283
150284 if ( this . PassThru )
151285 {
@@ -160,6 +294,17 @@ await DataMovementTransferHelper.DoTransfer(() =>
160294 }
161295 }
162296
297+ private long Finishedbytes = 0 ;
298+ private async Task UploadFileRangAsync ( ShareFileClient file , HttpRange range , Stream content , IProgress < long > progressHandler = null )
299+ {
300+ await file . UploadRangeAsync (
301+ range ,
302+ content ,
303+ cancellationToken : this . CmdletCancellationToken ) . ConfigureAwait ( false ) ;
304+ Finishedbytes += range . Length is null ? 0 : range . Length . Value ;
305+ progressHandler . Report ( Finishedbytes ) ;
306+ }
307+
163308 private async Task < CloudFile > BuildCloudFileInstanceFromPathAsync ( string defaultFileName , string [ ] path , bool pathIsDirectory )
164309 {
165310 CloudFileDirectory baseDirectory = null ;
0 commit comments