avoid chunked transfers in putBlob when possible

This is to avoid Downloads.jl from initiating chunked transfer when a file IOStream is supplied to `putBlob`.

In 0.4.0, we have switched to using Downloads.jl as the HTTP client instead of HTTP.jl. This issue happens because Downloads.jl
is not able to determine the content length when an IO instance is provided to upload data from, and switches to using a chunked transfer encoding. And the putBlob Azure API endpoint does not support chunked transfers.

The solution is to not use chunked transfers for putBlob. This PR (JuliaLang/Downloads.jl#167) to Downloads.jl would let it also look into the content-length header that we supply to determine the data size to expect in the IO instance we pass.

But until the Downloads.jl PR is merged (and backported), we need this change in Azure.jl to detect IO handles that point to locally mappable files and supply the correct content length and `IOBuffer` of memory mapped file contents as data to upload instead.

fixes: #25

Author: tanmaykm

Project.toml

@@ -10,6 +10,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
+Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Swagger = "2d69052b-6a58-5cd9-a030-a48559c324ac"
 XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5"

src/StorageServices/StorageServices.jl

@@ -8,6 +8,7 @@ using URIs
 using MbedTLS
 using Dates
 using Base64
+using Mmap
 
 include("common.jl")
 include("blob.jl")

src/StorageServices/blob.jl

@@ -107,6 +107,19 @@ end
 
 const valid_blob_types = ("BlockBlob", "PageBlob", "AppendBlob")
 
+function check_mappable_file(io)
+    if isa(io, IOStream)
+        try
+            fsz = filesize(io)
+            data = Mmap.mmap(io, Vector{UInt8}, fsz)
+            return IOBuffer(data), fsz
+        catch
+            return io, nothing
+        end
+    end
+    return io, nothing
+end
+
 """
 Creates a new block, page, or append blob, or updates the content of an existing block blob.
 
@@ -164,8 +177,22 @@ function putBlob(ctx, subscription_id::String, resource_group_name::String, uri:
                     content_length = length(codeunits(block_blob_contents))
                 elseif isa(block_blob_contents, Vector{UInt8})
                     content_length = length(block_blob_contents)
+                elseif isa(block_blob_contents, IOBuffer)
+                    content_length = block_blob_contents.size
+                else
+                    wrapped_contents, content_length = check_mappable_file(block_blob_contents)
+                    if content_length === nothing
+                        error("content_length must be specified for blob contents of type $(typeof(block_blob_contents))")
+                    else
+                        block_blob_contents = wrapped_contents
+                    end
+                end
+            elseif isa(block_blob_contents, IOStream)
+                wrapped_contents, wrapped_content_length = check_mappable_file(block_blob_contents)
+                if content_length !== wrapped_content_length
+                    error("content_length must match the length of the stream")
                 else
-                    error("content_length must be specified for blob contents of type $(typeof(block_blob_contents))")
+                    block_blob_contents = wrapped_contents
                 end
             end
         end