Merge pull request #45 from RelationalAI/ag-multipart-metrics-and-padding

Multipart PUT InvalidBlockList fix, new Rust metrics and padding behavior

Author: andrebsguedes

Project.toml

@@ -1,6 +1,6 @@
 name = "RustyObjectStore"
 uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610"
-version = "0.9.1"
+version = "0.10.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -20,7 +20,7 @@ ReTestItems = "1"
 Sockets = "1"
 Test = "1"
 julia = "1.8"
-object_store_ffi_jll = "0.9.1"
+object_store_ffi_jll = "0.10.0"
 
 [extras]
 CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"

src/RustyObjectStore.jl

@@ -799,8 +799,8 @@ function safe_message(e::Exception)
     if e isa RequestException
         msg = message(e)
         r = reason(e)
-        if contains(msg, "<Error>") || contains(msg, "http")
-            # Contains unreadacted payload from backend or urls, try extracting safe information
+        if contains(msg, "<Error>")
+            # Contains unreadacted payload from backend, try extracting safe information
             code, backend_msg, report = extract_safe_parts(message(e))
             reason_str = reason_description(r)
 
@@ -809,6 +809,9 @@ function safe_message(e::Exception)
             retry_report = isnothing(report) ? "" : "\n\n$(report)"
 
             return "$(backend_msg) (code: $(code), reason: $(reason_str))$(retry_report)"
+        elseif contains(msg, "http")
+            # Contains urls, redact them
+            return replace(msg, r"https?://[^\n )]+" => "<redacted url>")
         else
             # Assuming it safe as it does not come from backend or has url, return message directly
             return msg
@@ -2004,12 +2007,11 @@ function invalidate_config(conf::Option{AbstractConfig}=nothing)
     end
 end
 
-struct Metrics
-    live_bytes::Int64
-end
-
 function current_metrics()
-    return @ccall rust_lib.current_metrics()::Metrics
+    metrics_ptr = @ccall rust_lib.current_metrics()::Ptr{Cchar}
+    metrics_string = unsafe_string(metrics_ptr)
+    metrics = JSON3.read(metrics_string)
+    return metrics
 end
 
 module Test

test/azure_blobs_exception_tests.jl

@@ -594,6 +594,63 @@ end # @testitem
         return nrequests[]
     end
 
+    function test_invalid_block_list()
+        nrequests = Ref(0)
+        uploadid = nothing
+        (port, tcp_server) = Sockets.listenany(8081)
+        http_server = HTTP.serve!(tcp_server) do request::HTTP.Request
+            nrequests[] += 1
+            if request.method == "PUT"
+                if occursin("comp=blocklist", request.target)
+                    uploadid_value = HTTP.header(request, "x-ms-meta-uploadid")
+                    if !isnothing(uploadid_value)
+                        uploadid = "x-ms-meta-uploadid" => uploadid_value
+                    end
+                    return HTTP.Response(400, "InvalidBlockList")
+                else
+                    return HTTP.Response(200, [
+                        "Content-Length" => "0",
+                        "Last-Modified" => "Tue, 15 Oct 2019 12:45:26 GMT",
+                        "ETag" => "123"
+                    ], "")
+                end
+            elseif request.method == "HEAD"
+                return HTTP.Response(200, [
+                    uploadid,
+                    "Content-Length" => "0",
+                    "Last-Modified" => "Tue, 15 Oct 2019 12:45:26 GMT",
+                    "ETag" => "123"
+                ], "")
+            else
+                return HTTP.Response(404, "Not Found")
+            end
+        end
+
+        baseurl = "http://127.0.0.1:$port/$account/$container/"
+        conf = AzureConfig(;
+            storage_account_name=account,
+            container_name=container,
+            storage_account_key=shared_key_from_azurite,
+            host=baseurl,
+            opts=ClientOptions(;
+                max_retries=max_retries,
+                retry_timeout_secs=retry_timeout_secs,
+                request_timeout_secs
+            )
+        )
+
+        try
+            put_object(zeros(UInt8, 11 * 1024 * 1024), "blob", conf)
+            @test true
+        catch e
+            @test false
+        finally
+            Threads.@spawn HTTP.forceclose(http_server)
+        end
+        # wait(http_server)
+        return nrequests[]
+    end
+
     @testset "400: Bad Request" begin
         # Returned when there's an error in the request URI, headers, or body. The response body
         # contains an error message explaining what the specific problem is.
@@ -762,4 +819,9 @@ end # @testitem
         nrequests = test_cancellation()
         @test nrequests == 1
     end
+
+    @testset "InvalidBlockList" begin
+        nrequests = test_invalid_block_list()
+        @test nrequests == 4
+    end
 end

test/basic_unified_tests.jl

@@ -589,7 +589,7 @@ function run_list_test_cases(config::AbstractConfig; strict_entry_size=true)
 
         entries = list_objects("list_empty/", config)
         @test length(entries) == 3
-        @test map(x -> x.size, entries) == [0, 0, 0]
+        @test within_margin(sort(map(x -> x.size, entries)), [0, 0, 0], margin)
         @test map(x -> x.location, entries) == ["list_empty/10.csv", "list_empty/20.csv", "list_empty/30.csv"]
     end
 

test/snowflake_stage_exception_tests.jl

@@ -89,15 +89,15 @@
             @test false # Should have thrown an error
         catch e
             @test e isa RustyObjectStore.PutException
-            @test occursin("Connection refused", e.msg)
+            @test occursin("Connection refused", e.msg) || occursin("Unable to access master token file", e.msg)
         end
 
         try
             get_object!(buffer, "still_doesnt_exist.csv", _stale_config)
             @test false # Should have thrown an error
         catch e
             @test e isa RustyObjectStore.GetException
-            @test occursin("Connection refused", e.msg)
+            @test occursin("Connection refused", e.msg) || occursin("Unable to access master token file", e.msg)
         end
     end