Skip to content

Commit f80f52c

Browse files
samtkaplansamtkaplan
committed
use copy api
1 parent 5ab726f commit f80f52c

File tree

4 files changed

+205
-18
lines changed

4 files changed

+205
-18
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ jobs:
3737
run: |
3838
az group create -l southcentralus -n "AzureBackupRG-azstorage-${{ matrix.os }}-${{ matrix.version }}-${{ github.run_id }}"
3939
az storage account create --min-tls-version TLS1_2 -n "s${{ steps.uuid.outputs.uuid }}" -g "AzureBackupRG-azstorage-${{ matrix.os }}-${{ matrix.version }}-${{ github.run_id }}" -l southcentralus
40+
az storage account create --min-tls-version TLS1_2 -n "s${{ steps.uuid.outputs.uuid }}2" -g "AzureBackupRG-azstorage-${{ matrix.os }}-${{ matrix.version }}-${{ github.run_id }}" -l southcentralus
4041
- uses: julia-actions/cache@v1
4142
- uses: julia-actions/julia-buildpkg@v1
4243
- uses: julia-actions/julia-runtest@v1
@@ -45,6 +46,7 @@ jobs:
4546
CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
4647
TENANT: ${{ secrets.TENANT_ID }}
4748
STORAGE_ACCOUNT: "s${{ steps.uuid.outputs.uuid }}"
49+
STORAGE_ACCOUNT_TOO: "s${{ steps.uuid.outputs.uuid }}2"
4850
- uses: julia-actions/julia-processcoverage@v1
4951
- uses: codecov/codecov-action@v5
5052
with:

‎Project.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "AzStorage"
22
uuid = "c6697862-1611-5eae-9ef8-48803c85c8d6"
3-
version = "2.7.2"
3+
version = "2.7.3"
44

55
[deps]
66
AbstractStorage = "14dbef02-f468-5f15-853e-5ec8dee7b899"
@@ -12,6 +12,7 @@ DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
1212
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
1313
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
1414
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
15+
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
1516
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
1617
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
1718
XML = "72c71f33-b9b6-44de-8c94-c961784809e2"
@@ -20,8 +21,10 @@ XML = "72c71f33-b9b6-44de-8c94-c961784809e2"
2021
AbstractStorage = "^1.3"
2122
AzSessions = "2"
2223
AzStorage_jll = "0.9"
24+
Base64 = "1"
2325
DelimitedFiles = "1"
2426
HTTP = "1"
2527
ProgressMeter = "1"
28+
SHA = "0.7"
2629
XML = "0.3"
2730
julia = "^1.6"

‎src/AzStorage.jl‎

Lines changed: 183 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module AzStorage
22

3-
using AbstractStorage, AzSessions, AzStorage_jll, Base64, Dates, DelimitedFiles, XML, HTTP, Printf, ProgressMeter, Serialization, Sockets
3+
using AbstractStorage, AzSessions, AzStorage_jll, Base64, Dates, DelimitedFiles, XML, HTTP, Printf, ProgressMeter, SHA, Serialization, Sockets
44

55
# https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
66
# https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling
@@ -854,9 +854,186 @@ function Base.cp(inc::AzContainer, inb::AbstractString, out::AbstractString; buf
854854
close(io)
855855
end
856856

857-
function Base.cp(inc::AzContainer, inb::AbstractString, outc::AzContainer, outb::AbstractString)
858-
bytes = read!(inc, inb, Vector{UInt8}(undef, filesize(inc, inb)))
859-
write(outc, outb, bytes)
857+
#=
858+
If the source and destination storage accounts for a blob copy are different, then the Azure storage API does not allow us
859+
to use OAuth2/RBAC directly for the source blob. But, we can use a user delegation SAS token which is built in the following
860+
two methods: 'generate_user_delegation_key' and 'get_user_delegation_sas'.
861+
=#
862+
function get_user_delegation_key(c::AzContainer; start=now(UTC), expiry=now(UTC)+Hour(1))
863+
start_str = Dates.format(start, "yyyy-mm-ddTHH:MM:SSZ")
864+
expiry_str = Dates.format(expiry, "yyyy-mm-ddTHH:MM:SSZ")
865+
866+
r = @retry c.nretry HTTP.request(
867+
"POST",
868+
"https://$(c.storageaccount).blob.core.windows.net/?restype=service&comp=userdelegationkey",
869+
[
870+
"Authorization" => "Bearer $(token(c.session))",
871+
"x-ms-version" => API_VERSION,
872+
"Content-Type" => "application/xml"
873+
],
874+
"""
875+
<?xml version="1.0" encoding="utf-8"?>
876+
<KeyInfo>
877+
<Start>$start_str</Start>
878+
<Expiry>$expiry_str</Expiry>
879+
</KeyInfo>
880+
""";
881+
retry = false,
882+
verbose = c.verbose,
883+
connect_timeout = c.connect_timeout,
884+
readtimeout = c.read_timeout)
885+
886+
b = XML.parse(String(r.body), LazyNode)
887+
delegation_key = Dict{String,String}()
888+
for child in children(b)
889+
if tag(child) == "UserDelegationKey"
890+
for grandchild in children(child)
891+
if tag(grandchild) in ("SignedOid", "SignedTid", "SignedStart", "SignedExpiry", "SignedService", "SignedVersion", "Value")
892+
delegation_key[string(tag(grandchild))] = value(first(children(grandchild)))
893+
end
894+
end
895+
end
896+
end
897+
898+
delegation_key
899+
end
900+
901+
function generate_user_delegation_sas(c::AzContainer, b::AbstractString; permissions="r", start=now(UTC), expiry=now(UTC)+Hour(6))
902+
delegation_key = get_user_delegation_key(c; start, expiry)
903+
904+
signedPermissions = permissions
905+
signedStart = Dates.format(start, "yyyy-mm-ddTHH:MM:SSZ")
906+
signedExpiry = Dates.format(expiry, "yyyy-mm-ddTHH:MM:SSZ")
907+
canonicalizedResource = "/blob/$(c.storageaccount)/$(c.containername)/$(addprefix(c,b))"
908+
signedKeyObjectId = delegation_key["SignedOid"]
909+
signedKeyTenantId = delegation_key["SignedTid"]
910+
signedKeyStart = delegation_key["SignedStart"]
911+
signedKeyExpiry = delegation_key["SignedExpiry"]
912+
signedKeyService = delegation_key["SignedService"]
913+
signedKeyVersion = delegation_key["SignedVersion"]
914+
signedAuthorizedUserObjectId = ""
915+
signedUnauthorizedUserObjectId = ""
916+
signedCorrelationId = ""
917+
signedIP = ""
918+
signedProtocol = "https"
919+
signedVersion = API_VERSION
920+
signedResource = "b"
921+
signedSnapshotTime = ""
922+
signedEncryptionScope = ""
923+
rscc = ""
924+
rscd = ""
925+
rsce = ""
926+
rscl = ""
927+
rsct = ""
928+
929+
string_to_sign =
930+
signedPermissions * "\n" *
931+
signedStart * "\n" *
932+
signedExpiry * "\n" *
933+
canonicalizedResource * "\n" *
934+
signedKeyObjectId * "\n" *
935+
signedKeyTenantId * "\n" *
936+
signedKeyStart * "\n" *
937+
signedKeyExpiry * "\n" *
938+
signedKeyService * "\n" *
939+
signedKeyVersion * "\n" *
940+
signedAuthorizedUserObjectId * "\n" *
941+
signedUnauthorizedUserObjectId * "\n" *
942+
signedCorrelationId * "\n" *
943+
"\n" *
944+
"\n" *
945+
signedIP * "\n" *
946+
signedProtocol * "\n" *
947+
signedVersion * "\n" *
948+
signedResource * "\n" *
949+
signedSnapshotTime * "\n" *
950+
signedEncryptionScope * "\n" *
951+
rscc * "\n" *
952+
rscd * "\n" *
953+
rsce * "\n" *
954+
rscl * "\n" *
955+
rsct
956+
957+
# sign the string using the delegation key
958+
key = base64decode(delegation_key["Value"])
959+
message = collect(codeunits(string_to_sign))
960+
signed_string = HTTP.escapeuri(base64encode(hmac_sha256(key, message)))
961+
962+
# sas token
963+
"sp=$signedPermissions&" *
964+
"st=$signedStart&" *
965+
"se=$signedExpiry&" *
966+
"skoid=$signedKeyObjectId&" *
967+
"sktid=$signedKeyTenantId&" *
968+
"skt=$signedKeyStart&" *
969+
"ske=$signedKeyExpiry&" *
970+
"sks=$signedKeyService&" *
971+
"skv=$signedKeyVersion&" *
972+
(isempty(signedIP) ? "" : "sip=$signedIP&") *
973+
"spr=$signedProtocol&" *
974+
"sv=$signedVersion&" *
975+
"sr=$signedResource&" *
976+
"sig=$signed_string"
977+
end
978+
979+
function Base.cp(inc::AzContainer, inb::AbstractString, outc::AzContainer, outb::AbstractString; showprogress=false)
980+
source_url = "https://$(inc.storageaccount).blob.core.windows.net/$(inc.containername)/$(addprefix(inc,inb))"
981+
982+
if inc.storageaccount != outc.storageaccount
983+
sas = generate_user_delegation_sas(inc, inb; permissions="r", start=now(UTC), expiry=now(UTC)+Hour(1))
984+
source_url *= "?$sas"
985+
end
986+
987+
headers = [
988+
"Authorization" => "Bearer $(token(outc.session))",
989+
"x-ms-version" => API_VERSION,
990+
"x-ms-copy-source" => source_url
991+
]
992+
993+
r = @retry inc.nretry HTTP.request(
994+
"PUT",
995+
"https://$(outc.storageaccount).blob.core.windows.net/$(outc.containername)/$(addprefix(outc,outb))",
996+
headers;
997+
retry = false,
998+
verbose = inc.verbose,
999+
connect_timeout = inc.connect_timeout,
1000+
readtimeout = inc.read_timeout
1001+
)
1002+
1003+
if r.status == 202
1004+
local copy_progress
1005+
while true
1006+
r_status = HTTP.request(
1007+
"GET",
1008+
"https://$(outc.storageaccount).blob.core.windows.net/$(outc.containername)/$(addprefix(outc,outb))",
1009+
[
1010+
"Authorization" => "Bearer $(token(outc.session))",
1011+
"x-ms-version" => API_VERSION
1012+
];
1013+
retry = false,
1014+
verbose = inc.verbose,
1015+
connect_timeout = inc.connect_timeout,
1016+
readtimeout = inc.read_timeout
1017+
)
1018+
1019+
copy_status = HTTP.header(r_status, "x-ms-copy-status")
1020+
copy_progress = HTTP.header(r_status, "x-ms-copy-progress")
1021+
if copy_status == "success"
1022+
break
1023+
end
1024+
if copy_status == "aborted"
1025+
reason = HTTP.header(r_status, "x-ms-copy-status-description")
1026+
error("blob copy aborted, src=$(inc.storageaccount): $(inc.containername)/$(addprefix(inc,inb)), dest=$(outc.storageaccount): $(outc.containername)/$(addprefix(outc,outb)), reason=$reason")
1027+
break
1028+
end
1029+
1030+
showprogress && print("copy progress: $copy_progress bytes\r")
1031+
sleep(.1)
1032+
end
1033+
showprogress && print("copy progress: $copy_progress bytes\r\n")
1034+
end
1035+
1036+
nothing
8601037
end
8611038

8621039
"""
@@ -1193,19 +1370,8 @@ function Base.cp(src::AzContainer, dst::AzContainer)
11931370
mkpath(dst)
11941371

11951372
blobs = readdir(src)
1196-
for blob in blobs
1197-
@retry dst.nretry HTTP.request(
1198-
"PUT",
1199-
"https://$(dst.storageaccount).blob.core.windows.net/$(dst.containername)/$(addprefix(dst,blob))",
1200-
[
1201-
"Authorization" => "Bearer $(token(dst.session))",
1202-
"x-ms-version" => API_VERSION,
1203-
"x-ms-copy-source" => "https://$(src.storageaccount).blob.core.windows.net/$(src.containername)/$(addprefix(src,blob))"
1204-
],
1205-
retry = false,
1206-
verbose = src.verbose,
1207-
connect_timeout = src.connect_timeout,
1208-
readtimeout = src.read_timeout)
1373+
@sync for blob in blobs
1374+
@async cp(src, blob, dst, blob)
12091375
end
12101376
nothing
12111377
end

‎test/runtests.jl‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ end
4848
storageaccount = ENV["STORAGE_ACCOUNT"]
4949
@info "storageaccount=$storageaccount"
5050

51+
storageaccount_too = get(ENV, "STORAGE_ACCOUNT_TOO", "")
52+
@info "storageaccount_too=$storageaccount_too"
53+
5154
for container in containers(;storageaccount=storageaccount,session=session)
5255
rm(AzContainer(container;storageaccount=storageaccount,session=session))
5356
end
@@ -548,6 +551,19 @@ end
548551
rm(c)
549552
end
550553

554+
@testset "Container, copy blob to blob, different storage accounts" begin
555+
r = uuid4()
556+
c1 = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10)
557+
c1 = robust_mkpath(c1)
558+
write(c1, "foo.txt", "Hello world")
559+
c2 = AzContainer("foo-$r-o", storageaccount=storageaccount_too, session=session, nthreads=2, nretry=10)
560+
c2 = robust_mkpath(c2)
561+
cp(c1, "foo.txt", c2, "bar.txt")
562+
@test read(c2, "bar.txt", String) == "Hello world"
563+
rm(c1)
564+
rm(c2)
565+
end
566+
551567
@testset "Object, copy blob to local file" begin
552568
r = uuid4()
553569
c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10)

0 commit comments

Comments
 (0)