Saving prange result from S3 fails with strange zarr error #126
Description
I just got the following error while trying to merge the prange results for orbit D156.
The data that is mentioned in the error below seems to be ok, because I can load this zarr dataset separately just fine.
I am wondering whether this might be a memory error from my machine.
Stacktrace:
[1] try_yieldto(undo::typeof(Base.ensure_rescheduled))
@ Base ./task.jl:1128
[2] wait()
@ Base ./task.jl:1200
[3] wait(c::Base.GenericCondition{ReentrantLock}; first::Bool)
@ Base ./condition.jl:141
[4] wait
@ ./condition.jl:136 [inlined]
[5] take_buffered(c::Channel{Pair{CartesianIndex{2}, Union{Nothing, Vector{UInt8}}}})
@ Base ./channels.jl:532
[6] take!
@ ./channels.jl:526 [inlined]
[7] readblock!(aout::Matrix{…}, z::ZArray{…}, r::CartesianIndices{…})
@ Zarr ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/ZArray.jl:184
[8] readblock!
@ ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/ZArray.jl:259 [inlined]
[9] readblock_checked!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:294 [inlined]
[10] getindex_disk_nobatch!(out::Nothing, a::ZArray{…}, i::Tuple{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:106
[11] getindex_disk!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:69 [inlined]
[12] getindex_disk
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:58 [inlined]
[13] getindex
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:312 [inlined]
[14] readblock!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayTools/src/DiskArrayTools.jl:266 [inlined]
[15] readblock_checked!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:294 [inlined]
[16] getindex_disk_nobatch!(out::Nothing, a::CFDiskArray{…}, i::Tuple{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:106
[17] getindex_disk!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:69 [inlined]
[18] getindex_disk
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:58 [inlined]
[19] getindex(::CFDiskArray{Union{…}, 2, Float64, ZArray{…}, Float32}, ::UnitRange{Int64}, ::UnitRange{Int64})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:312
[20] #readblock!##2
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/cat.jl:87 [inlined]
[21] (::DiskArrays.var"#_concat_diskarray_block_io##2#_concat_diskarray_block_io##3"{…})(cI::CartesianIndex{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/cat.jl:115
[22] iterate
@ ./generator.jl:48 [inlined]
[23] collect_to!(dest::Matrix{…}, itr::Base.Generator{…}, offs::Int64, st::CartesianIndex{…})
@ Base ./array.jl:848
[24] collect_to_with_first!(dest::Matrix{…}, v1::Matrix{…}, itr::Base.Generator{…}, st::CartesianIndex{…})
@ Base ./array.jl:826
[25] _collect(c::CartesianIndices{…}, itr::Base.Generator{…}, ::Base.EltypeUnknown, isz::Base.HasShape{…})
@ Base ./array.jl:820
[26] collect_similar(cont::CartesianIndices{…}, itr::Base.Generator{…})
@ Base ./array.jl:732
[27] map(f::Function, A::CartesianIndices{2, Tuple{UnitRange{Int64}, UnitRange{Int64}}})
@ Base ./abstractarray.jl:3372
[28] _concat_diskarray_block_io(::DiskArrays.var"#readblock!##2#readblock!##3"{…}, ::DiskArrays.ConcatDiskArray{…}, ::UnitRange{…}, ::Vararg{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/cat.jl:105
[29] readblock!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/cat.jl:86 [inlined]
[30] readblock_checked!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:294 [inlined]
[31] getindex_disk_nobatch!(out::Nothing, a::DiskArrays.ConcatDiskArray{…}, i::Tuple{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:106
[32] getindex_disk!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:69 [inlined]
[33] getindex_disk
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:58 [inlined]
[34] getindex
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:312 [inlined]
[35] read_range(r::Tuple{…}, ia::DiskArrayEngine.InputArray{…}, buffer::Matrix{…})
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/buffers.jl:130
[36] _broadcast_getindex_evalf
@ ./broadcast.jl:699 [inlined]
[37] _broadcast_getindex
@ ./broadcast.jl:672 [inlined]
[38] #copy##0
@ ./broadcast.jl:1124 [inlined]
[39] ntuple
@ ./ntuple.jl:50 [inlined]
[40] copy
@ ./broadcast.jl:1124 [inlined]
[41] materialize
@ ./broadcast.jl:894 [inlined]
[42] default_loopbody(inow::Tuple{…}, op::DiskArrayEngine.GMDWop{…}, inbuffers_pure::Tuple{…}, outbuffers::Tuple{…}, threaded::Bool, outars::Tuple{…}, cb::Tuple{…}, runfilter::Tuple{}, piddir::Nothing)
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/runner.jl:174
[43] run_loop(::DiskArrayEngine.LocalRunner, op::DiskArrayEngine.GMDWop{…}, inbuffers_pure::Tuple{…}, outbuffers::Tuple{…}, threaded::Bool, outars::Tuple{…}, loopranges::DiskArrayEngine.ProductArray{…}, cb::Tuple{…}, runfilter::Tuple{}; groupspecs::Nothing)
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/runner.jl:186
[44] run_loop(runner::DiskArrayEngine.LocalRunner, loopranges::DiskArrayEngine.ProductArray{…}; groupspecs::Nothing)
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/runner.jl:166
[45] run_loop(runner::DiskArrayEngine.LocalRunner, loopranges::DiskArrayEngine.ProductArray{Tuple{…}, 2, Tuple{…}})
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/runner.jl:165
[46] readblock!(::DiskArrayEngine.GMWOPResult{…}, ::Matrix{…}, ::UnitRange{…}, ::UnitRange{…})
@ DiskArrayEngine ~/Documents/baresoilfraction_opengeohub/dev/DiskArrayEngine/src/diskarrayresults.jl:52
[47] readblock_checked!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:294 [inlined]
[48] getindex_disk_nobatch!(out::Nothing, a::DiskArrayEngine.GMWOPResult{…}, i::Tuple{…})
@ DiskArrays ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:106
[49] getindex_disk!
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:69 [inlined]
[50] getindex_disk
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:58 [inlined]
[51] getindex
@ ~/Documents/baresoilfraction_opengeohub/dev/DiskArrays/src/indexing.jl:312 [inlined]
[52] macro expansion
@ ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/Cubes/Rechunker.jl:101 [inlined]
[53] macro expansion
@ ~/.julia/packages/ProgressMeter/N660J/src/ProgressMeter.jl:1026 [inlined]
[54] copydata(outar::CFDiskArray{…}, inar::DiskArrayEngine.GMWOPResult{…}, copybuf::DiskArrays.GridChunks{…})
@ YAXArrays.Cubes ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/Cubes/Rechunker.jl:100
[55] copy_diskarray(incube::DiskArrayEngine.GMWOPResult{…}, outcube::CFDiskArray{…}; writefac::Float64, maxbuf::Float64, align_output::Bool)
@ YAXArrays.Cubes ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/Cubes/Rechunker.jl:113
[56] copydataset!(diskds::Dataset, ds::Dataset; writefac::Float64, maxbuf::Float64)
@ YAXArrays.Datasets ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/DatasetAPI/Datasets.jl:627
[57] copydataset!
@ ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/DatasetAPI/Datasets.jl:624 [inlined]
[58] savedataset(ds::Dataset; path::String, persist::Nothing, overwrite::Bool, append::Bool, skeleton::Bool, backend::Symbol, driver::Symbol, max_cache::Float64, writefac::Float64, kwargs::@Kwargs{})
@ YAXArrays.Datasets ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/DatasetAPI/Datasets.jl:761
[59] savedataset
@ ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/DatasetAPI/Datasets.jl:688 [inlined]
[60] savecube(c::YAXArray{…}, path::String; layername::String, datasetaxis::String, max_cache::Float64, backend::Symbol, driver::Symbol, chunks::Nothing, overwrite::Bool, append::Bool, skeleton::Bool, writefac::Float64, kwargs::@Kwargs{})
@ YAXArrays.Datasets ~/Documents/baresoilfraction_opengeohub/dev/YAXArrays/src/DatasetAPI/Datasets.jl:805
[61] macro expansion
@ ./timing.jl:689 [inlined]
nested task error: HTTP.RequestError:
HTTP.Request:
HTTP.Messages.Request:
"""
GET /rqa-equi7-south-america/E066N063T3_prange_VH_D156_2021-07-01_2023-07-01.zarr/layer/0.2 HTTP/1.1
User-Agent: AWS.jl/1.97.1
Host: s3.nexus.mpcdf.mpg.de
x-amz-date: 20251215T102707Z
x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
Content-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==
Authorization: ******
Accept: */*
Content-Length: 0
Accept-Encoding: gzip
"""Underlying error:
EOFError: read end of file
Stacktrace:
[1] getindex(s::S3Store, i::String)
@ Zarr ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/s3store.jl:27
[2] getindex
@ ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/Storage.jl:78 [inlined]
[3] getindex
@ ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/Storage.jl:77 [inlined]
[4] (::Zarr.var"#read_items!##0#read_items!##1"{S3Store, Channel{Pair{…}}, String})(ii::CartesianIndex{2})
@ Zarr ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/Storage.jl:156
[5] (::Base.var"#936#937"{…})(r::Base.RefValue{…}, args::Tuple{…})
@ Base ./asyncmap.jl:96
[6] (::Base.var"#start_worker_task!##0#start_worker_task!##1"{Base.var"#936#937"{…}, Channel{…}, Nothing})()
@ Base ./asyncmap.jl:230
Stacktrace:
[1] (::Base.var"#maptwice##0#maptwice##1")(x::Task)
@ Base ./asyncmap.jl:173
[2] foreach(f::Base.var"#maptwice##0#maptwice##1", itr::Vector{Any})
@ Base ./abstractarray.jl:3188
[3] maptwice(wrapped_f::Function, chnl::Channel{Any}, worker_tasks::Vector{Any}, c::CartesianIndices{2, Tuple{…}})
@ Base ./asyncmap.jl:173
[4] wrap_n_exec_twice
@ ./asyncmap.jl:149 [inlined]
[5] async_usemap(f::Zarr.var"#read_items!##0#read_items!##1"{…}, c::CartesianIndices{…}; ntasks::Int64, batch_size::Nothing)
@ Base ./asyncmap.jl:99
[6] async_usemap
@ ./asyncmap.jl:80 [inlined]
[7] #asyncmap#921
@ ./asyncmap.jl:77 [inlined]
[8] asyncmap
@ ./asyncmap.jl:76 [inlined]
[9] read_items!
@ ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/Storage.jl:154 [inlined]
[10] read_items!
@ ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/Storage/Storage.jl:144 [inlined]
[11] (::Zarr.var"#readblock!##2#readblock!##3"{Channel{…}, ZArray{…}, ZArray{…}, CartesianIndices{…}})()
@ Zarr ~/Documents/baresoilfraction_opengeohub/dev/Zarr/src/ZArray.jl:177
Some type information was truncated. Use `show(err)` to see complete types.