Skip to content

Commit ec75b56

Browse files
lazarusAlazarusA
committed
how to do getloopchunks and getCubeCache with DAE?
1 parent 000489d commit ec75b56

File tree

2 files changed

+86
-85
lines changed

2 files changed

+86
-85
lines changed

‎docs/Project.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
2626
PlotUtils = "995b91a9-d308-5afd-9ec6-746e21dbc043"
2727
SkipNan = "aed68c70-c8b0-4309-8cd1-d392a74f991a"
2828
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
29+
TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
2930
TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e"
3031
WeightedOnlineStats = "bbac0a1f-7c9d-5672-960b-c6ca726e5d5d"
3132
YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14"

‎test/DAT/loopchunks.jl

Lines changed: 85 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,101 @@
11

2-
@testset "Loop chunk distribution" begin
3-
using DiskArrays: DiskArrays, GridChunks, RegularChunks, IrregularChunks, AbstractDiskArray
4-
using YAXArrayBase: YAXArrayBase
5-
using YAXArrays
6-
struct LargeDiskArray{N,CT<:GridChunks{N}} <: AbstractDiskArray{Float64,N}
7-
size::NTuple{N,Int}
8-
chunks::CT
9-
compressed::Bool
10-
end
11-
Base.size(a::LargeDiskArray) = a.size
12-
DiskArrays.eachchunk(a::LargeDiskArray) = a.chunks
13-
DiskArrays.haschunks(::LargeDiskArray) = DiskArrays.Chunked()
14-
YAXArrayBase.iscompressed(a::LargeDiskArray) = a.compressed
15-
s = (4000,2000,1500)
16-
cs = (100,100,700)
17-
YAXArrays.YAXDefaults.max_cache[] = 1.0e8
18-
a1 = YAXArray(LargeDiskArray(s, GridChunks(s,cs),true))
2+
# @testset "Loop chunk distribution" begin
3+
# using DiskArrays: DiskArrays, GridChunks, RegularChunks, IrregularChunks, AbstractDiskArray
4+
# using YAXArrayBase: YAXArrayBase
5+
# using YAXArrays
6+
# struct LargeDiskArray{N,CT<:GridChunks{N}} <: AbstractDiskArray{Float64,N}
7+
# size::NTuple{N,Int}
8+
# chunks::CT
9+
# compressed::Bool
10+
# end
11+
# Base.size(a::LargeDiskArray) = a.size
12+
# DiskArrays.eachchunk(a::LargeDiskArray) = a.chunks
13+
# DiskArrays.haschunks(::LargeDiskArray) = DiskArrays.Chunked()
14+
# YAXArrayBase.iscompressed(a::LargeDiskArray) = a.compressed
15+
# s = (4000,2000,1500)
16+
# cs = (100,100,700)
17+
# YAXArrays.YAXDefaults.max_cache[] = 1.0e8
18+
# a1 = YAXArray(LargeDiskArray(s, GridChunks(s,cs),true))
1919

20-
#Test case where chunk has to be split
21-
dc = mapslices(sum, a1, dims="Dim_1", debug = true)
22-
ch = YAXArrays.DAT.getloopchunks(dc)
20+
# #Test case where chunk has to be split
21+
# dc = mapslices(sum, a1, dims="Dim_1", debug = true)
22+
# ch = YAXArrays.DAT.getloopchunks(dc)
2323

24-
@test length(ch) == 2
25-
@test ch[1] == RegularChunks(4,0,2000)
26-
@test ch[2] == RegularChunks(700,0,1500)
27-
dc.outcubes[1].cube
28-
# Test that the allocated buffer is close to what the prescribes size
29-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
30-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;
24+
# @test length(ch) == 2
25+
# @test ch[1] == RegularChunks(4,0,2000)
26+
# @test ch[2] == RegularChunks(700,0,1500)
27+
# dc.outcubes[1].cube
28+
# # Test that the allocated buffer is close to what the prescribes size
29+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
30+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;
3131

3232

33-
#Test subsets and offset
34-
a2 = a1[Dim_3=201..1300.5]
35-
dc = mapslices(sum, a2, dims="Dim_1", debug = true)
36-
ch = YAXArrays.DAT.getloopchunks(dc)
37-
@test ch == (RegularChunks(4,0,2000), RegularChunks(700,200,1100))
38-
# Test that the allocated buffer is close to what the prescribes size
39-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
40-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;
33+
# #Test subsets and offset
34+
# a2 = a1[Dim_3=201..1300.5]
35+
# dc = mapslices(sum, a2, dims="Dim_1", debug = true)
36+
# ch = YAXArrays.DAT.getloopchunks(dc)
37+
# @test ch == (RegularChunks(4,0,2000), RegularChunks(700,200,1100))
38+
# # Test that the allocated buffer is close to what the prescribes size
39+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
40+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;
4141

42-
#Test with different max_cache
43-
YAXArrays.YAXDefaults.max_cache[] = 2.0e8
44-
dc = mapslices(sum, a2, dims="Dim_1", debug = true)
45-
ch = YAXArrays.DAT.getloopchunks(dc)
46-
#Test loop chunk sizes
47-
@test ch == (RegularChunks(5,0,2000), RegularChunks(700,200,1100))
48-
# Test that the allocated buffer is close to what the prescribes size
49-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
50-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
42+
# #Test with different max_cache
43+
# YAXArrays.YAXDefaults.max_cache[] = 2.0e8
44+
# dc = mapslices(sum, a2, dims="Dim_1", debug = true)
45+
# ch = YAXArrays.DAT.getloopchunks(dc)
46+
# #Test loop chunk sizes
47+
# @test ch == (RegularChunks(5,0,2000), RegularChunks(700,200,1100))
48+
# # Test that the allocated buffer is close to what the prescribes size
49+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
50+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
5151

5252

53-
# Test case that is chunk-friendly
54-
YAXArrays.YAXDefaults.max_cache[] = 1.5e8
55-
dc = mapslices(sum, a1, dims="Dim_3", debug = true)
56-
ch = YAXArrays.DAT.getloopchunks(dc)
57-
@test ch == (RegularChunks(100,0,4000), RegularChunks(100,0,2000))
58-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
59-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
53+
# # Test case that is chunk-friendly
54+
# YAXArrays.YAXDefaults.max_cache[] = 1.5e8
55+
# dc = mapslices(sum, a1, dims="Dim_3", debug = true)
56+
# ch = YAXArrays.DAT.getloopchunks(dc)
57+
# @test ch == (RegularChunks(100,0,4000), RegularChunks(100,0,2000))
58+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
59+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
6060

61-
#With offset
62-
a2 = a1[Dim_1=51..3050.5]
63-
dc = mapslices(sum, a2, dims="Dim_3", debug = true)
64-
ch = YAXArrays.DAT.getloopchunks(dc)
65-
@test ch == (RegularChunks(100,50,3000), RegularChunks(100,0,2000))
66-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
67-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
61+
# #With offset
62+
# a2 = a1[Dim_1=51..3050.5]
63+
# dc = mapslices(sum, a2, dims="Dim_3", debug = true)
64+
# ch = YAXArrays.DAT.getloopchunks(dc)
65+
# @test ch == (RegularChunks(100,50,3000), RegularChunks(100,0,2000))
66+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
67+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
6868

69-
#With more working memory
70-
YAXArrays.YAXDefaults.max_cache[] = 4.5e8
71-
a2 = a1[Dim_1=51..3050.5]
72-
dc = mapslices(sum, a1, dims="Dim_3", debug = true);
73-
ch = YAXArrays.DAT.getloopchunks(dc)
74-
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000))
75-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
76-
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
69+
# #With more working memory
70+
# YAXArrays.YAXDefaults.max_cache[] = 4.5e8
71+
# a2 = a1[Dim_1=51..3050.5]
72+
# dc = mapslices(sum, a1, dims="Dim_3", debug = true);
73+
# ch = YAXArrays.DAT.getloopchunks(dc)
74+
# @test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000))
75+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
76+
# @test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size
7777

78-
#Now a completely different test, this is more for DataFrame functionality
79-
# and tests if we can proagate irregular axes into the cubes
80-
using Dates
81-
yearchunks = map(y->isleapyear(y) ? 366 : 365, 2001:2008)
82-
c = GridChunks(RegularChunks(100,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
83-
s = last.(last(c))
84-
YAXArrays.YAXDefaults.max_cache[] = 1.0e8
85-
a1 = YAXArray(LargeDiskArray(s, c,true))
78+
# #Now a completely different test, this is more for DataFrame functionality
79+
# # and tests if we can proagate irregular axes into the cubes
80+
# using Dates
81+
# yearchunks = map(y->isleapyear(y) ? 366 : 365, 2001:2008)
82+
# c = GridChunks(RegularChunks(100,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
83+
# s = last.(last(c))
84+
# YAXArrays.YAXDefaults.max_cache[] = 1.0e8
85+
# a1 = YAXArray(LargeDiskArray(s, c,true))
8686

87-
dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true)
88-
ch = YAXArrays.DAT.getloopchunks(dc)
89-
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), RegularChunks(365,0,2922))
90-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
91-
@test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#
87+
# dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true)
88+
# ch = YAXArrays.DAT.getloopchunks(dc)
89+
# @test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), RegularChunks(365,0,2922))
90+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
91+
# @test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#
9292

93-
dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true,irregular_loopranges = true)
94-
ch = YAXArrays.DAT.getloopchunks(dc)
95-
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
96-
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
97-
@test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#
98-
end
93+
# dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true,irregular_loopranges = true)
94+
# ch = YAXArrays.DAT.getloopchunks(dc)
95+
# @test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
96+
# incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
97+
# @test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#
98+
# end
9999

100100
@testitem "Map Cubes with Different Chunks Issue #182" begin
101101
using YAXArrays

0 commit comments

Comments
 (0)