Skip to content

Commit 3d323af

Browse files
jishnubjishnub
committed
Add nelements
1 parent 271dd7b commit 3d323af

File tree

3 files changed

+171
-23
lines changed

3 files changed

+171
-23
lines changed

‎Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "ParallelUtilities"
22
uuid = "fad6cfc8-4f83-11e9-06cc-151124046ad0"
33
authors = ["Jishnu Bhattacharya <[email protected]>"]
4-
version = "0.7.3"
4+
version = "0.7.4"
55

66
[deps]
77
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"

‎src/productsplit.jl

Lines changed: 78 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -333,8 +333,73 @@ function _checknorollover(t,firstindchild,lastindchild)
333333
end
334334
_checknorollover(::Tuple{},::Tuple{},::Tuple{}) = true
335335

336+
function _nrollovers(ps::AbstractConstrainedProduct{<:Any,N},dim::Integer) where {N}
337+
dim == N && return 0
338+
nelements(ps,dim+1) - 1
339+
end
340+
341+
"""
342+
nelements(ps::AbstractConstrainedProduct; dim::Integer)
343+
344+
Compute the number of unique values in the element number `dim` of the tuples
345+
that are returned when `ps` is iterated over.
346+
347+
# Examples
348+
```jldoctest
349+
julia> ps = ProductSplit((1:5,2:4,1:3),7,3);
350+
351+
julia> collect(ps)
352+
7-element Array{Tuple{Int64,Int64,Int64},1}:
353+
(5, 4, 1)
354+
(1, 2, 2)
355+
(2, 2, 2)
356+
(3, 2, 2)
357+
(4, 2, 2)
358+
(5, 2, 2)
359+
(1, 3, 2)
360+
361+
julia> ParallelUtilities.nelements(ps,3)
362+
2
363+
364+
julia> ParallelUtilities.nelements(ps,2)
365+
3
366+
367+
julia> ParallelUtilities.nelements(ps,1)
368+
5
369+
```
370+
"""
371+
nelements(ps::AbstractConstrainedProduct; dim::Integer) = nelements(ps,dim)
372+
function nelements(ps::AbstractConstrainedProduct{<:Any,N},dim::Integer) where {N}
373+
1 <= dim <= N || throw(ArgumentError("1 ⩽ dim ⩽ N=$N not satisfied for dim=$dim"))
374+
375+
iter = ps.iterators[dim]
376+
377+
if _nrollovers(ps,dim) == 0
378+
st = first(ps)[dim]
379+
en = last(ps)[dim]
380+
stind = searchsortedfirst(iter,st)
381+
enind = searchsortedfirst(iter,en)
382+
nel = length(stind:enind)
383+
elseif _nrollovers(ps,dim) > 1
384+
nel = length(iter)
385+
else
386+
st = first(ps)[dim]
387+
en = last(ps)[dim]
388+
stind = searchsortedfirst(iter,st)
389+
enind = searchsortedfirst(iter,en)
390+
if stind > enind
391+
# some elements are missed out
392+
nel = length(stind:length(iter)) + length(1:enind)
393+
else
394+
nel = length(iter)
395+
end
396+
end
397+
return nel
398+
end
399+
400+
336401
"""
337-
maximum(ps::ProductSplit; dim::Integer)
402+
maximum(ps::AbstractConstrainedProduct; dim::Integer)
338403
339404
Compute the maximum value of the range number `dim` that is
340405
contained in `ps`.
@@ -355,7 +420,7 @@ julia> maximum(ps,dim=2)
355420
4
356421
```
357422
"""
358-
function Base.maximum(ps::AbstractConstrainedProduct{<:Any,N};dim::Integer) where {N}
423+
function Base.maximum(ps::AbstractConstrainedProduct{<:Any,N},dim::Integer) where {N}
359424

360425
@boundscheck (1 <= dim <= N) || throw(BoundsError(ps.iterators,dim))
361426

@@ -390,7 +455,7 @@ function Base.maximum(ps::AbstractConstrainedProduct{<:Any,1})
390455
end
391456

392457
"""
393-
minimum(ps::ProductSplit; dim::Integer)
458+
minimum(ps::AbstractConstrainedProduct; dim::Integer)
394459
395460
Compute the minimum value of the range number `dim` that is
396461
contained in `ps`.
@@ -411,7 +476,7 @@ julia> minimum(ps,dim=2)
411476
4
412477
```
413478
"""
414-
function Base.minimum(ps::AbstractConstrainedProduct{<:Any,N};dim::Integer) where {N}
479+
function Base.minimum(ps::AbstractConstrainedProduct{<:Any,N},dim::Integer) where {N}
415480

416481
@boundscheck (1 <= dim <= N) || throw(BoundsError(ps.iterators,dim))
417482

@@ -446,7 +511,7 @@ function Base.minimum(ps::AbstractConstrainedProduct{<:Any,1})
446511
end
447512

448513
"""
449-
extrema(ps::ProductSplit; dim::Integer)
514+
extrema(ps::AbstractConstrainedProduct; dim::Integer)
450515
451516
Compute the minimum and maximum of the range number `dim` that is
452517
contained in `ps`.
@@ -467,7 +532,7 @@ julia> extrema(ps,dim=2)
467532
(4, 4)
468533
```
469534
"""
470-
function Base.extrema(ps::AbstractConstrainedProduct{<:Any,N};dim::Integer) where {N}
535+
function Base.extrema(ps::AbstractConstrainedProduct{<:Any,N},dim::Integer) where {N}
471536

472537
@boundscheck (1 <= dim <= N) || throw(BoundsError(ps.iterators,dim))
473538

@@ -503,6 +568,12 @@ function Base.extrema(ps::AbstractConstrainedProduct{<:Any,1})
503568
(iter[fic_dim],iter[lic_dim])
504569
end
505570

571+
for f in [:maximum,:minimum,:extrema]
572+
@eval function Base.$f(ps::AbstractConstrainedProduct;dim::Integer)
573+
$f(ps,dim)
574+
end
575+
end
576+
506577
"""
507578
extremadims(ps::AbstractConstrainedProduct)
508579
@@ -693,7 +764,7 @@ function whichproc(iterators, val, np::Integer)
693764
# If np is greater than the number of ntasks then it's possible
694765
# that ps is empty. In this case the value must be somewhere in
695766
# the previous workers. Otherwise each worker has some tasks and
696-
# these are sorted, so carry out a binary seaarch
767+
# these are sorted, so carry out a binary search
697768

698769
if isempty(ps) || val_t < ReverseLexicographicTuple(first(ps))
699770
right = mid - 1

‎test/tests.jl

Lines changed: 92 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
parentnoderank, nchildren,
1010
maybepvalput!, createbranchchannels, nworkersactive, workersactive,
1111
procs_node, leafrankfoldedtree,
12-
TopTreeNode, SubTreeNode, ProductSection, indexinproduct, dropleading
12+
TopTreeNode, SubTreeNode, ProductSection, indexinproduct, dropleading,
13+
nelements
1314
end
1415

1516
macro testsetwithinfo(str,ex)
@@ -414,7 +415,43 @@ end
414415
@test dropleading(ps) isa ProductSection
415416
@test collect(dropleading(ps)) == [(4,1),(2,2),(3,2)]
416417
@test collect(dropleading(dropleading(ps))) == [(1,),(2,)]
418+
419+
ps = ProductSection((1:5,2:4,1:3),5:8);
420+
@test dropleading(ps) isa ProductSection
421+
@test collect(dropleading(ps)) == [(2,1),(3,1)]
422+
@test collect(dropleading(dropleading(ps))) == [(1,)]
423+
end
424+
@testset "nelements" begin
425+
ps = ProductSplit((1:5,2:4,1:3),7,3);
426+
@test nelements(ps,dim=1) == 5
427+
@test nelements(ps,dim=2) == 3
428+
@test nelements(ps,dim=3) == 2
429+
@test_throws ArgumentError nelements(ps,dim=0)
430+
@test_throws ArgumentError nelements(ps,dim=4)
431+
432+
ps = ProductSection((1:5,2:4,1:3),5:8);
433+
@test nelements(ps,1) == 4
434+
@test nelements(ps,2) == 2
435+
@test nelements(ps,3) == 1
436+
437+
ps = ProductSection((1:5,2:4,1:3),5:11);
438+
@test nelements(ps,1) == 5
439+
@test nelements(ps,2) == 3
440+
@test nelements(ps,3) == 1
441+
442+
ps = ProductSection((1:5,2:4,1:3),4:8);
443+
@test nelements(ps,1) == 5
444+
@test nelements(ps,2) == 2
445+
@test nelements(ps,3) == 1
446+
447+
ps = ProductSection((1:5,2:4,1:3),4:9);
448+
@test nelements(ps,1) == 5
449+
@test nelements(ps,2) == 2
450+
@test nelements(ps,3) == 1
417451
end
452+
453+
@test ParallelUtilities._checknorollover((),(),())
454+
@test ParallelUtilities.c2l_rec(3,1,(),()) == 3
418455
end;
419456

420457
@testset "ReverseLexicographicTuple" begin
@@ -856,6 +893,13 @@ end;
856893
@test_throws BoundsError(tree,16) nchildren(tree,16)
857894
end;
858895
end;
896+
897+
@testset "fulltree-toptree indexing" begin
898+
procs = 1:5
899+
tree = SequentialBinaryTree(procs)
900+
@test ParallelUtilities.toptree_to_fulltree_index(tree,3) == 3
901+
@test ParallelUtilities.fulltree_to_toptree_index(tree,3) == 3
902+
end
859903
end
860904

861905
@testsetwithinfo "SegmentedOrderedBinaryTree" begin
@@ -1061,6 +1105,11 @@ end;
10611105
end;
10621106
end;
10631107
end
1108+
1109+
@testsetwithinfo "unsegmentedtree" begin
1110+
@test ParallelUtilities.unsegmentedtree(SegmentedSequentialBinaryTree) == SequentialBinaryTree
1111+
@test ParallelUtilities.unsegmentedtree(SegmentedOrderedBinaryTree) == OrderedBinaryTree
1112+
end
10641113
end
10651114

10661115
@testsetwithinfo "RemoteChannelContainer" begin
@@ -1496,10 +1545,9 @@ end;
14961545
@test take!(pipe.selfchannels.out) == pval(rank,Int[1])
14971546
end
14981547

1499-
function test_on_pipe(fn,iterator,pipe,result_expected)
1500-
progressrc = nothing
1548+
function test_on_pipe(fn,iterator,pipe,result_expected,progressrc=nothing)
15011549
rank = 1
1502-
@test_throws ErrorException mapTreeNode(x->error(""),iterator,rank,pipe,progressrc)
1550+
@test_throws ErrorException mapTreeNode(x->error("fmap"),iterator,rank,pipe,progressrc)
15031551
@test !isready(pipe.selfchannels.out) # should not have any result as there was an error
15041552
@test isready(pipe.selfchannels.err)
15051553
@test take!(pipe.selfchannels.err) # error flag should be true
@@ -1508,6 +1556,10 @@ end;
15081556
@test !isready(pipe.parentchannels.err)
15091557
@test !isready(pipe.childrenchannels.out)
15101558
@test !isready(pipe.childrenchannels.err)
1559+
if progressrc isa RemoteChannel
1560+
@test isready(progressrc)
1561+
@test take!(progressrc) == (false,false,rank)
1562+
end
15111563

15121564
mapTreeNode(fn,iterator,rank,pipe,progressrc)
15131565
@test isready(pipe.selfchannels.err)
@@ -1520,13 +1572,21 @@ end;
15201572
@test !isready(pipe.parentchannels.err)
15211573
@test !isready(pipe.childrenchannels.out)
15221574
@test !isready(pipe.childrenchannels.err)
1575+
if progressrc isa RemoteChannel
1576+
@test isready(progressrc)
1577+
@test take!(progressrc) == (true,false,rank)
1578+
end
15231579
end
15241580

15251581
@testset "range" begin
15261582
iterator = 1:10
15271583

15281584
pipe = BranchChannel{Int,Int}(myid(),0)
15291585
test_on_pipe(sum,iterator,pipe,sum(iterator))
1586+
1587+
pipe = BranchChannel{Int,Int}(myid(),0)
1588+
progress = RemoteChannel(()->Channel{Tuple{Bool,Bool,Int}}(1))
1589+
test_on_pipe(sum,iterator,pipe,sum(iterator),progress)
15301590
end
15311591

15321592
@testset "ProductSplit" begin
@@ -1652,7 +1712,7 @@ end;
16521712
end
16531713
end
16541714

1655-
end
1715+
end # everwhere
16561716

16571717
@testset "reducedvalue" begin
16581718

@@ -1769,31 +1829,35 @@ end;
17691829
@test pipe.childrenchannels.err.where == 0
17701830
end
17711831

1832+
strippedrank(t::ParallelUtilities.ReductionNode) = t.rank
1833+
strippedrank(t::Integer) = t
1834+
17721835
function testreduction(freduce::Function,pipe::BranchChannel,
1773-
ifsorted::Ordering,res_exp,rank,args...)
1836+
ifsorted::Ordering,res_exp,rank,
1837+
progressrc=nothing,args...)
17741838

17751839
@test !isready(pipe.parentchannels.out)
17761840
@test !isready(pipe.parentchannels.err)
17771841

1778-
progressrc = nothing
1842+
wait(@spawnat(pipe.p,
1843+
putselfchildren!(pipe,ifsorted,rank,args...) ) )
1844+
reduceTreeNode(freduce,rank,pipe,ifsorted,progressrc)
17791845

1780-
try
1781-
wait(@spawnat(pipe.p,
1782-
putselfchildren!(pipe,ifsorted,rank,args...) ) )
1783-
reduceTreeNode(freduce,rank,pipe,ifsorted,progressrc)
1784-
catch
1785-
rethrow()
1786-
end
17871846
@test isready(pipe.parentchannels.out)
17881847
@test isready(pipe.parentchannels.err)
17891848
@test !take!(pipe.parentchannels.err) # there should be no error
17901849
@test value(take!(pipe.parentchannels.out)) == res_exp
17911850

1851+
if progressrc isa RemoteChannel
1852+
@test isready(progressrc)
1853+
@test take!(progressrc) == (false,true,strippedrank(rank))
1854+
end
1855+
17921856
# The pipe should be finalized at this point
17931857
testfinalized(rank,pipe)
17941858
end
17951859

1796-
end
1860+
end # everywhere
17971861

17981862
for n = 1:2
17991863
@testset "Unsorted" begin
@@ -1807,6 +1871,19 @@ end;
18071871
pipe = BranchChannel{Int,Int}(myid(),n)
18081872
testreduction(sum,pipe,Unsorted(),res_exp,TopTreeNode(0))
18091873

1874+
pipe = BranchChannel{Int,Int}(myid(),n)
1875+
progress = RemoteChannel(()->Channel{Tuple{Bool,Bool,Int}}(1))
1876+
@test_throws ErrorException testreduction(
1877+
x->error("fred"),pipe,Unsorted(),
1878+
res_exp,TopTreeNode(0),progress)
1879+
@test isready(progress)
1880+
@test take!(progress) == (false,false,0)
1881+
1882+
pipe = BranchChannel{Int,Int}(myid(),n)
1883+
progress = RemoteChannel(()->Channel{Tuple{Bool,Bool,Int}}(1))
1884+
testreduction(sum,pipe,Unsorted(),
1885+
res_exp,TopTreeNode(0),progress)
1886+
18101887
rc_parent = RemoteChannelContainer{Int}(1)
18111888
p = workers()[1]
18121889

0 commit comments

Comments
 (0)