From 20bf76c1c97993fee60ec0b816dfe543ba232d9d Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Thu, 15 Mar 2018 09:33:45 +0100 Subject: [PATCH 01/10] nested op_name scope in reduce functions --- src/ops/math.jl | 57 +++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index 2e6d1daa..0c316940 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -176,36 +176,37 @@ end # TODO Clean this up for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] @eval @op function $(Symbol("reduce_", reduction))(n::AbstractTensor; axis=nothing, keep_dims=false, name=nothing) - if name === nothing - name = get_name("reduce") - end - if axis == nothing - n = Tensor(n) # TODO: rewrite this - range_start = constant(Int32(0)) - range_delta = constant(Int32(1)) - desc = NodeDescription("Rank", "$name/rank") - add_input(desc, n) - rank = Tensor(Operation(desc), 1) - desc = NodeDescription("Range", "$name/range") - add_input(desc, range_start) - add_input(desc, rank) - add_input(desc, range_delta) - range = Tensor(Operation(desc), 1) - desc = NodeDescription($(capitalize(reduction)), name) - add_input(desc, n) - add_input(desc, range) - Tensor(Operation(desc), 1) - else - if isa(axis, Number) - axis = [axis] + local desc + with_op_name(name, string("reduce_", $(reduction))) do + node_name = get_cur_node_name() + + if axis == nothing + n = Tensor(n) # TODO: rewrite this + range_start = constant(Int32(0)) + range_delta = constant(Int32(1)) + rank_desc = NodeDescription("Rank", "$node_name/rank") + add_input(rank_desc, n) + rank = Tensor(Operation(rank_desc), 1) + desc_range = NodeDescription("Range", "$node_name/range") + add_input(desc_range, range_start) + add_input(desc_range, rank) + add_input(desc_range, range_delta) + range = Tensor(Operation(desc_range), 1) + desc = NodeDescription($(capitalize(reduction)), string(node_name, "/", $(reduction))) + add_input(desc, n) + add_input(desc, range) + else + if isa(axis, Number) + axis = [axis] + end + axis = [Int32(idx-1) for idx in axis] + desc = NodeDescription($(capitalize(reduction)), string(node_name, "/", $(reduction))) + add_input(desc, Tensor(n)) + add_input(desc, Tensor(axis)) + desc["keep_dims"] = keep_dims end - axis = [Int32(idx-1) for idx in axis] - desc = NodeDescription($(capitalize(reduction)), name) - add_input(desc, Tensor(n)) - add_input(desc, Tensor(axis)) - desc["keep_dims"] = keep_dims - Tensor(Operation(desc), 1) end + Tensor(Operation(desc), 1) end end From aaa39f77ddbf706a32e6db01c23123633dc2ad7b Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Fri, 16 Mar 2018 10:33:50 +0100 Subject: [PATCH 02/10] Changed the name of output tensor from reduce blocks. The name for reduce_-functions names the scope for which the operations needed to perform a reduce. --- test/meta.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/meta.jl b/test/meta.jl index 952b0b75..336c7ce0 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -64,10 +64,10 @@ end @test Tensor(W.assign_node) == get_tensor_by_name(g, "logisitic_model/W/Assign") @test Y == get_tensor_by_name(g, "Y") - @test Ysum1 == get_tensor_by_name(g, "Ysum1") - @test Ysum2 == get_tensor_by_name(g, "Ysum2") - @test Ysum3 == get_tensor_by_name(g, "Ysum3") - @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4") + @test Ysum1 == get_tensor_by_name(g, "Ysum1/sum") + @test Ysum2 == get_tensor_by_name(g, "Ysum2/sum") + @test Ysum3 == get_tensor_by_name(g, "Ysum3/sum") + @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4/sum") end From 022b1b92c94d3cfcd06d5ad7b797d86333dfc063 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Fri, 16 Mar 2018 10:52:19 +0100 Subject: [PATCH 03/10] reduce_sum with axis doesn't need new op_name scope --- src/ops/math.jl | 2 +- test/meta.jl | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index 0c316940..d06088c3 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -200,7 +200,7 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] axis = [axis] end axis = [Int32(idx-1) for idx in axis] - desc = NodeDescription($(capitalize(reduction)), string(node_name, "/", $(reduction))) + desc = NodeDescription($(capitalize(reduction)), node_name) add_input(desc, Tensor(n)) add_input(desc, Tensor(axis)) desc["keep_dims"] = keep_dims diff --git a/test/meta.jl b/test/meta.jl index 336c7ce0..089f78b7 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -18,7 +18,7 @@ end @testset "Naming" begin let g = Graph() - local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4 + local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4, Ysum5 as_default(g) do @tf begin i = constant(1.0) @@ -46,6 +46,8 @@ end Ysum3 = reduce_sum(Y, keep_dims=true) # With a comma (issue #188) Ysum4 = reduce_sum(Y, keep_dims=true, name="namefor_Ysum4") # With a comma (issue #188) + + Ysum5 = reduce_sum(Y, axis=2) end end @@ -68,6 +70,7 @@ end @test Ysum2 == get_tensor_by_name(g, "Ysum2/sum") @test Ysum3 == get_tensor_by_name(g, "Ysum3/sum") @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4/sum") + @test Ysum5 == get_tensor_by_name(g, "Ysum5") end From 9d7f5c361c7147012e4b2b8c3f386e8e6fda5538 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Fri, 16 Mar 2018 14:41:16 +0100 Subject: [PATCH 04/10] respect node_name on output node --- src/ops/math.jl | 4 ++-- test/meta.jl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index d06088c3..f1a17fe4 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -192,7 +192,7 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] add_input(desc_range, rank) add_input(desc_range, range_delta) range = Tensor(Operation(desc_range), 1) - desc = NodeDescription($(capitalize(reduction)), string(node_name, "/", $(reduction))) + desc = NodeDescription($(capitalize(reduction))) add_input(desc, n) add_input(desc, range) else @@ -200,7 +200,7 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] axis = [axis] end axis = [Int32(idx-1) for idx in axis] - desc = NodeDescription($(capitalize(reduction)), node_name) + desc = NodeDescription($(capitalize(reduction))) add_input(desc, Tensor(n)) add_input(desc, Tensor(axis)) desc["keep_dims"] = keep_dims diff --git a/test/meta.jl b/test/meta.jl index 089f78b7..4fe9b224 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -66,10 +66,10 @@ end @test Tensor(W.assign_node) == get_tensor_by_name(g, "logisitic_model/W/Assign") @test Y == get_tensor_by_name(g, "Y") - @test Ysum1 == get_tensor_by_name(g, "Ysum1/sum") - @test Ysum2 == get_tensor_by_name(g, "Ysum2/sum") - @test Ysum3 == get_tensor_by_name(g, "Ysum3/sum") - @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4/sum") + @test Ysum1 == get_tensor_by_name(g, "Ysum1") + @test Ysum2 == get_tensor_by_name(g, "Ysum2") + @test Ysum3 == get_tensor_by_name(g, "Ysum3") + @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4") @test Ysum5 == get_tensor_by_name(g, "Ysum5") From 7fc20b3c1fb4c5716895fff4df83861ed9f1add7 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Tue, 20 Mar 2018 08:41:34 +0100 Subject: [PATCH 05/10] test nested with_op_name --- test/meta.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/meta.jl b/test/meta.jl index 4fe9b224..ac0fc1e8 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -18,7 +18,7 @@ end @testset "Naming" begin let g = Graph() - local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4, Ysum5 + local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4, Ysum5, Ysum6, Ysum7, Ysum8 as_default(g) do @tf begin i = constant(1.0) @@ -48,6 +48,14 @@ end Ysum4 = reduce_sum(Y, keep_dims=true, name="namefor_Ysum4") # With a comma (issue #188) Ysum5 = reduce_sum(Y, axis=2) + + nn.tf.with_op_name("level1") do + Ysum6 = reduce_sum(Y) + nn.tf.with_op_name("level2") do + Ysum7 = reduce_sum(Y) + Ysum8 = reduce_sum(Y, axis=1) + end + end end end @@ -71,8 +79,9 @@ end @test Ysum3 == get_tensor_by_name(g, "Ysum3") @test Ysum4 == get_tensor_by_name(g, "namefor_Ysum4") @test Ysum5 == get_tensor_by_name(g, "Ysum5") - - + @test Ysum6 == get_tensor_by_name(g, "level1/Ysum6") + @test Ysum7 == get_tensor_by_name(g, "level1/level2/Ysum7") + @test Ysum8 == get_tensor_by_name(g, "level1/level2/Ysum8") end end From 83afc1e897f09f2d5c70124abc35743e6a59c4c6 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Wed, 25 Apr 2018 17:49:26 +0200 Subject: [PATCH 06/10] scope and names in reduce functions --- src/ops/math.jl | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index f1a17fe4..44f1f746 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -176,34 +176,40 @@ end # TODO Clean this up for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] @eval @op function $(Symbol("reduce_", reduction))(n::AbstractTensor; axis=nothing, keep_dims=false, name=nothing) - local desc - with_op_name(name, string("reduce_", $(reduction))) do - node_name = get_cur_node_name() - - if axis == nothing - n = Tensor(n) # TODO: rewrite this - range_start = constant(Int32(0)) - range_delta = constant(Int32(1)) - rank_desc = NodeDescription("Rank", "$node_name/rank") - add_input(rank_desc, n) - rank = Tensor(Operation(rank_desc), 1) - desc_range = NodeDescription("Range", "$node_name/range") - add_input(desc_range, range_start) + if name == nothing + name = $(capitalize(reduction)) + end + + if axis == nothing + n = Tensor(n) # TODO: rewrite this + desc_rank = tf.with_op_name(nothing, "Rank") do + NodeDescription("Rank") + end + add_input(desc_rank, n) + rank = Tensor(Operation(desc_rank), 1) + range = tf.with_op_name(nothing, "range") do + @tf start = constant(Int32(0)) + @tf delta = constant(Int32(1)) + desc_range = NodeDescription("Range") + add_input(desc_range, start) add_input(desc_range, rank) - add_input(desc_range, range_delta) - range = Tensor(Operation(desc_range), 1) + add_input(desc_range, delta) + Tensor(Operation(desc_range), 1) + end + desc = tf.with_op_name(nothing, name) do desc = NodeDescription($(capitalize(reduction))) add_input(desc, n) add_input(desc, range) - else - if isa(axis, Number) - axis = [axis] - end - axis = [Int32(idx-1) for idx in axis] + desc + end + else + desc = tf.with_op_name(nothing, name) do + @tf reduction_indices = constant(Int32.(axis.-1)) desc = NodeDescription($(capitalize(reduction))) add_input(desc, Tensor(n)) - add_input(desc, Tensor(axis)) + add_input(desc, reduction_indices) desc["keep_dims"] = keep_dims + desc end end Tensor(Operation(desc), 1) From c68f188696e6a3c7ea2d340d18d5807192cbcebe Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Wed, 25 Apr 2018 18:24:48 +0200 Subject: [PATCH 07/10] skip rank query if known --- src/ops/math.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index 44f1f746..fd50a867 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -180,7 +180,8 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] name = $(capitalize(reduction)) end - if axis == nothing + shape = get_shape(n) + if axis == nothing && shape.rank_unknown n = Tensor(n) # TODO: rewrite this desc_rank = tf.with_op_name(nothing, "Rank") do NodeDescription("Rank") @@ -204,6 +205,9 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] end else desc = tf.with_op_name(nothing, name) do + if axis == nothing + axis = 1:length(shape.dims) + end @tf reduction_indices = constant(Int32.(axis.-1)) desc = NodeDescription($(capitalize(reduction))) add_input(desc, Tensor(n)) From 24eccc9098490f3ca2cac4f6e9efbc47250b295b Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Wed, 25 Apr 2018 18:43:55 +0200 Subject: [PATCH 08/10] local desc --- src/ops/math.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index fd50a867..93bcc03b 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -176,18 +176,20 @@ end # TODO Clean this up for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] @eval @op function $(Symbol("reduce_", reduction))(n::AbstractTensor; axis=nothing, keep_dims=false, name=nothing) + local desc + shape = get_shape(n) + if name == nothing name = $(capitalize(reduction)) end - shape = get_shape(n) if axis == nothing && shape.rank_unknown n = Tensor(n) # TODO: rewrite this - desc_rank = tf.with_op_name(nothing, "Rank") do - NodeDescription("Rank") + rank = tf.with_op_name(nothing, "Rank") do + desc_rank = NodeDescription("Rank") + add_input(desc_rank, n) + Tensor(Operation(desc_rank), 1) end - add_input(desc_rank, n) - rank = Tensor(Operation(desc_rank), 1) range = tf.with_op_name(nothing, "range") do @tf start = constant(Int32(0)) @tf delta = constant(Int32(1)) @@ -197,14 +199,13 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] add_input(desc_range, delta) Tensor(Operation(desc_range), 1) end - desc = tf.with_op_name(nothing, name) do + tf.with_op_name(nothing, name) do desc = NodeDescription($(capitalize(reduction))) add_input(desc, n) add_input(desc, range) - desc end else - desc = tf.with_op_name(nothing, name) do + tf.with_op_name(nothing, name) do if axis == nothing axis = 1:length(shape.dims) end @@ -213,7 +214,6 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] add_input(desc, Tensor(n)) add_input(desc, reduction_indices) desc["keep_dims"] = keep_dims - desc end end Tensor(Operation(desc), 1) From 6f99951b29398b79a60ba79c460d581f2577db24 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Wed, 25 Apr 2018 21:50:32 +0200 Subject: [PATCH 09/10] tests --- test/meta.jl | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/meta.jl b/test/meta.jl index ac0fc1e8..ca89eb40 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -18,7 +18,9 @@ end @testset "Naming" begin let g = Graph() - local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4, Ysum5, Ysum6, Ysum7, Ysum8 + local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, + Ysum1, Ysum2, Ysum3, Ysum4, Ysum5, Ysum6, Ysum7, Ysum8, + psum1, psum2, psum3, psum4, psum5 as_default(g) do @tf begin i = constant(1.0) @@ -56,6 +58,19 @@ end Ysum8 = reduce_sum(Y, axis=1) end end + + p = placeholder(Float32) + psum1 = reduce_sum(p) + psum2 = reduce_sum(p, axis=1) + + nn.tf.with_op_name("anotherlevel1") do + psum3 = reduce_sum(p) + + nn.tf.with_op_name("level2") do + psum4 = reduce_sum(p) + psum5 = reduce_sum(p, axis=1) + end + end end end @@ -82,6 +97,12 @@ end @test Ysum6 == get_tensor_by_name(g, "level1/Ysum6") @test Ysum7 == get_tensor_by_name(g, "level1/level2/Ysum7") @test Ysum8 == get_tensor_by_name(g, "level1/level2/Ysum8") + + @test psum1 == get_tensor_by_name(g, "psum1") + @test psum2 == get_tensor_by_name(g, "psum2") + @test psum3 == get_tensor_by_name(g, "anotherlevel1/psum3") + @test psum4 == get_tensor_by_name(g, "anotherlevel1/level2/psum4") + @test psum5 == get_tensor_by_name(g, "anotherlevel1/level2/psum5") end end From 5f640ca55bd8dbb0147ff56a7a3910eb2299ecd9 Mon Sep 17 00:00:00 2001 From: Johan Gustafsson Date: Fri, 27 Apr 2018 17:12:29 +0200 Subject: [PATCH 10/10] Reusing the same name twice should throw an error TensorFlow.jl typically doesn't allow the same name to be used twice whereas tensorflow.py adds a unique postfix. --- src/ops/math.jl | 13 +++++-------- test/meta.jl | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/ops/math.jl b/src/ops/math.jl index 93bcc03b..cf7909d9 100644 --- a/src/ops/math.jl +++ b/src/ops/math.jl @@ -178,10 +178,7 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] @eval @op function $(Symbol("reduce_", reduction))(n::AbstractTensor; axis=nothing, keep_dims=false, name=nothing) local desc shape = get_shape(n) - - if name == nothing - name = $(capitalize(reduction)) - end + nodetype = $(capitalize(reduction)) if axis == nothing && shape.rank_unknown n = Tensor(n) # TODO: rewrite this @@ -199,18 +196,18 @@ for reduction in [:sum, :prod, :min, :max, :all, :any, :mean] add_input(desc_range, delta) Tensor(Operation(desc_range), 1) end - tf.with_op_name(nothing, name) do - desc = NodeDescription($(capitalize(reduction))) + tf.with_op_name(name, nodetype) do + desc = NodeDescription(nodetype) add_input(desc, n) add_input(desc, range) end else - tf.with_op_name(nothing, name) do + tf.with_op_name(name, nodetype) do if axis == nothing axis = 1:length(shape.dims) end @tf reduction_indices = constant(Int32.(axis.-1)) - desc = NodeDescription($(capitalize(reduction))) + desc = NodeDescription(nodetype) add_input(desc, Tensor(n)) add_input(desc, reduction_indices) desc["keep_dims"] = keep_dims diff --git a/test/meta.jl b/test/meta.jl index ca89eb40..a0d4b2aa 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -20,7 +20,7 @@ end g = Graph() local i, j_jl, j, k, ijk, ij, ij2, fq, m, W, Y, Ysum1, Ysum2, Ysum3, Ysum4, Ysum5, Ysum6, Ysum7, Ysum8, - psum1, psum2, psum3, psum4, psum5 + p, psum1, psum2, psum3, psum4, psum5 as_default(g) do @tf begin i = constant(1.0) @@ -103,6 +103,8 @@ end @test psum3 == get_tensor_by_name(g, "anotherlevel1/psum3") @test psum4 == get_tensor_by_name(g, "anotherlevel1/level2/psum4") @test psum5 == get_tensor_by_name(g, "anotherlevel1/level2/psum5") + + @test_throws TensorFlow.TFException reduce_sum(p, name="Ysum1") end end