Automatically create nested BenchmarkGroup on access (#309)

* Create key when accessing

* Export `clear_empty!` command for BenchmarkGroup

* Describe use of `clear_empty!`

* Remove redundant `clear_empty!`

* Add test for EasyConfig-style BenchmarkGroups

* Fix GroupsTests.jl

* Fix docs

* Test tuple-based indexing also eagerly creates groups

* Add docstrings to `clear_empty!` and `isempty`

* Test setindex! for tuple keys

Author: MilesCranmer

docs/src/manual.md

@@ -547,6 +547,19 @@ julia> suite
 As you might imagine, `BenchmarkGroup` supports a subset of Julia's `Associative` interface. A full list of
 these supported functions can be found [in the reference document](reference.md#benchmarkgrouptagsvector-datadict).
 
+One can also create a nested `BenchmarkGroup` simply by indexing the keys:
+
+```julia
+suite2 = BenchmarkGroup()
+
+suite2["my"]["nested"]["benchmark"] = @benchmarkable sum(randn(32))
+```
+
+which will result in a hierarchical benchmark without us needing to create the `BenchmarkGroup` at each level ourselves.
+
+Note that keys are automatically created upon access, even if a key does not exist. Thus, if you wish
+to empty the unused keys, you can use `clear_empty!(suite)` to do so.
+
 ### Tuning and running a `BenchmarkGroup`
 
 Similarly to individual benchmarks, you can `tune!` and `run` whole `BenchmarkGroup` instances (following from the previous section):

src/BenchmarkTools.jl

@@ -55,7 +55,8 @@ export BenchmarkGroup,
        addgroup!,
        leaves,
        @benchmarkset,
-       @case
+       @case,
+       clear_empty!
 
 ######################
 # Execution Strategy #

src/groups.jl

@@ -23,16 +23,43 @@ function addgroup!(suite::BenchmarkGroup, id, args...)
     return g
 end
 
+"""
+    clear_empty!(group::BenchmarkGroup)
+
+Recursively remove any empty subgroups from `group`.
+
+Use this to prune a `BenchmarkGroup` after accessing the incorrect
+fields, such as `g=BenchmarkGroup(); g[1]`, without storing
+anything to `g[1]`, which will create an empty subgroup `g[1]`.
+"""
+function clear_empty!(group::BenchmarkGroup)
+    for (k, v) in pairs(group)
+        if v isa BenchmarkGroup && isempty(v)
+            delete!(group, k)
+        end
+    end
+    group
+end
+clear_empty!(x) = x
+
 # Dict-like methods #
 #-------------------#
 
 Base.:(==)(a::BenchmarkGroup, b::BenchmarkGroup) = a.tags == b.tags && a.data == b.data
 Base.copy(group::BenchmarkGroup) = BenchmarkGroup(copy(group.tags), copy(group.data))
 Base.similar(group::BenchmarkGroup) = BenchmarkGroup(copy(group.tags), empty(group.data))
-Base.isempty(group::BenchmarkGroup) = isempty(group.data)
+
+"""
+    isempty(group::BenchmarkGroup)
+
+Return `true` if `group` is empty. This will first
+run `clear_empty!` on `group` to recursively remove any empty subgroups.
+"""
+Base.isempty(group::BenchmarkGroup) = isempty(clear_empty!(group).data)
+
 Base.length(group::BenchmarkGroup) = length(group.data)
-Base.getindex(group::BenchmarkGroup, k) = getindex(group.data, makekey(k))
-Base.getindex(group::BenchmarkGroup, k...) = getindex(group.data, makekey(k))
+Base.getindex(group::BenchmarkGroup, k) = get!(group.data, makekey(k), BenchmarkGroup())
+Base.getindex(group::BenchmarkGroup, k...) = get!(group.data, makekey(k), BenchmarkGroup())
 Base.setindex!(group::BenchmarkGroup, v, k) = setindex!(group.data, v, makekey(k))
 Base.setindex!(group::BenchmarkGroup, v, k...) = setindex!(group.data, v, makekey(k))
 Base.delete!(group::BenchmarkGroup, k) = delete!(group.data, makekey(k))

test/GroupsTests.jl

@@ -324,4 +324,52 @@ g1["c"] = tc
   "c" => TrialEstimate(1.000 ns)
   ⋮"""
 
+# EasyConfig-style benchmark groups #
+#-----------------------------------#
+
+g1 = BenchmarkGroup()
+for T in [Float32, Float64], n in [10, 100], m in [5, 20]
+  g1["sum"][T][n][m] = @benchmarkable sum(x) setup=(x=randn($T, $n, $m))
+end
+
+# Test that the groups were created:
+for T in [Float32, Float64], n in [10, 100], m in [5, 20]
+  @test "sum" in keys(g1.data)
+  @test string(T) in keys(g1["sum"].data)
+  @test n in keys(g1["sum"][T].data)
+  @test m in keys(g1["sum"][T][n].data)
+  @test typeof(g1["sum"][T][n][m]) == BenchmarkTools.Benchmark
+end
+
+# Expected side effect is that accessing groups creates them:
+g1["ssum"]
+@test "ssum" in keys(g1.data)
+g1["ssum2"][Int32]
+@test "ssum2" in keys(g1.data)
+@test "Int32" in keys(g1["ssum2"].data)
+
+# So we can clear the empty groups with `clear_empty!`:
+clear_empty!(g1)
+
+# Now it is clean
+@test !("ssum" in keys(g1.data))
+@test !("ssum2" in keys(g1.data))
+
+# Likewise with multi-key groups:
+g1[1, 2, 3][1, 2, 3][1, 2, 3] = BenchmarkGroup()
+@test (1, 2, 3) in keys(g1.data)
+@test (1, 2, 3) in keys(g1[1, 2, 3].data)
+clear_empty!(g1)
+@test !((1, 2, 3) in keys(g1.data))
+
+# But other groups should still be present:
+for T in [Float32, Float64], n in [10, 100], m in [5, 20]
+  @test "sum" in keys(g1.data)
+  @test string(T) in keys(g1["sum"].data)
+  @test n in keys(g1["sum"][T].data)
+  @test m in keys(g1["sum"][T][n].data)
+  @test typeof(g1["sum"][T][n][m]) == BenchmarkTools.Benchmark
+end
+
+
 # end # module