Skip to content

Commit 5b4fdce

Browse files
ZenGround0ZenGround0
authored
Merge pull request #62 from filecoin-project/addntl-benchmarks
Additional benchmarks
2 parents a45e38f + a4c6bbe commit 5b4fdce

File tree

4 files changed

+243
-30
lines changed

4 files changed

+243
-30
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ os:
44
language: go
55

66
go:
7-
- 1.11.x
7+
- 1.13.x
88

99
env:
1010
global:

‎hamt.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -405,10 +405,16 @@ func loadNode(
405405
return &out, nil
406406
}
407407

408-
// Calculate the total _byte weight_ of the HAMT by fetching each node
409-
// from the IpldStore and adding its raw byte size to the total. This
410-
// operation will exhaustively load every node of the HAMT so should not
411-
// be used lightly.
408+
// checkSize computes the total serialized size of the entire HAMT.
409+
// It both puts and loads blocks as necesary to do this
410+
// (using the Put operation and a paired Get to discover the serial size,
411+
// and the load to move recursively as necessary).
412+
//
413+
// This is an expensive operation and should only be used in testing and analysis.
414+
//
415+
// Note that checkSize *does* actually *use the blockstore*: therefore it
416+
// will affect get and put counts (and makes no attempt to avoid duplicate puts!);
417+
// be aware of this if you are measuring those event counts.
412418
func (n *Node) checkSize(ctx context.Context) (uint64, error) {
413419
c, err := n.store.Put(ctx, n)
414420
if err != nil {

‎hamt_bench_test.go‎

Lines changed: 157 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -51,53 +51,184 @@ func BenchmarkSerializeNode(b *testing.B) {
5151
}
5252

5353
type benchSetCase struct {
54-
count int
54+
kcount int
5555
bitwidth int
5656
}
5757

58-
func BenchmarkSet(b *testing.B) {
59-
kCounts := []int{1, 10, 100}
60-
bitwidths := []int{5, 8}
58+
var benchSetCaseTable []benchSetCase
6159

62-
var table []benchSetCase
60+
func init() {
61+
kCounts := []int{
62+
1,
63+
5,
64+
10,
65+
50,
66+
100,
67+
500,
68+
1000, // aka 1M
69+
//10000, // aka 10M -- you'll need a lot of RAM for this. Also, some patience.
70+
}
71+
bitwidths := []int{
72+
3,
73+
4,
74+
5,
75+
6,
76+
7,
77+
8,
78+
}
79+
// bucketsize-aka-arraywidth? maybe someday.
6380
for _, c := range kCounts {
64-
6581
for _, bw := range bitwidths {
66-
table = append(table, benchSetCase{count: c * 1000, bitwidth: bw})
82+
benchSetCaseTable = append(benchSetCaseTable, benchSetCase{kcount: c, bitwidth: bw})
6783
}
84+
}
85+
}
6886

87+
// The benchmark results can be graphed. Here are some reasonable selections:
88+
/*
89+
benchdraw --filter=BenchmarkFill --plot=line --x=n "--y=blocks/entry" < sample > BenchmarkFill-blocks-per-entry-vs-scale.svg
90+
benchdraw --filter=BenchmarkFill --plot=line --x=n "--y=bytes(blockstoreAccnt)/entry" < sample > BenchmarkFill-totalBytes-per-entry-vs-scale.svg
91+
benchdraw --filter=BenchmarkSetBulk --plot=line --x=n "--y=addntlBlocks/addntlEntry" < sample > BenchmarkSetBulk-addntlBlocks-per-addntlEntry-vs-scale.svg
92+
benchdraw --filter=BenchmarkSetIndividual --plot=line --x=n "--y=addntlBlocks/addntlEntry" < sample > BenchmarkSetIndividual-addntlBlocks-per-addntlEntry-vs-scale.svg
93+
benchdraw --filter=BenchmarkFind --plot=line --x=n "--y=ns/op" < sample > BenchmarkFind-speed-vs-scale.svg
94+
benchdraw --filter=BenchmarkFind --plot=line --x=n "--y=getEvts/find" < sample > BenchmarkFind-getEvts-vs-scale.svg
95+
*/
96+
// (The 'benchdraw' command alluded to here is https://github.com/cep21/benchdraw .)
97+
98+
// Histograms of blocksizes can be logged from some of the following functions, but are commented out.
99+
// The main thing to check for in those is whether there are any exceptionally small blocks being produced:
100+
// less than 64 bytes is a bit concerning because we assume there's some overhead per block in most operations (even if the exact amount may vary situationally).
101+
// We do see some of these small blocks with small bitwidth parameters (e.g. 3), but almost none with larger bitwidth parameters.
102+
103+
// BenchmarkFill creates a large HAMT, and measures how long it takes to generate all of this many entries;
104+
// the number of entries is varied in sub-benchmarks, denoted by their "n=" label component.
105+
// Flush is done once for the entire structure, meaning the number of blocks generated per entry can be much fewer than 1.
106+
//
107+
// The number of blocks saved to the blockstore per entry is reported, and the total content size in bytes.
108+
// The nanoseconds-per-op report on this function is not very useful, because the size of "op" varies with "n" between benchmarks.
109+
//
110+
// See "BenchmarkSet*" for a probe of how long it takes to set additional entries in an already-large hamt
111+
// (this gives a more interesting and useful nanoseconds-per-op indicators).
112+
func BenchmarkFill(b *testing.B) {
113+
for _, t := range benchSetCaseTable {
114+
b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) {
115+
for i := 0; i < b.N; i++ {
116+
r := rander{rand.New(rand.NewSource(int64(i)))}
117+
blockstore := newMockBlocks()
118+
n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth))
119+
//b.ResetTimer()
120+
for j := 0; j < t.kcount*1000; j++ {
121+
if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil {
122+
b.Fatal(err)
123+
}
124+
}
125+
if err := n.Flush(context.Background()); err != nil {
126+
b.Fatal(err)
127+
}
128+
b.StopTimer()
129+
if i < 3 {
130+
//b.Logf("block size histogram: %v\n", blockstore.getBlockSizesHistogram())
131+
}
132+
if blockstore.stats.evtcntPutDup > 0 {
133+
b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats) // note: must refer to this before doing `n.checkSize`; that function has many effects.
134+
}
135+
b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.kcount*1000), "getEvts/entry")
136+
b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.kcount*1000), "putEvts/entry")
137+
b.ReportMetric(float64(len(blockstore.data))/float64(t.kcount*1000), "blocks/entry")
138+
binarySize, _ := n.checkSize(context.Background())
139+
b.ReportMetric(float64(binarySize)/float64(t.kcount*1000), "bytes(hamtAccnt)/entry")
140+
b.ReportMetric(float64(blockstore.totalBlockSizes())/float64(t.kcount*1000), "bytes(blockstoreAccnt)/entry")
141+
b.StartTimer()
142+
}
143+
})
69144
}
70-
r := rander{rand.New(rand.NewSource(int64(42)))}
71-
for _, t := range table {
72-
b.Run(fmt.Sprintf("%d/%d", t.count, t.bitwidth), func(b *testing.B) {
73-
ctx := context.Background()
74-
n := NewNode(cbor.NewCborStore(newMockBlocks()), UseTreeBitWidth(t.bitwidth))
75-
b.ResetTimer()
145+
}
146+
147+
// BenchmarkSetBulk creates a large HAMT, then resets the timer, and does another 1000 inserts,
148+
// measuring the time taken for this second batch of inserts.
149+
// Flushing happens once after all 1000 inserts.
150+
//
151+
// The number of *additional* blocks per entry is reported.
152+
// This number is usually less than one, because the bulk flush means changes might be amortized.
153+
func BenchmarkSetBulk(b *testing.B) {
154+
doBenchmarkSetSuite(b, false)
155+
}
156+
157+
// BenchmarkSetIndividual is the same as BenchmarkSetBulk, but flushes more.
158+
// Flush happens per insert.
159+
//
160+
// The number of *additional* blocks per entry is reported.
161+
// Since we flush each insert individually, this number should be at least 1 --
162+
// however, since we choose random keys, it can still turn out lower if keys happen to collide.
163+
// (The Set method does not make it possible to adjust our denominator to compensate for this: it does not yield previous values nor indicators of prior presense.)
164+
func BenchmarkSetIndividual(b *testing.B) {
165+
doBenchmarkSetSuite(b, true)
166+
}
167+
168+
func doBenchmarkSetSuite(b *testing.B, flushPer bool) {
169+
for _, t := range benchSetCaseTable {
170+
b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) {
76171
for i := 0; i < b.N; i++ {
77-
for j := 0; j < t.count; j++ {
78-
if err := n.Set(ctx, r.randString(), r.randValue()); err != nil {
172+
r := rander{rand.New(rand.NewSource(int64(i)))}
173+
blockstore := newMockBlocks()
174+
n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth))
175+
// Initial fill:
176+
for j := 0; j < t.kcount*1000; j++ {
177+
if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil {
79178
b.Fatal(err)
80179
}
81180
}
181+
if err := n.Flush(context.Background()); err != nil {
182+
b.Fatal(err)
183+
}
184+
initalBlockstoreSize := len(blockstore.data)
185+
b.ResetTimer()
186+
blockstore.stats = blockstoreStats{}
187+
// Additional inserts:
188+
b.ReportAllocs()
189+
for j := 0; j < 1000; j++ {
190+
if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil {
191+
b.Fatal(err)
192+
}
193+
if flushPer {
194+
if err := n.Flush(context.Background()); err != nil {
195+
b.Fatal(err)
196+
}
197+
}
198+
}
199+
if !flushPer {
200+
if err := n.Flush(context.Background()); err != nil {
201+
b.Fatal(err)
202+
}
203+
}
204+
b.StopTimer()
205+
if i < 3 {
206+
// b.Logf("block size histogram: %v\n", blockstore.getBlockSizesHistogram())
207+
}
208+
if blockstore.stats.evtcntPutDup > 0 {
209+
b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats)
210+
}
211+
b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.kcount*1000), "getEvts/entry")
212+
b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.kcount*1000), "putEvts/entry")
213+
b.ReportMetric(float64(len(blockstore.data)-initalBlockstoreSize)/float64(1000), "addntlBlocks/addntlEntry")
214+
b.StartTimer()
82215
}
83216
})
84217
}
85218
}
86219

87220
func BenchmarkFind(b *testing.B) {
88-
b.Run("find-10k", doBenchmarkEntriesCount(10000, 8))
89-
b.Run("find-100k", doBenchmarkEntriesCount(100000, 8))
90-
b.Run("find-1m", doBenchmarkEntriesCount(1000000, 8))
91-
b.Run("find-10k-bitwidth-5", doBenchmarkEntriesCount(10000, 5))
92-
b.Run("find-100k-bitwidth-5", doBenchmarkEntriesCount(100000, 5))
93-
b.Run("find-1m-bitwidth-5", doBenchmarkEntriesCount(1000000, 5))
94-
221+
for _, t := range benchSetCaseTable {
222+
b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth),
223+
doBenchmarkEntriesCount(t.kcount*1000, t.bitwidth))
224+
}
95225
}
96226

97227
func doBenchmarkEntriesCount(num int, bitWidth int) func(b *testing.B) {
98228
r := rander{rand.New(rand.NewSource(int64(num)))}
99229
return func(b *testing.B) {
100-
cs := cbor.NewCborStore(newMockBlocks())
230+
blockstore := newMockBlocks()
231+
cs := cbor.NewCborStore(blockstore)
101232
n := NewNode(cs, UseTreeBitWidth(bitWidth))
102233

103234
var keys []string
@@ -119,6 +250,7 @@ func doBenchmarkEntriesCount(num int, bitWidth int) func(b *testing.B) {
119250
}
120251

121252
runtime.GC()
253+
blockstore.stats = blockstoreStats{}
122254
b.ResetTimer()
123255
b.ReportAllocs()
124256

@@ -132,5 +264,7 @@ func doBenchmarkEntriesCount(num int, bitWidth int) func(b *testing.B) {
132264
b.Fatal(err)
133265
}
134266
}
267+
b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(b.N), "getEvts/find")
268+
b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(b.N), "putEvts/find") // surely this is zero, but for completeness.
135269
}
136270
}

‎hamt_test.go‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"encoding/hex"
88
"fmt"
99
"math/rand"
10+
"strconv"
1011
"strings"
1112
"testing"
1213
"time"
@@ -18,14 +19,16 @@ import (
1819
)
1920

2021
type mockBlocks struct {
21-
data map[cid.Cid]block.Block
22+
data map[cid.Cid]block.Block
23+
stats blockstoreStats
2224
}
2325

2426
func newMockBlocks() *mockBlocks {
25-
return &mockBlocks{make(map[cid.Cid]block.Block)}
27+
return &mockBlocks{make(map[cid.Cid]block.Block), blockstoreStats{}}
2628
}
2729

2830
func (mb *mockBlocks) Get(c cid.Cid) (block.Block, error) {
31+
mb.stats.evtcntGet++
2932
d, ok := mb.data[c]
3033
if ok {
3134
return d, nil
@@ -34,10 +37,80 @@ func (mb *mockBlocks) Get(c cid.Cid) (block.Block, error) {
3437
}
3538

3639
func (mb *mockBlocks) Put(b block.Block) error {
40+
mb.stats.evtcntPut++
41+
if _, exists := mb.data[b.Cid()]; exists {
42+
mb.stats.evtcntPutDup++
43+
}
3744
mb.data[b.Cid()] = b
3845
return nil
3946
}
4047

48+
type blockstoreStats struct {
49+
evtcntGet int
50+
evtcntPut int
51+
evtcntPutDup int
52+
}
53+
54+
func (mb *mockBlocks) totalBlockSizes() int {
55+
sum := 0
56+
for _, v := range mb.data {
57+
sum += len(v.RawData())
58+
}
59+
return sum
60+
}
61+
62+
type blockSizesHistogram [12]int
63+
64+
func (mb *mockBlocks) getBlockSizesHistogram() (h blockSizesHistogram) {
65+
for _, v := range mb.data {
66+
l := len(v.RawData())
67+
switch {
68+
case l <= 2<<2: // 8
69+
h[0]++
70+
case l <= 2<<3: // 16
71+
h[1]++
72+
case l <= 2<<4: // 32
73+
h[2]++
74+
case l <= 2<<5: // 64
75+
h[3]++
76+
case l <= 2<<6: // 128
77+
h[4]++
78+
case l <= 2<<7: // 256
79+
h[5]++
80+
case l <= 2<<8: // 512
81+
h[6]++
82+
case l <= 2<<9: // 1024
83+
h[7]++
84+
case l <= 2<<10: // 2048
85+
h[8]++
86+
case l <= 2<<11: // 4096
87+
h[9]++
88+
case l <= 2<<12: // 8192
89+
h[10]++
90+
default:
91+
h[11]++
92+
}
93+
}
94+
return
95+
}
96+
97+
func (h blockSizesHistogram) String() string {
98+
v := "["
99+
v += "<=" + strconv.Itoa(2<<2) + ":" + strconv.Itoa(h[0]) + ", "
100+
v += "<=" + strconv.Itoa(2<<3) + ":" + strconv.Itoa(h[1]) + ", "
101+
v += "<=" + strconv.Itoa(2<<4) + ":" + strconv.Itoa(h[2]) + ", "
102+
v += "<=" + strconv.Itoa(2<<5) + ":" + strconv.Itoa(h[3]) + ", "
103+
v += "<=" + strconv.Itoa(2<<6) + ":" + strconv.Itoa(h[4]) + ", "
104+
v += "<=" + strconv.Itoa(2<<7) + ":" + strconv.Itoa(h[5]) + ", "
105+
v += "<=" + strconv.Itoa(2<<8) + ":" + strconv.Itoa(h[6]) + ", "
106+
v += "<=" + strconv.Itoa(2<<9) + ":" + strconv.Itoa(h[7]) + ", "
107+
v += "<=" + strconv.Itoa(2<<10) + ":" + strconv.Itoa(h[8]) + ", "
108+
v += "<=" + strconv.Itoa(2<<11) + ":" + strconv.Itoa(h[9]) + ", "
109+
v += "<=" + strconv.Itoa(2<<12) + ":" + strconv.Itoa(h[10]) + ", "
110+
v += ">" + strconv.Itoa(2<<12) + ":" + strconv.Itoa(h[11])
111+
return v + "]"
112+
}
113+
41114
func randString() string {
42115
buf := make([]byte, 18)
43116
rand.Read(buf)

0 commit comments

Comments
 (0)