Skip to content

Commit f75ed44

Browse files
committed
metrics: count copy compactions
Currently copy compactions don't count towards the total number of compactions, even though the bytes they write do count towards the total bytes written by compactions. This change fixes this and adds a copy compaction metric.
1 parent 1a2a77b commit f75ed44

File tree

9 files changed

+43
-28
lines changed

9 files changed

+43
-28
lines changed

metrics.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ type Metrics struct {
157157
DefaultCount int64
158158
DeleteOnlyCount int64
159159
ElisionOnlyCount int64
160+
CopyCount int64
160161
MoveCount int64
161162
ReadCount int64
162163
RewriteCount int64
@@ -569,13 +570,14 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
569570
redact.Safe(m.Compact.NumInProgress),
570571
humanize.Bytes.Int64(m.Compact.InProgressBytes))
571572

572-
w.Printf(" default: %d delete: %d elision: %d move: %d read: %d rewrite: %d multi-level: %d\n",
573+
w.Printf(" default: %d delete: %d elision: %d move: %d read: %d rewrite: %d copy: %d multi-level: %d\n",
573574
redact.Safe(m.Compact.DefaultCount),
574575
redact.Safe(m.Compact.DeleteOnlyCount),
575576
redact.Safe(m.Compact.ElisionOnlyCount),
576577
redact.Safe(m.Compact.MoveCount),
577578
redact.Safe(m.Compact.ReadCount),
578579
redact.Safe(m.Compact.RewriteCount),
580+
redact.Safe(m.Compact.CopyCount),
579581
redact.Safe(m.Compact.MultiLevelCount))
580582

581583
w.Printf("MemTables: %d (%s) zombie: %d (%s)\n",

metrics_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ func exampleMetrics() Metrics {
4040
m.Compact.MoveCount = 30
4141
m.Compact.ReadCount = 31
4242
m.Compact.RewriteCount = 32
43-
m.Compact.MultiLevelCount = 33
43+
m.Compact.CopyCount = 33
44+
m.Compact.MultiLevelCount = 34
4445
m.Compact.EstimatedDebt = 6
4546
m.Compact.InProgressBytes = 7
4647
m.Compact.NumInProgress = 2

replay/replay.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ type Metrics struct {
104104
Move int64
105105
Read int64
106106
Rewrite int64
107+
Copy int64
107108
MultiLevel int64
108109
}
109110
EstimatedDebt SampledMetric
@@ -169,6 +170,7 @@ func (m *Metrics) WriteBenchmarkString(name string, w io.Writer) error {
169170
{Value: float64(m.CompactionCounts.Move), Unit: "move"},
170171
{Value: float64(m.CompactionCounts.Read), Unit: "read"},
171172
{Value: float64(m.CompactionCounts.Rewrite), Unit: "rewrite"},
173+
{Value: float64(m.CompactionCounts.Copy), Unit: "copy"},
172174
{Value: float64(m.CompactionCounts.MultiLevel), Unit: "multilevel"},
173175
}},
174176
// Total database sizes sampled after every workload step and
@@ -555,6 +557,7 @@ func (r *Runner) Wait() (Metrics, error) {
555557
m.CompactionCounts.Move = pm.Compact.MoveCount
556558
m.CompactionCounts.Read = pm.Compact.ReadCount
557559
m.CompactionCounts.Rewrite = pm.Compact.RewriteCount
560+
m.CompactionCounts.Copy = pm.Compact.CopyCount
558561
m.CompactionCounts.MultiLevel = pm.Compact.MultiLevelCount
559562
m.Ingest.BytesIntoL0 = pm.Levels[0].BytesIngested
560563
m.Ingest.BytesWeightedByLevel = ingestBytesWeighted

replay/replay_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ func TestBenchmarkString(t *testing.T) {
441441
var buf bytes.Buffer
442442
require.NoError(t, m.WriteBenchmarkString("tpcc", &buf))
443443
require.Equal(t, strings.TrimSpace(`
444-
BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 multilevel
444+
BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 copy 0 multilevel
445445
BenchmarkBenchmarkReplay/tpcc/DatabaseSize/mean 1 5.36870912e+09 bytes
446446
BenchmarkBenchmarkReplay/tpcc/DatabaseSize/max 1 5.36870912e+09 bytes
447447
BenchmarkBenchmarkReplay/tpcc/DurationWorkload 1 1 sec/op

testdata/event_listener

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ total | 3 1.7KB 0B 0 | - | 671B | 1 590B | 0 0
222222
WAL: 1 files (0B) in: 48B written: 81B (69% overhead)
223223
Flushes: 3
224224
Compactions: 1 estimated debt: 1.7KB in progress: 0 (0B)
225-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
225+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
226226
MemTables: 1 (256KB) zombie: 1 (256KB)
227227
Zombie tables: 0 (0B, local: 0B)
228228
Backing tables: 0 (0B)
@@ -322,7 +322,7 @@ total | 6 3.5KB 0B 0 | - | 1.8KB | 3 1.7KB | 0 0
322322
WAL: 1 files (0B) in: 82B written: 108B (32% overhead)
323323
Flushes: 6
324324
Compactions: 1 estimated debt: 3.5KB in progress: 0 (0B)
325-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
325+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
326326
MemTables: 1 (512KB) zombie: 1 (512KB)
327327
Zombie tables: 0 (0B, local: 0B)
328328
Backing tables: 0 (0B)

testdata/ingest

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ total | 1 569B 0B 0 | - | 569B | 1 569B | 0 0
4646
WAL: 1 files (0B) in: 0B written: 0B (0% overhead)
4747
Flushes: 0
4848
Compactions: 0 estimated debt: 0B in progress: 0 (0B)
49-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
49+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
5050
MemTables: 1 (256KB) zombie: 0 (0B)
5151
Zombie tables: 0 (0B, local: 0B)
5252
Backing tables: 0 (0B)

testdata/metrics

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ total | 2.8K 2.7KB 0B 2.8K | - | 2.8KB | 2.9K 2.8KB | 2.9K 2.8K
1515
WAL: 22 files (24B) in: 25B written: 26B (4% overhead)
1616
Flushes: 8
1717
Compactions: 5 estimated debt: 6B in progress: 2 (7B)
18-
default: 27 delete: 28 elision: 29 move: 30 read: 31 rewrite: 32 multi-level: 33
18+
default: 27 delete: 28 elision: 29 move: 30 read: 31 rewrite: 32 copy: 33 multi-level: 34
1919
MemTables: 12 (11B) zombie: 14 (13B)
2020
Zombie tables: 16 (15B, local: 30B)
2121
Backing tables: 1 (2.0MB)
@@ -66,7 +66,7 @@ total | 1 589B 0B 0 | - | 28B | 0 0B | 0 0
6666
WAL: 1 files (0B) in: 17B written: 28B (65% overhead)
6767
Flushes: 1
6868
Compactions: 0 estimated debt: 0B in progress: 0 (0B)
69-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
69+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
7070
MemTables: 1 (256KB) zombie: 1 (256KB)
7171
Zombie tables: 0 (0B, local: 0B)
7272
Backing tables: 0 (0B)
@@ -121,7 +121,7 @@ total | 1 595B 0B 0 | - | 56B | 0 0B | 0 0
121121
WAL: 1 files (0B) in: 34B written: 56B (65% overhead)
122122
Flushes: 2
123123
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
124-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
124+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
125125
MemTables: 1 (256KB) zombie: 2 (512KB)
126126
Zombie tables: 2 (1.2KB, local: 1.2KB)
127127
Backing tables: 0 (0B)
@@ -163,7 +163,7 @@ total | 1 595B 0B 0 | - | 56B | 0 0B | 0 0
163163
WAL: 1 files (0B) in: 34B written: 56B (65% overhead)
164164
Flushes: 2
165165
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
166-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
166+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
167167
MemTables: 1 (256KB) zombie: 2 (512KB)
168168
Zombie tables: 2 (1.2KB, local: 1.2KB)
169169
Backing tables: 0 (0B)
@@ -202,7 +202,7 @@ total | 1 595B 0B 0 | - | 56B | 0 0B | 0 0
202202
WAL: 1 files (0B) in: 34B written: 56B (65% overhead)
203203
Flushes: 2
204204
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
205-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
205+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
206206
MemTables: 1 (256KB) zombie: 2 (512KB)
207207
Zombie tables: 1 (589B, local: 589B)
208208
Backing tables: 0 (0B)
@@ -245,7 +245,7 @@ total | 1 595B 0B 0 | - | 56B | 0 0B | 0 0
245245
WAL: 1 files (0B) in: 34B written: 56B (65% overhead)
246246
Flushes: 2
247247
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
248-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
248+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
249249
MemTables: 1 (256KB) zombie: 1 (256KB)
250250
Zombie tables: 0 (0B, local: 0B)
251251
Backing tables: 0 (0B)
@@ -315,7 +315,7 @@ total | 4 2.6KB 38B 0 | - | 149B | 0 0B | 0 0
315315
WAL: 1 files (0B) in: 116B written: 149B (28% overhead)
316316
Flushes: 3
317317
Compactions: 1 estimated debt: 2.6KB in progress: 0 (0B)
318-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
318+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
319319
MemTables: 1 (256KB) zombie: 1 (256KB)
320320
Zombie tables: 0 (0B, local: 0B)
321321
Backing tables: 0 (0B)
@@ -369,7 +369,7 @@ total | 3 2.0KB 41B 0 | - | 149B | 0 0B | 0 0
369369
WAL: 1 files (0B) in: 116B written: 149B (28% overhead)
370370
Flushes: 3
371371
Compactions: 2 estimated debt: 0B in progress: 0 (0B)
372-
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
372+
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
373373
MemTables: 1 (256KB) zombie: 1 (256KB)
374374
Zombie tables: 0 (0B, local: 0B)
375375
Backing tables: 0 (0B)
@@ -472,7 +472,7 @@ total | 7 4.3KB 41B 0 | - | 1.9KB | 3 1.7KB | 0 0
472472
WAL: 1 files (0B) in: 176B written: 187B (6% overhead)
473473
Flushes: 8
474474
Compactions: 2 estimated debt: 4.3KB in progress: 0 (0B)
475-
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
475+
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
476476
MemTables: 1 (1.0MB) zombie: 1 (1.0MB)
477477
Zombie tables: 0 (0B, local: 0B)
478478
Backing tables: 0 (0B)
@@ -534,7 +534,7 @@ total | 10 6.1KB 41B 0 | - | 2.0KB | 3 1.7KB | 0 0
534534
WAL: 1 files (0B) in: 223B written: 245B (10% overhead)
535535
Flushes: 9
536536
Compactions: 2 estimated debt: 6.1KB in progress: 0 (0B)
537-
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
537+
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
538538
MemTables: 1 (1.0MB) zombie: 1 (1.0MB)
539539
Zombie tables: 0 (0B, local: 0B)
540540
Backing tables: 0 (0B)
@@ -610,7 +610,7 @@ total | 11 5.6KB 41B 2 | - | 2.5KB | 4 2.3KB | 0 0
610610
WAL: 1 files (0B) in: 223B written: 245B (10% overhead)
611611
Flushes: 9
612612
Compactions: 2 estimated debt: 5.6KB in progress: 0 (0B)
613-
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
613+
default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
614614
MemTables: 1 (1.0MB) zombie: 1 (1.0MB)
615615
Zombie tables: 0 (0B, local: 0B)
616616
Backing tables: 2 (1.2KB)
@@ -711,7 +711,7 @@ total | 6 3.8KB 41B 0 | - | 3.1KB | 5 2.9KB | 0 0
711711
WAL: 1 files (0B) in: 223B written: 245B (10% overhead)
712712
Flushes: 9
713713
Compactions: 3 estimated debt: 0B in progress: 0 (0B)
714-
default: 3 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
714+
default: 3 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
715715
MemTables: 1 (1.0MB) zombie: 1 (1.0MB)
716716
Zombie tables: 0 (0B, local: 0B)
717717
Backing tables: 0 (0B)
@@ -764,7 +764,7 @@ total | 1 604B 0B 0 | - | 38B | 0 0B | 0 0
764764
WAL: 1 files (0B) in: 27B written: 38B (41% overhead)
765765
Flushes: 1
766766
Compactions: 0 estimated debt: 0B in progress: 0 (0B)
767-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
767+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
768768
MemTables: 1 (256KB) zombie: 1 (256KB)
769769
Zombie tables: 0 (0B, local: 0B)
770770
Backing tables: 0 (0B)
@@ -800,8 +800,8 @@ total | 1 604B 0B 0 | - | 38B | 0 0B | 0 0
800800
-------------------------------------------------------------------------------------------------------------------
801801
WAL: 1 files (0B) in: 27B written: 38B (41% overhead)
802802
Flushes: 1
803-
Compactions: 0 estimated debt: 0B in progress: 0 (0B)
804-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
803+
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
804+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 1 multi-level: 0
805805
MemTables: 1 (256KB) zombie: 1 (256KB)
806806
Zombie tables: 0 (0B, local: 0B)
807807
Backing tables: 0 (0B)
@@ -846,8 +846,8 @@ total | 2 1.2KB 0B 0 | - | 627B | 1 589B | 0 0
846846
-------------------------------------------------------------------------------------------------------------------
847847
WAL: 1 files (0B) in: 27B written: 38B (41% overhead)
848848
Flushes: 1
849-
Compactions: 0 estimated debt: 1.2KB in progress: 0 (0B)
850-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
849+
Compactions: 1 estimated debt: 1.2KB in progress: 0 (0B)
850+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 1 multi-level: 0
851851
MemTables: 1 (256KB) zombie: 1 (256KB)
852852
Zombie tables: 0 (0B, local: 0B)
853853
Backing tables: 0 (0B)
@@ -893,8 +893,8 @@ total | 3 1.7KB 0B 0 | - | 655B | 1 589B | 0 0
893893
-------------------------------------------------------------------------------------------------------------------
894894
WAL: 1 files (0B) in: 44B written: 66B (50% overhead)
895895
Flushes: 2
896-
Compactions: 0 estimated debt: 1.7KB in progress: 0 (0B)
897-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
896+
Compactions: 1 estimated debt: 1.7KB in progress: 0 (0B)
897+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 1 multi-level: 0
898898
MemTables: 1 (256KB) zombie: 1 (256KB)
899899
Zombie tables: 0 (0B, local: 0B)
900900
Backing tables: 0 (0B)
@@ -932,7 +932,7 @@ total | 3 1.7KB 0B 0 | - | 0B | 0 0B | 0 0
932932
WAL: 1 files (0B) in: 0B written: 0B (0% overhead)
933933
Flushes: 0
934934
Compactions: 0 estimated debt: 1.7KB in progress: 0 (0B)
935-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
935+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
936936
MemTables: 1 (256KB) zombie: 0 (0B)
937937
Zombie tables: 0 (0B, local: 0B)
938938
Backing tables: 0 (0B)
@@ -969,7 +969,7 @@ total | 1 603B 0B 0 | - | 0B | 0 0B | 0 0
969969
WAL: 1 files (0B) in: 0B written: 0B (0% overhead)
970970
Flushes: 0
971971
Compactions: 1 estimated debt: 0B in progress: 0 (0B)
972-
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
972+
default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
973973
MemTables: 1 (256KB) zombie: 0 (0B)
974974
Zombie tables: 0 (0B, local: 0B)
975975
Backing tables: 0 (0B)

tool/testdata/db_lsm

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ total | 1 709B 0B 0 | - | 0B | 0 0B | 0 0
2525
WAL: 0 files (0B) in: 0B written: 0B (0% overhead)
2626
Flushes: 0
2727
Compactions: 0 estimated debt: 0B in progress: 0 (0B)
28-
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0
28+
default: 0 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 copy: 0 multi-level: 0
2929
MemTables: 1 (256KB) zombie: 0 (0B)
3030
Zombie tables: 0 (0B, local: 0B)
3131
Backing tables: 0 (0B)

version_set.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,15 @@ func (vs *versionSet) incrementCompactions(
845845
case compactionKindRewrite:
846846
vs.metrics.Compact.Count++
847847
vs.metrics.Compact.RewriteCount++
848+
849+
case compactionKindCopy:
850+
vs.metrics.Compact.Count++
851+
vs.metrics.Compact.CopyCount++
852+
853+
default:
854+
if invariants.Enabled {
855+
panic("unhandled compaction kind")
856+
}
848857
}
849858
if len(extraLevels) > 0 {
850859
vs.metrics.Compact.MultiLevelCount++

0 commit comments

Comments
 (0)