metrics: count copy compactions

Currently copy compactions don't count towards the total number of
compactions, even though the bytes they write do count towards the
total bytes written by compactions.

This change fixes this and adds a copy compaction metric.

Author: RaduBerinde

metrics.go

@@ -157,6 +157,7 @@ type Metrics struct {
 		DefaultCount      int64
 		DeleteOnlyCount   int64
 		ElisionOnlyCount  int64
+		CopyCount         int64
 		MoveCount         int64
 		ReadCount         int64
 		RewriteCount      int64
@@ -569,13 +570,14 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.Compact.NumInProgress),
 		humanize.Bytes.Int64(m.Compact.InProgressBytes))
 
-	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  multi-level: %d\n",
+	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  copy: %d  multi-level: %d\n",
 		redact.Safe(m.Compact.DefaultCount),
 		redact.Safe(m.Compact.DeleteOnlyCount),
 		redact.Safe(m.Compact.ElisionOnlyCount),
 		redact.Safe(m.Compact.MoveCount),
 		redact.Safe(m.Compact.ReadCount),
 		redact.Safe(m.Compact.RewriteCount),
+		redact.Safe(m.Compact.CopyCount),
 		redact.Safe(m.Compact.MultiLevelCount))
 
 	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",

metrics_test.go

@@ -40,7 +40,8 @@ func exampleMetrics() Metrics {
 	m.Compact.MoveCount = 30
 	m.Compact.ReadCount = 31
 	m.Compact.RewriteCount = 32
-	m.Compact.MultiLevelCount = 33
+	m.Compact.CopyCount = 33
+	m.Compact.MultiLevelCount = 34
 	m.Compact.EstimatedDebt = 6
 	m.Compact.InProgressBytes = 7
 	m.Compact.NumInProgress = 2

replay/replay.go

@@ -104,6 +104,7 @@ type Metrics struct {
 		Move        int64
 		Read        int64
 		Rewrite     int64
+		Copy        int64
 		MultiLevel  int64
 	}
 	EstimatedDebt SampledMetric
@@ -169,6 +170,7 @@ func (m *Metrics) WriteBenchmarkString(name string, w io.Writer) error {
 			{Value: float64(m.CompactionCounts.Move), Unit: "move"},
 			{Value: float64(m.CompactionCounts.Read), Unit: "read"},
 			{Value: float64(m.CompactionCounts.Rewrite), Unit: "rewrite"},
+			{Value: float64(m.CompactionCounts.Copy), Unit: "copy"},
 			{Value: float64(m.CompactionCounts.MultiLevel), Unit: "multilevel"},
 		}},
 		// Total database sizes sampled after every workload step and
@@ -555,6 +557,7 @@ func (r *Runner) Wait() (Metrics, error) {
 	m.CompactionCounts.Move = pm.Compact.MoveCount
 	m.CompactionCounts.Read = pm.Compact.ReadCount
 	m.CompactionCounts.Rewrite = pm.Compact.RewriteCount
+	m.CompactionCounts.Copy = pm.Compact.CopyCount
 	m.CompactionCounts.MultiLevel = pm.Compact.MultiLevelCount
 	m.Ingest.BytesIntoL0 = pm.Levels[0].BytesIngested
 	m.Ingest.BytesWeightedByLevel = ingestBytesWeighted

replay/replay_test.go

@@ -441,7 +441,7 @@ func TestBenchmarkString(t *testing.T) {
 	var buf bytes.Buffer
 	require.NoError(t, m.WriteBenchmarkString("tpcc", &buf))
 	require.Equal(t, strings.TrimSpace(`
-BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 multilevel
+BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 copy 0 multilevel
 BenchmarkBenchmarkReplay/tpcc/DatabaseSize/mean 1 5.36870912e+09 bytes
 BenchmarkBenchmarkReplay/tpcc/DatabaseSize/max 1 5.36870912e+09 bytes
 BenchmarkBenchmarkReplay/tpcc/DurationWorkload 1 1 sec/op

testdata/event_listener

@@ -222,7 +222,7 @@ total |     3  1.7KB     0B       0 |     - |  671B |     1   590B |     0     0
 WAL: 1 files (0B)  in: 48B  written: 81B (69% overhead)
 Flushes: 3
 Compactions: 1  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -322,7 +322,7 @@ total |     6  3.5KB     0B       0 |     - | 1.8KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 82B  written: 108B (32% overhead)
 Flushes: 6
 Compactions: 1  estimated debt: 3.5KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (512KB)  zombie: 1 (512KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)

testdata/ingest

@@ -46,7 +46,7 @@ total |     1   569B     0B       0 |     - |  569B |     1   569B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)

testdata/metrics

@@ -15,7 +15,7 @@ total |  2.8K  2.7KB     0B    2.8K |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8K
 WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
 Flushes: 8
 Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
-             default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
+             default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  copy: 33  multi-level: 34
 MemTables: 12 (11B)  zombie: 14 (13B)
 Zombie tables: 16 (15B, local: 30B)
 Backing tables: 1 (2.0MB)
@@ -66,7 +66,7 @@ total |     1   589B     0B       0 |     - |   28B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 17B  written: 28B (65% overhead)
 Flushes: 1
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -121,7 +121,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
@@ -163,7 +163,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
@@ -202,7 +202,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 1 (589B, local: 589B)
 Backing tables: 0 (0B)
@@ -245,7 +245,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -315,7 +315,7 @@ total |     4  2.6KB    38B       0 |     - |  149B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 116B  written: 149B (28% overhead)
 Flushes: 3
 Compactions: 1  estimated debt: 2.6KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -369,7 +369,7 @@ total |     3  2.0KB    41B       0 |     - |  149B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 116B  written: 149B (28% overhead)
 Flushes: 3
 Compactions: 2  estimated debt: 0B  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -472,7 +472,7 @@ total |     7  4.3KB    41B       0 |     - | 1.9KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 176B  written: 187B (6% overhead)
 Flushes: 8
 Compactions: 2  estimated debt: 4.3KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -534,7 +534,7 @@ total |    10  6.1KB    41B       0 |     - | 2.0KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 2  estimated debt: 6.1KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -610,7 +610,7 @@ total |    11  5.6KB    41B       2 |     - | 2.5KB |     4  2.3KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 2  estimated debt: 5.6KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 2 (1.2KB)
@@ -711,7 +711,7 @@ total |     6  3.8KB    41B       0 |     - | 3.1KB |     5  2.9KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 3  estimated debt: 0B  in progress: 0 (0B)
-             default: 3  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 3  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -764,7 +764,7 @@ total |     1   604B     0B       0 |     - |   38B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -800,8 +800,8 @@ total |     1   604B     0B       0 |     - |   38B |     0     0B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
-Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -846,8 +846,8 @@ total |     2  1.2KB     0B       0 |     - |  627B |     1   589B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
-Compactions: 0  estimated debt: 1.2KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 1.2KB  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -893,8 +893,8 @@ total |     3  1.7KB     0B       0 |     - |  655B |     1   589B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 44B  written: 66B (50% overhead)
 Flushes: 2
-Compactions: 0  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 1.7KB  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -932,7 +932,7 @@ total |     3  1.7KB     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -969,7 +969,7 @@ total |     1   603B     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)

tool/testdata/db_lsm

@@ -25,7 +25,7 @@ total |     1   709B     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 0 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)

version_set.go

@@ -845,6 +845,15 @@ func (vs *versionSet) incrementCompactions(
 	case compactionKindRewrite:
 		vs.metrics.Compact.Count++
 		vs.metrics.Compact.RewriteCount++
+
+	case compactionKindCopy:
+		vs.metrics.Compact.Count++
+		vs.metrics.Compact.CopyCount++
+
+	default:
+		if invariants.Enabled {
+			panic("unhandled compaction kind")
+		}
 	}
 	if len(extraLevels) > 0 {
 		vs.metrics.Compact.MultiLevelCount++