Skip to content

Commit 7fbd668

Browse files
MarinPostmaMarinPostma
authored
Merge pull request #1782 from tursodatabase/tiered-compaction
tiered compaction
2 parents 935a878 + 82dfa91 commit 7fbd668

12 files changed

+359
-15
lines changed

‎libsql-server/src/wal_toolkit.rs‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,11 @@ use libsql_wal::storage::backend::s3::S3Backend;
1212
use libsql_wal::storage::backend::Backend;
1313
use libsql_wal::storage::compaction::strategy::identity::IdentityStrategy;
1414
use libsql_wal::storage::compaction::strategy::log_strategy::LogReductionStrategy;
15-
use libsql_wal::storage::compaction::strategy::PartitionStrategy;
15+
use libsql_wal::storage::compaction::strategy::tiered::LevelsStrategy;
16+
use libsql_wal::storage::compaction::strategy::CompactionStrategy;
1617
use libsql_wal::storage::compaction::Compactor;
1718
use rusqlite::OpenFlags;
1819

19-
#[derive(Clone, Debug, clap::ValueEnum, Copy)]
20-
pub enum CompactStrategy {
21-
Logarithmic,
22-
CompactAll,
23-
}
24-
2520
#[derive(Debug, clap::Subcommand)]
2621
pub enum WalToolkitCommand {
2722
Monitor(MonitorCommand),
@@ -119,6 +114,13 @@ impl SyncCommand {
119114
}
120115
}
121116

117+
#[derive(Clone, Debug, clap::ValueEnum, Copy)]
118+
pub enum CompactStrategy {
119+
Logarithmic,
120+
CompactAll,
121+
Tiered,
122+
}
123+
122124
#[derive(Debug, clap::Args)]
123125
/// Compact segments into bigger segments
124126
pub struct CompactCommand {
@@ -168,10 +170,12 @@ impl CompactCommand {
168170
namespace: &NamespaceName,
169171
) -> anyhow::Result<()> {
170172
let analysis = compactor.analyze(&namespace)?;
171-
let strat: Box<dyn PartitionStrategy> = match self.strategy {
173+
let strat: Box<dyn CompactionStrategy> = match self.strategy {
172174
CompactStrategy::Logarithmic => Box::new(LogReductionStrategy),
173175
CompactStrategy::CompactAll => Box::new(IdentityStrategy),
176+
CompactStrategy::Tiered => Box::new(LevelsStrategy::new(self.threshold)),
174177
};
178+
175179
let set = analysis.shortest_restore_path();
176180
if set.len() <= self.threshold {
177181
println!(

‎libsql-wal/src/shared_wal.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ pub struct SharedWal<IO: Io> {
5151
pub(crate) db_file: IO::File,
5252
pub(crate) namespace: NamespaceName,
5353
pub(crate) registry: Arc<dyn SwapLog<IO>>,
54-
#[allow(dead_code)] // used by replication
5554
pub(crate) checkpointed_frame_no: AtomicU64,
5655
/// max frame_no acknowledged by the durable storage
5756
pub(crate) durable_frame_no: Arc<Mutex<u64>>,

‎libsql-wal/src/storage/compaction/mod.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -591,13 +591,23 @@ impl AnalyzedSegments {
591591

592592
/// A set of segments, with the guarantee that segments are non-overlapping and increasing in
593593
/// frameno
594-
#[derive(Clone)]
594+
#[derive(Clone, Debug)]
595595
pub struct SegmentSet {
596596
namespace: NamespaceName,
597597
segments: Vec<SegmentKey>,
598598
}
599599

600600
impl SegmentSet {
601+
/// return segments end - start
602+
pub fn span(&self) -> u64 {
603+
if self.is_empty() {
604+
0
605+
} else {
606+
self.segments.last().unwrap().end_frame_no
607+
- self.segments.first().unwrap().start_frame_no
608+
}
609+
}
610+
601611
pub fn range(&self) -> Option<(u64, u64)> {
602612
self.segments
603613
.first()

‎libsql-wal/src/storage/compaction/strategy/identity.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
use crate::storage::compaction::SegmentSet;
22

3-
use super::PartitionStrategy;
3+
use super::CompactionStrategy;
44

55
/// partition strategy that doesn't split the passed set
66
pub struct IdentityStrategy;
77

8-
impl PartitionStrategy for IdentityStrategy {
8+
impl CompactionStrategy for IdentityStrategy {
99
fn partition(&self, segments: &SegmentSet) -> Vec<SegmentSet> {
1010
let mut out = Vec::with_capacity(1);
1111
out.push(segments.clone());

‎libsql-wal/src/storage/compaction/strategy/log_strategy.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ use std::ops::Deref as _;
22

33
use crate::storage::compaction::SegmentSet;
44

5-
use super::PartitionStrategy;
5+
use super::CompactionStrategy;
66

77
/// partition the SegmentSet in logarithmically reducing sets
88
pub struct LogReductionStrategy;
99

10-
impl PartitionStrategy for LogReductionStrategy {
10+
impl CompactionStrategy for LogReductionStrategy {
1111
fn partition(&self, segments: &SegmentSet) -> Vec<SegmentSet> {
1212
let mut segs = segments.deref();
1313
let mut out = Vec::new();

‎libsql-wal/src/storage/compaction/strategy/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ use super::SegmentSet;
22

33
pub mod identity;
44
pub mod log_strategy;
5+
pub mod tiered;
56

6-
pub trait PartitionStrategy {
7+
pub trait CompactionStrategy {
78
fn partition(&self, segments: &SegmentSet) -> Vec<SegmentSet>;
89
}

‎libsql-wal/src/storage/compaction/strategy/snapshots/libsql_wal__storage__compaction__strategy__tier_reduction__test__partition_tiered-2.snap‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
---
2+
source: libsql-wal/src/storage/compaction/strategy/tier_reduction.rs
3+
expression: partition.first().unwrap()
4+
---
5+
SegmentSet {
6+
namespace: test,
7+
segments: [
8+
SegmentKey {
9+
start_frame_no: 101,
10+
end_frame_no: 105,
11+
timestamp: 1970-01-01T00:00:00Z,
12+
},
13+
SegmentKey {
14+
start_frame_no: 106,
15+
end_frame_no: 110,
16+
timestamp: 1970-01-01T00:00:00Z,
17+
},
18+
SegmentKey {
19+
start_frame_no: 111,
20+
end_frame_no: 115,
21+
timestamp: 1970-01-01T00:00:00Z,
22+
},
23+
SegmentKey {
24+
start_frame_no: 116,
25+
end_frame_no: 120,
26+
timestamp: 1970-01-01T00:00:00Z,
27+
},
28+
SegmentKey {
29+
start_frame_no: 121,
30+
end_frame_no: 122,
31+
timestamp: 1970-01-01T00:00:00Z,
32+
},
33+
],
34+
}

‎libsql-wal/src/storage/compaction/strategy/snapshots/libsql_wal__storage__compaction__strategy__tier_reduction__test__partition_tiered.snap‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
source: libsql-wal/src/storage/compaction/strategy/tier_reduction.rs
3+
expression: partition.first().unwrap()
4+
---
5+
SegmentSet {
6+
namespace: test,
7+
segments: [
8+
SegmentKey {
9+
start_frame_no: 1,
10+
end_frame_no: 20,
11+
timestamp: 1970-01-01T00:00:00Z,
12+
},
13+
SegmentKey {
14+
start_frame_no: 21,
15+
end_frame_no: 27,
16+
timestamp: 1970-01-01T00:00:00Z,
17+
},
18+
SegmentKey {
19+
start_frame_no: 28,
20+
end_frame_no: 41,
21+
timestamp: 1970-01-01T00:00:00Z,
22+
},
23+
SegmentKey {
24+
start_frame_no: 42,
25+
end_frame_no: 70,
26+
timestamp: 1970-01-01T00:00:00Z,
27+
},
28+
SegmentKey {
29+
start_frame_no: 71,
30+
end_frame_no: 81,
31+
timestamp: 1970-01-01T00:00:00Z,
32+
},
33+
SegmentKey {
34+
start_frame_no: 82,
35+
end_frame_no: 100,
36+
timestamp: 1970-01-01T00:00:00Z,
37+
},
38+
],
39+
}

‎libsql-wal/src/storage/compaction/strategy/snapshots/libsql_wal__storage__compaction__strategy__tiered__test__partition_tiered-2.snap‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
---
2+
source: libsql-wal/src/storage/compaction/strategy/tiered.rs
3+
expression: partition.first().unwrap()
4+
---
5+
SegmentSet {
6+
namespace: test,
7+
segments: [
8+
SegmentKey {
9+
start_frame_no: 101,
10+
end_frame_no: 105,
11+
timestamp: 1970-01-01T00:00:00Z,
12+
},
13+
SegmentKey {
14+
start_frame_no: 106,
15+
end_frame_no: 110,
16+
timestamp: 1970-01-01T00:00:00Z,
17+
},
18+
SegmentKey {
19+
start_frame_no: 111,
20+
end_frame_no: 115,
21+
timestamp: 1970-01-01T00:00:00Z,
22+
},
23+
SegmentKey {
24+
start_frame_no: 116,
25+
end_frame_no: 120,
26+
timestamp: 1970-01-01T00:00:00Z,
27+
},
28+
SegmentKey {
29+
start_frame_no: 121,
30+
end_frame_no: 122,
31+
timestamp: 1970-01-01T00:00:00Z,
32+
},
33+
],
34+
}

‎libsql-wal/src/storage/compaction/strategy/snapshots/libsql_wal__storage__compaction__strategy__tiered__test__partition_tiered.snap‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
source: libsql-wal/src/storage/compaction/strategy/tiered.rs
3+
expression: partition.first().unwrap()
4+
---
5+
SegmentSet {
6+
namespace: test,
7+
segments: [
8+
SegmentKey {
9+
start_frame_no: 1,
10+
end_frame_no: 20,
11+
timestamp: 1970-01-01T00:00:00Z,
12+
},
13+
SegmentKey {
14+
start_frame_no: 21,
15+
end_frame_no: 27,
16+
timestamp: 1970-01-01T00:00:00Z,
17+
},
18+
SegmentKey {
19+
start_frame_no: 28,
20+
end_frame_no: 41,
21+
timestamp: 1970-01-01T00:00:00Z,
22+
},
23+
SegmentKey {
24+
start_frame_no: 42,
25+
end_frame_no: 70,
26+
timestamp: 1970-01-01T00:00:00Z,
27+
},
28+
SegmentKey {
29+
start_frame_no: 71,
30+
end_frame_no: 81,
31+
timestamp: 1970-01-01T00:00:00Z,
32+
},
33+
SegmentKey {
34+
start_frame_no: 82,
35+
end_frame_no: 100,
36+
timestamp: 1970-01-01T00:00:00Z,
37+
},
38+
],
39+
}

0 commit comments

Comments
 (0)