Skip to content

Commit deb8e20

Browse files
Zhou-jwZhou-jw
authored
Merge pull request #2 from Zhou-jw/w3d2
impl w3d1 & w3d2
2 parents fef0ba9 + 7663822 commit deb8e20

File tree

14 files changed

+291
-126
lines changed

14 files changed

+291
-126
lines changed

‎.vscode/launch.json‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,125 @@
4040
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
4141
],
4242
"preRunCommands": [
43-
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py",
43+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
4444
],
4545
"args": [
4646
"tests::week2_day5",
4747
"--show-output"
4848
]
49+
},
50+
{
51+
"type": "lldb-dap",
52+
"request": "launch",
53+
"name": "test tests::week1_day2::test_task4_integration",
54+
"cwd": "${workspaceFolder}/mini-lsm-starter",
55+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
56+
"env": [
57+
"RUST_BACKTRACE=short",
58+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
59+
],
60+
"preRunCommands": [
61+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
62+
],
63+
"args": [
64+
"tests::week1_day2::test_task4_integration",
65+
"--exact",
66+
"--show-output"
67+
]
68+
},
69+
{
70+
"type": "lldb-dap",
71+
"request": "launch",
72+
"name": "test tests::week1_day1::test_task3_freeze_on_capacity",
73+
"cwd": "${workspaceFolder}/mini-lsm-starter",
74+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
75+
"env": [
76+
"RUST_BACKTRACE=short",
77+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
78+
],
79+
"preRunCommands": [
80+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
81+
],
82+
"args": [
83+
"tests::week1_day1::test_task3_freeze_on_capacity",
84+
"--exact",
85+
"--show-output"
86+
]
87+
},
88+
{
89+
"type": "lldb-dap",
90+
"request": "launch",
91+
"name": "test-mod tests::week1_day6",
92+
"cwd": "${workspaceFolder}/mini-lsm-starter",
93+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
94+
"env": [
95+
"RUST_BACKTRACE=short",
96+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
97+
],
98+
"preRunCommands": [
99+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
100+
],
101+
"args": [
102+
"tests::week1_day6",
103+
"--show-output"
104+
]
105+
},
106+
{
107+
"type": "lldb-dap",
108+
"request": "launch",
109+
"name": "test tests::week2_day1::test_task3_integration",
110+
"cwd": "${workspaceFolder}/mini-lsm-starter",
111+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
112+
"env": [
113+
"RUST_BACKTRACE=short",
114+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
115+
],
116+
"preRunCommands": [
117+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
118+
],
119+
"args": [
120+
"tests::week2_day1::test_task3_integration",
121+
"--exact",
122+
"--show-output"
123+
]
124+
},
125+
{
126+
"type": "lldb-dap",
127+
"request": "launch",
128+
"name": "test tests::week2_day2::test_integration",
129+
"cwd": "${workspaceFolder}/mini-lsm-starter",
130+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
131+
"env": [
132+
"RUST_BACKTRACE=short",
133+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
134+
],
135+
"preRunCommands": [
136+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
137+
],
138+
"args": [
139+
"tests::week2_day2::test_integration",
140+
"--exact",
141+
"--show-output"
142+
]
143+
},
144+
{
145+
"type": "lldb-dap",
146+
"request": "launch",
147+
"name": "test tests::week3_day1::test_sst_build_multi_version_hard",
148+
"cwd": "${workspaceFolder}/mini-lsm-starter",
149+
"program": "${workspaceFolder}/target/debug/deps/mini_lsm_starter-430c4aaead80470b",
150+
"env": [
151+
"RUST_BACKTRACE=short",
152+
"RUSTC_TOOLCHAIN=/home/zjw/.rustup/toolchains/stable-x86_64-unknown-linux-gnu"
153+
],
154+
"preRunCommands": [
155+
"command script import ${workspaceFolder}/util/rust_prettifier_for_lldb.py"
156+
],
157+
"args": [
158+
"tests::week3_day1::test_sst_build_multi_version_hard",
159+
"--exact",
160+
"--show-output"
161+
]
49162
}
50163
]
51164
}

‎mini-lsm-starter/src/block/iterator.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ impl BlockIterator {
105105
.append(&self.first_key.key_ref()[..key_overlap_len]);
106106
self.key.append(key);
107107
let ts = entry.get_u64();
108+
self.key.set_ts(ts);
108109
let value_len = entry.get_u16() as usize;
109110
let value_offset_begin = offset + SIZEOF_U16 * 2 + res_len + SIZEOF_U64 + SIZEOF_U16;
110111
let value_offset_end = value_offset_begin + value_len;
@@ -128,6 +129,7 @@ impl BlockIterator {
128129
/// Note: You should assume the key-value pairs in the block are sorted when being added by
129130
/// callers.
130131
pub fn seek_to_key(&mut self, key: KeySlice) {
132+
// if key doesn't exist, it will seek to empty key
131133
let mut low = 0;
132134
let mut high = self.block.offsets.len();
133135
while low < high {
@@ -143,6 +145,7 @@ impl BlockIterator {
143145
self.seek_to(low);
144146

145147
// if key doesn't exist, it will seek to the last_key in this block
148+
// O(n)
146149
// let mut offset: usize;
147150
// for i in 0..self.block.offsets.len() {
148151
// offset = self.block.offsets[i] as usize;

‎mini-lsm-starter/src/compact.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ impl LsmStorageInner {
134134
let builder_inner = sst_builder.as_mut().unwrap();
135135
builder_inner.add(iter.key(), iter.value());
136136

137-
if builder_inner.estimated_size() >= target_sst_size && !is_same_as_prevkey{
137+
if builder_inner.estimated_size() >= target_sst_size && !is_same_as_prevkey {
138138
let next_sst_id = self.next_sst_id();
139139
let block_cache = self.block_cache.clone();
140140
let builder = sst_builder.take().unwrap();
@@ -148,7 +148,7 @@ impl LsmStorageInner {
148148

149149
iter.next()?;
150150

151-
if !is_same_as_prevkey {
151+
if !is_same_as_prevkey && iter.is_valid() {
152152
prev_key.clear();
153153
prev_key.extend(iter.key().key_ref());
154154
}
@@ -357,6 +357,12 @@ impl LsmStorageInner {
357357
}
358358
snapshot.levels[0] = (1, level_1);
359359

360+
println!("===== After compaction =====");
361+
println!("L0 : {:?}", snapshot.l0_sstables);
362+
for (tier, sstables) in snapshot.levels.iter() {
363+
println!("L{:?} : {:?}", tier, sstables);
364+
}
365+
println!();
360366
// lock and update LsmStorageState
361367
{
362368
let _state_lock = self.state_lock.lock();

‎mini-lsm-starter/src/iterators/merge_iterator.rs‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,17 @@ impl<I: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>> StorageIt
9494
while let Some(mut inner_iter) = self.iters.peek_mut() {
9595
//cur.1.key is the latest data, when next.1.key==cur.1.key , inner_iter should call inner_iter.next()
9696
// eprintln!(
97-
// "cur.1.key is {:?}, cur.1.value is {:?}",
97+
// "cur.1.key is {:?}, ts is {:?}, cur.1.value is {:?}",
9898
// cur.1.key(),
99+
// cur.1.key().ts(),
99100
// cur.1.value()
100101
// );
101-
// eprintln!("inner_iter.1.key is {:?}", inner_iter.1.key());
102+
// eprintln!(
103+
// "inner_iter.1.key is {:?}, ts is {:?}, inner_iter.1.value is {:?}",
104+
// inner_iter.1.key(),
105+
// inner_iter.1.key().ts(),
106+
// inner_iter.1.value()
107+
// );
102108
// io::stdout().flush().unwrap();
103109
if cur.1.key() == inner_iter.1.key() {
104110
/*
@@ -160,9 +166,14 @@ impl<I: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>> StorageIt
160166
}
161167

162168
fn num_active_iterators(&self) -> usize {
163-
match self.current {
164-
Some(_) => self.iters.len() + 1,
165-
None => self.iters.len(),
166-
}
169+
self.iters
170+
.iter()
171+
.map(|x| x.1.num_active_iterators())
172+
.sum::<usize>()
173+
+ self
174+
.current
175+
.as_ref()
176+
.map(|x| x.1.num_active_iterators())
177+
.unwrap_or(0)
167178
}
168179
}

‎mini-lsm-starter/src/iterators/two_merge_iterator.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,16 @@ impl<
8181
}
8282

8383
fn num_active_iterators(&self) -> usize {
84-
if !self.aflag {
85-
if !self.b.is_valid() {
86-
return 0;
87-
}
88-
return self.b.num_active_iterators();
89-
}
84+
// if !self.aflag {
85+
// if !self.b.is_valid() {
86+
// return 0;
87+
// }
88+
// return self.b.num_active_iterators();
89+
// }
9090

91-
if !self.b.is_valid() {
92-
return self.a.num_active_iterators();
93-
}
91+
// if !self.b.is_valid() {
92+
// return self.a.num_active_iterators();
93+
// }
9494

9595
self.a.num_active_iterators() + self.b.num_active_iterators()
9696
}

‎mini-lsm-starter/src/key.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ use timestamped_key as key;
55

66
pub const TS_ENABLED: bool = key::TS_ENABLED;
77
pub const TS_DEFAULT: u64 = key::TS_DEFAULT;
8+
pub const TS_RANGE_BEGIN: u64 = key::TS_RANGE_BEGIN;
9+
pub const TS_RANGE_END: u64 = key::TS_RANGE_END;
810
pub const TS_MAX: u64 = key::TS_MAX;
911
pub const TS_MIN: u64 = key::TS_MIN;
1012

‎mini-lsm-starter/src/key/timestamped_key.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ pub(super) type KeyVec = Key<Vec<u8>>;
1212
pub(super) type KeyBytes = Key<Bytes>;
1313

1414
pub const TS_DEFAULT: u64 = 0;
15+
pub const TS_RANGE_BEGIN: u64 = std::u64::MAX;
16+
pub const TS_RANGE_END: u64 = std::u64::MIN;
1517
pub const TS_MAX: u64 = std::u64::MAX;
1618
pub const TS_MIN: u64 = std::u64::MIN;
1719

@@ -33,7 +35,7 @@ impl<T: AsRef<[u8]>> Key<T> {
3335
}
3436

3537
pub fn for_testing_ts(self) -> u64 {
36-
0
38+
self.1
3739
}
3840
}
3941

@@ -58,6 +60,11 @@ impl Key<Vec<u8>> {
5860
self.0.extend(data)
5961
}
6062

63+
/// set ts
64+
pub fn set_ts(&mut self, ts: u64) {
65+
self.1 = ts;
66+
}
67+
6168
/// Set the key from a slice without re-allocating. The signature will change in week 3.
6269
pub fn set_from_slice(&mut self, key_slice: KeySlice) {
6370
self.0.clear();
@@ -87,7 +94,7 @@ impl Key<Vec<u8>> {
8794
}
8895

8996
pub fn for_testing_from_vec_no_ts(key: Vec<u8>) -> Self {
90-
Self(key, u64::default())
97+
Self(key, TS_DEFAULT)
9198
}
9299
}
93100

@@ -168,7 +175,7 @@ impl<T: AsRef<[u8]> + Default> Default for Key<T> {
168175

169176
impl<T: AsRef<[u8]> + PartialEq> PartialEq for Key<T> {
170177
fn eq(&self, other: &Self) -> bool {
171-
self.0.eq(&other.0)
178+
(self.0.as_ref(), self.1).eq(&(other.0.as_ref(), other.1))
172179
}
173180
}
174181

‎mini-lsm-starter/src/lsm_iterator.rs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ impl LsmIterator {
3434
end_bound,
3535
prev_key: Vec::new(),
3636
};
37-
valid_iter.move_to_next_key()?;
37+
valid_iter.move_to_key()?;
3838
Ok(valid_iter)
3939
}
4040

@@ -49,7 +49,7 @@ impl LsmIterator {
4949
// Ok(())
5050
// }
5151

52-
fn move_to_next_key(&mut self) -> Result<()> {
52+
fn move_to_key(&mut self) -> Result<()> {
5353
// while self.inner.is_valid() && self.inner.value().is_empty() {
5454
// self.inner.next()?;
5555
// }
@@ -66,6 +66,10 @@ impl LsmIterator {
6666

6767
self.prev_key.clear();
6868
self.prev_key.extend(self.inner.key().key_ref());
69+
70+
if !self.inner.value().is_empty() {
71+
break;
72+
}
6973
}
7074
Ok(())
7175
}
@@ -76,7 +80,7 @@ impl LsmIterator {
7680
self.is_valid = false;
7781
return Ok(());
7882
}
79-
83+
8084
match self.end_bound.as_ref() {
8185
Bound::Included(x) => {
8286
self.is_valid =
@@ -110,7 +114,7 @@ impl StorageIterator for LsmIterator {
110114
fn next(&mut self) -> Result<()> {
111115
// self.inner.next()?;
112116
self.inner_next()?;
113-
self.move_to_next_key()?;
117+
self.move_to_key()?;
114118
Ok(())
115119
}
116120

0 commit comments

Comments
 (0)