From 234a84102b3230c725e2fb9bd5dd5f23fd2c0a0f Mon Sep 17 00:00:00 2001 From: Jake Roggenbuck Date: Thu, 6 Mar 2025 16:21:09 -0800 Subject: [PATCH 1/3] Make an internal select --- python/lstore/query.py | 11 +-------- src/query.rs | 54 ++++++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/python/lstore/query.py b/python/lstore/query.py index bac02fe..38a2bb9 100644 --- a/python/lstore/query.py +++ b/python/lstore/query.py @@ -2,14 +2,6 @@ from .lstore import RQuery, RTable -class ReturnRecord: - def __init__(self, columns: List[int]): - self.columns = columns[4:] - - def __str__(self): - return f"Record({self.columns})" - - class Query: def __init__(self, table: RTable): """Creates a Query object that can perform different queries on the @@ -65,8 +57,7 @@ def select( ] """ res = self.rquery.select(search_key, search_key_index, projected_columns_index) - # If res is not None, it should be a list of records. - return [ReturnRecord(r) for r in res] if res is not None else [] + return res def select_version( self, diff --git a/src/query.rs b/src/query.rs index d2ee808..a0c986a 100644 --- a/src/query.rs +++ b/src/query.rs @@ -5,6 +5,11 @@ use pyo3::prelude::*; use std::iter::zip; use std::sync::atomic::{AtomicBool, Ordering}; +#[pyclass] +pub struct RReturnRecord { + columns: Vec>, +} + #[pyclass] pub struct RQuery { // pub table: RTable, @@ -70,6 +75,31 @@ impl RQuery { search_key: i64, search_key_index: i64, projected_columns_index: Vec, + ) -> Option>>> { + // Cut off the first 4 columns + let ret = self.internal_select(search_key, search_key_index, projected_columns_index); + + let mut out = vec![]; + + match ret { + Some(rows) => { + for row in rows { + let mut a = row; + a.drain(0..4); + out.push(Some(RReturnRecord { columns: a })) + } + } + None => out.push(None) + } + + return out; + } + + fn internal_select( + &mut self, + search_key: i64, + search_key_index: i64, + projected_columns_index: Vec, ) -> Option>>> { let table = self.handle.table.read().unwrap(); @@ -319,7 +349,7 @@ impl RQuery { // Select the value of the column before we increment let cols = vec![1i64; num_cols]; - let ret = self.select(primary_key, 0, cols); + let ret = self.internal_select(primary_key, 0, cols); if let Some(records) = ret { let record = &records[0]; @@ -347,7 +377,7 @@ mod tests { q.insert(vec![1, 2, 3]); // Use primary_key of 1 - let vals = q.select(1, 0, vec![1, 1, 1]); + let vals = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals.unwrap()[0], vec![ @@ -373,7 +403,7 @@ mod tests { // Increment the first user column (column 1) q.increment(1, 1); - let vals = q.select(1, 0, vec![1, 1, 1]); // Select entire row + let vals = q.internal_select(1, 0, vec![1, 1, 1]); // Select entire row assert_eq!( vals.unwrap()[0], vec![ @@ -389,7 +419,7 @@ mod tests { q.increment(1, 1); - let vals2 = q.select(1, 0, vec![1, 1, 1]); + let vals2 = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals2.unwrap()[0], vec![ @@ -405,7 +435,7 @@ mod tests { q.increment(1, 1); - let vals3 = q.select(1, 0, vec![1, 1, 1]); + let vals3 = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals3.unwrap()[0], vec![ @@ -455,7 +485,7 @@ mod tests { q.insert(vec![1, 2, 3]); // Use primary_key of 1 - let vals = q.select(1, 0, vec![1, 1, 1]); + let vals = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals.unwrap()[0], vec![ @@ -472,7 +502,7 @@ mod tests { let success = q.update(1, vec![Some(1), Some(5), Some(6)]); assert!(success); - let vals2 = q.select(1, 0, vec![1, 1, 1]); + let vals2 = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals2.unwrap()[0], vec![ @@ -512,7 +542,7 @@ mod tests { q.update(1, vec![Some(1), Some(6), Some(7)]); q.update(1, vec![Some(1), Some(8), Some(9)]); - let vals = q.select(1, 0, vec![1, 1, 1]); + let vals = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals.unwrap()[0], vec![ @@ -536,7 +566,7 @@ mod tests { q.insert(vec![1, 2, 3]); q.delete(1); - assert_eq!(q.select(1, 0, vec![1, 1, 1]), None); + assert_eq!(q.internal_select(1, 0, vec![1, 1, 1]), None); } #[test] @@ -624,7 +654,7 @@ mod tests { assert!(result.is_none()); // Verify that the original record is still intact - let vals = q.select(1, 0, vec![1, 1, 1]); + let vals = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vals.unwrap()[0], vec![ @@ -655,7 +685,7 @@ mod tests { q = RQuery::new(table_ref); - let v = q.select(1, 0, vec![1, 1, 1]); + let v = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vec![ Some(3), @@ -685,7 +715,7 @@ mod tests { q = RQuery::new(table_ref.clone()); - let v = q.select(1, 0, vec![1, 1, 1]); + let v = q.internal_select(1, 0, vec![1, 1, 1]); assert_eq!( vec![ Some(600), From 322e2e63e7775c6babf68b305cb04680fe066786 Mon Sep 17 00:00:00 2001 From: Jake Roggenbuck Date: Thu, 6 Mar 2025 16:37:23 -0800 Subject: [PATCH 2/3] Remove py ReturnRecord stuff and move it to Rust Fix for select_version --- python/lstore/query.py | 33 +++++++-------------------------- src/query.rs | 33 ++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/python/lstore/query.py b/python/lstore/query.py index 38a2bb9..f24319b 100644 --- a/python/lstore/query.py +++ b/python/lstore/query.py @@ -43,21 +43,8 @@ def select( Returns a list of Record objects upon success Returns False if record locked by TPL Assume that select will never be called on a key that doesn't exist - - return [ - ReturnRecord( - list( - self.rquery.select( - search_key, - search_key_index, - projected_columns_index, - ) - ) - ) - ] """ - res = self.rquery.select(search_key, search_key_index, projected_columns_index) - return res + return self.rquery.select(search_key, search_key_index, projected_columns_index) def select_version( self, @@ -75,18 +62,12 @@ def select_version( Returns False if record locked by TPL Assume that select will never be called on a key that doesn't exist """ - return [ - ReturnRecord( - list( - self.rquery.select_version( - search_key, - search_key_index, - projected_columns_index, - relative_version, - ) - ) - ) - ] + return self.rquery.select_version( + search_key, + search_key_index, + projected_columns_index, + relative_version, + ) def update(self, primary_key: int, *columns): """Update a record with specified key and columns diff --git a/src/query.rs b/src/query.rs index a0c986a..6e0aed4 100644 --- a/src/query.rs +++ b/src/query.rs @@ -7,6 +7,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; #[pyclass] pub struct RReturnRecord { + #[pyo3(get)] columns: Vec>, } @@ -75,7 +76,7 @@ impl RQuery { search_key: i64, search_key_index: i64, projected_columns_index: Vec, - ) -> Option>>> { + ) -> Option>> { // Cut off the first 4 columns let ret = self.internal_select(search_key, search_key_index, projected_columns_index); @@ -89,10 +90,10 @@ impl RQuery { out.push(Some(RReturnRecord { columns: a })) } } - None => out.push(None) + None => out.push(None), } - return out; + return Some(out); } fn internal_select( @@ -159,6 +160,24 @@ impl RQuery { _search_key_index: i64, projected_columns_index: Vec, relative_version: i64, + ) -> Option { + match self.internal_select_version( + primary_key, + 0, + projected_columns_index, + relative_version, + ) { + Some(columns) => Some(RReturnRecord { columns }), + None => None, + } + } + + pub fn internal_select_version( + &mut self, + primary_key: i64, + _search_key_index: i64, + projected_columns_index: Vec, + relative_version: i64, ) -> Option>> { let table = self.handle.table.read().unwrap(); let Some(ret) = table.read_relative(primary_key, relative_version) else { @@ -584,7 +603,7 @@ mod tests { q.update(1, vec![Some(1), Some(8), Some(9)]); // Version 3 // Test different versions - let latest = q.select_version(1, 0, vec![1, 1, 1], 0); + let latest = q.internal_select_version(1, 0, vec![1, 1, 1], 0); assert_eq!( latest.unwrap(), vec![ @@ -598,7 +617,7 @@ mod tests { ] ); // Most recent version - let one_back = q.select_version(1, 0, vec![1, 1, 1], 1); + let one_back = q.internal_select_version(1, 0, vec![1, 1, 1], 1); assert_eq!( one_back.unwrap(), vec![ @@ -612,7 +631,7 @@ mod tests { ] ); // One version back - let two_back = q.select_version(1, 0, vec![1, 1, 1], 2); + let two_back = q.internal_select_version(1, 0, vec![1, 1, 1], 2); assert_eq!( two_back.unwrap(), vec![ @@ -626,7 +645,7 @@ mod tests { ] ); // Two versions back - let original = q.select_version(1, 0, vec![1, 1, 1], 3); + let original = q.internal_select_version(1, 0, vec![1, 1, 1], 3); assert_eq!( original.unwrap(), vec![ From 91e7fa8be3dab0b412393b8d1a352a12d4eef37d Mon Sep 17 00:00:00 2001 From: Jake Roggenbuck Date: Thu, 6 Mar 2025 16:48:28 -0800 Subject: [PATCH 3/3] Add comments --- src/query.rs | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/query.rs b/src/query.rs index 6e0aed4..c487df5 100644 --- a/src/query.rs +++ b/src/query.rs @@ -18,7 +18,10 @@ pub struct RQuery { merging: AtomicBool, } +/// Use the projected vector to decide which columns to set to None fn filter_projected(column_values: Vec, projected: Vec) -> Vec> { + // TODO: HUGE SPEED IMPROVEMENT AVAILABLE - Don't read columns that will be None anyway!!! + // Add the 4 columns used internally let mut projected_cols: Vec = vec![1, 1, 1, 1]; projected_cols.extend(projected.clone()); @@ -77,7 +80,6 @@ impl RQuery { search_key_index: i64, projected_columns_index: Vec, ) -> Option>> { - // Cut off the first 4 columns let ret = self.internal_select(search_key, search_key_index, projected_columns_index); let mut out = vec![]; @@ -86,7 +88,9 @@ impl RQuery { Some(rows) => { for row in rows { let mut a = row; + // Remove the first 4 columns that are used only internally a.drain(0..4); + // Return the columns encased in the RReturnRecord struct out.push(Some(RReturnRecord { columns: a })) } } @@ -96,6 +100,7 @@ impl RQuery { return Some(out); } + /// Formerly just `select` does a select on the database fn internal_select( &mut self, search_key: i64, @@ -154,6 +159,22 @@ impl RQuery { } } + /// Formerly just `select_version` does a select and picks which version of the record + fn internal_select_version( + &mut self, + primary_key: i64, + _search_key_index: i64, + projected_columns_index: Vec, + relative_version: i64, + ) -> Option>> { + let table = self.handle.table.read().unwrap(); + let Some(ret) = table.read_relative(primary_key, relative_version) else { + return None; + }; + + Some(filter_projected(ret, projected_columns_index)) + } + pub fn select_version( &mut self, primary_key: i64, @@ -167,26 +188,12 @@ impl RQuery { projected_columns_index, relative_version, ) { + // Return the columns encased in the RReturnRecord struct Some(columns) => Some(RReturnRecord { columns }), None => None, } } - pub fn internal_select_version( - &mut self, - primary_key: i64, - _search_key_index: i64, - projected_columns_index: Vec, - relative_version: i64, - ) -> Option>> { - let table = self.handle.table.read().unwrap(); - let Some(ret) = table.read_relative(primary_key, relative_version) else { - return None; - }; - - Some(filter_projected(ret, projected_columns_index)) - } - pub fn update(&mut self, primary_key: i64, columns: Vec>) -> bool { let mut table = self.handle.table.write().unwrap();