Merge pull request #2612 from TheBlueMatt/2023-09-namespace-split

Refer to top-level persistence namespaces as primary_namespace

Author: TheBlueMatt

lightning-background-processor/src/lib.rs

@@ -506,10 +506,10 @@ use core::task;
 /// # use lightning_background_processor::{process_events_async, GossipSync};
 /// # struct MyStore {}
 /// # impl lightning::util::persist::KVStore for MyStore {
-/// #     fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
-/// #     fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
-/// #     fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
-/// #     fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
+/// #     fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
+/// #     fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
+/// #     fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
+/// #     fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
 /// # }
 /// # struct MyEventHandler {}
 /// # impl MyEventHandler {
@@ -868,7 +868,10 @@ mod tests {
 	use lightning::util::config::UserConfig;
 	use lightning::util::ser::Writeable;
 	use lightning::util::test_utils;
-	use lightning::util::persist::{KVStore, CHANNEL_MANAGER_PERSISTENCE_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY, SCORER_PERSISTENCE_NAMESPACE, SCORER_PERSISTENCE_SUB_NAMESPACE, SCORER_PERSISTENCE_KEY};
+	use lightning::util::persist::{KVStore,
+		CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY,
+		NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY,
+		SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY};
 	use lightning_persister::fs_store::FilesystemStore;
 	use std::collections::VecDeque;
 	use std::{fs, env};
@@ -983,22 +986,22 @@ mod tests {
 	}
 
 	impl KVStore for Persister {
-		fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
-			self.kv_store.read(namespace, sub_namespace, key)
+		fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
+			self.kv_store.read(primary_namespace, secondary_namespace, key)
 		}
 
-		fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
-			if namespace == CHANNEL_MANAGER_PERSISTENCE_NAMESPACE &&
-				sub_namespace == CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE &&
+		fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
+			if primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE &&
+				secondary_namespace == CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE &&
 				key == CHANNEL_MANAGER_PERSISTENCE_KEY
 			{
 				if let Some((error, message)) = self.manager_error {
 					return Err(std::io::Error::new(error, message))
 				}
 			}
 
-			if namespace == NETWORK_GRAPH_PERSISTENCE_NAMESPACE &&
-				sub_namespace == NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE &&
+			if primary_namespace == NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE &&
+				secondary_namespace == NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE &&
 				key == NETWORK_GRAPH_PERSISTENCE_KEY
 			{
 				if let Some(sender) = &self.graph_persistence_notifier {
@@ -1013,24 +1016,24 @@ mod tests {
 				}
 			}
 
-			if namespace == SCORER_PERSISTENCE_NAMESPACE &&
-				sub_namespace == SCORER_PERSISTENCE_SUB_NAMESPACE &&
+			if primary_namespace == SCORER_PERSISTENCE_PRIMARY_NAMESPACE &&
+				secondary_namespace == SCORER_PERSISTENCE_SECONDARY_NAMESPACE &&
 				key == SCORER_PERSISTENCE_KEY
 			{
 				if let Some((error, message)) = self.scorer_error {
 					return Err(std::io::Error::new(error, message))
 				}
 			}
 
-			self.kv_store.write(namespace, sub_namespace, key, buf)
+			self.kv_store.write(primary_namespace, secondary_namespace, key, buf)
 		}
 
-		fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
-			self.kv_store.remove(namespace, sub_namespace, key, lazy)
+		fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
+			self.kv_store.remove(primary_namespace, secondary_namespace, key, lazy)
 		}
 
-		fn list(&self, namespace: &str, sub_namespace: &str) -> lightning::io::Result<Vec<String>> {
-			self.kv_store.list(namespace, sub_namespace)
+		fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> lightning::io::Result<Vec<String>> {
+			self.kv_store.list(primary_namespace, secondary_namespace)
 		}
 	}
 

lightning-persister/src/fs_store.rs

@@ -67,7 +67,7 @@ impl FilesystemStore {
 		}
 	}
 
-	fn get_dest_dir_path(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<PathBuf> {
+	fn get_dest_dir_path(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<PathBuf> {
 		let mut dest_dir_path = {
 			#[cfg(target_os = "windows")]
 			{
@@ -81,20 +81,20 @@ impl FilesystemStore {
 			}
 		};
 
-		dest_dir_path.push(namespace);
-		if !sub_namespace.is_empty() {
-			dest_dir_path.push(sub_namespace);
+		dest_dir_path.push(primary_namespace);
+		if !secondary_namespace.is_empty() {
+			dest_dir_path.push(secondary_namespace);
 		}
 
 		Ok(dest_dir_path)
 	}
 }
 
 impl KVStore for FilesystemStore {
-	fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
-		check_namespace_key_validity(namespace, sub_namespace, Some(key), "read")?;
+	fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
+		check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "read")?;
 
-		let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
+		let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
 		dest_file_path.push(key);
 
 		let mut buf = Vec::new();
@@ -114,10 +114,10 @@ impl KVStore for FilesystemStore {
 		Ok(buf)
 	}
 
-	fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
-		check_namespace_key_validity(namespace, sub_namespace, Some(key), "write")?;
+	fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
+		check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "write")?;
 
-		let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
+		let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
 		dest_file_path.push(key);
 
 		let parent_directory = dest_file_path
@@ -201,10 +201,10 @@ impl KVStore for FilesystemStore {
 		res
 	}
 
-	fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
-		check_namespace_key_validity(namespace, sub_namespace, Some(key), "remove")?;
+	fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
+		check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "remove")?;
 
-		let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
+		let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
 		dest_file_path.push(key);
 
 		if !dest_file_path.is_file() {
@@ -290,10 +290,10 @@ impl KVStore for FilesystemStore {
 		Ok(())
 	}
 
-	fn list(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<Vec<String>> {
-		check_namespace_key_validity(namespace, sub_namespace, None, "list")?;
+	fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<Vec<String>> {
+		check_namespace_key_validity(primary_namespace, secondary_namespace, None, "list")?;
 
-		let prefixed_dest = self.get_dest_dir_path(namespace, sub_namespace)?;
+		let prefixed_dest = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
 		let mut keys = Vec::new();
 
 		if !Path::new(&prefixed_dest).exists() {
@@ -320,17 +320,17 @@ impl KVStore for FilesystemStore {
 
 			let metadata = p.metadata()?;
 
-			// We allow the presence of directories in the empty namespace and just skip them.
+			// We allow the presence of directories in the empty primary namespace and just skip them.
 			if metadata.is_dir() {
 				continue;
 			}
 
 			// If we otherwise don't find a file at the given path something went wrong.
 			if !metadata.is_file() {
 				debug_assert!(false, "Failed to list keys of {}/{}: file couldn't be accessed.",
-					PrintableString(namespace), PrintableString(sub_namespace));
+					PrintableString(primary_namespace), PrintableString(secondary_namespace));
 				let msg = format!("Failed to list keys of {}/{}: file couldn't be accessed.",
-					PrintableString(namespace), PrintableString(sub_namespace));
+					PrintableString(primary_namespace), PrintableString(secondary_namespace));
 				return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 			}
 
@@ -342,17 +342,17 @@ impl KVStore for FilesystemStore {
 						}
 					} else {
 						debug_assert!(false, "Failed to list keys of {}/{}: file path is not valid UTF-8",
-							PrintableString(namespace), PrintableString(sub_namespace));
+							PrintableString(primary_namespace), PrintableString(secondary_namespace));
 						let msg = format!("Failed to list keys of {}/{}: file path is not valid UTF-8",
-							PrintableString(namespace), PrintableString(sub_namespace));
+							PrintableString(primary_namespace), PrintableString(secondary_namespace));
 						return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 					}
 				}
 				Err(e) => {
 					debug_assert!(false, "Failed to list keys of {}/{}: {}",
-						PrintableString(namespace), PrintableString(sub_namespace), e);
+						PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
 					let msg = format!("Failed to list keys of {}/{}: {}",
-						PrintableString(namespace), PrintableString(sub_namespace), e);
+						PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
 					return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 				}
 			}

lightning-persister/src/test_utils.rs

@@ -12,34 +12,35 @@ use std::panic::RefUnwindSafe;
 pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_store: &K) {
 	let data = [42u8; 32];
 
-	let namespace = "testspace";
-	let sub_namespace = "testsubspace";
+	let primary_namespace = "testspace";
+	let secondary_namespace = "testsubspace";
 	let key = "testkey";
 
 	// Test the basic KVStore operations.
-	kv_store.write(namespace, sub_namespace, key, &data).unwrap();
+	kv_store.write(primary_namespace, secondary_namespace, key, &data).unwrap();
 
-	// Test empty namespace/sub_namespace is allowed, but not empty namespace and non-empty
-	// sub-namespace, and not empty key.
+	// Test empty primary_namespace/secondary_namespace is allowed, but not empty primary_namespace
+	// and non-empty secondary_namespace, and not empty key.
 	kv_store.write("", "", key, &data).unwrap();
-	let res = std::panic::catch_unwind(|| kv_store.write("", sub_namespace, key, &data));
+	let res = std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, &data));
 	assert!(res.is_err());
-	let res = std::panic::catch_unwind(|| kv_store.write(namespace, sub_namespace, "", &data));
+	let res = std::panic::catch_unwind(|| kv_store.write(primary_namespace, secondary_namespace, "", &data));
 	assert!(res.is_err());
 
-	let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
+	let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
 	assert_eq!(listed_keys.len(), 1);
 	assert_eq!(listed_keys[0], key);
 
-	let read_data = kv_store.read(namespace, sub_namespace, key).unwrap();
+	let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap();
 	assert_eq!(data, &*read_data);
 
-	kv_store.remove(namespace, sub_namespace, key, false).unwrap();
+	kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap();
 
-	let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
+	let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
 	assert_eq!(listed_keys.len(), 0);
 
-	// Ensure we have no issue operating with namespace/sub_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN
+	// Ensure we have no issue operating with primary_namespace/secondary_namespace/key being
+	// KVSTORE_NAMESPACE_KEY_MAX_LEN
 	let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect();
 	kv_store.write(&max_chars, &max_chars, &max_chars, &data).unwrap();
 

lightning-persister/src/utils.rs

@@ -6,51 +6,53 @@ pub(crate) fn is_valid_kvstore_str(key: &str) -> bool {
 	key.len() <= KVSTORE_NAMESPACE_KEY_MAX_LEN && key.chars().all(|c| KVSTORE_NAMESPACE_KEY_ALPHABET.contains(c))
 }
 
-pub(crate) fn check_namespace_key_validity(namespace: &str, sub_namespace: &str, key: Option<&str>, operation: &str) -> Result<(), std::io::Error> {
+pub(crate) fn check_namespace_key_validity(
+	primary_namespace: &str, secondary_namespace: &str, key: Option<&str>, operation: &str)
+-> Result<(), std::io::Error> {
 	if let Some(key) = key {
 		if key.is_empty() {
 			debug_assert!(false, "Failed to {} {}/{}/{}: key may not be empty.", operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
 			let msg = format!("Failed to {} {}/{}/{}: key may not be empty.", operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
 			return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 		}
 
-		if namespace.is_empty() && !sub_namespace.is_empty() {
+		if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
 			debug_assert!(false,
-				"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
+				"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
 				operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
 			let msg = format!(
-				"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.", operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
+				"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.", operation,
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
 			return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 		}
 
-		if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) || !is_valid_kvstore_str(key) {
-			debug_assert!(false, "Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
+		if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) || !is_valid_kvstore_str(key) {
+			debug_assert!(false, "Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
 				operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
-			let msg = format!("Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
+			let msg = format!("Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
 				operation,
-				PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
+				PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
 			return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 		}
 	} else {
-		if namespace.is_empty() && !sub_namespace.is_empty() {
+		if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
 			debug_assert!(false,
-				"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
-				operation, PrintableString(namespace), PrintableString(sub_namespace));
+				"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
+				operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
 			let msg = format!(
-				"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
-				operation, PrintableString(namespace), PrintableString(sub_namespace));
+				"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
+				operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
 			return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 		}
-		if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) {
-			debug_assert!(false, "Failed to {} {}/{}: namespace and sub-namespace must be valid.",
-				operation, PrintableString(namespace), PrintableString(sub_namespace));
-			let msg = format!("Failed to {} {}/{}: namespace and sub-namespace must be valid.",
-				operation, PrintableString(namespace), PrintableString(sub_namespace));
+		if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) {
+			debug_assert!(false, "Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
+				operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
+			let msg = format!("Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
+				operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
 			return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 		}
 	}