(More) proper(?) handling of static text.

- Generally cleaner dfa and lexer simulation interface/usage.

Author: Adrian

rust/Cargo.toml

@@ -10,5 +10,4 @@ crate-type = ["lib", "staticlib"]
 cbindgen = "0.29"
 
 [dependencies]
-libc = "0.2"
 nom = "8"

rust/Makefile

@@ -5,8 +5,11 @@ default:
 # https://www.gnu.org/software/make/manual/html_node/Force-Targets.html
 FORCE:
 
-examples:
+examples: target/debug/liblog_mechanic.a
 	$(MAKE) --directory $@
 
 include/log_mechanic.generated.hpp: FORCE
 	cbindgen --config cbindgen.toml --crate log-mechanic > $@
+
+target/debug/liblog_mechanic.a: FORCE
+	cargo build

rust/cbindgen.toml

@@ -75,6 +75,7 @@ renaming_overrides_prefixing = false
 
 [export.pre_body]
 "CSlice" = """
+// Custom
 CSlice(std::string_view const& view)
 	requires std::is_same_v<T, char>
 	: CSlice(view.data(), view.size())
@@ -84,10 +85,15 @@ CSlice(char const* c_str)
 	requires std::is_same_v<T, char>
 	: CSlice(std::string_view { c_str })
 	{}
+
+// Generated
 """
 
-"LogComponent" = """
-LogComponent() = default;
+"CLogFragment" = """
+// Custom
+CLogFragment() = default;
+
+// Generated
 """
 
 [export.body]

rust/examples/cpp_usage/usage.cpp

@@ -6,18 +6,17 @@
 using namespace clp::log_mechanic;
 
 int main() {
-	Schema* schema { clp_log_mechanic_schema_new() };
+	Box<Schema> schema { clp_log_mechanic_schema_new() };
 
 	clp_log_mechanic_schema_add_rule(schema, "hello", "abc|def");
 
-	Lexer* lexer { clp_log_mechanic_lexer_new(schema) };
+	Box<Lexer> lexer { clp_log_mechanic_lexer_new(schema) };
 
-	LogComponent component {};
 	size_t pos { 0 };
 
-	clp_log_mechanic_lexer_next_token(lexer, "def", &pos, &component);
-	assert(component.rule == 1);
-	assert(component.start + 3 == component.end);
+	CLogFragment fragment { clp_log_mechanic_lexer_next_fragment(lexer, "def", &pos) };
+	assert(fragment.rule == 1);
+	assert(fragment.start + 3 == fragment.end);
 
 	printf("good!\n");
 

rust/include/log_mechanic.generated.hpp

@@ -14,6 +14,7 @@ struct Schema;
 
 template<typename T>
 struct CSlice {
+// Custom
 CSlice(std::string_view const& view)
 	requires std::is_same_v<T, char>
 	: CSlice(view.data(), view.size())
@@ -23,6 +24,8 @@ CSlice(char const* c_str)
 	requires std::is_same_v<T, char>
 	: CSlice(std::string_view { c_str })
 	{}
+
+// Generated
   const T *pointer;
   size_t length;
 
@@ -38,28 +41,28 @@ using CStringView = CSlice<char>;
 
 struct Capture {
   CStringView name;
-  const uint8_t *start;
-  const uint8_t *end;
+  CStringView lexeme;
 
   Capture(CStringView const& name,
-          const uint8_t *const& start,
-          const uint8_t *const& end)
+          CStringView const& lexeme)
     : name(name),
-      start(start),
-      end(end)
+      lexeme(lexeme)
   {}
 
 };
 
-struct LogComponent {
-LogComponent() = default;
+struct CLogFragment {
+// Custom
+CLogFragment() = default;
+
+// Generated
   size_t rule;
   const uint8_t *start;
   const uint8_t *end;
   const Capture *captures;
   size_t captures_count;
 
-  LogComponent(size_t const& rule,
+  CLogFragment(size_t const& rule,
                const uint8_t *const& start,
                const uint8_t *const& end,
                const Capture *const& captures,
@@ -80,10 +83,13 @@ void clp_log_mechanic_lexer_delete(Box<Lexer> lexer);
 
 Box<Lexer> clp_log_mechanic_lexer_new(const Schema *schema);
 
-bool clp_log_mechanic_lexer_next_token(Lexer *lexer,
-                                       CStringView input,
-                                       size_t *pos,
-                                       LogComponent *log_component);
+/// Very unsafe!
+///
+/// The returned [`CLogFragment`] includes a hidden exclusive borrow of `lexer`
+/// (it contains a pointer into an interal buffer of `lexer`),
+/// so it is nolonger valid/you must not use it after a subsequent exclusive borrow of `lexer`
+/// (i.e. this borrow has ended).
+CLogFragment clp_log_mechanic_lexer_next_fragment(Lexer *lexer, CStringView input, size_t *pos);
 
 void clp_log_mechanic_schema_add_rule(Schema *schema, CStringView name, CStringView pattern);
 

rust/include/log_mechanic.hpp

@@ -4,7 +4,6 @@
 namespace clp::log_mechanic {
 
 	template<typename T> using Box = T*;
-
 }
 
 #include "log_mechanic.generated.hpp"

rust/src/c_interface.rs

@@ -2,12 +2,14 @@ use std::ffi::c_char;
 use std::marker::PhantomData;
 use std::str::Utf8Error;
 
+use crate::lexer::Capture;
+use crate::lexer::Fragment;
 use crate::lexer::Lexer;
-use crate::lexer::LogComponent;
 use crate::regex::Regex;
 use crate::schema::Schema;
 
 #[repr(C)]
+#[derive(Debug, Clone, Copy)]
 pub struct CSlice<'lifetime, T> {
 	pointer: *const T,
 	length: usize,
@@ -16,14 +18,14 @@ pub struct CSlice<'lifetime, T> {
 
 pub type CStringView<'lifetime> = CSlice<'lifetime, c_char>;
 
-// #[unsafe(no_mangle)]
-// unsafe extern "C" fn clp_log_mechanic_c_string_view<'unknown>(pointer: *const c_char) -> CSlice<'unknown, c_char> {
-// 	CSlice {
-// 		pointer,
-// 		length: unsafe { libc::strlen(pointer) },
-// 		_lifetime: PhantomData,
-// 	}
-// }
+#[repr(C)]
+pub struct CLogFragment<'schema, 'input> {
+	pub rule: usize,
+	pub start: *const u8,
+	pub end: *const u8,
+	pub captures: *const Capture<'schema, 'input>,
+	pub captures_count: usize,
+}
 
 #[unsafe(no_mangle)]
 extern "C" fn clp_log_mechanic_schema_new() -> Box<Schema> {
@@ -53,28 +55,35 @@ unsafe extern "C" fn clp_log_mechanic_schema_add_rule(
 }
 
 #[unsafe(no_mangle)]
-unsafe extern "C" fn clp_log_mechanic_lexer_new<'schema>(schema: &'schema Schema) -> Box<Lexer<'schema>> {
-	let lexer: Lexer<'_> = Lexer::new(schema).unwrap();
+unsafe extern "C" fn clp_log_mechanic_lexer_new<'schema, 'a>(schema: &'schema Schema) -> Box<Lexer<'schema, 'a>> {
+	let lexer: Lexer<'_, '_> = Lexer::new(schema).unwrap();
 	Box::new(lexer)
 }
 
 #[unsafe(no_mangle)]
-unsafe extern "C" fn clp_log_mechanic_lexer_delete(lexer: Box<Lexer<'_>>) {
+unsafe extern "C" fn clp_log_mechanic_lexer_delete<'a>(lexer: Box<Lexer<'_, 'a>>) {
 	std::mem::drop(lexer);
 }
 
+/// Very unsafe!
+///
+/// The returned [`CLogFragment`] includes a hidden exclusive borrow of `lexer`
+/// (it contains a pointer into an interal buffer of `lexer`),
+/// so it is nolonger valid/you must not use it after a subsequent exclusive borrow of `lexer`
+/// (i.e. this borrow has ended).
 #[unsafe(no_mangle)]
-unsafe extern "C" fn clp_log_mechanic_lexer_next_token<'schema, 'lexer>(
-	lexer: &'lexer mut Lexer<'schema>,
-	input: CStringView<'_>,
+unsafe extern "C" fn clp_log_mechanic_lexer_next_fragment<'schema, 'lexer, 'input>(
+	lexer: &'lexer mut Lexer<'schema, 'input>,
+	input: CStringView<'input>,
 	pos: &mut usize,
-	log_component: &mut LogComponent<'schema, 'lexer>,
-) -> bool {
-	if let Some(component) = lexer.next_token(input.as_utf8().unwrap(), pos) {
-		*log_component = component;
-		true
-	} else {
-		false
+) -> CLogFragment<'schema, 'input> {
+	let fragment: Fragment<'_, '_, '_> = lexer.next_fragment(input.as_utf8().unwrap(), pos);
+	CLogFragment {
+		rule: fragment.rule,
+		start: fragment.lexeme.as_bytes().as_ptr_range().start,
+		end: fragment.lexeme.as_bytes().as_ptr_range().end,
+		captures: fragment.captures.as_ptr(),
+		captures_count: fragment.captures.len(),
 	}
 }
 
@@ -98,3 +107,45 @@ impl<'lifetime> CStringView<'lifetime> {
 		str::from_utf8(bytes)
 	}
 }
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use crate::regex::Regex;
+
+	#[test]
+	fn basic() {
+		let mut schema: Schema = Schema::new();
+		schema.set_delimiters(" ");
+		schema.add_rule("hello", Regex::from_pattern("hello world").unwrap());
+		schema.add_rule("bye", Regex::from_pattern("goodbye").unwrap());
+
+		let mut lexer: Lexer<'_, '_> = Lexer::new(&schema).unwrap();
+		let input: CStringView<'_> = CStringView::from_utf8("hello world goodbye hello world  goodbye  ");
+		let mut pos: usize = 0;
+
+		unsafe {
+			assert_eq!(
+				clp_log_mechanic_lexer_next_fragment(&mut lexer, input, &mut pos).rule,
+				1
+			);
+			assert_eq!(
+				clp_log_mechanic_lexer_next_fragment(&mut lexer, input, &mut pos).rule,
+				2
+			);
+			assert_eq!(
+				clp_log_mechanic_lexer_next_fragment(&mut lexer, input, &mut pos).rule,
+				1
+			);
+			assert_eq!(
+				clp_log_mechanic_lexer_next_fragment(&mut lexer, input, &mut pos).rule,
+				2
+			);
+			assert_eq!(
+				clp_log_mechanic_lexer_next_fragment(&mut lexer, input, &mut pos).rule,
+				0
+			);
+		}
+		assert_eq!(pos, input.length);
+	}
+}

rust/src/dfa.rs

@@ -23,6 +23,12 @@ pub struct Dfa<'schema> {
 	number_of_registers: usize,
 }
 
+#[derive(Debug, Clone)]
+pub struct GotRule<'input> {
+	pub rule: usize,
+	pub lexeme: &'input str,
+}
+
 #[derive(Debug)]
 struct DfaState<'schema> {
 	kernel: Kernel<'schema>,
@@ -98,14 +104,14 @@ impl<'schema> Dfa<'schema> {
 		self.simulate_with_captures(input, |name, lexeme| {
 			println!("- captured {name}, {lexeme}");
 		})
-		.is_some()
+		.is_ok()
 	}
 
 	pub fn simulate_with_captures<'input, F>(
 		&self,
 		input: &'input str,
 		mut on_capture: F,
-	) -> Option<(usize, &'input str)>
+	) -> Result<GotRule<'input>, usize>
 	where
 		F: FnMut(&'schema str, &'input str),
 	{
@@ -117,6 +123,8 @@ impl<'schema> Dfa<'schema> {
 
 		let mut maybe_backup: Option<(usize, usize, Vec<usize>)> = None;
 
+		let mut consumed: usize = 0;
+
 		for (i, ch) in input.char_indices() {
 			println!("=== step {i} (state {current_state}), ch {ch:?} ({})", u32::from(ch));
 			/*
@@ -130,6 +138,7 @@ impl<'schema> Dfa<'schema> {
 			}
 			*/
 			if let Some(transition) = self.states[current_state].transitions.lookup(u32::from(ch)) {
+				consumed = i;
 				current_state = transition.destination;
 				self.apply_operations(
 					&mut registers,
@@ -138,22 +147,20 @@ impl<'schema> Dfa<'schema> {
 					&transition.operations,
 					&self.states[current_state].tag_for_register,
 				);
+				if self.states[current_state].is_final {
+					maybe_backup = Some((consumed, current_state, registers.clone()));
+				}
 			} else {
 				break;
 			}
-			if self.states[current_state].is_final {
-				maybe_backup = Some((i, current_state, registers.clone()));
-			}
 		}
 
-		println!("ended at {current_state}");
-		let (consumed, state, mut registers): (usize, usize, Vec<usize>) = maybe_backup?;
-		println!("had final at {state}");
+		let (consumed, state, mut registers): (usize, usize, Vec<usize>) = maybe_backup.ok_or(consumed)?;
 
 		self.apply_operations(
 			&mut registers,
 			&mut prefix_tree,
-			consumed + 1,
+			consumed,
 			&self.states[state].final_operations,
 			&self.states[state].tag_for_register,
 		);
@@ -188,11 +195,14 @@ impl<'schema> Dfa<'schema> {
 				maybe_rule = Some(var.rule);
 			}
 			for (&i, &j) in std::iter::zip(starts.iter(), ends.iter()) {
-				on_capture(var.name, &input[i..j]);
+				on_capture(var.name, &input[i..=j]);
 			}
 		}
 
-		return Some((maybe_rule.unwrap(), &input[..consumed + 1]));
+		Ok(GotRule {
+			rule: maybe_rule.unwrap(),
+			lexeme: &input[..=consumed],
+		})
 	}
 
 	fn apply_operations(