More regex parser cleanup and improved diagnostics.

Author: Adrian

rust/include/log_mechanic.generated.hpp

@@ -56,10 +56,15 @@ struct CLogFragment {
 CLogFragment() = default;
 
 // Generated
+  /// `0` iff no variable found (static text until end of input).
   size_t rule;
+  /// Start of variable (if found).
   const uint8_t *start;
+  /// End of variable (if found).
   const uint8_t *end;
+  /// Pointer to an array of captures (if variable found).
   const Capture *captures;
+  /// Number of captures.
   size_t captures_count;
 
   CLogFragment(size_t const& rule,
@@ -87,7 +92,7 @@ Box<Lexer> clp_log_mechanic_lexer_new(const Schema *schema);
 ///
 /// The returned [`CLogFragment`] includes a hidden exclusive borrow of `lexer`
 /// (it contains a pointer into an interal buffer of `lexer`),
-/// so it is nolonger valid/you must not use it after a subsequent exclusive borrow of `lexer`
+/// so it is nolonger valid (you must not touch it) after any subsequent borrow of `lexer`
 /// (i.e. this borrow has ended).
 CLogFragment clp_log_mechanic_lexer_next_fragment(Lexer *lexer, CStringView input, size_t *pos);
 

rust/src/c_interface.rs

@@ -19,12 +19,19 @@ pub struct CSlice<'lifetime, T> {
 pub type CStringView<'lifetime> = CSlice<'lifetime, c_char>;
 
 #[repr(C)]
-pub struct CLogFragment<'schema, 'input> {
+pub struct CLogFragment<'schema, 'input, 'buffer> {
+	/// `0` iff no variable found (static text until end of input).
 	pub rule: usize,
+	/// Start of variable (if found).
 	pub start: *const u8,
+	/// End of variable (if found).
 	pub end: *const u8,
+	/// Pointer to an array of captures (if variable found).
 	pub captures: *const Capture<'schema, 'input>,
+	/// Number of captures.
 	pub captures_count: usize,
+	/// Indicates that `captures` points into (borrows from) some external `'buffer`.
+	pub _captures_lifetime: PhantomData<&'buffer [Capture<'schema, 'input>]>,
 }
 
 #[unsafe(no_mangle)]
@@ -69,21 +76,22 @@ unsafe extern "C" fn clp_log_mechanic_lexer_delete<'a>(lexer: Box<Lexer<'_, 'a>>
 ///
 /// The returned [`CLogFragment`] includes a hidden exclusive borrow of `lexer`
 /// (it contains a pointer into an interal buffer of `lexer`),
-/// so it is nolonger valid/you must not use it after a subsequent exclusive borrow of `lexer`
+/// so it is nolonger valid (you must not touch it) after any subsequent borrow of `lexer`
 /// (i.e. this borrow has ended).
 #[unsafe(no_mangle)]
 unsafe extern "C" fn clp_log_mechanic_lexer_next_fragment<'schema, 'lexer, 'input>(
 	lexer: &'lexer mut Lexer<'schema, 'input>,
 	input: CStringView<'input>,
 	pos: &mut usize,
-) -> CLogFragment<'schema, 'input> {
+) -> CLogFragment<'schema, 'input, 'lexer> {
 	let fragment: Fragment<'_, '_, '_> = lexer.next_fragment(input.as_utf8().unwrap(), pos);
 	CLogFragment {
 		rule: fragment.rule,
 		start: fragment.lexeme.as_bytes().as_ptr_range().start,
 		end: fragment.lexeme.as_bytes().as_ptr_range().end,
 		captures: fragment.captures.as_ptr(),
 		captures_count: fragment.captures.len(),
+		_captures_lifetime: PhantomData,
 	}
 }
 

rust/src/lexer.rs

@@ -75,7 +75,6 @@ impl<'schema, 'input> Lexer<'schema, 'input> {
 	}
 
 	fn glob_static_text(&self, input: &'input str, pos: &mut usize) {
-		// TODO use automata?
 		for ch in input[*pos..].chars() {
 			*pos += ch.len_utf8();
 			if self.is_delimiter(ch) {
@@ -85,7 +84,9 @@ impl<'schema, 'input> Lexer<'schema, 'input> {
 	}
 
 	fn is_delimiter(&self, ch: char) -> bool {
-		self.schema.delimiters.contains(ch)
+		// XXX: Needs benchmarking, but given a small set of delimiter characters,
+		// a linear search may be faster than a theoretically $O(1)$ hash lookup.
+		self.schema.delimiters().contains(ch)
 	}
 }
 
@@ -104,11 +105,17 @@ mod test {
 		let mut lexer: Lexer<'_, '_> = Lexer::new(&schema).unwrap();
 		let input: &str = "hello world goodbye hello world  goodbye  ";
 		let mut pos: usize = 0;
-		assert_eq!(lexer.next_fragment(input, &mut pos).rule, 1);
-		assert_eq!(lexer.next_fragment(input, &mut pos).rule, 2);
-		assert_eq!(lexer.next_fragment(input, &mut pos).rule, 1);
-		assert_eq!(lexer.next_fragment(input, &mut pos).rule, 2);
-		assert_eq!(lexer.next_fragment(input, &mut pos).rule, 0);
+		assert_eq!(lexer.next_fragment(input, &mut pos).projection(), (1, "hello world"));
+		assert_eq!(lexer.next_fragment(input, &mut pos).projection(), (2, "goodbye"));
+		assert_eq!(lexer.next_fragment(input, &mut pos).projection(), (1, "hello world"));
+		assert_eq!(lexer.next_fragment(input, &mut pos).projection(), (2, "goodbye"));
+		assert_eq!(lexer.next_fragment(input, &mut pos).projection(), (0, ""));
 		assert_eq!(pos, input.len());
 	}
+
+	impl<'schema, 'input, 'buffer> Fragment<'schema, 'input, 'buffer> {
+		fn projection(&self) -> (usize, &'input str) {
+			(self.rule, self.lexeme)
+		}
+	}
 }