feat: add timestamp rules and event boundary detection

Add is_timestamp field to Schema::Rule and add_timestamp_rule() method.
Add is_event_start field to Fragment, set to true when a timestamp rule
matches at byte offset 0 or immediately after a newline. This enables
downstream consumers to split multi-line log events.

Also add schema_add_timestamp_rule FFI and is_event_start to CLogFragment.

Author: Jack Luo

rust/include/log_mechanic.generated.hpp

@@ -66,17 +66,21 @@ CLogFragment() = default;
   const Capture *captures;
   /// Number of captures.
   size_t captures_count;
+  /// Whether this fragment starts a new log event (timestamp at start of line).
+  bool is_event_start;
 
   CLogFragment(size_t const& rule,
                const uint8_t *const& start,
                const uint8_t *const& end,
                const Capture *const& captures,
-               size_t const& captures_count)
+               size_t const& captures_count,
+               bool const& is_event_start)
     : rule(rule),
       start(start),
       end(end),
       captures(captures),
-      captures_count(captures_count)
+      captures_count(captures_count),
+      is_event_start(is_event_start)
   {}
 
 };
@@ -98,6 +102,10 @@ CLogFragment clp_log_mechanic_lexer_next_fragment(Lexer *lexer, CStringView inpu
 
 bool clp_log_mechanic_schema_add_rule(Schema *schema, CStringView name, CStringView pattern);
 
+bool clp_log_mechanic_schema_add_timestamp_rule(Schema *schema,
+                                                CStringView name,
+                                                CStringView pattern);
+
 void clp_log_mechanic_schema_delete(Box<Schema> schema);
 
 Box<Schema> clp_log_mechanic_schema_new();

rust/src/c_interface.rs

@@ -30,6 +30,8 @@ pub struct CLogFragment<'schema, 'input, 'buffer> {
 	pub captures: *const Capture<'schema, 'input>,
 	/// Number of captures.
 	pub captures_count: usize,
+	/// Whether this fragment starts a new log event (timestamp at start of line).
+	pub is_event_start: bool,
 	/// Indicates that `captures` points into (borrows from) some external `'buffer`.
 	pub _captures_lifetime: PhantomData<&'buffer [Capture<'schema, 'input>]>,
 }
@@ -77,6 +79,25 @@ unsafe extern "C" fn clp_log_mechanic_schema_add_rule(
 	true
 }
 
+#[unsafe(no_mangle)]
+unsafe extern "C" fn clp_log_mechanic_schema_add_timestamp_rule(
+	schema: &mut Schema,
+	name: CStringView<'_>,
+	pattern: CStringView<'_>,
+) -> bool {
+	let Ok(name) = name.as_utf8() else {
+		return false;
+	};
+	let Ok(pattern) = pattern.as_utf8() else {
+		return false;
+	};
+	let Ok(regex) = Regex::from_pattern(pattern) else {
+		return false;
+	};
+	schema.add_timestamp_rule(name, regex);
+	true
+}
+
 #[unsafe(no_mangle)]
 unsafe extern "C" fn clp_log_mechanic_schema_rule_count(schema: &Schema) -> usize {
 	schema.rules().len()
@@ -121,6 +142,7 @@ unsafe extern "C" fn clp_log_mechanic_lexer_next_fragment<'schema, 'lexer, 'inpu
 		end: fragment.lexeme.as_bytes().as_ptr_range().end,
 		captures: fragment.captures.as_ptr(),
 		captures_count: fragment.captures.len(),
+		is_event_start: fragment.is_event_start,
 		_captures_lifetime: PhantomData,
 	}
 }

rust/src/lexer.rs

@@ -15,6 +15,7 @@ pub struct Fragment<'schema, 'input, 'buffer> {
 	pub rule: usize,
 	pub lexeme: &'input str,
 	pub captures: &'buffer [Capture<'schema, 'input>],
+	pub is_event_start: bool,
 }
 
 #[repr(C)]
@@ -54,10 +55,14 @@ impl<'schema, 'input> Lexer<'schema, 'input> {
 							continue;
 						}
 					}
+					let at_line_start = static_text_end == 0
+						|| input.as_bytes().get(static_text_end - 1) == Some(&b'\n');
+					let is_event_start = at_line_start && self.schema.rules()[rule].is_timestamp;
 					return Fragment {
 						rule,
 						lexeme,
 						captures: &self.captures_buffer,
+						is_event_start,
 					};
 				},
 				Err(consumed) => {
@@ -71,6 +76,7 @@ impl<'schema, 'input> Lexer<'schema, 'input> {
 			rule: 0,
 			lexeme: &input[*pos..],
 			captures: &[],
+			is_event_start: false,
 		}
 	}
 
@@ -113,6 +119,34 @@ mod test {
 		assert_eq!(pos, input.len());
 	}
 
+	#[test]
+	fn event_boundary() {
+		let mut schema: Schema = Schema::new();
+		schema.set_delimiters(" \n");
+		schema.add_timestamp_rule("ts", Regex::from_pattern("[0-9][0-9][0-9][0-9]\\-[0-9][0-9]\\-[0-9][0-9]").unwrap());
+		schema.add_rule("word", Regex::from_pattern("[a-zA-Z]+").unwrap());
+
+		let mut lexer: Lexer<'_, '_> = Lexer::new(&schema).unwrap();
+		let input: &str = "2024-01-15 hello\n2024-01-16 world\n";
+		let mut pos: usize = 0;
+
+		let f1 = lexer.next_fragment(input, &mut pos);
+		assert_eq!(f1.rule, 1); // ts
+		assert!(f1.is_event_start); // at start of input
+
+		let f2 = lexer.next_fragment(input, &mut pos);
+		assert_eq!(f2.rule, 2); // word "hello"
+		assert!(!f2.is_event_start);
+
+		let f3 = lexer.next_fragment(input, &mut pos);
+		assert_eq!(f3.rule, 1); // ts
+		assert!(f3.is_event_start); // after \n
+
+		let f4 = lexer.next_fragment(input, &mut pos);
+		assert_eq!(f4.rule, 2); // word "world"
+		assert!(!f4.is_event_start);
+	}
+
 	#[test]
 	fn capture_boundaries() {
 		let mut schema: Schema = Schema::new();

rust/src/schema.rs

@@ -11,6 +11,7 @@ pub struct Rule {
 	pub idx: usize,
 	pub name: String,
 	pub regex: Regex,
+	pub is_timestamp: bool,
 }
 
 impl Schema {
@@ -22,6 +23,7 @@ impl Schema {
 				idx: 0,
 				name: "static".to_owned(),
 				regex: Self::pattern_for_delimiters(Self::DEFAULT_DELIMITERS),
+				is_timestamp: false,
 			}],
 			delimiters: Self::DEFAULT_DELIMITERS.to_owned(),
 		}
@@ -41,6 +43,20 @@ impl Schema {
 			idx,
 			name: name.into(),
 			regex,
+			is_timestamp: false,
+		});
+	}
+
+	pub fn add_timestamp_rule<LikeString>(&mut self, name: LikeString, regex: Regex)
+	where
+		LikeString: Into<String>,
+	{
+		let idx: usize = self.rules.len();
+		self.rules.push(Rule {
+			idx,
+			name: name.into(),
+			regex,
+			is_timestamp: true,
 		});
 	}