Big chungus

Author: Adrian

rust/.gitignore

@@ -1,5 +1,7 @@
 /target
 
+__pycache__/
+
 /examples/**/*.o
 
 *.vim

rust/Cargo.toml

@@ -4,10 +4,20 @@ version = "0.1.0"
 edition = "2024"
 
 [lib]
-crate-type = ["lib", "staticlib", "cdylib"]
+crate-type = [
+	"lib",
+	"staticlib",
+	"cdylib"
+]
 
 [features]
-python = []
+default = [
+	"python",
+]
+python = [
+	"pyo3/extension-module",
+	"pyo3/macros",
+]
 
 [build-dependencies]
 cbindgen = "0.29"
@@ -18,3 +28,4 @@ tracing = "0.1"
 
 [dependencies.pyo3]
 version = "0.27"
+default-features = false

rust/Makefile

@@ -12,4 +12,4 @@ include/log_mechanic.generated.hpp: FORCE
 	cbindgen --config cbindgen.toml --crate log-mechanic > $@
 
 target/debug/liblog_mechanic.a: FORCE
-	cargo build
+	cargo build --no-default-features

rust/cbindgen.toml

@@ -62,7 +62,10 @@ include = [
 	"type_traits",
 	"string_view",
 ]
-exclude = ["Box"]
+exclude = [
+	"Box",
+	"Option",
+]
 # prefix = "CAPI_"
 item_types = []
 renaming_overrides_prefixing = false

rust/examples/cpp_usage/usage.cpp

@@ -1,5 +1,6 @@
 #include <cassert>
 #include <cstdio>
+#include <cstring>
 
 #include "log_mechanic.hpp"
 
@@ -10,17 +11,22 @@ int main() {
 
 	clp_log_mechanic_schema_add_rule(schema, "hello", "abc|def");
 
-	Box<Lexer> lexer { clp_log_mechanic_lexer_new(schema) };
+	Box<Parser> parser { clp_log_mechanic_parser_new(schema) };
 
 	size_t pos { 0 };
 
-	CLogFragment fragment { clp_log_mechanic_lexer_next_fragment(lexer, "def", &pos, nullptr, nullptr) };
-	assert(fragment.rule == 1);
-	assert(fragment.start + 3 == fragment.end);
+	char const* input = "def";
+
+	Box<LogEvent> event { clp_log_mechanic_parser_next_event(parser, input, &pos) };
+	assert(clp_log_mechanic_event_token_count(event) == 1);
+	Token const* token { clp_log_mechanic_event_token(event, 0) };
+	assert(clp_log_mechanic_event_token_rule(token) == 1);
+	assert(pos == strlen(input));
 
 	printf("good!\n");
 
-	clp_log_mechanic_lexer_delete(lexer);
+	clp_log_mechanic_event_delete(event);
+	clp_log_mechanic_parser_delete(parser);
 	clp_log_mechanic_schema_delete(schema);
 
 	return 0;

rust/examples/python_usage/extract_logtypes.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+from schema import parser
+
+log_file = 'cassandra-node2.log'
+
+with open(log_file) as f:
+	contents = f.read()
+
+	parser.set_input_stream(contents)
+
+	lines = 0
+	log_types = {}
+	logs = []
+
+	while True:
+		event = parser.next_log_event()
+
+		if event is None:
+			break
+
+		template = ""
+		variables = []
+
+		for token in event.tokens:
+			if token.rule is None:
+				template += token.text
+			else:
+				template += f"%{token.rule}%"
+				variables.append((token.text, token.captures))
+
+		if template in log_types:
+			x = log_types[template]
+		else:
+			x = len(log_types)
+			log_types[template] = x
+
+		logs.append((x, variables))
+
+templates = list(log_types.items())
+templates.sort(key= lambda x: x[1])
+
+for i in range(len(templates)):
+	print(f"{i}. {templates[i]}")
+
+print(f"{len(logs)} logs, {len(log_types)} log types")

rust/examples/python_usage/schema.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+from logmech import ReaderParser
+from pattern import PATTERN
+
+parser = ReaderParser()
+
+# Custom delimiters can be specified, though the default is usually sufficient
+parser.set_delimiters(" \t\r\n:,!;%@/()[].")
+
+# Step 1 - Timestamp pattern: Extract full HH:MM:SS as components
+parser.add_variable_pattern("TIMESTAMP", rf"(?<hour>\d{{2}}):(?<minute>\d{{2}}):(?<second>\d{{2}})")
+
+# Step 2 - Verbosity level: INFO, WARN, ERROR
+parser.add_variable_pattern("LEVEL", rf"(?<level>(INFO)|(WARN)|(ERROR))")
+
+# Step 3 - Java exception pattern
+parser.add_variable_pattern(
+	"SYSTEM_EXCEPTION",
+	rf"(?<system_exception_type>({PATTERN.JAVA_PACKAGE_SEGMENT})+[{PATTERN.JAVA_IDENTIFIER_CHARSET}]*Exception): "
+	rf"(?<system_exception_msg>{PATTERN.LOG_LINE})"
+)
+
+# Stack trace patterns - simplified to avoid issues
+parser.add_variable_pattern(
+	"SYSTEM_STACK_TRACE",
+	rf"\s+at (?<class_method>[a-zA-Z0-9_$\.]+)\((?<source_file>[a-zA-Z0-9_]+\.(java|kt|scala)):(?<line_num>\d+)\)"
+)
+
+# Stack trace with jar info
+parser.add_variable_pattern(
+	"STACK_WITH_JAR",
+	rf"\s+at (?<class_method>[a-zA-Z0-9_$\.]+)\((?<source_file>[a-zA-Z0-9_]+\.java):(?<line_num>\d+)\) ~?\[(?<jar>[^\]]+\.jar):(?<version>[^\]]+)\]"
+)
+
+# Stack trace with na: prefix
+parser.add_variable_pattern(
+	"STACK_WITH_NA",
+	rf"\s+at (?<class_method>[a-zA-Z0-9_$\.]+)\((?<source_file>[a-zA-Z0-9_]+\.java):(?<line_num>\d+)\) ~?\[na:(?<version>\d+\.\d+[\._]\d+)\]"
+)
+
+# Cassandra-specific patterns
+parser.add_variable_pattern("STREAM_ID", rf"Stream #(?<stream_id>{PATTERN.UUID})")
+parser.add_variable_pattern("HINT_FILE", rf"(?<hint_file>{PATTERN.UUID}\-\d+\-\d+\.hints)")
+parser.add_variable_pattern("KEYSPACE_TABLE", rf"Initializing (?<keyspace>[a-z0-9_]+)\.(?<table>[a-z0-9_]+)")
+parser.add_variable_pattern("CASSANDRA_HOST", rf"cassandra\-(?<hostname>[a-z0-9\-]+)")
+parser.add_variable_pattern("BOOTSTRAP_TOKEN", rf"tokens \[(?<tokens>\-?\d+(, \-?\d+)*)\]")
+
+# Memory patterns
+parser.add_variable_pattern("MEMORY_MB", rf"(?<memory>{PATTERN.INT})MB")
+parser.add_variable_pattern("MEMORY_BYTES", rf"(?<bytes>\d+)\((?<kb>\d+)K\)")
+parser.add_variable_pattern("MEMORY_TYPE", rf"Global memtable (?<memtype>(on\-heap)|(off\-heap)) threshold")
+
+# Duration
+parser.add_variable_pattern("DURATION_MS", rf"(?<duration>{PATTERN.INT})\s*ms")
+
+# Netty channel patterns
+parser.add_variable_pattern("NETTY_CHANNEL_FULL", rf"channel = \[id: (?<channel_id>0x[a-f0-9]+), /(?<src_ip>{PATTERN.IPV4}):(?<src_port>{PATTERN.PORT}) =\> /(?<dst_ip>{PATTERN.IPV4}):(?<dst_port>{PATTERN.PORT})\]")
+parser.add_variable_pattern("NETTY_CHANNEL_SHORT", rf"channel = \[id: (?<channel_id>0x[a-f0-9]+), L:/(?<local_ip>{PATTERN.IPV4})")
+
+# IP patterns
+parser.add_variable_pattern("HANDSHAKING_IP", rf"Handshaking version with (?<hostname>[\w\-]+)/(?<ip>{PATTERN.IPV4})")
+parser.add_variable_pattern("SESSION_WITH_IP", rf"Session with /(?<ip>{PATTERN.IPV4})")
+parser.add_variable_pattern("STREAMING_TO_IP", rf"streaming to /(?<ip>{PATTERN.IPV4})")
+parser.add_variable_pattern("CQL_LISTENING", rf"Starting listening for CQL clients on /(?<ip>{PATTERN.IPV4}):(?<port>{PATTERN.PORT})")
+
+# Hinted handoff
+parser.add_variable_pattern("HANDOFF_FINISHED", rf"Finished hinted handoff of file (?<file>{PATTERN.UUID}\-\d+\-\d+\.hints) to endpoint /(?<ip>{PATTERN.IPV4}): (?<uuid>{PATTERN.UUID})")
+
+# Directory paths
+parser.add_variable_pattern("CASSANDRA_DIR", rf"Directory /var/lib/cassandra/(?<dirtype>(commitlog)|(data)|(saved_caches)|(hints))")
+
+# CompilerOracle patterns
+parser.add_variable_pattern("ORACLE_METHOD", rf"CompilerOracle: (?<action>(inline)|(dontinline)) (?<classname>[a-zA-Z0-9_/$]+)\.(?<method>\w+)")
+parser.add_variable_pattern("ORACLE_VARIANT_METHOD", rf"org/apache/cassandra/db/transform/StoppingTransformation\.(?<stopmethod>(stop)|(stopInPartition))")
+parser.add_variable_pattern("MEMORY_CLASS_METHOD", rf"org/apache/cassandra/io/util/(?<memclass>(Memory)|(SafeMemory))\.checkBounds")
+
+# Status patterns
+parser.add_variable_pattern("JOINING_STATUS", rf"JOINING: (?<status>(schema)|(calculation)) complete")
+
+# Thread names
+parser.add_variable_pattern("THREAD_NAME", rf"Thread\[(?<thread>[^\]]+)\]")
+
+# Unknown column
+parser.add_variable_pattern("UNKNOWN_COLUMN", rf"Unknown column (?<column>\w+)")
+
+# Frames omitted
+parser.add_variable_pattern("FRAMES_OMITTED", rf"\.\.\. (?<frames>\d+) common frames omitted")
+
+# General paths
+parser.add_variable_pattern("PATH", rf"(?<path>/[\w/\-\.]+)")
+parser.add_variable_pattern("FILE_PATH", rf"(?<filepath>[^\s]+\.(jar|properties|yaml|log))")
+
+# Class names
+parser.add_variable_pattern("CLASS_NAME", rf"(?<classname>[a-zA-Z0-9_$]+)")
+
+# Long numbers (7-20 digits)
+parser.add_variable_pattern("LONG_NUMBER", rf"(?<long>\-?\d{{7,20}})")
+
+# Generic patterns - should be last
+parser.add_variable_pattern("SYSTEM_IP", rf"(?<ip>{PATTERN.IPV4})")
+parser.add_variable_pattern("SYSTEM_UUID", rf"(?<uuid>{PATTERN.UUID})")
+parser.add_variable_pattern("GENERIC_FLOAT", rf"(?<float>{PATTERN.FLOAT})")
+parser.add_variable_pattern("GENERIC_INT", rf"(?<int>{PATTERN.INT})")
+parser.add_variable_pattern("HEX_NUMBER", rf"(?<hex>0x[a-f0-9]+)")
+parser.add_variable_pattern("PORT_NUMBER", rf"(?<port>{PATTERN.PORT})")
+
+parser.compile()

rust/examples/python_usage/usage.py

@@ -4,23 +4,33 @@
 
 a = ReaderParser()
 
-a.add_var("number", r"[0-9]+")
-a.add_var("username", r"@(?<inside>[a-z]+)(?<parts>\.[a-z]+)*")
+a.add_variable_pattern("number", r"[0-9]+")
+a.add_variable_pattern("username", r"@(?<inside>[a-z]+)(?<parts>\.[a-z]+)*")
 
 a.compile()
 
-text = "123 awesrgesrgesrg 6346346 @someone foo@username @someone.foo.bar.baz"
+text = """
+123 awesrgesrgesrg 6346346 @someone foo@username @someone.foo.bar.baz
+foo
+bar
+baz
+"""
+
 a.set_input_stream(text)
 
 print(f"py: input [{text}]")
+print()
+
 while True:
 	event = a.next_log_event()
 
 	if event is None:
 		break
 
-	if event.variable is not None:
-		print(f"py: static text: [{event.static_text}] {event.variable.name}: [{event.variable.text}] [{event.variable.captures!r}]")
-	else:
-		# Last static text before eof
-		print(f"py: static text: [{event.static_text}] (end)")
+	print(f"header is: '{event.header!r}'")
+	print(f"tokens are:")
+	for t in event.tokens:
+		# print(f"- ({t!r}): {t.rule}, '{t.text}', {t.captures}")
+		print(f"- {t.rule}, '{t.text}'")
+
+	print()

rust/include/log_mechanic.generated.hpp

@@ -8,31 +8,13 @@
 
 namespace clp::log_mechanic {
 
-struct Lexer;
+struct LogEvent;
 
-struct Schema;
-
-struct CLogFragment {
-// Custom
-CLogFragment() = default;
+struct Parser;
 
-// Generated
-  /// `0` iff no variable found (static text until end of input).
-  size_t rule;
-  /// Start of variable (if found).
-  const uint8_t *start;
-  /// End of variable (if found).
-  const uint8_t *end;
-
-  CLogFragment(size_t const& rule,
-               const uint8_t *const& start,
-               const uint8_t *const& end)
-    : rule(rule),
-      start(start),
-      end(end)
-  {}
+struct Schema;
 
-};
+struct Token;
 
 template<typename T>
 struct CSlice {
@@ -61,20 +43,26 @@ CSlice(char const* c_str)
 
 using CStringView = CSlice<char>;
 
-using LogFragmentOnCapture = void(*)(const void *data, CStringView name, CStringView lexeme);
-
 
 extern "C" {
 
-void clp_log_mechanic_lexer_delete(Box<Lexer> lexer);
+CStringView clp_log_mecahnic_event_token_name(const Token *token);
+
+void clp_log_mechanic_event_delete(Box<LogEvent> event);
+
+const Token *clp_log_mechanic_event_token(const LogEvent *event, size_t i);
+
+size_t clp_log_mechanic_event_token_count(const LogEvent *event);
+
+size_t clp_log_mechanic_event_token_rule(const Token *token);
+
+void clp_log_mechanic_parser_delete(Box<Parser> parser);
 
-Box<Lexer> clp_log_mechanic_lexer_new(const Schema *schema);
+Box<Parser> clp_log_mechanic_parser_new(const Schema *schema);
 
-CLogFragment clp_log_mechanic_lexer_next_fragment(Lexer *lexer,
-                                                  CStringView input,
-                                                  size_t *pos,
-                                                  LogFragmentOnCapture maybe_closure,
-                                                  const void *data);
+Option<Box<LogEvent>> clp_log_mechanic_parser_next_event(Parser *parser,
+                                                         CStringView input,
+                                                         size_t *pos);
 
 void clp_log_mechanic_schema_add_rule(Schema *schema, CStringView name, CStringView pattern);
 

rust/include/log_mechanic.hpp

@@ -1,9 +1,16 @@
 #ifndef LOG_MECHANIC_HPP
 #define LOG_MECHANIC_HPP
 
+#include <type_traits>
+
 namespace clp::log_mechanic {
 
 	template<typename T> using Box = T*;
+
+	template <typename T> struct is_rust_box_t: std::false_type {};
+	template <typename T> struct is_rust_box_t<Box<T>>: std::true_type {};
+
+	template<typename T> using Option = std::enable_if_t<is_rust_box_t<T>::value, T>;
 }
 
 #include "log_mechanic.generated.hpp"