Skip to content

Commit 6e799f7

Browse files
proditisproditis
committed
Merge pull request #37 from ardichoke/master
MariaDB 10.0.5+ PCRE Support
2 parents 574841d + c88e973 commit 6e799f7

File tree

2 files changed

+212
-2
lines changed

2 files changed

+212
-2
lines changed

‎INSTALL.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ reasonably recent versions too).
2020

2121
#### 1.2.1 - MySQL User Defined Functions
2222

23-
Echofish is dependent on [lib_mysql_udf_preg](https://github.com/mysqludf/lib_mysqludf_preg/) for PCRE pattern matching.
23+
Echofish is dependent on [lib_mysql_udf_preg](https://github.com/mysqludf/lib_mysqludf_preg/) OR MariaDB 10.0.5+ for PCRE pattern matching.
2424

2525
### 1.3 - Syslog
2626

@@ -56,10 +56,22 @@ cd echofish/
5656
mysql -u root -p ETS_echofish < schema/00_echofish-schema.sql
5757
mysql -u root -p ETS_echofish < schema/echofish-dataonly.sql
5858
mysql -u root -p ETS_echofish < schema/echofish-functions.sql
59-
mysql -u root -p ETS_echofish < schema/echofish-procedures.sql
6059
mysql -u root -p ETS_echofish < schema/echofish-triggers.sql
6160
mysql -u root -p ETS_echofish < schema/echofish-events.sql
6261
```
62+
Import the appropriate procedures for your database server
63+
64+
MySQL with udf_preg
65+
66+
```sh
67+
mysql -u root -p ETS_echofish < schema/echofish-procedures.sql
68+
```
69+
70+
MariaDB 10.0.5+
71+
72+
```sh
73+
mysql -u root -p ETS_echofish < schema/echofish-procedures.mariadb10.sql
74+
```
6375

6476
For events to run, make sure you set `event_scheduler=on` somewhere under the
6577
`[mysqld]` section in the default mysql config file, usually `/etc/my.cnf` or

‎schema/echofish-procedures.mariadb10.sql‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/* $Id$ */
2+
3+
SET FOREIGN_KEY_CHECKS=0;
4+
SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
5+
SET time_zone = "+00:00";
6+
7+
8+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
9+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
10+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
11+
/*!40101 SET NAMES utf8 */;
12+
13+
DELIMITER //
14+
15+
DROP PROCEDURE IF EXISTS delete_duplicate_whitelist//
16+
CREATE PROCEDURE delete_duplicate_whitelist()
17+
BEGIN
18+
DECLARE wid,wfacility,wlevel,done BIGINT DEFAULT 0;
19+
DECLARE whost,wprogram VARCHAR(255) DEFAULT '';
20+
DECLARE wpattern VARCHAR(512) DEFAULT '';
21+
DECLARE uwp CURSOR FOR SELECT id,host,program,facility,`level`,pattern FROM whitelist;
22+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
23+
START TRANSACTION WITH CONSISTENT SNAPSHOT;
24+
OPEN uwp;
25+
26+
read_loop: LOOP
27+
FETCH uwp INTO wid,whost,wprogram,wfacility,wlevel,wpattern;
28+
IF done = -1 THEN
29+
LEAVE read_loop;
30+
END IF;
31+
delete_segment: BEGIN
32+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET @x='OUPS';
33+
DELETE FROM whitelist WHERE
34+
pattern LIKE wpattern AND
35+
program LIKE if(wprogram='' or wprogram is null,'%',wprogram) AND
36+
facility like if(wfacility<0,'%',wfacility) AND
37+
`level` like if(wlevel<0,'%',wlevel) AND
38+
host LIKE if(whost='0','%',whost) AND
39+
id!=wid;
40+
END delete_segment;
41+
END LOOP read_loop;
42+
CLOSE uwp;
43+
COMMIT;
44+
END;
45+
//
46+
47+
DROP PROCEDURE IF EXISTS extract_ipaddr//
48+
CREATE PROCEDURE extract_ipaddr(IN msg VARCHAR(5000))
49+
BEGIN
50+
DECLARE matching INT default 1;
51+
DECLARE ipaddr VARCHAR(255);
52+
SET ipaddr=(SELECT REGEXP_SUBSTR(msg, '/(([0-9]+)(?:\.[0-9]+){3})/'));
53+
tfer_loop:WHILE (ipaddr IS NOT NULL and length(ipaddr)>0 ) DO
54+
SELECT ipaddr;
55+
set matching=matching+1;
56+
SET ipaddr=(SELECT REGEXP_SUBSTR(msg, '/(([0-9]+)(?:\.[0-9]+){3})/'));
57+
END WHILE tfer_loop;
58+
END;
59+
//
60+
61+
62+
DROP PROCEDURE IF EXISTS archive_parser_trigger//
63+
CREATE PROCEDURE archive_parser_trigger(IN aid BIGINT UNSIGNED,IN ahost BIGINT UNSIGNED,IN aprogram VARCHAR(255),IN afacility INT,in alevel INT,IN apid BIGINT,in amsg TEXT,in areceived_ts TIMESTAMP,IN ttype VARCHAR(10))
64+
BEGIN
65+
DECLARE apid,done INT;
66+
DECLARE apptype,apname VARCHAR(255);
67+
DECLARE uwp CURSOR FOR SELECT id,name FROM archive_parser WHERE ptype=ttype ORDER BY weight,name,id;
68+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
69+
OPEN uwp;
70+
71+
read_loop: LOOP
72+
FETCH uwp INTO apid,apname;
73+
IF done = -1 THEN
74+
LEAVE read_loop;
75+
END IF;
76+
77+
SET @callquery=concat('CALL ',apname,'(?,?,?,?,?,?,?,?)');
78+
PREPARE stmtcall FROM @callquery;
79+
set @aid=aid;
80+
set @ahost=ahost;
81+
set @aprogram=aprogram;
82+
set @afacility=afacility;
83+
set @alevel=alevel;
84+
set @apid=apid;
85+
set @amsg=amsg;
86+
set @areceived_ts=areceived_ts;
87+
EXECUTE stmtcall USING @aid,@ahost,@aprogram,@afacility,@alevel,@apid,@amsg,@areceived_ts;
88+
DEALLOCATE PREPARE stmtcall;
89+
END LOOP read_loop;
90+
CLOSE uwp;
91+
END;
92+
//
93+
94+
95+
DROP PROCEDURE IF EXISTS archive_parse_unparsed//
96+
CREATE PROCEDURE archive_parse_unparsed()
97+
BEGIN
98+
DECLARE deadlock,done INT DEFAULT 0;
99+
DECLARE attempts INT DEFAULT 0;
100+
DECLARE auid BIGINT UNSIGNED DEFAULT 0;
101+
DECLARE uwp CURSOR FOR SELECT id FROM archive_unparse WHERE pending=1 ORDER BY id LIMIT 10000;
102+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
103+
SET SESSION time_zone='+00:00';
104+
START TRANSACTION;
105+
OPEN uwp;
106+
read_loop: LOOP
107+
FETCH uwp INTO auid;
108+
IF done = -1 THEN
109+
LEAVE read_loop;
110+
END IF;
111+
DELETE FROM archive_unparse WHERE id=auid;
112+
SELECT host,facility,`level`,program,pid,msg,received_ts INTO @ahost,@afacility,@alevel,@aprogram,@apid,@amsg,@areceived_ts FROM archive WHERE id=auid;
113+
IF @ahost IS NOT NULL AND @afacility IS NOT NULL AND @alevel IS NOT NULL AND @aprogram IS NOT NULL AND @apid IS NOT NULL AND @amsg IS NOT NULL THEN
114+
CALL archive_parser_trigger(auid,@ahost,@aprogram,@afacility,@alevel,@apid,@amsg,@areceived_ts,'archive');
115+
SET @hostexists=(SELECT count(*) FROM `host` WHERE id=@ahost);
116+
IF @hostexists IS NULL OR @hostexists = 0 and @ahost is not null THEN
117+
INSERT INTO `host` (fqdn,short) values (@ahost,@ahost);
118+
END IF;
119+
END IF;
120+
END LOOP read_loop;
121+
CLOSE uwp;
122+
COMMIT;
123+
END;
124+
//
125+
126+
/*
127+
* Simple wrapper around the insert for the log of abuser evidence
128+
*/
129+
DROP PROCEDURE IF EXISTS abuser_log_evidence//
130+
CREATE PROCEDURE abuser_log_evidence(IN abuser_id BIGINT UNSIGNED,IN entry_id BIGINT UNSIGNED)
131+
BEGIN
132+
INSERT INTO abuser_evidence (incident_id,archive_id) VALUES (abuser_id,entry_id);
133+
END;
134+
//
135+
136+
/*
137+
* Parse given entry through the abuser trigger rules.
138+
*/
139+
DROP PROCEDURE IF EXISTS abuser_parser//
140+
CREATE PROCEDURE abuser_parser(IN aid BIGINT UNSIGNED,IN ahost BIGINT UNSIGNED,IN aprogram VARCHAR(255),IN afacility INT,in alevel INT,IN apid BIGINT,in amsg TEXT,in areceived_ts TIMESTAMP)
141+
BEGIN
142+
DECLARE done,mts,Ccapture INT DEFAULT 0;
143+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
144+
145+
SELECT id,pattern,grouping,capture INTO mts,@pattern,@grouping,Ccapture FROM abuser_trigger WHERE
146+
amsg LIKE msg AND
147+
aprogram LIKE if(program='' or program is null,'%',program) AND
148+
afacility like if(facility<0,'%',facility) AND
149+
alevel like if(`severity`<0,'%',`severity`) and active=1
150+
LIMIT 1;
151+
IF mts>0 AND Ccapture IS NOT NULL AND INET_ATON(REGEXP_REPLACE(amsg,@pattern,CONCAT('\\' COLLATE utf8_general_ci,@grouping))) IS NOT NULL THEN
152+
INSERT INTO abuser_incident (ip,trigger_id,counter,first_occurrence,last_occurrence)
153+
VALUES (INET_ATON(REGEXP_REPLACE(amsg,@pattern,CONCAT('\\' COLLATE utf8_general_ci,@grouping))),
154+
mts,1,areceived_ts,areceived_ts)
155+
ON DUPLICATE KEY UPDATE counter=counter+1,last_occurrence=areceived_ts;
156+
SELECT id INTO @incident_id FROM abuser_incident WHERE ip=INET_ATON(REGEXP_REPLACE(amsg,@pattern,CONCAT('\\' COLLATE utf8_general_ci,@grouping))) AND trigger_id=mts;
157+
CALL abuser_log_evidence(@incident_id,aid);
158+
END IF;
159+
END;//
160+
161+
162+
/*
163+
* Procedure to process old archive log entries and delete them
164+
*/
165+
DROP PROCEDURE IF EXISTS eproc_rotate_archive//
166+
CREATE PROCEDURE eproc_rotate_archive()
167+
BEGIN
168+
DROP TABLE IF EXISTS archive_ids;
169+
SET @archive_days=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_days'),7);
170+
SET @archive_limit=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_limit'),0);
171+
SET @use_mem=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_use_mem'),'no');
172+
IF @archive_days>0 THEN
173+
IF @use_mem != 'yes' THEN
174+
CREATE TEMPORARY TABLE IF NOT EXISTS archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY);
175+
ELSE
176+
CREATE TEMPORARY TABLE IF NOT EXISTS archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY) ENGINE=MEMORY;
177+
END IF;
178+
179+
SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
180+
START TRANSACTION;
181+
IF @archive_limit > 0 THEN
182+
PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ? DAY LIMIT ?';
183+
EXECUTE choose_archive_ids USING @archive_days, @archive_limit;
184+
ELSE
185+
PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ? DAY';
186+
EXECUTE choose_archive_ids USING @archive_days;
187+
END IF;
188+
DEALLOCATE PREPARE choose_archive_ids;
189+
-- Ignore ID's from entries that exist on archive_unparse
190+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN archive_unparse AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
191+
-- Ignore ID's from entries that exist on syslog
192+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN syslog AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
193+
-- Ignore ID's from entries that exist on abuser_evidense
194+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN abuser_evidence AS t2 ON t1.id=t2.archive_id WHERE t2.archive_id IS NOT NULL;
195+
DELETE t1.* FROM `archive` AS t1 LEFT JOIN archive_ids AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
196+
COMMIT;
197+
END IF;
198+
END;//

0 commit comments

Comments
 (0)