Skip to content

Commit 95e7bde

Browse files
proditisproditis
committed
Merge remote-tracking branch 'upstream/master'
2 parents 754e182 + 58decf3 commit 95e7bde

File tree

2 files changed

+222
-2
lines changed

2 files changed

+222
-2
lines changed

‎INSTALL.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ reasonably recent versions too).
2020

2121
#### 1.2.1 - MySQL User Defined Functions
2222

23-
Echofish is dependent on [lib_mysql_udf_preg](https://github.com/mysqludf/lib_mysqludf_preg/) for PCRE pattern matching.
23+
Echofish is dependent on [lib_mysql_udf_preg](https://github.com/mysqludf/lib_mysqludf_preg/) OR MariaDB 10.0.5+ for PCRE pattern matching.
2424

2525
### 1.3 - Syslog
2626

@@ -56,10 +56,22 @@ cd echofish/
5656
mysql -u root -p ETS_echofish < schema/00_echofish-schema.sql
5757
mysql -u root -p ETS_echofish < schema/echofish-dataonly.sql
5858
mysql -u root -p ETS_echofish < schema/echofish-functions.sql
59-
mysql -u root -p ETS_echofish < schema/echofish-procedures.sql
6059
mysql -u root -p ETS_echofish < schema/echofish-triggers.sql
6160
mysql -u root -p ETS_echofish < schema/echofish-events.sql
6261
```
62+
Import the appropriate procedures for your database server
63+
64+
MySQL with udf_preg
65+
66+
```sh
67+
mysql -u root -p ETS_echofish < schema/echofish-procedures.sql
68+
```
69+
70+
MariaDB 10.0.5+
71+
72+
```sh
73+
mysql -u root -p ETS_echofish < schema/echofish-procedures.mariadb10.sql
74+
```
6375

6476
For events to run, make sure you set `event_scheduler=on` somewhere under the
6577
`[mysqld]` section in the default mysql config file, usually `/etc/my.cnf` or

‎schema/echofish-procedures.mariadb10.sql‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
/* $Id$ */
2+
3+
SET FOREIGN_KEY_CHECKS=0;
4+
SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
5+
SET time_zone = "+00:00";
6+
7+
8+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
9+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
10+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
11+
/*!40101 SET NAMES utf8 */;
12+
13+
DELIMITER //
14+
15+
DROP PROCEDURE IF EXISTS delete_duplicate_whitelist//
16+
CREATE PROCEDURE delete_duplicate_whitelist()
17+
BEGIN
18+
DECLARE wid,wfacility,wlevel,done BIGINT DEFAULT 0;
19+
DECLARE whost,wprogram VARCHAR(255) DEFAULT '';
20+
DECLARE wpattern VARCHAR(512) DEFAULT '';
21+
DECLARE uwp CURSOR FOR SELECT id,host,program,facility,`level`,pattern FROM whitelist;
22+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
23+
START TRANSACTION WITH CONSISTENT SNAPSHOT;
24+
OPEN uwp;
25+
26+
read_loop: LOOP
27+
FETCH uwp INTO wid,whost,wprogram,wfacility,wlevel,wpattern;
28+
IF done = -1 THEN
29+
LEAVE read_loop;
30+
END IF;
31+
delete_segment: BEGIN
32+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET @x='OUPS';
33+
DELETE FROM whitelist WHERE
34+
pattern LIKE wpattern AND
35+
program LIKE if(wprogram='' or wprogram is null,'%',wprogram) AND
36+
facility like if(wfacility<0,'%',wfacility) AND
37+
`level` like if(wlevel<0,'%',wlevel) AND
38+
host LIKE if(whost='0','%',whost) AND
39+
id!=wid;
40+
END delete_segment;
41+
END LOOP read_loop;
42+
CLOSE uwp;
43+
COMMIT;
44+
END;
45+
//
46+
47+
DROP PROCEDURE IF EXISTS extract_ipaddr//
48+
CREATE PROCEDURE extract_ipaddr(IN msg VARCHAR(5000))
49+
BEGIN
50+
DECLARE matching INT default 1;
51+
DECLARE ipaddr VARCHAR(255);
52+
SET ipaddr=(SELECT REGEXP_SUBSTR(msg, '\\d{1,3}(?:\.\\d{1,3}){3}'));
53+
tfer_loop:WHILE ( ipaddr IS NOT NULL and length(ipaddr)>0 ) DO
54+
SELECT ipaddr;
55+
SET matching=matching+1;
56+
SET msg=(SELECT REPLACE( msg, @ipaddr, '' ));
57+
SET ipaddr=(SELECT REGEXP_SUBSTR(msg, '?:\\d{1,3}(?:\.\\d{1,3}){3})'));
58+
END WHILE tfer_loop;
59+
END;
60+
//
61+
62+
63+
DROP PROCEDURE IF EXISTS archive_parser_trigger//
64+
CREATE PROCEDURE archive_parser_trigger(IN aid BIGINT UNSIGNED,IN ahost BIGINT UNSIGNED,IN aprogram VARCHAR(255),IN afacility INT,in alevel INT,IN apid BIGINT,in amsg TEXT,in areceived_ts TIMESTAMP,IN ttype VARCHAR(10))
65+
BEGIN
66+
DECLARE apid,done INT;
67+
DECLARE apptype,apname VARCHAR(255);
68+
DECLARE uwp CURSOR FOR SELECT id,name FROM archive_parser WHERE ptype=ttype ORDER BY weight,name,id;
69+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
70+
OPEN uwp;
71+
72+
read_loop: LOOP
73+
FETCH uwp INTO apid,apname;
74+
IF done = -1 THEN
75+
LEAVE read_loop;
76+
END IF;
77+
78+
SET @callquery=concat('CALL ',apname,'(?,?,?,?,?,?,?,?)');
79+
PREPARE stmtcall FROM @callquery;
80+
set @aid=aid;
81+
set @ahost=ahost;
82+
set @aprogram=aprogram;
83+
set @afacility=afacility;
84+
set @alevel=alevel;
85+
set @apid=apid;
86+
set @amsg=amsg;
87+
set @areceived_ts=areceived_ts;
88+
EXECUTE stmtcall USING @aid,@ahost,@aprogram,@afacility,@alevel,@apid,@amsg,@areceived_ts;
89+
DEALLOCATE PREPARE stmtcall;
90+
END LOOP read_loop;
91+
CLOSE uwp;
92+
END;
93+
//
94+
95+
96+
DROP PROCEDURE IF EXISTS archive_parse_unparsed//
97+
CREATE PROCEDURE archive_parse_unparsed()
98+
BEGIN
99+
DECLARE deadlock,done INT DEFAULT 0;
100+
DECLARE attempts INT DEFAULT 0;
101+
DECLARE auid BIGINT UNSIGNED DEFAULT 0;
102+
DECLARE uwp CURSOR FOR SELECT id FROM archive_unparse WHERE pending=1 ORDER BY id LIMIT 10000;
103+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
104+
SET SESSION time_zone='+00:00';
105+
START TRANSACTION;
106+
OPEN uwp;
107+
read_loop: LOOP
108+
FETCH uwp INTO auid;
109+
IF done = -1 THEN
110+
LEAVE read_loop;
111+
END IF;
112+
DELETE FROM archive_unparse WHERE id=auid;
113+
SELECT host,facility,`level`,program,pid,msg,received_ts INTO @ahost,@afacility,@alevel,@aprogram,@apid,@amsg,@areceived_ts FROM archive WHERE id=auid;
114+
IF @ahost IS NOT NULL AND @afacility IS NOT NULL AND @alevel IS NOT NULL AND @aprogram IS NOT NULL AND @apid IS NOT NULL AND @amsg IS NOT NULL THEN
115+
CALL archive_parser_trigger(auid,@ahost,@aprogram,@afacility,@alevel,@apid,@amsg,@areceived_ts,'archive');
116+
SET @hostexists=(SELECT count(*) FROM `host` WHERE id=@ahost);
117+
IF @hostexists IS NULL OR @hostexists = 0 and @ahost is not null THEN
118+
INSERT INTO `host` (fqdn,short) values (@ahost,@ahost);
119+
END IF;
120+
END IF;
121+
END LOOP read_loop;
122+
CLOSE uwp;
123+
COMMIT;
124+
END;
125+
//
126+
127+
/*
128+
* Simple wrapper around the insert for the log of abuser evidence
129+
*/
130+
DROP PROCEDURE IF EXISTS abuser_log_evidence//
131+
CREATE PROCEDURE abuser_log_evidence(IN abuser_id BIGINT UNSIGNED,IN entry_id BIGINT UNSIGNED)
132+
BEGIN
133+
INSERT INTO abuser_evidence (incident_id,archive_id) VALUES (abuser_id,entry_id);
134+
END;
135+
//
136+
137+
/*
138+
* Parse given entry through the abuser trigger rules.
139+
*/
140+
DROP PROCEDURE IF EXISTS abuser_parser//
141+
CREATE PROCEDURE abuser_parser(IN aid BIGINT UNSIGNED,IN ahost BIGINT UNSIGNED,IN aprogram VARCHAR(255),IN afacility INT,in alevel INT,IN apid BIGINT,in amsg TEXT,in areceived_ts TIMESTAMP)
142+
BEGIN
143+
DECLARE done,mts,Ccapture INT DEFAULT 0;
144+
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = -1;
145+
146+
SELECT id,pattern,grouping,capture INTO mts,@pattern,@grouping,Ccapture FROM abuser_trigger WHERE
147+
amsg LIKE msg AND
148+
aprogram LIKE if(program='' or program is null,'%',program) AND
149+
afacility like if(facility<0,'%',facility) AND
150+
alevel like if(`severity`<0,'%',`severity`) and active=1
151+
LIMIT 1;
152+
153+
SET @grouping = (CONVERT(CONCAT('\\',@grouping) USING utf8) COLLATE utf8_unicode_ci);
154+
IF @pattern REGEXP '^\\^' != '1' THEN
155+
SET @pattern = (CONCAT('^.*',@pattern));
156+
END IF;
157+
if @pattern REGEXP '\\$$' != '1' THEN
158+
SET @pattern = (CONCAT(@pattern,'.*$'));
159+
END IF;
160+
161+
IF mts>0 AND Ccapture IS NOT NULL AND INET_ATON(REGEXP_REPLACE(amsg,@pattern,@grouping)) IS NOT NULL THEN
162+
INSERT INTO abuser_incident (ip,trigger_id,counter,first_occurrence,last_occurrence)
163+
VALUES (INET_ATON(REGEXP_REPLACE(amsg,@pattern,@grouping)),
164+
mts,1,areceived_ts,areceived_ts)
165+
ON DUPLICATE KEY UPDATE counter=counter+1,last_occurrence=areceived_ts;
166+
SELECT id INTO @incident_id FROM abuser_incident WHERE ip=INET_ATON(REGEXP_REPLACE(amsg,@pattern,@grouping)) AND trigger_id=mts;
167+
CALL abuser_log_evidence(@incident_id,aid);
168+
END IF;
169+
END;//
170+
171+
172+
/*
173+
* Procedure to process old archive log entries and delete them
174+
*/
175+
DROP PROCEDURE IF EXISTS eproc_rotate_archive//
176+
CREATE PROCEDURE eproc_rotate_archive()
177+
BEGIN
178+
DROP TABLE IF EXISTS archive_ids;
179+
SET @archive_days=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_days'),7);
180+
SET @archive_limit=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_limit'),0);
181+
SET @use_mem=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_use_mem'),'no');
182+
IF @archive_days>0 THEN
183+
IF @use_mem != 'yes' THEN
184+
CREATE TEMPORARY TABLE IF NOT EXISTS archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY);
185+
ELSE
186+
CREATE TEMPORARY TABLE IF NOT EXISTS archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY) ENGINE=MEMORY;
187+
END IF;
188+
189+
SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
190+
START TRANSACTION;
191+
IF @archive_limit > 0 THEN
192+
PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ? DAY LIMIT ?';
193+
EXECUTE choose_archive_ids USING @archive_days, @archive_limit;
194+
ELSE
195+
PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ? DAY';
196+
EXECUTE choose_archive_ids USING @archive_days;
197+
END IF;
198+
DEALLOCATE PREPARE choose_archive_ids;
199+
-- Ignore ID's from entries that exist on archive_unparse
200+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN archive_unparse AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
201+
-- Ignore ID's from entries that exist on syslog
202+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN syslog AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
203+
-- Ignore ID's from entries that exist on abuser_evidense
204+
DELETE t1.* FROM archive_ids as t1 LEFT JOIN abuser_evidence AS t2 ON t1.id=t2.archive_id WHERE t2.archive_id IS NOT NULL;
205+
DELETE t1.* FROM `archive` AS t1 LEFT JOIN archive_ids AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL;
206+
COMMIT;
207+
END IF;
208+
END;//

0 commit comments

Comments
 (0)