Skip to content

Commit 9c15409

Browse files
author
Sebastian Wagner
committed
Merge branch 'maintenance' into develop
2 parents 835e31a + 7a750f4 commit 9c15409

File tree

6 files changed

+173
-48
lines changed

6 files changed

+173
-48
lines changed

CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
8282
#### Collectors
8383

8484
#### Parsers
85+
- `intelmq.bots.parsers.cymru.parser_cap_program`:
86+
- Adapt parser to new upstream format for events of category "bruteforce" (PR#1795 by Sebastian Wagner, fixes 1794).
8587

8688
#### Experts
8789

@@ -90,13 +92,18 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
9092
### Documentation
9193
- Add missing newlines at end of `docs/_static/intelmq-manager/*.png.license` files (PR#1785 by Sebastian Wagner, fixes #1777).
9294
- Ecosystem: Revise sections on intelmq-cb-mailgen and fody (PR#1792 by Bernhard Reiter).
95+
- intelmq-api: Add documentation about necessary write permission for the session database file (PR#1798 by Birger Schacht, fixes intelmq-api#23).
9396

9497
### Packaging
9598

9699
### Tests
97100
- Add missing newlines at end of various test input files (PR#1785 by Sebastian Wagner, fixes #1777).
98101

99102
### Tools
103+
- `intelmqsetup`:
104+
- Also cover required directory layout and file permissions for `intelmq-api` (PR#1787 by Sebastian Wagner, fixes #1783).
105+
- `intelmqctl`:
106+
- Do not log an error message if logging to file is explicitly disabled, e.g. in calls from `intelmsetup`. The error message would not be useful for the user and is not necessary.
100107

101108
### Contrib
102109

intelmq/bin/intelmqctl.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,7 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
710710

711711
try:
712712
if no_file_logging:
713-
raise FileNotFoundError
713+
raise FileNotFoundError('Logging to file disabled.')
714714
logger = utils.log('intelmqctl', log_level=self.logging_level,
715715
log_format_stream=utils.LOG_FORMAT_SIMPLE,
716716
logging_level_stream=logging_level_stream,
@@ -720,7 +720,8 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
720720
logger = utils.log('intelmqctl', log_level=self.logging_level, log_path=False,
721721
log_format_stream=utils.LOG_FORMAT_SIMPLE,
722722
logging_level_stream=logging_level_stream)
723-
logger.error('Not logging to file: %s', exc)
723+
if not isinstance(exc, FileNotFoundError) and exc.args[0] != 'Logging to file disabled.':
724+
logger.error('Not logging to file: %s', exc)
724725
self.logger = logger
725726
if defaults_loading_exc:
726727
self.logger.exception('Loading the defaults configuration failed!',

intelmq/bin/intelmqsetup.py

Lines changed: 129 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,162 @@
11
# -*- coding: utf-8 -*-
22
"""
3-
© 2019 Sebastian Wagner <wagner@cert.at>
3+
© 2019-2021 nic.at GmbH <intelmq-team@cert.at>
44
5-
SPDX-License-Identifier: AGPL-3.0
5+
SPDX-License-Identifier: AGPL-3.0-only
66
77
Sets up an intelmq environment after installation or upgrade by
88
* creating needed directories
99
* set intelmq as owner for those
1010
* providing example configuration files if not already existing
1111
12+
If intelmq-api is installed, the similar steps are performed:
13+
* creates needed directories
14+
* sets the webserver as group for them
15+
* sets group write permissions
16+
1217
Reasoning:
1318
Pip does not (and cannot) create `/opt/intelmq`/user-given ROOT_DIR, as described in
1419
https://github.com/certtools/intelmq/issues/819
1520
"""
1621
import argparse
17-
import glob
1822
import os
1923
import shutil
24+
import stat
2025
import sys
2126
import pkg_resources
2227

23-
from pwd import getpwuid
28+
from grp import getgrnam
29+
from pathlib import Path
30+
from pwd import getpwnam
31+
from typing import Optional
32+
33+
try:
34+
import intelmq_api
35+
except ImportError:
36+
intelmq_api = None
2437

38+
from termstyle import red
2539
from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
2640
VAR_STATE_PATH, BOTS_FILE, STATE_FILE_PATH)
2741
from intelmq.bin.intelmqctl import IntelMQController
2842

2943

30-
def intelmqsetup(ownership=True, state_file=STATE_FILE_PATH):
31-
if os.geteuid() != 0 and ownership:
32-
sys.exit('You need to run this program as root (for setting file ownership)')
44+
MANAGER_CONFIG_DIR = Path(CONFIG_DIR) / 'manager/'
45+
FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/'
3346

47+
48+
def basic_checks(skip_ownership):
49+
if os.geteuid() != 0 and not skip_ownership:
50+
sys.exit(red('You need to run this program as root for setting file ownership!'))
3451
if not ROOT_DIR:
35-
sys.exit('Not a pip-installation of IntelMQ, nothing to initialize.')
36-
37-
create_dirs = ('%s/file-output' % VAR_STATE_PATH,
38-
VAR_RUN_PATH,
39-
DEFAULT_LOGGING_PATH,
40-
CONFIG_DIR)
41-
for create_dir in create_dirs:
42-
if not os.path.isdir(create_dir):
43-
os.makedirs(create_dir, mode=0o755,
44-
exist_ok=True)
45-
print('Created directory %r.' % create_dir)
46-
47-
example_confs = glob.glob(pkg_resources.resource_filename('intelmq', 'etc/*.conf'))
52+
sys.exit(red('Not a pip-installation of IntelMQ, nothing to initialize.'))
53+
54+
if skip_ownership:
55+
return
56+
try:
57+
getpwnam('intelmq')
58+
except KeyError:
59+
sys.exit(red("User 'intelmq' does not exist. Please create it and then re-run this program."))
60+
try:
61+
getgrnam('intelmq')
62+
except KeyError:
63+
sys.exit(red("Group 'intelmq' does not exist. Please create it and then re-run this program."))
64+
65+
66+
def create_directory(directory: str, octal_mode: int):
67+
directory = Path(directory)
68+
readable_mode = stat.filemode(octal_mode)
69+
if not directory.is_dir():
70+
directory.mkdir(mode=octal_mode, exist_ok=True, parents=True)
71+
print(f'Created directory {directory!s} with permissions {readable_mode}.')
72+
else:
73+
current_mode = directory.stat().st_mode
74+
if current_mode != octal_mode:
75+
current_mode_readable = stat.filemode(current_mode)
76+
print(f'Fixed wrong permissions of {directory!s}: {current_mode_readable!r} -> {readable_mode!r}.')
77+
directory.chmod(octal_mode)
78+
79+
80+
def change_owner(file: str, owner=None, group=None, log: bool = True):
81+
if owner and Path(file).owner() != owner:
82+
if log:
83+
print(f'Fixing owner of {file!s}.')
84+
shutil.chown(file, user=owner)
85+
if group and Path(file).group() != group:
86+
if log:
87+
print(f'Fixing group of {file!s}.')
88+
shutil.chown(file, group=group)
89+
90+
91+
def find_webserver_user():
92+
candidates = ('www-data', 'wwwrun', 'httpd', 'apache')
93+
for candidate in candidates:
94+
try:
95+
getpwnam(candidate)
96+
except KeyError:
97+
pass
98+
else:
99+
print(f'Detected webserver username {candidate!r}.')
100+
return candidate
101+
102+
103+
def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
104+
create_directory(FILE_OUTPUT_PATH, 0o40755)
105+
create_directory(VAR_RUN_PATH, 0o40755)
106+
create_directory(DEFAULT_LOGGING_PATH, 0o40755)
107+
create_directory(CONFIG_DIR, 0o40775)
108+
109+
example_confs = Path(pkg_resources.resource_filename('intelmq', 'etc')).glob('*.conf')
48110
for example_conf in example_confs:
49-
fname = os.path.split(example_conf)[-1]
50-
if os.path.exists(os.path.join(CONFIG_DIR, fname)):
51-
print('Not overwriting existing %r with example.' % fname)
111+
fname = Path(example_conf).name
112+
destination_file = Path(CONFIG_DIR) / fname
113+
if destination_file.exists():
114+
print(f'Not overwriting existing {fname!r} with example.')
115+
log_ownership_change = True
52116
else:
53117
shutil.copy(example_conf, CONFIG_DIR)
54-
print('Use example %r.' % fname)
55-
56-
print('Writing BOTS file.')
57-
shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
58-
BOTS_FILE)
118+
print(f'Installing example {fname!r} to {CONFIG_DIR}.')
119+
log_ownership_change = False # For installing the new files, we don't need to inform the admin that the permissions have been "fixed"
120+
if ownership:
121+
change_owner(destination_file, owner='intelmq', group='intelmq', log=log_ownership_change)
122+
123+
if Path(BOTS_FILE).is_symlink():
124+
print('Skip writing BOTS file as it is a link.')
125+
else:
126+
print('Writing BOTS file.')
127+
shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
128+
BOTS_FILE)
59129

60130
if ownership:
61131
print('Setting intelmq as owner for it\'s directories.')
62132
for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
63-
VAR_STATE_PATH, VAR_STATE_PATH + 'file-output'):
64-
if getpwuid(os.stat(obj).st_uid).pw_name != 'intelmq':
65-
shutil.chown(obj, user='intelmq')
133+
VAR_STATE_PATH, FILE_OUTPUT_PATH):
134+
change_owner(obj, owner='intelmq')
66135

67-
print('Calling `intelmqctl upgrade-config to update/create state file')
136+
print('Calling `intelmqctl upgrade-config` to update/create state file.')
68137
controller = IntelMQController(interactive=False, no_file_logging=True,
69138
drop_privileges=False)
70139
controller.upgrade_conf(state_file=state_file, no_backup=True)
71140

72141

142+
def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = None):
143+
if ownership:
144+
change_owner(CONFIG_DIR, group='intelmq')
145+
146+
# Manager configuration directory
147+
create_directory(MANAGER_CONFIG_DIR, 0o40775)
148+
if ownership:
149+
change_owner(MANAGER_CONFIG_DIR, group='intelmq')
150+
151+
intelmq_group = getgrnam('intelmq')
152+
webserver_user = webserver_user or find_webserver_user()
153+
if webserver_user not in intelmq_group.gr_mem:
154+
sys.exit(red(f"Webserver user {webserver_user} is not a member of the 'intelmq' group. "
155+
f"Please add it with: 'usermod -aG intelmq {webserver_user}'."))
156+
157+
print('Setup of intelmq-api successful.')
158+
159+
73160
def main():
74161
parser = argparse.ArgumentParser("Set's up directories and example "
75162
"configurations for IntelMQ.")
@@ -78,9 +165,17 @@ def main():
78165
parser.add_argument('--state-file',
79166
help='The state file location to use.',
80167
default=STATE_FILE_PATH)
168+
parser.add_argument('--webserver-user',
169+
help='The webserver to use instead of auto-detection.')
81170
args = parser.parse_args()
82-
intelmqsetup(ownership=not args.skip_ownership,
83-
state_file=args.state_file)
171+
172+
basic_checks(skip_ownership=args.skip_ownership)
173+
intelmqsetup_core(ownership=not args.skip_ownership,
174+
state_file=args.state_file)
175+
if intelmq_api:
176+
print('Running setup for intelmq-api.')
177+
intelmqsetup_api(ownership=not args.skip_ownership,
178+
webserver_user=args.webserver_user)
84179

85180

86181
if __name__ == '__main__':

intelmq/bots/parsers/cymru/parser_cap_program.py

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66

77
MAPPING_STATIC = {'bot': {
88
'classification.type': 'infected-system'},
9-
'bruteforce': {
10-
'classification.type': 'brute-force'},
9+
'bruteforce': {'classification.type': 'brute-force'},
1110
'controller': {
1211
'classification.type': 'c2-server'},
1312
'darknet': {'classification.type': 'scanner',
@@ -33,8 +32,6 @@
3332
'classification.identifier': 'conficker',
3433
'malware.name': 'conficker'},
3534
}
36-
MAPPING_COMMENT = {'bruteforce': ('classification.identifier', 'protocol.application'),
37-
'phishing': ('source.url', )}
3835
PROTOCOL_MAPPING = { # TODO: use `getent protocols <number>`, maybe in harmonization
3936
'1': 'icmp',
4037
'6': 'tcp',
@@ -229,6 +226,24 @@ def parse_line_old(self, line, report):
229226
yield event
230227

231228
def parse_line_new(self, line, report):
229+
"""
230+
The format is two following:
231+
category|address|asn|timestamp|optional_information|asninfo
232+
Therefore very similar to CSV, just with the pipe as separator
233+
category: the type (resulting in classification.*) and optional_information needs to be parsed differently per category
234+
address: source.ip
235+
asn: source.asn
236+
timestamp: time.source
237+
optional_information: needs special care.
238+
For some categories it needs parsing, as it contains a mapping of keys to values, whereas the meaning of the keys can differ between the categories
239+
For categories in MAPING_COMMENT, this field only contains one value.
240+
For the category 'bruteforce' *both* situations apply.
241+
Previously, the bruteforce events only had the protocol in the comment,
242+
while most other categories had a mapping. Now, the bruteforce categories also uses
243+
the type-value syntax. So we need to support both formats, the old and the new.
244+
See also https://github.com/certtools/intelmq/issues/1794
245+
asninfo: source.as_name
246+
"""
232247
category, ip, asn, timestamp, notes, asninfo = line.split('|')
233248

234249
# to detect bogous lines like 'hostname: sub.example.comport: 80'
@@ -252,11 +267,6 @@ def parse_line_new(self, line, report):
252267
event.add('time.source', timestamp + ' GMT')
253268
event.add('source.as_name', ', '.join(asninfo_split[:-1])) # contains CC at the end
254269
event.add('source.geolocation.cc', asninfo_split[-1])
255-
if category in MAPPING_COMMENT:
256-
# if the comment is missing, we can't add that information
257-
if comment_split:
258-
for field in MAPPING_COMMENT[category]:
259-
event.add(field, comment_split[0])
260270

261271
try:
262272
for key, value in MAPPING_STATIC[category].items():
@@ -266,11 +276,16 @@ def parse_line_new(self, line, report):
266276
destination_ports = []
267277

268278
for comment in comment_split:
269-
if category in MAPPING_COMMENT:
270-
break
271279
if ': ' not in comment:
272280
if category == 'proxy':
273281
comment = 'proxy_type: %s' % comment
282+
elif category == 'bruteforce': # optional_information can just be 'ssh;'
283+
event.add('classification.identifier', comment)
284+
event.add('protocol.application', comment)
285+
break
286+
elif category == 'phishing':
287+
event.add('source.url', comment)
288+
break
274289
else:
275290
if category == 'bot':
276291
try:

intelmq/tests/bots/parsers/cymru/certname_20190327.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ scanner|172.16.0.21|64496|2020-07-09 03:40:15|username: pm;|Example AS Name, AT
3333
darknet|172.16.0.21|64496|2020-10-08 02:21:26|protocol: 47;|Example AS Name, AT
3434
darknet|172.16.0.21|64496|2020-10-15 09:22:10|protocol: 59;|Example AS Name, AT
3535
proxy|172.16.0.21|64496|2020-12-14 08:28:01|httpconnect-51915; additional_asns: 212682;|Example AS Name, AT
36+
bruteforce|172.16.0.21|64496|2021-03-09 00:11:21|destination_port_numbers: 22;port: 16794;protocol: 6;|Example AS Name, AT

intelmq/tests/bots/parsers/cymru/test_cap_program_new.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,17 @@
221221
'protocol.application': 'httpconnect',
222222
'source.port': 51915,
223223
},
224+
{'classification.type': 'brute-force',
225+
'protocol.transport': 'tcp',
226+
'destination.port': 22,
227+
'source.port': 16794,
228+
'time.source': '2021-03-09T00:11:21+00:00',
229+
},
224230
]
225231

226232
# The number of events a single line in the raw data produces
227233
NUM_EVENTS = (1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
228-
1, 1, 10, 1, 1, 1, 1, 1, 1, 1, 1)
234+
1, 1, 10, 1, 1, 1, 1, 1, 1, 1, 1, 1)
229235
RAWS = []
230236
for i, line in enumerate(RAW_LINES[3:]):
231237
for count in range(NUM_EVENTS[i]):

0 commit comments

Comments
 (0)