Skip to content

Commit 320a875

Browse files
Merge pull request #239 from GalaxP/geolite
Added new module - geolite
2 parents c475786 + 3221f8a commit 320a875

File tree

3 files changed

+270
-0
lines changed

3 files changed

+270
-0
lines changed

geolite/Makefile.am

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
EXTRA_DIST=geolite.py readme.md
2+
bin_SCRIPTS=geolite.py
3+
4+
pkgdocdir=${docdir}/geolite
5+
pkgdoc_DATA=readme.md
6+
7+
pylint:
8+
pylint-3 geolite.py
9+
10+
flake8:
11+
flake8 geolite.py
12+
13+
pycodestyle:
14+
pycodestyle-3 geolite.py
15+
16+
lint: pylint flake8 pycodestyle
17+
18+
include ../aminclude.am

geolite/README.md

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Geolite
2+
3+
## Module description
4+
5+
This module outputs flow records with geolocation data using a [geolite database](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data/).
6+
7+
8+
## Input data
9+
10+
This module expects flow records in Unirec format. The required fields
11+
are determined by run time parameters.
12+
13+
14+
## Output data
15+
16+
Flows are sent on the output interface, also in Unirec format, they
17+
contain geolocation data. The fields included in the output interface will vary depending on the selected database type.
18+
19+
Below are the fields that will be set for each database type:
20+
* `country`
21+
* ipaddr `ip`
22+
* string `name`
23+
* string `iso_code`
24+
* uint32 `geoname_id`
25+
* uint32 `is_in_european_union`
26+
27+
* `city`
28+
* ipaddr `ip`
29+
* string `country_name`
30+
* string `country_iso_code`
31+
* uint32 `country_geoname_id`
32+
* uint32 `is_in_european_union`
33+
* string `city_name`
34+
* float `latitude`
35+
* float `longitude`
36+
* uint32 `accuracy_radius`
37+
38+
* `asn`
39+
* ipaddr `ip`
40+
* uint32 `asn`
41+
* string `string autonomous_system_organization`
42+
43+
44+
## Module parameters
45+
46+
In addition to the implicit *libtrap* parameters `-i IFC_SPEC`, `-h`
47+
and `-v` (see [Execute a
48+
module](https://github.com/CESNET/Nemea#try-out-nemea-modules)) this
49+
module takes the following parameters:
50+
51+
* `-d` `--db` path
52+
53+
* Specify path to the database file.
54+
55+
* `-f` `--fields` field1,field2,...
56+
57+
* Specify the name of field(s) from the input interface, which will be used for geolocation and lookup in the database (case sensitive).
58+
If multiple fields are specified, they must be separated by a comma.
59+
60+
* `-t` `--type` {country, city, asn}
61+
62+
* Specify the type of GeoLite database. The default value is `country`.
63+
64+
* `-c` `--cache` number
65+
66+
* Specify the number of lookup calls that will be cached. Set to `0` to disable caching. The default value is `128`.
67+
68+
69+
## Example
70+
The following command :
71+
72+
`./geolite.py -i f:/etc/nemea/data/data.dan.trapcap,f:test.trapcap -d '/usr/share/GeoIP/GeoLite2-Country.mmdb' -t country -f "SRC_IP,DST_IP"`
73+
74+
will be interpreted as follows:
75+
76+
* `-i f:/etc/nemea/data/data.dan.trapcap,f:test.trapcap`
77+
sets the input and output interfaces to a file.
78+
79+
* `-d '/usr/share/GeoIP/GeoLite2-Country.mmdb'` sets the path to the database file.
80+
81+
* `-t country` sets the database type to `country` (can be omitted as it is the default).
82+
83+
* `-f "SRC_IP,DST_IP"` specifies the names of the fields containing IP addresses to be used for geolocation.
84+
85+
<!--- Local variables: -->
86+
<!--- mode: markdown; -->
87+
<!--- mode: auto-fill; -->
88+
<!--- mode: flyspell; -->
89+
<!--- ispell-local-dictionary: "british"; -->
90+
<!--- End: -->

geolite/geolite.py

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
#!/usr/bin/python3
2+
#
3+
# Copyright (C) 2025 CESNET
4+
#
5+
# LICENSE TERMS
6+
#
7+
# Redistribution and use in source and binary forms, with or without
8+
# modification, are permitted provided that the following conditions
9+
# are met:
10+
# 1. Redistributions of source code must retain the above copyright
11+
# notice, this list of conditions and the following disclaimer.
12+
# 2. Redistributions in binary form must reproduce the above copyright
13+
# notice, this list of conditions and the following disclaimer in
14+
# the documentation and/or other materials provided with the
15+
# distribution.
16+
# 3. Neither the name of the Company nor the names of its contributors
17+
# may be used to endorse or promote products derived from this
18+
# software without specific prior written permission.
19+
#
20+
# ALTERNATIVELY, provided that this notice is retained in full, this
21+
# product may be distributed under the terms of the GNU General Public
22+
# License (GPL) version 2 or later, in which case the provisions
23+
# of the GPL apply INSTEAD OF those given above.
24+
#
25+
# This software is provided ``as is'', and any express or implied
26+
# warranties, including, but not limited to, the implied warranties of
27+
# merchantability and fitness for a particular purpose are disclaimed.
28+
# In no event shall the company or contributors be liable for any
29+
# direct, indirect, incidental, special, exemplary, or consequential
30+
# damages (including, but not limited to, procurement of substitute
31+
# goods or services; loss of use, data, or profits; or business
32+
# interruption) however caused and on any theory of liability, whether
33+
# in contract, strict liability, or tort (including negligence or
34+
# otherwise) arising in any way out of the use of this software, even
35+
# if advised of the possibility of such damage.
36+
37+
38+
import argparse
39+
from functools import lru_cache
40+
import pytrap
41+
import geoip2.database
42+
43+
44+
CITY_OUTPUTSPEC = "ipaddr ip, string country_name, string country_iso_code, uint32 country_geoname_id, uint32 is_in_european_union, string city_name, float latitude, float longitude, uint32 accuracy_radius"
45+
COUNTRY_OUTPUTSPEC = "ipaddr ip, string name, string iso_code, uint32 geoname_id, uint32 is_in_european_union"
46+
ASN_OUTPUTSPEC = "ipaddr ip, uint32 asn, string autonomous_system_organization"
47+
48+
49+
parser = argparse.ArgumentParser(description='Module for geolocation using GeoLite2 database')
50+
parser.add_argument('-i', "--ifcspec",
51+
help="select TRAP IFC specifier")
52+
parser.add_argument('-d', "--db",
53+
help="path to the GeoLite2 database file",
54+
required=True)
55+
parser.add_argument('-f', "--fields",
56+
help="input fields to use for geolocation seperated by comma",
57+
default="SRC_IP")
58+
parser.add_argument('-t', "--type",
59+
help="type of GeoLite database",
60+
choices=['country', 'city', 'asn'],
61+
default="country")
62+
parser.add_argument('-c', "--cache",
63+
type=int,
64+
help="number of cached lookups. If set to 0, caching is disabled.",
65+
default=128)
66+
67+
# parse command line arguments
68+
args = parser.parse_args()
69+
fields = args.fields.split(',')
70+
71+
72+
# initialize TRAP context
73+
trap = pytrap.TrapCtx()
74+
trap.init(['-i', args.ifcspec], 1, 1)
75+
76+
# output interface
77+
fmttype = pytrap.FMT_UNIREC
78+
79+
# set the correct output specification based on the type of GeoLite database
80+
if args.type == 'asn':
81+
outputspec = ASN_OUTPUTSPEC
82+
elif args.type == 'city':
83+
outputspec = CITY_OUTPUTSPEC
84+
else:
85+
outputspec = COUNTRY_OUTPUTSPEC
86+
87+
trap.setDataFmt(0, fmttype, outputspec)
88+
output = pytrap.UnirecTemplate(outputspec)
89+
output.createMessage()
90+
91+
# input interface
92+
fmtspec = ""
93+
trap.setRequiredFmt(0, fmttype, fmtspec)
94+
rec = pytrap.UnirecTemplate(fmtspec)
95+
96+
# open GeoLite2 database reader
97+
with geoip2.database.Reader(args.db) as reader:
98+
@lru_cache(maxsize=args.cache)
99+
def lookup_in_database(lookup_ip):
100+
#Function to look up the IP address in the GeoLite2 database.
101+
if args.type == 'city':
102+
return reader.city(lookup_ip)
103+
elif args.type == 'country':
104+
return reader.country(lookup_ip)
105+
elif args.type == 'asn':
106+
return reader.asn(lookup_ip)
107+
108+
109+
# main loop
110+
while True:
111+
try:
112+
data = trap.recv()
113+
except pytrap.FormatChanged as e:
114+
fmttype, fmtspec = trap.getDataFmt(0)
115+
rec = pytrap.UnirecTemplate(fmtspec)
116+
data = e.data
117+
except pytrap.Terminated:
118+
print("Terminated trap.")
119+
break
120+
except pytrap.TrapError:
121+
print("Trap error, exiting.")
122+
break
123+
if len(data) <= 1:
124+
break
125+
126+
else:
127+
rec.setData(data)
128+
for field in fields:
129+
ip = rec.get(data, field)
130+
output.ip = ip
131+
try:
132+
geolocation = lookup_in_database(str(ip))
133+
except:
134+
continue
135+
if geolocation is None:
136+
continue
137+
138+
# fill output fields based on geolocation type
139+
if args.type == 'country':
140+
output.name = geolocation.country.name or "unkown"
141+
output.iso_code = geolocation.country.iso_code or "unkown"
142+
output.geoname_id = geolocation.country.geoname_id or 0
143+
output.is_in_european_union = geolocation.country.is_in_european_union or False
144+
elif args.type == 'city':
145+
output.country_name = geolocation.country.name or "unkown"
146+
output.country_iso_code = geolocation.country.iso_code or "unkown"
147+
output.country_geoname_id = geolocation.country.geoname_id or 0
148+
output.is_in_european_union = geolocation.country.is_in_european_union or False
149+
output.city_name = geolocation.city.name or "unknown"
150+
output.latitude = geolocation.location.latitude or 0.0
151+
output.longitude = geolocation.location.longitude or 0.0
152+
output.accuracy_radius = geolocation.location.accuracy_radius or 0
153+
elif args.type == 'asn':
154+
output.asn = geolocation.autonomous_system_number or 0
155+
output.autonomous_system_organization = geolocation.autonomous_system_organization or "unknown"
156+
157+
# send output data
158+
trap.send(output.getData(), 0)
159+
160+
# send end-of-stream message and exit
161+
trap.sendFlush(0)
162+
trap.finalize()

0 commit comments

Comments
 (0)