Skip to content

introducing elasticsearch #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkind wants to merge 8 commits into
base: master
Choose a base branch
from
Open

introducing elasticsearch #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
45c9aec
This commit removes analyzers and switches to elasticsearch. Now, every
mkind Dec 17, 2017
eb7e059
using bulk API request to improve performance
mkind Dec 17, 2017
5b600a9
removing debug limitation
mkind Dec 17, 2017
36a7710
also support non tcp/udp protocols. also get proper MAC address
mkind Dec 17, 2017
1fb557e
storing human friendly protocol string doesn't work in elastic, so we…
mkind Dec 17, 2017
2f1e71c
fetching OUIs and storing vendor information
mkind Dec 21, 2017
c081f62
setting elasticserver via command line argument
mkind Dec 21, 2017
74a90c7
adding missing requirment
mkind Dec 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file removed pcapscanner/analyzers/__init__.py
View file
Open in desktop
Empty file.
66 changes: 0 additions & 66 deletions pcapscanner/analyzers/conversations.py
View file
Open in desktop

This file was deleted.

41 changes: 0 additions & 41 deletions pcapscanner/analyzers/hosts.py
View file
Open in desktop

This file was deleted.

32 changes: 10 additions & 22 deletions pcapscanner/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,10 @@
import time
from multiprocessing import Pool

from analyzers import hosts, conversations
import pcap

NUM_THREADS = 4

ANALYZERS = [
hosts,
conversations
]

ASCII_LOGO = """

@@@@@@@ @@@@@@@ @@@@@@ @@@@@@@ @@@@@@ @@@@@@@ @@@@@@ @@@ @@@
Expand All @@ -38,9 +32,10 @@

"""


class Main:

def __init__(self, outputdir, inputdir, parser):
def __init__(self, outputdir, inputdir, elastic):

# log files
self.outputdir = outputdir
Expand All @@ -58,12 +53,7 @@ def __init__(self, outputdir, inputdir, parser):
.format(inputdir)
)
self.inputdir = inputdir

# initialize all analyzers
for a in ANALYZERS:
a.init()

self.parser = parser
self.elastic = elastic

def _log_errors(self):
if not self.ignoredFiles:
Expand All @@ -75,9 +65,6 @@ def _log_errors(self):

print("ignored {} files".format(len(self.ignoredFiles)))

def _log_results(self):
for a in ANALYZERS:
a.log(self.outputdir)

def start(self):
pcapfiles = pcap.walk(self.inputdir)
Expand All @@ -86,6 +73,8 @@ def start(self):
format(len(pcapfiles), self.inputdir)
)

ouis = pcap.fetch_ouis()

with Pool(processes=NUM_THREADS) as pool:
c = 0
# async map the process_pcap function to the list of files
Expand All @@ -98,7 +87,7 @@ def start(self):
# asynchronously
pool.apply_async(
pcap.process_pcap,
(fn, [a.analyze for a in ANALYZERS], progressbar_position, self.parser)
(fn, progressbar_position, ouis, self.elastic)
)

# close pool
Expand All @@ -108,7 +97,6 @@ def start(self):
pool.join()

self._log_errors()
self._log_results()

# return number of pcap files
return len(pcapfiles)
Expand All @@ -129,10 +117,10 @@ def start(self):
help='path to the output directory'
)
parser.add_argument(
'-p', '--parser',
'-e', '--elastic',
nargs='?',
default=pcap.Parser.DPKT.name,
choices=[p.name for p in pcap.Parser]
default='localhost:9200',
help='elasticsearch location scheme'
)

args = parser.parse_args()
Expand All @@ -141,7 +129,7 @@ def start(self):
scanner = Main(
outputdir=args.outputdir,
inputdir=args.inputdir,
parser=args.parser
elastic=args.elastic
)
# measure time
startTime = time.time()
Expand Down
Loading