From 0a2e024be0b58d8b9b7008cca2d832fcd628e3cb Mon Sep 17 00:00:00 2001 From: gokupwn Date: Sun, 12 May 2024 11:33:17 +0200 Subject: [PATCH 1/3] Added new options: - The new option can be used to add the list of files to exclude from the file parsing process --- man_spider/lib/spider.py | 6 +++++- man_spider/lib/spiderling.py | 14 +++++++++++--- man_spider/manspider.py | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/man_spider/lib/spider.py b/man_spider/lib/spider.py index 3c77c71..8f9f775 100644 --- a/man_spider/lib/spider.py +++ b/man_spider/lib/spider.py @@ -41,6 +41,10 @@ def __init__(self, options): self.or_logic = options.or_logic self.extension_blacklist= options.exclude_extensions + + #! List of files to exclude from parsing + self.exclude_files = options.exclude_files + self.file_extensions = options.extensions if self.file_extensions: extensions_str = '"' + '", "'.join(list(self.file_extensions)) + '"' @@ -84,7 +88,7 @@ def start(self): if process is None or not process.is_alive(): # start spiderling self.spiderling_pool[i] = multiprocessing.Process( - target=Spiderling, args=(target, self), daemon=False + target=Spiderling, args=(target, self.exclude_files, self), daemon=False ) self.spiderling_pool[i].start() # success, break out of infinite loop diff --git a/man_spider/lib/spiderling.py b/man_spider/lib/spiderling.py index 9e5973b..4698351 100644 --- a/man_spider/lib/spiderling.py +++ b/man_spider/lib/spiderling.py @@ -53,13 +53,17 @@ class Spiderling: '.xz', ] - def __init__(self, target, parent): + def __init__(self, target, files_toskip, parent): try: self.parent = parent self.target = target + #! Files to skip from parsing (excluded files) + self.files_toskip = files_toskip + + # unless we're only searching local files, connect to target if type(self.target) == pathlib.PosixPath: self.local = True @@ -115,6 +119,11 @@ def go(self): # remote files for file in self.files: + #! Skip parsing for excluded files + if str(file).split("\\")[-1] in self.files_toskip: + log.debug(f"{self.target}: Skipping {str(file)}: match filenames to skip filters") + continue + # if content searching is enabled, parse the file if self.parent.parser.content_filters: try: @@ -169,8 +178,7 @@ def parse_file(self, file): For sole purpose of threading ''' - try: - + try: if type(file) == RemoteFile: matches = self.parent.parser.parse_file(str(file.tmp_filename), pretty_filename=str(file)) if matches and not self.parent.no_download: diff --git a/man_spider/manspider.py b/man_spider/manspider.py index c0479e5..1e19e1f 100755 --- a/man_spider/manspider.py +++ b/man_spider/manspider.py @@ -90,6 +90,8 @@ def main(): parser.add_argument('-f', '--filenames', nargs='+', default=[], help=f'filter filenames using regex (space-separated)', metavar='REGEX') parser.add_argument('-e', '--extensions',nargs='+', default=[], help='only show filenames with these extensions (space-separated, e.g. `docx xlsx` for only word & excel docs)', metavar='EXT') parser.add_argument('--exclude-extensions',nargs='+', default=[], help='ignore files with these extensions', metavar='EXT') + # Argparser option to enter a list of excluded files from parsing + parser.add_argument('--exclude-files',nargs='+', default=[], help='dont parse files with these names (space-separated, e.g. `office.exe junk.bin` to skip parsing for office.exe & junk.data)', metavar='EXCLUDEDFILES') parser.add_argument('-c', '--content', nargs='+', default=[], help='search for file content using regex (multiple supported)', metavar='REGEX') parser.add_argument('--sharenames', nargs='+', default=[], help='only search shares with these names (multiple supported)', metavar='SHARE') parser.add_argument('--exclude-sharenames', nargs='*', default=['IPC$', 'C$', 'ADMIN$', 'PRINT$'],help='don\'t search shares with these names (multiple supported)', metavar='SHARE') From 202f2c8fa85f8f05449edd1c37fdcdd8e8e9572d Mon Sep 17 00:00:00 2001 From: gokupwn Date: Sun, 12 May 2024 17:08:30 +0200 Subject: [PATCH 2/3] Added new option: - The new option can be used to add a list of files to exclude from the file parsing process - Re-reviewed this option and now it will work with both mode local/remote --- man_spider/lib/parser/parser.py | 46 +++++++++++++++++++++++---------- man_spider/lib/spider.py | 5 ++-- man_spider/lib/spiderling.py | 10 +------ 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/man_spider/lib/parser/parser.py b/man_spider/lib/parser/parser.py index 7051893..a2e5b28 100644 --- a/man_spider/lib/parser/parser.py +++ b/man_spider/lib/parser/parser.py @@ -6,6 +6,7 @@ from ..logger import * import subprocess as sp from ..logger import ColoredFormatter +from pathlib import PosixPath log = logging.getLogger('manspider.parser') @@ -38,9 +39,12 @@ class FileParser: ] - def __init__(self, filters, quiet=False): + def __init__(self, filters, files_toskip, quiet=False): self.init_content_filters(filters) + #! list of files to skip from parsing + self.files_toskip = files_toskip + self.quiet = quiet @@ -126,23 +130,37 @@ def parse_file(self, file, pretty_filename=None): if pretty_filename is None: pretty_filename = str(file) + + #! We are in remote mode + if type(pretty_filename) == str: + filename_tocheck = str(pretty_filename).split("\\")[-1] + #! We are in local mode + elif type(pretty_filename) == PosixPath: + filename_tocheck = pretty_filename.name + + #! Skip the file if it's excluded from the parsing (with option: (--exclude-files ...)) + if filename_tocheck in self.files_toskip: + log.debug(f"Skipping {str(pretty_filename)}: one of the filenames to skip") + return None + + else: + #! Parse the file + log.debug(f'Parsing file: {pretty_filename}') - log.debug(f'Parsing file: {pretty_filename}') - - matches = dict() + matches = dict() - try: + try: - matches = self.textract(file, pretty_filename=pretty_filename) + matches = self.textract(file, pretty_filename=pretty_filename) - except Exception as e: - #except (BadZipFile, textract.exceptions.CommandLineError) as e: - if log.level <= logging.DEBUG: - log.warning(f'Error extracting text from {pretty_filename}: {e}') - else: - log.warning(f'Error extracting text from {pretty_filename} (-v to debug)') - - return matches + except Exception as e: + #except (BadZipFile, textract.exceptions.CommandLineError) as e: + if log.level <= logging.DEBUG: + log.warning(f'Error extracting text from {pretty_filename}: {e}') + else: + log.warning(f'Error extracting text from {pretty_filename} (-v to debug)') + + return matches def textract(self, file, pretty_filename): diff --git a/man_spider/lib/spider.py b/man_spider/lib/spider.py index 8f9f775..20704d3 100644 --- a/man_spider/lib/spider.py +++ b/man_spider/lib/spider.py @@ -51,7 +51,8 @@ def __init__(self, options): log.info(f'Searching by file extension: {extensions_str}') self.init_filename_filters(options.filenames) - self.parser = FileParser(options.content, quiet=self.quiet) + #! excluded files will not be parsed (work with both: remote and local manspider mode) + self.parser = FileParser(options.content, self.exclude_files, quiet=self.quiet) self.failed_logons = 0 @@ -88,7 +89,7 @@ def start(self): if process is None or not process.is_alive(): # start spiderling self.spiderling_pool[i] = multiprocessing.Process( - target=Spiderling, args=(target, self.exclude_files, self), daemon=False + target=Spiderling, args=(target, self), daemon=False ) self.spiderling_pool[i].start() # success, break out of infinite loop diff --git a/man_spider/lib/spiderling.py b/man_spider/lib/spiderling.py index 4698351..7e44801 100644 --- a/man_spider/lib/spiderling.py +++ b/man_spider/lib/spiderling.py @@ -53,16 +53,13 @@ class Spiderling: '.xz', ] - def __init__(self, target, files_toskip, parent): + def __init__(self, target, parent): try: self.parent = parent self.target = target - #! Files to skip from parsing (excluded files) - self.files_toskip = files_toskip - # unless we're only searching local files, connect to target if type(self.target) == pathlib.PosixPath: @@ -119,11 +116,6 @@ def go(self): # remote files for file in self.files: - #! Skip parsing for excluded files - if str(file).split("\\")[-1] in self.files_toskip: - log.debug(f"{self.target}: Skipping {str(file)}: match filenames to skip filters") - continue - # if content searching is enabled, parse the file if self.parent.parser.content_filters: try: From 8def27a51e06901e43667560d41961f7561435f6 Mon Sep 17 00:00:00 2001 From: gokupwn Date: Sun, 12 May 2024 17:17:43 +0200 Subject: [PATCH 3/3] Updated The README File To Reflect The New Help Menu: - The new option: --exclude-files --- README.md | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index feeb8c5..ee4869a 100644 --- a/README.md +++ b/README.md @@ -133,18 +133,19 @@ For example, you could specify any or all of these: ## Usage: ~~~ -usage: manspider [-h] [-u USERNAME] [-p PASSWORD] [-d DOMAIN] [-m MAXDEPTH] [-H HASH] [-t THREADS] [-f REGEX [REGEX ...]] [-e EXT [EXT ...]] [--exclude-extensions EXT [EXT ...]] - [-c REGEX [REGEX ...]] [--sharenames SHARE [SHARE ...]] [--exclude-sharenames [SHARE ...]] [--dirnames DIR [DIR ...]] [--exclude-dirnames DIR [DIR ...]] [-q] [-n] - [-mfail INT] [-o] [-s SIZE] [-v] +usage: manspider [-h] [-u USERNAME] [-p PASSWORD] [-d DOMAIN] [-l LOOT_DIR] [-m MAXDEPTH] [-H HASH] [-t THREADS] [-f REGEX [REGEX ...]] + [-e EXT [EXT ...]] [--exclude-extensions EXT [EXT ...]] [--exclude-files EXCLUDEDFILES [EXCLUDEDFILES ...]] + [-c REGEX [REGEX ...]] [--sharenames SHARE [SHARE ...]] [--exclude-sharenames [SHARE ...]] [--dirnames DIR [DIR ...]] + [--exclude-dirnames DIR [DIR ...]] [-q] [-n] [-mfail INT] [-o] [-s SIZE] [-v] targets [targets ...] Scan for juicy data on SMB shares. Matching files and logs are stored in $HOME/.manspider. All filters are case-insensitive. positional arguments: - targets IPs, Hostnames, CIDR ranges, or files containing targets to spider (NOTE: local searching also supported, specify directory name or keyword "loot" to search - downloaded files) + targets IPs, Hostnames, CIDR ranges, or files containing targets to spider (NOTE: local searching also supported, specify + directory name or keyword "loot" to search downloaded files) -optional arguments: +options: -h, --help show this help message and exit -u USERNAME, --username USERNAME username for authentication @@ -152,6 +153,8 @@ optional arguments: password for authentication -d DOMAIN, --domain DOMAIN domain for authentication + -l LOOT_DIR, --loot-dir LOOT_DIR + loot directory (default ~/.manspider/) -m MAXDEPTH, --maxdepth MAXDEPTH maximum depth to spider (default: 10) -H HASH, --hash HASH NTLM hash for authentication @@ -163,6 +166,8 @@ optional arguments: only show filenames with these extensions (space-separated, e.g. `docx xlsx` for only word & excel docs) --exclude-extensions EXT [EXT ...] ignore files with these extensions + --exclude-files EXCLUDEDFILES [EXCLUDEDFILES ...] + dont parse files with these names (space-separated, e.g. `office.exe junk.bin` to skip parsing for office.exe & junk.data) -c REGEX [REGEX ...], --content REGEX [REGEX ...] search for file content using regex (multiple supported) --sharenames SHARE [SHARE ...] @@ -181,4 +186,19 @@ optional arguments: -s SIZE, --max-filesize SIZE don't retrieve files over this size, e.g. "500K" or ".5M" (default: 10M) -v, --verbose show debugging messages + + + # EXAMPLES + + Example 1: Search the network for filenames that may contain creds + $ manspider 192.168.0.0/24 -f passw user admin account network login logon cred -d evilcorp -u bob -p Passw0rd + + Example 2: Search for XLSX files containing "password" + $ manspider share.evilcorp.local -c password -e xlsx -d evilcorp -u bob -p Passw0rd + + Example 3: Search for interesting file extensions + $ manspider share.evilcorp.local -e bat com vbs ps1 psd1 psm1 pem key rsa pub reg txt cfg conf config -d evilcorp -u bob -p Passw0rd + + Example 4: Search for finance-related files + $ manspider share.evilcorp.local --dirnames bank financ payable payment reconcil remit voucher vendor eft swift -f '[0-9]{5,}' -d evilcorp -u bob -p Passw0rd ~~~