Skip to content

Commit fc8e0b8

Browse files
smicallefsmicallef
committed
Merge branch 'master' of https://github.com/smicallef/spiderfoot 
2 parents ec507ba + 2f1d7eb commit fc8e0b8

File tree

229 files changed

+269
-238
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

229 files changed

+269
-238
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ language: python
55
python:
66
- "3.6"
77
- "3.8"
8+
- "3.9"
89

910
cache: pip
1011

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
SpiderFoot 3.3-DEV
1+
SpiderFoot 3.3

‎dyn/HEADER.tmpl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<html lang="en">
33
<head>
44
<meta charset="utf-8">
5-
<title>SpiderFoot v3.3-DEV</title>
5+
<title>SpiderFoot v3.3</title>
66
<meta name="viewport" content="width=device-width, initial-scale=1.0">
77
<meta name="description" content="">
88
<meta name="author" content="">
@@ -64,13 +64,13 @@ class="nav-item"><a href="${docroot}/opts" class="nav-link"><i class="glyphicon
6464
<div class="modal-content">
6565

6666
<div class="modal-header">
67-
<h3 class="modal-title">SpiderFoot 3.3-DEV</h3>
67+
<h3 class="modal-title">SpiderFoot 3.3</h3>
6868
</div>
6969

7070
<div class="modal-body">
7171
<p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='https://twitter.com/binarypool' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>
7272
<p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:support@spiderfoot.net>support@spiderfoot.net</a>.
73-
<p>You are running version <b>3.3-DEV</b>.</p>
73+
<p>You are running version <b>3.3</b>.</p>
7474
<p> Visit the project website at <a href='https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQv&s=os_int' target=_new>https://www.spiderfoot.net/</a>.</p>
7575
<p> View the documentation at <a href='https://www.spiderfoot.net/documentation/' target=_new>https://www.spiderfoot.net/documentation/</a>.</p>
7676
<p> Check out the Github repo at <a href='https://github.com/smicallef/spiderfoot' target=_new>https://github.com/smicallef/spiderfoot</a>.</p>

‎modules/sfp_archiveorg.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def handleEvent(self, event):
165165
ret = None
166166

167167
if not ret:
168-
self.sf.error("Unable to process empty response from archive.org: {eventData}")
168+
self.sf.debug("Empty response from archive.org for {eventData}")
169169
continue
170170

171171
if len(ret['archived_snapshots']) < 1:

‎modules/sfp_binaryedge.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,10 @@ def handleEvent(self, event):
438438

439439
try:
440440
banner = prec['result']['data']['service']['banner']
441+
if '\\r\\n\\r\\n' in banner and "HTTP/" in banner:
442+
# We don't want the content after HTTP banners
443+
banner = banner.split('\\r\\n\\r\\n')[0]
444+
banner = banner.replace("\\r\\n", "\n")
441445
except Exception:
442446
self.sf.debug("No banner information found.")
443447
continue

‎modules/sfp_pageinfo.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,15 @@ def handleEvent(self, event):
8585
self.sf.debug("Not gathering page info for external site " + eventSource)
8686
return
8787

88+
# Ignore javascript and CSS
89+
if ".css?" in eventSource or eventSource.endswith(".css"):
90+
self.sf.debug("Not attempting to match CSS content.")
91+
return
92+
93+
if ".js?" in eventSource or eventSource.endswith(".js"):
94+
self.sf.debug("Not attempting to match JS content.")
95+
return
96+
8897
if eventSource in self.results:
8998
self.sf.debug("Already checked this page for a page type, skipping.")
9099
return

‎modules/sfp_s3bucket.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,8 @@ def handleEvent(self, event):
161161
if eventName == "LINKED_URL_EXTERNAL":
162162
if ".amazonaws.com" in eventData:
163163
b = self.sf.urlFQDN(eventData)
164+
if b in self.opts['endpoints']:
165+
b += "/" + eventData.split(b + "/")[1].split("/")[0]
164166
evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
165167
self.notifyListeners(evt)
166168
return None

‎modules/sfp_scylla.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@ class sfp_scylla(SpiderFootPlugin):
2727
'useCases': ["Footprint", "Investigate", "Passive"],
2828
'categories': ["Leaks, Dumps and Breaches"],
2929
'dataSource': {
30-
'website': "https://scylla.sh/",
30+
'website': "https://scylla.so/",
3131
'model': "FREE_NOAUTH_UNLIMITED",
3232
'references': [
33-
"https://scylla.sh/crowdsource"
33+
"https://scylla.so/crowdsource"
3434
],
3535
'favIcon': "",
3636
'logo': "",
37-
'description': "scylla.sh has two major goals. One is to have a community-oriented database leak community "
37+
'description': "scylla.so has two major goals. One is to have a community-oriented database leak community "
3838
"that is a useful tool for security researchers.\n"
3939
"The other major goal is to undercut those people that are selling databases.",
4040
}
@@ -76,15 +76,15 @@ def producedEvents(self):
7676
# Query Scylla API
7777
def query(self, qry, per_page=20, start=0):
7878
params = {
79-
'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
79+
'q': 'email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
8080
'size': str(per_page),
8181
'start': str(start)
8282
}
8383

8484
headers = {
8585
'Accept': 'application/json'
8686
}
87-
res = self.sf.fetchUrl('https://scylla.sh/search?' + urllib.parse.urlencode(params),
87+
res = self.sf.fetchUrl('https://scylla.so/search?' + urllib.parse.urlencode(params),
8888
headers=headers,
8989
timeout=15,
9090
useragent=self.opts['_useragent'],
@@ -99,7 +99,7 @@ def query(self, qry, per_page=20, start=0):
9999
return None
100100

101101
if res['content'] is None:
102-
self.sf.debug('No response from Scylla.sh')
102+
self.sf.debug('No response from Scylla.so')
103103
return None
104104

105105
try:

‎modules/sfp_shodan.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,11 @@ def query(self, qry):
9898
return None
9999

100100
try:
101-
return json.loads(res['content'])
101+
r = json.loads(res['content'])
102+
if "error" in r:
103+
self.sf.error("Error returned form SHODAN: {r['error']}")
104+
return None
105+
return r
102106
except Exception as e:
103107
self.sf.error(f"Error processing JSON response from SHODAN: {e}")
104108
return None
@@ -115,7 +119,11 @@ def searchHosts(self, qry):
115119
return None
116120

117121
try:
118-
return json.loads(res['content'])
122+
r = json.loads(res['content'])
123+
if "error" in r:
124+
self.sf.error("Error returned form SHODAN: {r['error']}")
125+
return None
126+
return r
119127
except Exception as e:
120128
self.sf.error(f"Error processing JSON response from SHODAN: {e}")
121129
return None
@@ -136,7 +144,14 @@ def searchHtml(self, qry):
136144
return None
137145

138146
try:
139-
return json.loads(res['content'])
147+
r = json.loads(res['content'])
148+
if "error" in r:
149+
self.sf.error("Error returned form SHODAN: {r['error']}")
150+
return None
151+
if r.get('total', 0) == 0:
152+
self.sf.info("No SHODAN info found for {qry}")
153+
return None
154+
return r
140155
except Exception as e:
141156
self.sf.error(f"Error processing JSON response from SHODAN: {e}")
142157
return None

‎modules/sfp_spider.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ def processUrl(self, url):
136136
self.getTarget().getNames())
137137

138138
if not links:
139-
self.sf.info(f"No links found at {url}")
139+
self.sf.debug(f"No links found at {url}")
140140
return None
141141

142142
# Notify modules about the links found
@@ -301,7 +301,7 @@ def handleEvent(self, event):
301301
if spiderTarget is None:
302302
return None
303303

304-
self.sf.info("Initiating spider of " + spiderTarget + " from " + srcModuleName)
304+
self.sf.debug("Initiating spider of " + spiderTarget + " from " + srcModuleName)
305305

306306
# Link the spidered URL to the event that triggered it
307307
self.urlEvents[spiderTarget] = event
@@ -368,7 +368,7 @@ def spiderFrom(self, startingPoint):
368368
break
369369

370370
nextLinks = self.cleanLinks(links)
371-
self.sf.info(f"Found links: {nextLinks}")
371+
self.sf.debug(f"Found links: {nextLinks}")
372372

373373
# We've scanned through another layer of the site
374374
levelsTraversed += 1
@@ -379,7 +379,7 @@ def spiderFrom(self, startingPoint):
379379

380380
# We've reached the end of our journey..
381381
if len(nextLinks) == 0:
382-
self.sf.info("No more links found to spider, finishing..")
382+
self.sf.debug("No more links found to spider, finishing..")
383383
keepSpidering = False
384384

385385
# We've been asked to stop scanning

0 commit comments

Comments
 (0)