Land rapid7#3677 - Updated shodan_search for new API

Author: wchen-r7

modules/auxiliary/gather/shodan_search.rb

@@ -5,7 +5,8 @@
 
 require 'msf/core'
 require 'rex'
-require 'net/dns'
+require 'net/https'
+require 'uri'
 
 class Metasploit4 < Msf::Auxiliary
 
@@ -16,170 +17,171 @@ def initialize(info = {})
     super(update_info(info,
       'Name' => 'Shodan Search',
       'Description' => %q{
-        This module uses the SHODAN API to query the database and
-        returns the first 50 IPs. SHODAN accounts are free & output
-        can be sent to a file for use by another program. Results
-        can also populated into the services table in the database.
-        NOTE: SHODAN filters (port, hostname, os, geo, city) can be
-        used in queries, but the free API does not allow net, country,
-        before, and after filters. An unlimited API key can be
-        purchased from the Shodan site to use those queries. The 50
-        result limit can also be raised to 10,000 for a small fee.
-        API: http://www.shodanhq.com/api_doc
-        FILTERS: http://www.shodanhq.com/help/filters
+        This module uses the Shodan API to search Shodan. Accounts are free
+        and an API key is required to used this module. Output from the module
+        is displayed to the screen and can be saved to a file or the MSF database.
+        NOTE: SHODAN filters (i.e. port, hostname, os, geo, city) can be used in
+        queries, but there are limitations when used with a free API key. Please
+        see the Shodan site for more information.
+        Shodan website: https://www.shodan.io/
+        API: https://developer.shodan.io/api
       },
       'Author' =>
         [
-          'John Sawyer <johnhsawyer[at]gmail.com>',  #sploitlab.com
-          'sinn3r'  #Metasploit-fu plus other features
+          'John H Sawyer <john[at]sploitlab.com>', # InGuardians, Inc.
+          'sinn3r'  # Metasploit-fu plus other features
         ],
       'License' => MSF_LICENSE
-    ))
+      )
+    )
 
-    # disabling all the unnecessary options that someone might set to break our query
-    deregister_options('RPORT','RHOST', 'DOMAIN',
-      'DigestAuthIIS', 'SSLVersion', 'NTLM::SendLM', 'NTLM::SendNTLM',
-      'NTLM::SendSPN', 'NTLM::UseLMKey', 'NTLM::UseNTLM2_session',
-      'NTLM::UseNTLMv2','SSL')
+    deregister_options('RHOST', 'DOMAIN', 'DigestAuthIIS', 'NTLM::SendLM',
+      'NTLM::SendNTLM', 'VHOST', 'RPORT', 'NTLM::SendSPN', 'NTLM::UseLMKey',
+      'NTLM::UseNTLM2_session', 'NTLM::UseNTLMv2')
 
     register_options(
       [
-        OptString.new('SHODAN_APIKEY', [true, "The SHODAN API key"]),
-        OptString.new('QUERY', [true, "Keywords you want to search for"]),
-        OptString.new('OUTFILE', [false, "A filename to store the list of IPs"]),
-        OptBool.new('DATABASE', [false, "Add search results to the database", false]),
-        OptInt.new('MAXPAGE', [true, "Max amount of pages to collect", 1]),
-        OptString.new('FILTER', [false, 'Search for a specific IP/City/Country/Hostname']),
-        OptString.new('VHOST', [true, 'The virtual host name to use in requests', 'www.shodanhq.com']),
+        OptString.new('SHODAN_APIKEY', [true, 'The SHODAN API key']),
+        OptString.new('QUERY', [true, 'Keywords you want to search for']),
+        OptString.new('OUTFILE', [false, 'A filename to store the list of IPs']),
+        OptBool.new('DATABASE', [false, 'Add search results to the database', false]),
+        OptInt.new('MAXPAGE', [true, 'Max amount of pages to collect', 1]),
+        OptRegexp.new('REGEX', [true, 'Regex search for a specific IP/City/Country/Hostname', '.*'])
+
       ], self.class)
   end
 
   # create our Shodan query function that performs the actual web request
   def shodan_query(query, apikey, page)
     # send our query to Shodan
-    uri = "/api/search?&q=" + Rex::Text.uri_encode(query) + "&key=" + apikey + "&page=" + page.to_s
-    res = send_request_raw(
-      {
-        'rhost'    => shodan_rhost,
-        'rport'    => shodan_rport,
-        'vhost'    => vhost,
-        'method'   => 'GET',
-        'uri'      => uri
-    }, 25)
-
-    # Check if we got a response, parse the JSON, and return it
-    if (res)
+    uri = URI.parse('https://api.shodan.io/shodan/host/search?query=' +
+      Rex::Text.uri_encode(query) + '&key=' + apikey + '&page=' + page.to_s)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    request = Net::HTTP::Get.new(uri.request_uri)
+    res = http.request(request)
+
+    if res and res.body =~ /<title>401 Unauthorized<\/title>/
+      fail_with(Failure::BadConfig, '401 Unauthorized. Your SHODAN_APIKEY is invalid')
+    end
+
+    # Check if we can resolve host, got a response,
+    # then parse the JSON, and return it
+    if res
       results = ActiveSupport::JSON.decode(res.body)
       return results
     else
-      return 'server_error'
+      return 'server_response_error'
     end
   end
 
+  # save output to file
   def save_output(data)
-    f = ::File.open(datastore['OUTFILE'], "wb")
-    f.write(data)
-    f.close
-    print_status("Save results in #{datastore['OUTFILE']}")
+    ::File.open(datastore['OUTFILE'], 'wb') do |f|
+      f.write(data)
+      print_status("Saved results in #{datastore['OUTFILE']}")
+    end
   end
 
+  # Check to see if api.shodan.io resolves properly
   def shodan_rhost
-    @res = Net::DNS::Resolver.new()
-    dns_query = @res.query("#{datastore['VHOST']}", "A")
+    @res = Net::DNS::Resolver.new
+    dns_query = @res.query('api.shodan.io', 'A')
     if dns_query.answer.length == 0
-      print_error("Could not resolve #{datastore['VHOST']}")
-      raise ::Rex::ConnectError(vhost, shodan_port)
+      print_error('Could not resolve api.shodan.io')
+      raise ::Rex::ConnectError('api.shodan.io', '443')
     end
     dns_query.answer[0].to_s.split(/[\s,]+/)[4]
   end
 
-  def shodan_rport
-    80
-  end
-
   def run
+    # check to ensure api.shodan.io is resolvable
+    shodan_rhost
 
     # create our Shodan request parameters
     query = datastore['QUERY']
     apikey = datastore['SHODAN_APIKEY']
-
     page = 1
+    maxpage = datastore['MAXPAGE']
 
     # results gets our results from shodan_query
     results = []
     results[page] = shodan_query(query, apikey, page)
 
-    if results[page].empty?
-      print_error("No Results Found!")
-      return
+    if results[page]['total'].nil? || results[page]['total'] == 0
+      print_error('No Results Found!')
     end
 
     # Determine page count based on total results
-    if results[page]['total']%50 == 0
-      tpages = results[page]['total']/50
+    if results[page]['total'] % 100 == 0
+      tpages = results[page]['total'] / 100
     else
-      tpages = results[page]['total']/50 + 1
+      tpages = results[page]['total'] / 100 + 1
+      maxpage = tpages if datastore['MAXPAGE'] > tpages
     end
 
     # start printing out our query statistics
-    print_status("Total: #{results[page]['total']} on #{tpages} pages. Showing: #{datastore['MAXPAGE']}")
-    print_status("Country Statistics:")
-    results[page]['countries'].each { |ctry|
-      print_status "\t#{ctry['name']} (#{ctry['code']}): #{ctry['count']}"
-    }
-
-    # If search results greater than 50, loop & get all results
-    print_status("Collecting data, please wait...")
-    if (results[page]['total'] > 50)
+    print_status("Total: #{results[page]['total']} on #{tpages} " +
+      "pages. Showing: #{maxpage} page(s)")
+
+    # If search results greater than 100, loop & get all results
+    print_status('Collecting data, please wait...')
+    if results[page]['total'] > 100
       page += 1
-      while page <= tpages
-        results[page] = shodan_query(query, apikey, page)
-        page +=1
+      while page <= maxpage
         break if page > datastore['MAXPAGE']
+        results[page] = shodan_query(query, apikey, page)
+        page += 1
       end
     end
 
     # Save the results to this table
     tbl = Rex::Ui::Text::Table.new(
-      'Header'  => 'IP Results',
+      'Header'  => 'Search Results',
       'Indent'  => 1,
-      'Columns' => ['IP', 'City', 'Country', 'Hostname']
+      'Columns' => ['IP:Port', 'City', 'Country', 'Hostname']
     )
 
-    # Organize results and put them into the table
-    page = 1
-    my_filter = datastore['FILTER']
-    for i in page..tpages
-      next if results[i].nil? or results[i]['matches'].nil?
-      results[i]['matches'].each { |host|
-
-        city = host['city'] || 'N/A'
-        ip   = host['ip'] || 'N/A'
+    # Organize results and put them into the table and database
+    p = 1
+    regex = datastore['REGEX'] if datastore['REGEX']
+    while p <= maxpage
+      break if p > maxpage
+      results[p]['matches'].each do |host|
+        city = host['location']['city'] || 'N/A'
+        ip   = host['ip_str'] || 'N/A'
         port = host['port'] || ''
-        country = host['country_name'] || 'N/A'
+        country = host['location']['country_name'] || 'N/A'
         hostname = host['hostnames'][0]
         data = host['data']
 
-        if  ip =~ /#{my_filter}/ or
-          city =~ /#{my_filter}/i or
-          country =~ /#{my_filter}/i or
-          hostname =~ /#{my_filter}/i or
-          data =~ /#{my_filter}/i
-          # Unfortunately we cannot display the banner properly,
-          # because it messes with our output format
-          tbl << ["#{ip}:#{port}", city, country, hostname]
+        report_host(:host     => ip,
+                    :name     => hostname,
+                    :comments => 'Added from Shodan',
+                    :info     => host['info']
+                    ) if datastore['DATABASE']
+
+        report_service(:host => ip,
+                       :port => port,
+                       :info => 'Added from Shodan'
+                       ) if datastore['DATABASE']
+
+        if ip =~ regex ||
+           city =~ regex ||
+           country =~ regex ||
+           hostname =~ regex ||
+           data =~ regex
+           # Unfortunately we cannot display the banner properly,
+           # because it messes with our output format
+           tbl << ["#{ip}:#{port}", city, country, hostname]
         end
-      }
+      end
+      p += 1
     end
 
-    #Show data and maybe save it if needed
-    print_line("\n#{tbl.to_s}")
-
-    report_note(
-      :type => 'shodan',
-      :data => tbl.to_csv
-    ) if datastore['DATABASE']
-
-    save_output(tbl.to_s) if not datastore['OUTFILE'].nil?
+    # Show data and maybe save it if needed
+    print_line
+    print_line("#{tbl}")
+    save_output(tbl) if datastore['OUTFILE']
   end
 end