Land rapid7#4138, @wchen-r7's reference cheking for module_reference.rb

* Fixes rapid7#4039

Author: jvazquez-r7

lib/msf/core/module/reference.rb

@@ -87,6 +87,9 @@ def self.from_a(ary)
 
   #
   # Initialize the site reference.
+  # If you're updating the references, please also update:
+  # * tools/module_reference.rb
+  # * https://github.com/rapid7/metasploit-framework/wiki/Metasploit-module-reference-identifiers
   #
   def initialize(in_ctx_id = 'Unknown', in_ctx_val = '')
     self.ctx_id  = in_ctx_id

tools/module_reference.rb

@@ -1,11 +1,7 @@
 #!/usr/bin/env ruby
 #
-# $Id$
-#
 # This script lists each module with its references
 #
-# $Revision$
-#
 
 msfbase = __FILE__
 while File.symlink?(msfbase)
@@ -20,64 +16,151 @@
 require 'rex'
 require 'msf/ui'
 require 'msf/base'
+require 'uri'
+
+
+# See lib/msf/core/module/reference.rb
+# We gsub '#{in_ctx_val}' with the actual value
+def types
+  {
+    'ALL'        => '',
+    'OSVDB'      => 'http://www.osvdb.org/#{in_ctx_val}',
+    'CVE'        => 'http://cvedetails.com/cve/#{in_ctx_val}/',
+    'CWE'        => 'http://cwe.mitre.org/data/definitions/#{in_ctx_val}.html',
+    'BID'        => 'http://www.securityfocus.com/bid/#{in_ctx_val}',
+    'MSB'        => 'http://technet.microsoft.com/en-us/security/bulletin/#{in_ctx_val}',
+    'EDB'        => 'http://www.exploit-db.com/exploits/#{in_ctx_val}',
+    'US-CERT-VU' => 'http://www.kb.cert.org/vuls/id/#{in_ctx_val}',
+    'ZDI'        => 'http://www.zerodayinitiative.com/advisories/ZDI-#{in_ctx_val}',
+    'WPVDB'      => 'https://wpvulndb.com/vulnerabilities/#{in_ctx_val}',
+    'URL'        => '#{in_ctx_val}'
+  }
+end
 
+STATUS_ALIVE       = 'Alive'
+STATUS_DOWN        = 'Down'
+STATUS_UNSUPPORTED = 'Unsupported'
 
-sort=0
-filter= 'All'
+sort    = 0
+filter  = 'All'
 filters = ['all','exploit','payload','post','nop','encoder','auxiliary']
-types = ['All','URL','CVE','OSVDB','BID','MSB','NSS','US-CERT-VU']
-type='All'
-match= nil
+type    ='ALL'
+match   = nil
+check   = false
+save    = nil
 
 opts = Rex::Parser::Arguments.new(
   "-h" => [ false, "Help menu." ],
+  "-c" => [ false, "Check reference status"],
   "-s" => [ false, "Sort by Reference instead of Module Type."],
   "-r" => [ false, "Reverse Sort"],
-  "-f" => [ true, "Filter based on Module Type [All,Exploit,Payload,Post,NOP,Encoder,Auxiliary] (Default = All)."],
-  "-t" => [ true, "Type of Reference to sort by [All,URL,CVE,OSVDB,BID,MSB,NSS,US-CERT-VU]"],
-  "-x" => [ true, "String or RegEx to try and match against the Reference Field"]
-
+  "-f" => [ true, "Filter based on Module Type [All,Exploit,Payload,Post,NOP,Encoder,Auxiliary] (Default = ALL)."],
+  "-t" => [ true, "Type of Reference to sort by #{types.keys}"],
+  "-x" => [ true, "String or RegEx to try and match against the Reference Field"],
+  "-o" => [ true, "Save the results to a file"]
 )
 
+flags = []
+
 opts.parse(ARGV) { |opt, idx, val|
   case opt
   when "-h"
     puts "\nMetasploit Script for Displaying Module Reference information."
     puts "=========================================================="
     puts opts.usage
     exit
+  when "-c"
+    flags << "URI Check: Yes"
+    check = true
   when "-s"
-    puts "Sorting by License"
+    flags << "Order: Sorting by License"
     sort = 1
   when "-r"
-    puts "Reverse Sorting"
+    flags << "Order: Reverse Sorting"
     sort = 2
   when "-f"
     unless filters.include?(val.downcase)
       puts "Invalid Filter Supplied: #{val}"
       puts "Please use one of these: #{filters.map{|f|f.capitalize}.join(", ")}"
       exit
     end
-    puts "Module Filter: #{val}"
+    flags << "Module Filter: #{val}"
     filter = val
   when "-t"
-    unless types.include?(val)
+    val = (val || '').upcase
+    unless types.has_key(val)
       puts "Invalid Type Supplied: #{val}"
-      puts "Please use one of these: [All,URL,CVE,OSVDB,BID,MSB,NSS,US-CERT-VU]"
+      puts "Please use one of these: #{types.keys.inspect}"
       exit
     end
-    puts "Type: #{val}"
     type = val
   when "-x"
-    puts "Regex: #{val}"
+    flags << "Regex: #{val}"
     match = Regexp.new(val)
+  when "-o"
+    flags << "Output to file: Yes"
+    save = val
   end
-
 }
 
-puts "Type: #{type}"
+flags << "Type: #{type}"
+
+puts flags * " | "
+
+def get_ipv4_addr(hostname)
+  Rex::Socket::getaddresses(hostname, false)[0]
+end
+
+def is_url_alive?(uri)
+  #puts "URI: #{uri}"
+
+  begin
+    uri = URI(uri)
+    rhost = get_ipv4_addr(uri.host)
+  rescue SocketError, URI::InvalidURIError => e
+    #puts "Return false 1: #{e.message}"
+    return false
+  end
+
+  rport = uri.port || 80
+  path  = uri.path.blank? ? '/' : uri.path
+  vhost = rport == 80 ? uri.host : "#{uri.host}:#{rport}"
+  if uri.scheme == 'https'
+    cli = ::Rex::Proto::Http::Client.new(rhost, 443, {}, true, 'TLS1')
+  else
+    cli = ::Rex::Proto::Http::Client.new(rhost, rport)
+  end
+
+  begin
+    cli.connect
+    req = cli.request_raw('uri'=>path, 'vhost'=>vhost)
+    res = cli.send_recv(req)
+  rescue Errno::ECONNRESET, Rex::ConnectionError, Rex::ConnectionRefused, Rex::HostUnreachable, Rex::ConnectionTimeout, Rex::UnsupportedProtocol, ::Timeout::Error, Errno::ETIMEDOUT => e
+    #puts "Return false 2: #{e.message}"
+    return false
+  ensure
+    cli.close
+  end
+
+  if res.nil? || res.code == 404 || res.body =~ /<title>.*not found<\/title>/i
+    #puts "Return false 3: HTTP #{res.code}"
+    #puts req.to_s
+    return false 
+  end
 
-Indent = '    '
+  true
+end
+
+def save_results(path, results)
+  begin
+    File.open(path, 'wb') do |f|
+      f.write(results)
+    end
+    puts "Results saved to: #{path}"
+  rescue Exception => e
+    puts "Failed to save the file: #{e.message}"
+  end
+end
 
 # Always disable the database (we never need it just to list module
 # information).
@@ -91,21 +174,50 @@
 # Initialize the simplified framework instance.
 $framework = Msf::Simple::Framework.create(framework_opts)
 
+if check
+  columns = [ 'Module', 'Status', 'Reference' ]
+else
+  columns = [ 'Module', 'Reference' ]
+end
 
 tbl = Rex::Ui::Text::Table.new(
   'Header'  => 'Module References',
-  'Indent'  => Indent.length,
-  'Columns' => [ 'Module', 'Reference' ]
+  'Indent'  => 2,
+  'Columns' => columns
 )
 
+bad_refs_count  = 0
+
 $framework.modules.each { |name, mod|
   next if match and not name =~ match
 
   x = mod.new
   x.references.each do |r|
-    if type=='All' or type==r.ctx_id
+    ctx_id = r.ctx_id.upcase
+    if type == 'ALL' || type == ctx_id
+
+      if check
+        if types.has_key?(ctx_id)
+          uri = types[r.ctx_id.upcase].gsub(/\#{in_ctx_val}/, r.ctx_val)
+          if is_url_alive?(uri)
+            status = STATUS_ALIVE
+          else
+            bad_refs_count += 1
+            status = STATUS_DOWN
+          end
+        else
+          # The reference ID isn't supported so we don't know how to check this
+          bad_refs_count += 1
+          status = STATUS_UNSUPPORTED
+        end
+      end
+
       ref = "#{r.ctx_id}-#{r.ctx_val}"
-      tbl << [ x.fullname, ref ]
+      new_column = []
+      new_column << x.fullname
+      new_column << status if check
+      new_column << ref
+      tbl << new_column
     end
   end
 }
@@ -120,5 +232,9 @@
   tbl.rows.reverse
 end
 
-
+puts
 puts tbl.to_s
+puts
+
+puts "Number of bad references found: #{bad_refs_count}" if check
+save_results(save, tbl.to_s) if save