Land rapid7#3373, recog

Conflicts:
	Gemfile
	Gemfile.lock
	data/js/detect/os.js
	lib/msf/core/exploit/remote/browser_exploit_server.rb
	modules/exploits/android/browser/webview_addjavascriptinterface.rb

Author: egypt

Gemfile

@@ -10,7 +10,7 @@ group :db do
   # Metasploit::Credential database models
   gem 'metasploit-credential', '~> 0.12.0'
   # Database models shared between framework and Pro.
-  gem 'metasploit_data_models', '~> 0.21.0'
+  gem 'metasploit_data_models', '~> 0.21.1'
   # Needed for module caching in Mdm::ModuleDetails
   gem 'pg', '>= 0.11'
 end

Gemfile.lock

@@ -14,6 +14,7 @@ PATH
       nokogiri
       packetfu (= 1.1.9)
       railties
+      recog (~> 1.0)
       robots
       rubyzip (~> 1.1)
       sqlite3
@@ -111,14 +112,15 @@ GEM
     metasploit-model (0.28.0)
       activesupport
       railties (< 4.0.0)
-    metasploit_data_models (0.21.0)
+    metasploit_data_models (0.21.1)
       activerecord (>= 3.2.13, < 4.0.0)
       activesupport
       arel-helpers
       metasploit-concern (~> 0.3.0)
       metasploit-model (~> 0.28.0)
       pg
       railties (< 4.0.0)
+      recog (~> 1.0)
     meterpreter_bins (0.0.7)
     method_source (0.8.2)
     mime-types (1.25.1)
@@ -161,6 +163,8 @@ GEM
     rake (10.3.2)
     rdoc (3.12.2)
       json (~> 1.4)
+    recog (1.0.0)
+      nokogiri
     redcarpet (3.1.2)
     rkelly-remix (0.0.6)
     robots (0.10.1)
@@ -220,7 +224,7 @@ DEPENDENCIES
   fivemat (= 1.2.1)
   metasploit-credential (~> 0.12.0)
   metasploit-framework!
-  metasploit_data_models (~> 0.21.0)
+  metasploit_data_models (~> 0.21.1)
   network_interface (~> 0.0.1)
   pcaprub
   pg (>= 0.11)

data/js/detect/os.js

@@ -29,7 +29,7 @@ class Metasploit4 < Msf::Exploit::Remote
     :ua_minver  => "8.0",
     :ua_maxver  => "10.0",
     :javascript => true,
-    :os_name    => OperatingSystems::WINDOWS,
+    :os_name    => OperatingSystems::Match::WINDOWS,
     :rank       => NormalRanking
   })
 
@@ -85,6 +85,8 @@ def get_target(agent)
       os_name = 'Windows 7'
     when '6.2'
       os_name = 'Windows 8'
+    when '6.3'
+      os_name = 'Windows 8.1'      
     end
 
     targets.each do |t|

documentation/samples/modules/exploits/ie_browser.rb

@@ -347,7 +347,8 @@ def load_session_info()
             self.db_record.save!
           end
 
-          framework.db.update_host_via_sysinfo(:host => self, :workspace => wspace, :info => sysinfo)
+          # XXX: This is obsolete given the Mdm::Host.normalize_os() support for host.os.session_fingerprint
+          # framework.db.update_host_via_sysinfo(:host => self, :workspace => wspace, :info => sysinfo)
 
           if nhost
             framework.db.report_note({

lib/msf/base/sessions/meterpreter.rb

@@ -58,32 +58,66 @@ module HttpClients
 
   UNKNOWN = "Unknown"
 end
+
 module OperatingSystems
   LINUX   = "Linux"
   MAC_OSX = "Mac OS X"
-  WINDOWS = "Microsoft Windows"
+  WINDOWS = "Windows"
   FREEBSD = "FreeBSD"
   NETBSD  = "NetBSD"
   OPENBSD = "OpenBSD"
   VMWARE  = "VMware"
+  ANDROID = "Android"
+  APPLE_IOS = "iOS"
 
   module VmwareVersions
     ESX   = "ESX"
     ESXI  = "ESXi"
   end
 
   module WindowsVersions
+    NINE5 = "95"
+    NINE8 = "98"
     NT    = "NT"
     XP    = "XP"
     TWOK  = "2000"
     TWOK3 = "2003"
     VISTA = "Vista"
     TWOK8 = "2008"
+    TWOK12 = "2012"
     SEVEN = "7"
     EIGHT = "8"
+    EIGHTONE = "8.1"
   end
 
   UNKNOWN = "Unknown"
+
+  module Match
+    WINDOWS         = /^(?:Microsoft )?Windows/
+    WINDOWS_95      = /^(?:Microsoft )?Windows 95/
+    WINDOWS_98      = /^(?:Microsoft )?Windows 98/
+    WINDOWS_ME      = /^(?:Microsoft )?Windows ME/
+    WINDOWS_NT3     = /^(?:Microsoft )?Windows NT 3/
+    WINDOWS_NT4     = /^(?:Microsoft )?Windows NT 4/
+    WINDOWS_2000    = /^(?:Microsoft )?Windows 2000/
+    WINDOWS_XP      = /^(?:Microsoft )?Windows XP/
+    WINDOWS_2003    = /^(?:Microsoft )?Windows 2003/
+    WINDOWS_VISTA   = /^(?:Microsoft )?Windows Vista/
+    WINDOWS_2008    = /^(?:Microsoft )?Windows 2008/
+    WINDOWS_7       = /^(?:Microsoft )?Windows 7/
+    WINDOWS_2012    = /^(?:Microsoft )?Windows 2012/
+    WINDOWS_8       = /^(?:Microsoft )?Windows 8/
+    WINDOWS_81      = /^(?:Microsoft )?Windows 8\.1/
+
+    LINUX      = /^Linux/i
+    MAC_OSX    = /^(?:Apple )?Mac OS X/
+    FREEBSD    = /^FreeBSD/
+    NETBSD     = /^NetBSD/
+    OPENBSD    = /^OpenBSD/
+    VMWARE     = /^VMware/
+    ANDROID    = /^(?:Google )?Android/
+    APPLE_IOS  = /^(?:Apple )?iOS/
+  end
 end
 end
 
@@ -104,5 +138,4 @@ module WindowsVersions
     BSD_LICENSE,
     ARTISTIC_LICENSE,
     UNKNOWN_LICENSE
-  ]
-
+  ]

lib/msf/core/constants.rb

@@ -304,8 +304,8 @@ def find_or_create_host(opts)
   #
   # The opts parameter can contain:
   # +:state+::        -- one of the Msf::HostState constants
-  # +:os_name+::      -- one of the Msf::OperatingSystems constants
-  # +:os_flavor+::    -- something like "XP" or "Gentoo"
+  # +:os_name+::      -- something like "Windows", "Linux", or "Mac OS X"
+  # +:os_flavor+::    -- something like "Enterprise", "Pro", or "Home"
   # +:os_sp+::        -- something like "SP2"
   # +:os_lang+::      -- something like "English", "French", or "en-US"
   # +:arch+::         -- one of the ARCH_* constants
@@ -452,14 +452,11 @@ def update_host_via_sysinfo(opts)
     end
 
     if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i
-      res[:os_name]   = "Microsoft Windows"
-      res[:os_flavor] = $1.strip
+      res[:os_name]   = "Windows #{$1.strip}"
       build = $2.strip
 
       if build =~ /Service Pack (\d+)/
         res[:os_sp] = "SP" + $1
-      else
-        res[:os_sp] = "SP0"
       end
     end
 
@@ -3531,7 +3528,18 @@ def import_spiceworks_csv(args={}, &block)
       :task      => args[:task]
       }
 
-      conf[:os_name] = os if os
+
+      if os
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => ip,
+          :type => 'host.os.spiceworks_fingerprint',
+          :data => {
+            :os => os.to_s.strip
+          }
+        )
+      end
 
       info = []
       info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name)

lib/msf/core/db.rb

@@ -439,53 +439,100 @@ def proxies
     datastore['Proxies']
   end
 
+
+  #
+  # Lookup HTTP fingerprints from the database that match the current
+  # destination host and port. This method falls back to using the old
+  # service.info field to represent the HTTP Server header.
+  #
+  # @option opts [String] :uri ('/') An HTTP URI to request in order to generate
+  #   a fingerprint
+  # @option opts [String] :method ('GET') An HTTP method to use in the fingerprint
+  #   request
+  def lookup_http_fingerprints(opts={})
+    uri     = opts[:uri] || '/'
+    method  = opts[:method] || 'GET'
+    fprints = []
+
+    return fprints unless framework.db.active
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      wspace = datastore['WORKSPACE'] ?
+        framework.db.find_workspace(datastore['WORKSPACE']) : framework.db.workspace
+
+      service = framework.db.get_service(wspace, rhost, 'tcp', rport)
+      return fprints unless service
+
+      # Order by note_id descending so the first value is the most recent
+      service.notes.where(:ntype => 'http.fingerprint').order("notes.id DESC").each do |n|
+        next unless n.data && n.data.kind_of?(::Hash)
+        next unless n.data[:uri] == uri && n.data[:method] == method
+        # Append additional fingerprints to the results as found
+        fprints.unshift n.data.dup
+      end
+    }
+
+    fprints
+  end
+
   #
   # Record various things about an HTTP server that we can glean from the
   # response to a single request.  If this method is passed a response, it
   # will use it directly, otherwise it will check the database for a previous
   # fingerprint.  Failing that, it will make a request for /.
   #
-  # Options:
-  #	:response   an Http::Packet as returned from any of the send_* methods
+  # Other options are passed directly to {#connect} if :response is not given
   #
-  # Other options are passed directly to +connect+ if :response is not given
+  # @option opts [Rex::Proto::Http::Packet] :response The return value from any
+  #   of the send_* methods
+  # @option opts [String] :uri ('/') An HTTP URI to request in order to generate
+  #   a fingerprint
+  # @option opts [String] :method ('GET') An HTTP method to use in the fingerprint
+  #   request
+  # @option opts [Boolean] :full (false) Request the full HTTP fingerprint, not
+  #   just the signature
   #
+  # @return [String]
   def http_fingerprint(opts={})
+    res    = nil
+    uri    = opts[:uri] || '/'
+    method = opts[:method] || 'GET'
 
-    if (opts[:response])
+    # Short-circuit the fingerprint lookup and HTTP request if a response has
+    # already been provided by the caller.
+    if opts[:response]
       res = opts[:response]
     else
-      # Check to see if we already have a fingerprint before going out to
-      # the network.
-      if (framework.db.active)
-        ::ActiveRecord::Base.connection_pool.with_connection {
-          wspace = framework.db.workspace
-          if datastore['WORKSPACE']
-            wspace = framework.db.find_workspace(datastore['WORKSPACE'])
-          end
+      fprints = lookup_http_fingerprints(opts)
 
-          s = framework.db.get_service(wspace, rhost, 'tcp', rport)
-          if (s and s.info)
-            return s.info
-          end
-        }
+      if fprints.length > 0
+
+        # Grab the most recent fingerprint available for this service, uri, and method
+        fprint = fprints.last
+
+        # Return the full HTTP fingerprint if requested by the caller
+        return fprint if opts[:full]
+
+        # Otherwise just return the signature string for compatibility
+        return fprint[:signature]
       end
 
+      # Go ahead and send a request to the target for fingerprinting
       connect(opts)
-      uri = opts[:uri] || '/'
-      method = opts[:method] || 'GET'
       res = send_request_raw(
         {
           'uri'     => uri,
           'method'  => method
         })
     end
 
-    # Bail if we don't have anything to fingerprint
+    # Bail if the request did not receive a readable response
     return if not res
 
-    # From here to the end simply does some pre-canned combining and custom matches
-    # to build a human-readable string to store in service.info
+    # This section handles a few simple cases of pattern matching and service
+    # classification. This logic should be deprecated in favor of Recog-based
+    # fingerprint databases, but has been left in place for backward compat.
+
     extras = []
 
     if res.headers['Set-Cookie'] =~ /^vmware_soap_session/
@@ -537,6 +584,11 @@ def http_fingerprint(opts={})
       end
     end
 
+    #
+    # This HTTP response code tracking is used by a few modules and the MSP logic
+    # to identify and bruteforce certain types of servers. In the long run we
+    # should deprecate this and use the http.fingerprint fields instead.
+    #
     case res.code
     when 301,302
       extras << "#{res.code}-#{res.headers['Location']}"
@@ -548,12 +600,51 @@ def http_fingerprint(opts={})
       extras << "#{res.code}-#{res.message}"
     end
 
-    info  = "#{res.headers['Server']}"
+    # Build a human-readable string to store in service.info and http.fingerprint[:signature]
+    info = res.headers['Server'].to_s.dup
     info << " ( #{extras.join(", ")} )" if extras.length > 0
+
+    # Create a new fingerprint structure to track this response
+    fprint = {
+      :uri => uri, :method => method,
+      :code => res.code.to_s, :message => res.message.to_s,
+      :signature => info
+    }
+
+    res.headers.each_pair do |k,v|
+      hname = k.to_s.downcase.gsub('-', '_').gsub(/[^a-z0-9_]+/, '')
+      next unless hname.length > 0
+
+      # Set-Cookie >        :header_set_cookie       => JSESSIONID=AAASD23423452
+      # Server >            :header_server           => Apache/1.3.37
+      # WWW-Authenticate >  :header_www_authenticate => basic realm='www'
+
+      fprint["header_#{hname}".intern] = v
+    end
+
+    # Store the first 64k of the HTTP body as well
+    fprint[:content] = res.body.to_s[0,65535]
+
+    # Report a new http.fingerprint note
+    report_note(
+      :host   => rhost,
+      :port   => rport,
+      :proto  => 'tcp',
+      :ntype  => 'http.fingerprint',
+      :data   => fprint,
+      # Limit reporting to one stored note per host/service combination
+      :update => :unique
+    )
+
     # Report here even if info is empty since the fact that we didn't
     # return early means we at least got a connection and the service is up
     report_web_site(:host => rhost, :port => rport, :ssl => ssl, :vhost => vhost, :info => info.dup)
-    info
+
+    # Return the full HTTP fingerprint if requested by the caller
+    return fprint if opts[:full]
+
+    # Otherwise just return the signature string for compatibility
+    fprint[:signature]
   end
 
   def make_cnonce