Land rapid7#9424, Add SharknAT&To external scanner

Author: mkienow-r7

LICENSE

@@ -75,6 +75,10 @@ Files: lib/metasm.rb lib/metasm/* data/cpuinfo/*
 Copyright: 2006-2010 Yoann GUILLOT
 License: LGPL-2.1
 
+Files: lib/msf/core/modules/external/python/async_timeout/*
+Copyright: 2016-2017 Andrew Svetlov
+License: Apache 2.0
+
 Files: lib/net/dns.rb lib/net/dns/*
 Copyright: 2006 Marco Ceresa
 License: Ruby

lib/msf/core/data_store.rb

@@ -170,6 +170,20 @@ def to_h
     datastore_hash
   end
 
+  # Hack on a hack for the external modules
+  def to_nested_values
+    datastore_hash = {}
+    self.keys.each do |k|
+      # TODO arbitrary depth
+      if self[k].is_a? Array
+        datastore_hash[k.to_s] = self[k].map(&:to_s)
+      else
+        datastore_hash[k.to_s] = self[k].to_s
+      end
+    end
+    datastore_hash
+  end
+
   #
   # Persists the contents of the data store to a file
   #

lib/msf/core/module/external.rb

@@ -3,18 +3,15 @@ module Msf::Module::External
 
   def wait_status(mod)
     begin
-      while mod.running
-        m = mod.get_status
-        if m
-          case m.method
-          when :message
-            log_output(m)
-          when :report
-            process_report(m)
-          when :reply
-            # we're done
-            break
-          end
+      while m = mod.get_status
+        case m.method
+        when :message
+          log_output(m)
+        when :report
+          process_report(m)
+        when :reply
+          # we're done
+          break
         end
       end
     rescue Interrupt => e
@@ -72,6 +69,20 @@ def process_report(m)
       service[:name] = data['name'] if data['name']
 
       report_service(service)
+    when 'vuln'
+      # Required
+      vuln = {host: data['host'], name: data['name']}
+
+      # Optional
+      vuln[:info] = data['info'] if data['info']
+      vuln[:refs] = data['refs'] if data['refs']
+      vuln[:port] = data['port'] if data['port']
+      vuln[:proto] = data['port'] if data['port']
+
+      # Metasploit magic
+      vuln[:refs] = self.references
+
+      report_vuln(vuln)
     else
       print_warning "Skipping unrecognized report type #{m.params['type']}"
     end

lib/msf/core/modules/external/bridge.rb

@@ -26,10 +26,11 @@ def run(datastore)
   end
 
   def get_status
-    if self.running
+    if self.running || !self.messages.empty?
       m = receive_notification
       if m.nil?
         close_ios
+        self.messages.close
         self.running = false
       end
 
@@ -130,8 +131,9 @@ def recv(filter_id=nil, timeout=600)
           raise EOFError.new
         else
           fds = res[0]
-          # Preferentially drain and log stderr
-          if fds.include? err
+          # Preferentially drain and log stderr, EOF counts as activity, but
+          # stdout might have some buffered data left, so carry on
+          if fds.include?(err) && !err.eof?
             errbuf = err.readpartial(4096)
             elog "Unexpected output running #{self.path}:\n#{errbuf}"
           end

lib/msf/core/modules/external/message.rb

@@ -29,7 +29,13 @@ def initialize(m)
   end
 
   def to_json
-    JSON.generate({jsonrpc: '2.0', id: self.id, method: self.method, params: self.params.to_h})
+    params =
+      if self.params.respond_to? :to_nested_values
+        self.params.to_nested_values
+      else
+        self.params.to_h
+      end
+    JSON.generate({jsonrpc: '2.0', id: self.id, method: self.method, params: params})
   end
 
   protected

lib/msf/core/modules/external/python/async_timeout/__init__.py

@@ -0,0 +1,101 @@
+# Vendored from https://github.com/aio-libs/async-timeout
+# Copyright: 2016-2017 Andrew Svetlov
+# License: Apache 2.0
+
+import asyncio
+
+
+__version__ = '2.0.0'
+
+
+class timeout:
+    """timeout context manager.
+
+    Useful in cases when you want to apply timeout logic around block
+    of code or in cases when asyncio.wait_for is not suitable. For example:
+
+    >>> async with timeout(0.001):
+    ...     async with aiohttp.get('https://github.com') as r:
+    ...         await r.text()
+
+
+    timeout - value in seconds or None to disable timeout logic
+    loop - asyncio compatible event loop
+    """
+    def __init__(self, timeout, *, loop=None):
+        self._timeout = timeout
+        if loop is None:
+            loop = asyncio.get_event_loop()
+        self._loop = loop
+        self._task = None
+        self._cancelled = False
+        self._cancel_handler = None
+        self._cancel_at = None
+
+    def __enter__(self):
+        return self._do_enter()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._do_exit(exc_type)
+
+    @asyncio.coroutine
+    def __aenter__(self):
+        return self._do_enter()
+
+    @asyncio.coroutine
+    def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._do_exit(exc_type)
+
+    @property
+    def expired(self):
+        return self._cancelled
+
+    @property
+    def remaining(self):
+        if self._cancel_at is not None:
+            return max(self._cancel_at - self._loop.time(), 0.0)
+        else:
+            return None
+
+    def _do_enter(self):
+        # Support Tornado 5- without timeout
+        # Details: https://github.com/python/asyncio/issues/392
+        if self._timeout is None:
+            return self
+
+        self._task = current_task(self._loop)
+        if self._task is None:
+            raise RuntimeError('Timeout context manager should be used '
+                               'inside a task')
+
+        if self._timeout <= 0:
+            self._loop.call_soon(self._cancel_task)
+            return self
+
+        self._cancel_at = self._loop.time() + self._timeout
+        self._cancel_handler = self._loop.call_at(
+            self._cancel_at, self._cancel_task)
+        return self
+
+    def _do_exit(self, exc_type):
+        if exc_type is asyncio.CancelledError and self._cancelled:
+            self._cancel_handler = None
+            self._task = None
+            raise asyncio.TimeoutError
+        if self._timeout is not None and self._cancel_handler is not None:
+            self._cancel_handler.cancel()
+            self._cancel_handler = None
+        self._task = None
+
+    def _cancel_task(self):
+        self._task.cancel()
+        self._cancelled = True
+
+
+def current_task(loop):
+    task = asyncio.Task.current_task(loop=loop)
+    if task is None:
+        if hasattr(loop, 'current_task'):
+            task = loop.current_task()
+
+    return task

lib/msf/core/modules/external/python/metasploit/module.py

@@ -1,24 +1,31 @@
-import sys, os, json
+import json
+import os
+import sys
+
 
 def log(message, level='info'):
     rpc_send({'jsonrpc': '2.0', 'method': 'message', 'params': {
         'level': level,
         'message': message
     }})
 
-def report_host(ip, opts={}):
+
+def report_host(ip, **opts):
     host = opts.copy()
     host.update({'host': ip})
-    rpc_send({'jsonrpc': '2.0', 'method': 'report', 'params': {
-        'type': 'host', 'data': host
-    }})
+    report('host', host)
 
-def report_service(ip, opts={}):
+
+def report_service(ip, **opts):
     service = opts.copy()
     service.update({'host': ip})
-    rpc_send({'jsonrpc': '2.0', 'method': 'report', 'params': {
-        'type': 'service', 'data': service
-    }})
+    report('service', service)
+
+
+def report_vuln(ip, name, **opts):
+    vuln = opts.copy()
+    vuln.update({'host': ip, 'name': name})
+    report('vuln', vuln)
 
 
 def run(metadata, module_callback):
@@ -32,6 +39,13 @@ def run(metadata, module_callback):
             'message': 'Module completed'
         }})
 
+
+def report(kind, data):
+    rpc_send({'jsonrpc': '2.0', 'method': 'report', 'params': {
+        'type': kind, 'data': data
+    }})
+
+
 def rpc_send(req):
     print(json.dumps(req))
     sys.stdout.flush()

lib/msf/core/modules/external/python/metasploit/probe_scanner.py

@@ -0,0 +1,101 @@
+import asyncio
+import functools
+import re
+
+from async_timeout import timeout
+from metasploit import module
+
+
+def make_scanner(payload='', pattern='', onmatch=None, connect_timeout=3, read_timeout=10):
+    return lambda args: start_scanner(payload, pattern, args, onmatch, connect_timeout=connect_timeout, read_timeout=read_timeout)
+
+
+def start_scanner(payload, pattern, args, onmatch, **timeouts):
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run_scanner(payload, pattern, args, onmatch, **timeouts))
+
+
+async def run_scanner(payload, pattern, args, onmatch, **timeouts):
+    probes = [probe_host(host, int(args['rport']), payload, **timeouts) for host in args['rhosts']]
+    async for (target, res) in Scan(probes):
+        if isinstance(res, Exception):
+            module.log('{}:{} - Error connecting: {}'.format(*target, res), level='error')
+        elif res and re.search(pattern, res):
+            module.log('{}:{} - Matches'.format(*target), level='good')
+            module.log('{}:{} - Matches with: {}'.format(*target, res), level='debug')
+            onmatch(target, res)
+        else:
+            module.log('{}:{} - Does not match'.format(*target), level='info')
+            module.log('{}:{} - Does not match with: {}'.format(*target, res), level='debug')
+
+
+class Scan:
+    def __init__(self, runs):
+        self.queue = asyncio.queues.Queue()
+        self.total = len(runs)
+        self.done = 0
+
+        for r in runs:
+            f = asyncio.ensure_future(r)
+            args = r.cr_frame.f_locals
+            target = (args['host'], args['port'])
+            f.add_done_callback(functools.partial(self.__queue_result, target))
+
+    def __queue_result(self, target, f):
+        res = None
+
+        try:
+            res = f.result()
+        except Exception as e:
+            res = e
+
+        self.queue.put_nowait((target, res))
+
+    async def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if self.done == self.total:
+            raise StopAsyncIteration
+
+        res = await self.queue.get()
+        self.done += 1
+        return res
+
+
+async def probe_host(host, port, payload, connect_timeout, read_timeout):
+    buf = bytearray()
+
+    try:
+        async with timeout(connect_timeout):
+            r, w = await asyncio.open_connection(host, port)
+            remote = w.get_extra_info('peername')
+            if remote[0] == host:
+                module.log('{}:{} - Connected'.format(host, port), level='debug')
+            else:
+                module.log('{}({}):{} - Connected'.format(host, *remote), level='debug')
+            w.write(payload)
+            await w.drain()
+
+        async with timeout(read_timeout):
+            while len(buf) < 4096:
+                data = await r.read(4096)
+                if data:
+                    module.log('{}:{} - Received {} bytes'.format(host, port, len(data)), level='debug')
+                    buf.extend(data)
+                else:
+                    break
+    except asyncio.TimeoutError:
+        if buf:
+            pass
+        else:
+            raise
+    finally:
+        try:
+            w.close()
+        except Exception:
+            # Either we got something and the socket got in a bad state, or the
+            # original error will point to the root cause
+            pass
+
+    return buf