add license, fix style violations, log with dlog

Author: Brent Cook

lib/robots.rb

@@ -1,19 +1,44 @@
+#
+#  Copyright (c) 2008 Kyle Maxwell, contributors
+#
+#  Permission is hereby granted, free of charge, to any person
+#  obtaining a copy of this software and associated documentation
+#  files (the "Software"), to deal in the Software without
+#  restriction, including without limitation the rights to use,
+#  copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following
+#  conditions:
+#
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#  OTHER DEALINGS IN THE SOFTWARE.
+#
+
 require "open-uri"
 require "uri"
-require "rubygems"
 require "timeout"
+require 'rex/logging/log_dispatcher'
 
+# https://github.com/fizx/robots
 class Robots
-  
   DEFAULT_TIMEOUT = 3
-  
+
+  # Represents a parsed robots.txt file
   class ParsedRobots
-    
     def initialize(uri, user_agent)
       @last_accessed = Time.at(1)
-      
+
       io = Robots.get_robots_txt(uri, user_agent)
-      
+
       if !io || io.content_type != "text/plain" || io.status.first != "200"
         io = StringIO.new("User-agent: *\nAllow: /\n")
       end
@@ -45,27 +70,25 @@ def initialize(uri, user_agent)
           @other[key] << value
         end
       end
-      
+
       @parsed = true
     end
-    
+
     def allowed?(uri, user_agent)
       return true unless @parsed
       allowed = true
       path = uri.request_uri
-      
+
       @disallows.each do |key, value|
         if user_agent =~ key
           value.each do |rule|
-            if path =~ rule
-              allowed = false
-            end
+            allowed = false if path =~ rule
           end
         end
       end
-      
+
       @allows.each do |key, value|
-        unless allowed      
+        unless allowed
           if user_agent =~ key
             value.each do |rule|
               if path =~ rule
@@ -75,59 +98,61 @@ def allowed?(uri, user_agent)
           end
         end
       end
-      
+
       if allowed && @delays[user_agent]
         sleep @delays[user_agent] - (Time.now - @last_accessed)
         @last_accessed = Time.now
       end
-      
+
       return allowed
     end
-    
+
     def other_values
       @other
     end
-    
-  protected
-    
+
+    protected
+
     def to_regex(pattern)
       return /should-not-match-anything-123456789/ if pattern.strip.empty?
       pattern = Regexp.escape(pattern)
       pattern.gsub!(Regexp.escape("*"), ".*")
       Regexp.compile("^#{pattern}")
     end
   end
-  
+
   def self.get_robots_txt(uri, user_agent)
     begin
-      Timeout::timeout(Robots.timeout) do
-        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
-      end 
+      Timeout.timeout(Robots.timeout) do
+        begin
+          URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent)
+        rescue StandardError
+          nil
+        end
+      end
     rescue Timeout::Error
-      STDERR.puts "robots.txt request timed out"
+      dlog("robots.txt request timed out")
     end
   end
-  
-  def self.timeout=(t)
-    @timeout = t
-  end
-  
+
+  attr_writer :timeout
+
   def self.timeout
     @timeout || DEFAULT_TIMEOUT
   end
-  
+
   def initialize(user_agent)
     @user_agent = user_agent
     @parsed = {}
   end
-  
+
   def allowed?(uri)
     uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
     host = uri.host
     @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
     @parsed[host].allowed?(uri, @user_agent)
   end
-  
+
   def other_values(uri)
     uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
     host = uri.host