Fixing mix case of user-agent.

From bfbe173cd6bf91be477ef0affc2c4c86ca75bc1d Mon Sep 17 00:00:00 2001
From: Jason Kim <[email protected]>
Date: Thu, 3 Nov 2011 15:43:14 -0700
Subject: [PATCH 1/2] Fixing mix case of user-agent. Adding/Fixing test.

Author: Brent Cook

lib/robots.rb

@@ -26,19 +26,19 @@ def initialize(uri, user_agent)
       io.each do |line|
         next if line =~ /^\s*(#.*|$)/
         arr = line.split(":")
-        key = arr.shift
+        key = arr.shift.to_s.downcase
         value = arr.join(":").strip
         value.strip!
         case key
-        when "User-agent"
+        when "user-agent"
           agent = to_regex(value)
-        when "Allow"
+        when "allow"
           @allows[agent] ||= []
           @allows[agent] << to_regex(value)
-        when "Disallow"
+        when "disallow"
           @disallows[agent] ||= []
           @disallows[agent] << to_regex(value)
-        when "Crawl-delay"
+        when "crawl-delay"
           @delays[agent] = value.to_i
         else
           @other[key] ||= []