Merge pull request #15 from nickmoline/master

URL paths in robots.txt are case sensitive

Author: tomverran

src/Robot/RobotsFile.php

@@ -22,7 +22,7 @@ class RobotsFile implements \Iterator
      */
     public function __construct($content)
     {
-        $withoutComments = preg_replace( '/#.*/', '', strtolower($content));
+        $withoutComments = preg_replace( '/#.*/', '', $content);
         $lines = [];
 
         foreach(preg_split("/[\r\n]+/", $withoutComments) as $line) {
@@ -92,4 +92,4 @@ public function rewind()
     {
         $this->lineIterator->rewind();
     }
-}
+}

src/Robot/RobotsTxt.php

@@ -42,7 +42,8 @@ private function parseFile(RobotsFile $robotFile)
         $lastDirective = '';
 
         foreach($robotFile as $directive => $value) {
-
+            $directive = strtolower($directive);
+            
             // add a new record if we have the beginning of a new set of UAs
             if ($directive == self::USER_AGENT && $lastDirective != $directive) {
                 $fileRecords[] = $emptyRecord;
@@ -113,4 +114,4 @@ public function isDisallowed($userAgent, $path)
     {
         return !$this->isAllowed($userAgent,$path);
     }
-}
+}

tests/RobotTest.php

@@ -158,4 +158,17 @@ public function testGooglePrecedenceSpec()
         $this->assertTrue($g->isAllowed('Googlebot-Images', '/bot'));
         $this->assertTrue($g->isDisallowed('Googlebot-Images', '/news'));
     }
-} 
+
+    public function testCaseSensitivePaths()
+    {
+        $g = self::getRobotsTxt('caseSensitive');
+        $this->assertFalse($g->isAllowed('Googlebot', '/lowercase.html'));
+        $this->assertTrue($g->isAllowed('Googlebot', '/LOWERCASE.HTML'));
+
+        $this->assertTrue($g->isAllowed('Googlebot', '/camelcase.html'));
+        $this->assertFalse($g->isAllowed('Googlebot', '/CamelCase.html'));
+
+        $this->assertTrue($g->isAllowed('Googlebot', '/uppercase.html'));
+        $this->assertFalse($g->isAllowed('Googlebot', '/UPPERCASE.HTML'));
+    }
+}

tests/RobotsFileTest.php

@@ -27,8 +27,8 @@ public function newLines()
     public function givenCrOnlyLineBreaks_pickUpTwoLines($newline)
     {
         $test = $this->getTestFile($newline);
-        $this->assertEquals(['user-agent', '*'], [$test->key(), $test->current()]);
+        $this->assertEquals(['User-Agent', '*'], [$test->key(), $test->current()]);
         $test->next();
-        $this->assertEquals(['disallow', '/'], [$test->key(), $test->current()]);
+        $this->assertEquals(['Disallow', '/'], [$test->key(), $test->current()]);
     }
-}
+}

tests/files/caseSensitive.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /lowercase.html
+Disallow: /CamelCase.html
+Disallow: /UPPERCASE.HTML
+Allow: /