Merge pull request #5 from tomverran/1.02-hotfix

Fix infinite loop when coming across an unrecognised line

Author: Tom

src/Robot/RobotsFile.php

@@ -16,21 +16,33 @@ class RobotsFile
      */
     private $lines;
 
+    const USER_AGENT = 'user-agent';
+
+    const DISALLOW = 'disallow';
+
+    const ALLOW = 'allow';
+
     /**
      * Construct this Robots file
      * @param String $content
      */
     public function __construct($content)
     {
         $withoutComments = preg_replace( '/#.*/', '', strtolower($content));
+
         foreach(explode("\n", $withoutComments) as $line) {
             $lineParts = array_filter(array_map('trim', explode(':', $line)));
-            if (count($lineParts) == 2) {
+            if ($this->lineIsValid($lineParts)) {
                 $this->lines[] = $lineParts;
             }
         }
     }
 
+    private function lineIsValid($line) {
+        $validDirectives = [self::USER_AGENT, self::DISALLOW, self::ALLOW];
+        return count($line) == 2 && in_array($line[0], $validDirectives);
+    }
+
     /**
      * Get the first directive in the file
      */

src/Robot/RobotsTxt.php

@@ -31,11 +31,11 @@ private function parseFile(RobotsFile $robotFile)
     {
         while($robotFile->hasLines()) {
             $currentUserAgents = [];
-            while ($robotFile->firstDirectiveIs('user-agent')) {
+            while ($robotFile->firstDirectiveIs(RobotsFile::USER_AGENT)) {
                 $currentUserAgents[] = $robotFile->shiftArgument();
             }
-            while ($robotFile->firstDirectiveIs('allow', 'disallow')) {
-                $isAllowed = $robotFile->firstDirective() == 'allow';
+            while ($robotFile->firstDirectiveIs(RobotsFile::ALLOW, RobotsFile::DISALLOW)) {
+                $isAllowed = $robotFile->firstDirective() == RobotsFile::ALLOW;
                 $urlParts = array_filter(explode('/', $robotFile->shiftArgument()));
                 $this->tree->getNode($urlParts)->addRule($currentUserAgents, $isAllowed);
             }

tests/RobotTest.php

@@ -105,4 +105,9 @@ public function testMultipleNonConsecutiveUserAgents()
         $this->assertFalse(self::getRobotsTxt('multiUserAgent')->isAllowed('robot2', '/some/other'));
         $this->assertTrue(self::getRobotsTxt('multiUserAgent')->isAllowed('Googlebot', '/some/other'));
     }
+
+    public function testFileWithInvalidLines()
+    {
+        $this->assertTrue(self::getRobotsTxt('corrupted')->isAllowed('Googlebot', '/some/other'));
+    }
 } 

tests/files/corrupted.txt

@@ -0,0 +1,22 @@
+User-agent: *
+User-agent: UA
+User-agent: Googlebot
+
+
+
+Allow: /
+Disallow: /private/
+Disallow: /secret/page.html
+User-agent: robot2
+Something: 10
+Allow: /
+Whatever: 50
+
+
+
+
+Disallow: /some/other/
+
+
+
+