Merge pull request #16 from DavidGoodwin/master

invalid User-Agent causes strpos needle E_NOTICE

Author: tomverran

src/Robot/RobotsTxt.php

@@ -65,6 +65,9 @@ private function parseFile(RobotsFile $robotFile)
 
         $this->records = [];
         foreach($fileRecords as $record) {
+            if (!empty($record['ua'])) {
+                $record['ua'] = array_filter($record['ua']);
+            }
             if (!empty($record['ua']) && !empty($record['rules'])) {
                 $this->records[] = new Record(new UserAgent($record['ua']), new AccessRules($record['rules']));
             }

tests/RobotTest.php

@@ -171,4 +171,22 @@ public function testCaseSensitivePaths()
         $this->assertTrue($g->isAllowed('Googlebot', '/uppercase.html'));
         $this->assertFalse($g->isAllowed('Googlebot', '/UPPERCASE.HTML'));
     }
+
+    /**
+     * Do not emit an E_NOTICE with an empty needle from strpos when a user-agent line is invalid (empty).
+     * This effectively ignores the line.
+     */
+    public function testHandlesEmptyUserAgentLine()
+    {
+        $invalid_text = <<<EOF
+User-agent: foo
+User-agent:
+Disallow: /
+EOF;
+
+        $g = new RobotsTxt($invalid_text);
+
+        $this->assertFalse($g->isAllowed('foo', '/something'));
+        $this->assertTrue($g->isAllowed('Googlebot', '/something'));
+    }
 }