Merge pull request #12 from tomverran/fix-exact-match-precedence

Fix exact user-agent matches not always being chosen above partial matches

Author: tomverran

src/Robot/Record.php

@@ -38,4 +38,9 @@ public function isAllowed($userAgent, $url)
     {
         return !$this->ua->getMatches($userAgent) || $this->ar->isAllowed($url);
     }
+
+    public function matchesExactly($userAgent)
+    {
+        return in_array(strtolower($userAgent), $this->ua->getMatches($userAgent));
+    }
 }

src/Robot/RobotsTxt.php

@@ -82,6 +82,16 @@ public function isAllowed($userAgent, $path)
             return true;
         }
 
+        /** @var Record[] $exactMatches */
+        $exactMatches = array_filter($this->records, function(Record $r) use ($userAgent) {
+            return $r->matchesExactly($userAgent);
+        });
+
+        if (!empty($exactMatches)) {
+            $firstMatch = array_shift($exactMatches);
+            return $firstMatch->isAllowed($userAgent, $path);
+        }
+
         $matching = array_filter($this->records, function(Record $r) use ($userAgent) {
             return $r->matches($userAgent);
         });

tests/RecordTest.php

@@ -54,4 +54,17 @@ public function givenMatch_returnLengthOfMatchedUAAsMatchStrength()
         $googleOnly = new Record(new UserAgent(['Googlebot']), new AccessRules([]));
         $this->assertTrue($googleOnly->getMatchStrength('G') == 9, 'Length of the matched UA is the strength');
     }
+
+    /**
+     * @test
+     */
+    public function givenExactMatch_flagAsBeingExact()
+    {
+        $googleOnly = new Record(new UserAgent(['Googlebot']), new AccessRules([]));
+        $this->assertTrue($googleOnly->matchesExactly('Googlebot'));
+        $this->assertTrue($googleOnly->matchesExactly('googlebot'));
+
+        $this->assertFalse($googleOnly->matchesExactly('google'));
+        $this->assertFalse($googleOnly->matchesExactly('googlebot-news'));
+    }
 }

tests/RobotTest.php

@@ -141,7 +141,11 @@ public function testRfcExample()
         $this->assertRfcExample('/org/plans.html', [$webcrawler, $excite], [$unhipBot, $other]);
         $this->assertRfcExample('/%7Ejim/jim.html', [$webcrawler, $excite], [$unhipBot, $other]);
         $this->assertRfcExample('/%7Emak/mak.html', [$webcrawler, $excite, $other], [$unhipBot]);
+    }
 
-
+    public function testExactMatchesBeatPartial()
+    {
+        $this->assertTrue(self::getRobotsTxt('match')->isAllowed('Googlebot-News', '/'));
+        $this->assertTrue(self::getRobotsTxt('match')->isDisallowed('Googlebot', '/'));
     }
 } 

tests/files/match.txt

@@ -0,0 +1,5 @@
+User-Agent: Googlebot
+Disallow: /
+
+User-Agent: Googlebot-News
+Allow: /