Merge pull request #9 from tomverran/issue-8

Issue 8

Author: Tom

README.md

@@ -6,9 +6,7 @@ Robots.txt checker
 Given a robots.txt file this library will give you a straight forward yes/no as to whether you're allowed to access
 a given resource with a given user agent. Internally it organises the file into a tree.
 
-Wildcards are supported in a basic way:
-
-`Disallow: /hello/*/world` will disallow `/hello/whatever/world` but won't disallow `/hello/what/who/world`
+Wildcards are supported.
 
 If you find any bugs with this library please don't hesitate to let me know, either create an issue on GitHub or submit a pull request.
 

Vagrantfile

@@ -0,0 +1,8 @@
+Vagrant.configure(2) do |config|
+  config.vm.box = "landregistry/centos"
+  config.vm.network "public_network"
+  config.vm.provision "shell", inline: <<-SHELL
+     yum install -y epel-release curl php
+     curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer
+  SHELL
+end

composer.json

@@ -22,5 +22,10 @@
         "psr-4": {
             "tomverran\\Robot\\": "src/Robot/"
         }
+    },
+    "autoload-dev": {
+        "psr-4": {
+            "tomverran\\Robot\\": ["src/Robot/", "tests/"]
+        }
     }
 }

src/Robot/AccessRules.php

@@ -0,0 +1,48 @@
+<?php
+namespace tomverran\Robot;
+
+
+class AccessRules
+{
+    private $rules;
+
+    /**
+     * @param Boolean[] $rules A map of path => allowed
+     */
+    public function __construct(array $rules)
+    {
+        uksort($rules, function($r1, $r2) {
+            return strlen($r2) - strlen($r1);
+        });
+        $this->rules = $rules;
+    }
+
+    private function convertPathToRegex($path) {
+        $pathWithoutTrailingDollar = rtrim($path, '$');
+
+        $quotedWithWildcards = implode(array_map(function($input) {
+            return preg_quote($input, '#');
+        }, explode('*', $pathWithoutTrailingDollar)),'.*?');
+
+        $trailingDollar = $path == $pathWithoutTrailingDollar ? '' : '$';
+        return "#^${quotedWithWildcards}{$trailingDollar}#";
+    }
+
+    private function urlDecodeNonSlashes($str)
+    {
+        return implode(array_map(function($input) {
+            return strtolower($input) == '%2f' ? $input : urldecode($input);
+        }, preg_split('/(%2F)/i', $str, -1, PREG_SPLIT_DELIM_CAPTURE)), '');
+    }
+
+    public function isAllowed($url)
+    {
+        $matches = [];
+        foreach($this->rules as $ruleUrl => $allowed) {
+            if ($ruleUrl && preg_match($this->convertPathToRegex($this->urlDecodeNonSlashes($ruleUrl)), $this->urlDecodeNonSlashes($url))) {
+                $matches[$ruleUrl] = $allowed;
+            }
+        }
+        return empty($matches) || array_shift($matches);
+    }
+}

src/Robot/Leaf.php

@@ -0,0 +1,41 @@
+<?php
+namespace tomverran\Robot;
+use tomverran\Robot\UserAgent;
+
+class Record
+{
+    /**
+     * @var UserAgent
+     */
+    private $ua;
+
+    /**
+     * @var AccessRules
+     */
+    private $ar;
+
+    public function __construct(UserAgent $ua, AccessRules $ar)
+    {
+        $this->ua = $ua;
+        $this->ar = $ar;
+    }
+
+    public function matches($userAgent)
+    {
+        $matches = $this->ua->getMatches($userAgent);
+        return !empty($matches);
+    }
+
+    public function getMatchStrength($userAgent)
+    {
+        if (!$this->matches($userAgent)) {
+            return 0;
+        }
+        return max(array_map('strlen', $this->ua->getMatches($userAgent)));
+    }
+
+    public function isAllowed($userAgent, $url)
+    {
+        return !$this->ua->getMatches($userAgent) || $this->ar->isAllowed($url);
+    }
+}

src/Robot/Record.php

@@ -9,18 +9,12 @@
 namespace tomverran\Robot;
 
 
-class RobotsFile
+class RobotsFile implements \Iterator
 {
     /**
-     * @var String[]
+     * @var \ArrayIterator
      */
-    private $lines;
-
-    const USER_AGENT = 'user-agent';
-
-    const DISALLOW = 'disallow';
-
-    const ALLOW = 'allow';
+    private $lineIterator;
 
     /**
      * Construct this Robots file
@@ -29,51 +23,73 @@ class RobotsFile
     public function __construct($content)
     {
         $withoutComments = preg_replace( '/#.*/', '', strtolower($content));
+        $lines = [];
 
         foreach(explode("\n", $withoutComments) as $line) {
             $lineParts = array_filter(array_map('trim', explode(':', $line)));
-            if ($this->lineIsValid($lineParts)) {
-                $this->lines[] = $lineParts;
+            if (!empty($lineParts)) {
+                $lines[] = $lineParts;
             }
         }
+
+        $this->lineIterator = new \ArrayIterator($lines);
     }
 
-    private function lineIsValid($line) {
-        $validDirectives = [self::USER_AGENT, self::DISALLOW, self::ALLOW];
-        return count($line) == 2 && in_array($line[0], $validDirectives);
+    /**
+     * Return the current element
+     * @link http://php.net/manual/en/iterator.current.php
+     * @return mixed Can return any type.
+     * @since 5.0.0
+     */
+    public function current()
+    {
+        $cur = $this->lineIterator->current();
+        return count($cur) > 1 ? $cur[1] : '';
     }
 
     /**
-     * Get the first directive in the file
+     * Move forward to next element
+     * @link http://php.net/manual/en/iterator.next.php
+     * @return void Any returned value is ignored.
+     * @since 5.0.0
      */
-    public function firstDirective()
+    public function next()
     {
-        return $this->lines[0][0];
+        $this->lineIterator->next();
     }
 
-    public function firstDirectiveIs($args)
+    /**
+     * Return the key of the current element
+     * @link http://php.net/manual/en/iterator.key.php
+     * @return mixed scalar on success, or null on failure.
+     * @since 5.0.0
+     */
+    public function key()
     {
-        if (!$this->hasLines()) {
-            return false;
-        }
-        return in_array($this->firstDirective(), func_get_args());
+        $cur = $this->lineIterator->current();
+        return $cur[0];
     }
 
     /**
-     * Get the argument of the first directive,
-     * and shift the file to remove it
+     * Checks if current position is valid
+     * @link http://php.net/manual/en/iterator.valid.php
+     * @return boolean The return value will be casted to boolean and then evaluated.
+     * Returns true on success or false on failure.
+     * @since 5.0.0
      */
-    public function shiftArgument()
+    public function valid()
     {
-        return array_shift($this->lines)[1];
+        return $this->lineIterator->valid();
     }
 
     /**
-     * Does this file have any remaining lines
-     * @return bool
+     * Rewind the Iterator to the first element
+     * @link http://php.net/manual/en/iterator.rewind.php
+     * @return void Any returned value is ignored.
+     * @since 5.0.0
      */
-    public function hasLines()
+    public function rewind()
     {
-        return !empty($this->lines);
+        $this->lineIterator->rewind();
     }
 }