Merge pull request #34 from octoper/block-resource-types

[2.x] Added Blocked resource types

Author: rennokki

server.js

@@ -108,6 +108,16 @@ app.use('/healthcheck', require('express-healthcheck')());
         }
       }
 
+      // Allow to block certain resource types.
+      // For example: ?blocked_resource_types=image,media
+      if (query.blocked_resource_types) {
+        const blockedResourceTypes= query.blocked_resource_types.split(',');
+
+        if (blockedResourceTypes.includes(request.resourceType())) {
+          return request.abort();
+        }
+      }
+
       if (query.triggered_requests) {
         triggeredRequests.push({
           type: request.resourceType(),

src/Clusteer.php

@@ -117,6 +117,17 @@ public function blockExtensions(array $extensions)
         return $this->setParameter('blocked_extensions', implode(',', $extensions));
     }
 
+    /**
+     * Set the resource types to block.
+     *
+     * @param  array  $types
+     * @return $this
+     */
+    public function blockResourceTypes(array $types)
+    {
+        return $this->setParameter('blocked_resource_types', implode(',', $types));
+    }
+
     /**
      * Set the timeout.
      *

tests/CrawlingTest.php

@@ -75,6 +75,25 @@ public function test_block_extensions()
         }
     }
 
+    public function test_block_resource_types()
+    {
+        $blocked_types = [
+            'image',
+        ];
+
+        $clusteer = Clusteer::to('https://renoki.org')
+            ->blockResourceTypes($blocked_types)
+            ->waitUntilAllRequestsFinish()
+            ->withTriggeredRequests()
+            ->get();
+
+        foreach ($clusteer->getTriggeredRequests() as $request) {
+            $this->assertFalse(
+                (bool) in_array($request['type'], $blocked_types)
+            );
+        }
+    }
+
     public function test_cookies()
     {
         $this->markTestIncomplete(