Implement chunking of records before sending to ES (elasticquent#148)

* Resolve a problem whereby a large amount of data for a given model would result in a "Request size exceeded 10485760 bytes" error in AWS Elastic Search (and potentially other configurations)

Author: specialtactics

.gitignore

@@ -2,4 +2,5 @@
 composer.phar
 composer.lock
 .DS_Store
-Thumbs.db
+Thumbs.db
+.idea

src/ElasticquentCollectionTrait.php

@@ -10,6 +10,12 @@ trait ElasticquentCollectionTrait
 {
     use ElasticquentClientTrait;
 
+    /**
+     * @var int The number of records (ie. models) to send to Elasticsearch in one go
+     * Also, the number of models to get from the database at a time using Eloquent's chunk()
+     */
+    static public $entriesToSendToElasticSearchInOneGo = 500;
+
     /**
      * Add To Index
      *
@@ -23,21 +29,44 @@ public function addToIndex()
             return null;
         }
 
-        $params = array();
+        // Use an stdClass to store result of elasticsearch operation
+        $result = new \stdClass;
 
-        foreach ($this->all() as $item) {
-            $params['body'][] = array(
-                'index' => array(
-                    '_id' => $item->getKey(),
-                    '_type' => $item->getTypeName(),
-                    '_index' => $item->getIndexName(),
-                ),
-            );
+        // Iterate according to the amount configured, and put that iteration's worth of records into elastic search
+        // This is done so that we do not exceed the maximum request size
+        $chunkingResult = $this->chunk(static::$entriesToSendToElasticSearchInOneGo, function ($collectionChunk) use ($result) {
+            $params = array();
+            foreach ($collectionChunk as $item) {
+                $params['body'][] = array(
+                    'index' => array(
+                        '_id' => $item->getKey(),
+                        '_type' => $item->getTypeName(),
+                        '_index' => $item->getIndexName(),
+                    ),
+                );
+
+                $params['body'][] = $item->getIndexDocumentData();
+            }
+
+            $result->result = $this->getElasticSearchClient()->bulk($params);
 
-            $params['body'][] = $item->getIndexDocumentData();
+            // Check for errors
+            if ( (array_key_exists('errors', $result) && $result['errors'] != false ) || (array_key_exists('Message', $result) && stristr('Request size exceeded', $result['Message']) !== false)) {
+                return false;
+            }
+
+            // Remove vars immediately to prevent them hanging around in memory, in case we have a large number of iterations
+            unset($collectionChunk, $params);
+        });
+
+        // Get the result or null it
+        if ($chunkingResult && property_exists($result, 'result')) {
+            $result = $result->result;
+        } else {
+            $result = null;
         }
 
-        return $this->getElasticSearchClient()->bulk($params);
+        return $result;
     }
 
     /**