Skip to content

Commit 0281bb0

Browse files
specialtacticstimgws
authored andcommitted
Bugfix/chunk es payload (#198)
* Resolve a problem whereby a large amount of data for a given model would result in a "Request size exceeded 10485760 bytes" error in AWS Elastic Search (and potentially other configurations) * Re-implement without chunk (bulk sending data to elasticsearch)
1 parent 3936446 commit 0281bb0

File tree

1 file changed

+12
-13
lines changed

1 file changed

+12
-13
lines changed

src/ElasticquentCollectionTrait.php

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,13 @@ public function addToIndex()
3434

3535
// Iterate according to the amount configured, and put that iteration's worth of records into elastic search
3636
// This is done so that we do not exceed the maximum request size
37-
$chunkingResult = $this->chunk(static::$entriesToSendToElasticSearchInOneGo, function ($collectionChunk) use ($result) {
37+
$all = $this->all();
38+
$iteration = 0;
39+
do {
40+
$chunk = array_slice($all, (0 + ($iteration * static::$entriesToSendToElasticSearchInOneGo)), static::$entriesToSendToElasticSearchInOneGo);
41+
3842
$params = array();
39-
foreach ($collectionChunk as $item) {
43+
foreach ($chunk as $item) {
4044
$params['body'][] = array(
4145
'index' => array(
4246
'_id' => $item->getKey(),
@@ -48,23 +52,18 @@ public function addToIndex()
4852
$params['body'][] = $item->getIndexDocumentData();
4953
}
5054

51-
$result->result = $this->getElasticSearchClient()->bulk($params);
55+
$result = $this->getElasticSearchClient()->bulk($params);
5256

5357
// Check for errors
5458
if ( (array_key_exists('errors', $result) && $result['errors'] != false ) || (array_key_exists('Message', $result) && stristr('Request size exceeded', $result['Message']) !== false)) {
55-
return false;
59+
break;
5660
}
5761

5862
// Remove vars immediately to prevent them hanging around in memory, in case we have a large number of iterations
59-
unset($collectionChunk, $params);
60-
});
61-
62-
// Get the result or null it
63-
if ($chunkingResult && property_exists($result, 'result')) {
64-
$result = $result->result;
65-
} else {
66-
$result = null;
67-
}
63+
unset($chunk, $params);
64+
65+
++$iteration;
66+
} while (count($all) > ($iteration * static::$entriesToSendToElasticSearchInOneGo) );
6867

6968
return $result;
7069
}

0 commit comments

Comments
 (0)