Skip to content

Commit 101ab5a

Browse files
daniellienertdaniellienert
authored
Merge pull request #41 from daniellienert/feature/es-package-v7
!!! TASK: Make package compatible to upcomming ES Adapter v7
2 parents 1fffc83 + fea765e commit 101ab5a

File tree

5 files changed

+71
-43
lines changed

5 files changed

+71
-43
lines changed

‎Classes/Command/NodeIndexQueueCommandController.php‎

Lines changed: 46 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
* source code.
1414
*/
1515

16+
use Doctrine\Common\Collections\ArrayCollection;
1617
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
1718
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\ConfigurationException;
1819
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
@@ -98,26 +99,29 @@ class NodeIndexQueueCommandController extends CommandController
9899
*/
99100
protected $batchSize;
100101

102+
/**
103+
* @Flow\Inject
104+
* @var \Neos\ContentRepository\Domain\Service\ContentDimensionCombinator
105+
*/
106+
protected $contentDimensionCombinator;
107+
101108
/**
102109
* Index all nodes by creating a new index and when everything was completed, switch the index alias.
103110
*
104111
* @param string $workspace
105-
* @throws ConfigurationException
106112
* @throws Exception
107113
* @throws StopCommandException
108114
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
109115
* @throws \Flowpack\ElasticSearch\Exception
110-
* @throws \Neos\Flow\Http\Exception
111116
* @throws \Exception
112117
*/
113118
public function buildCommand(string $workspace = null): void
114119
{
115-
$indexPostfix = (string) time();
116-
$indexName = $this->createNextIndex($indexPostfix);
120+
$indexPostfix = (string)time();
117121
$this->updateMapping($indexPostfix);
118122

119123
$this->outputLine();
120-
$this->outputLine('<b>Indexing on %s ...</b>', [$indexName]);
124+
$this->outputLine('<b>Indexing on %s ...</b>', [$indexPostfix]);
121125

122126
$pendingJobs = $this->queueManager->getQueue(self::BATCH_QUEUE_NAME)->countReady();
123127
if ($pendingJobs !== 0) {
@@ -136,8 +140,11 @@ public function buildCommand(string $workspace = null): void
136140
$this->indexWorkspace($workspace, $indexPostfix);
137141
}
138142

139-
$updateAliasJob = new UpdateAliasJob($indexPostfix);
140-
$this->jobManager->queue(self::BATCH_QUEUE_NAME, $updateAliasJob);
143+
$combinations = new ArrayCollection($this->contentDimensionCombinator->getAllAllowedCombinations());
144+
$combinations->map(function (array $dimensionValues) use ($indexPostfix) {
145+
$updateAliasJob = new UpdateAliasJob($indexPostfix, $dimensionValues);
146+
$this->jobManager->queue(self::BATCH_QUEUE_NAME, $updateAliasJob);
147+
});
141148

142149
$this->outputLine('Indexing jobs created for queue %s with success ...', [self::BATCH_QUEUE_NAME]);
143150
$this->outputSystemReport();
@@ -159,9 +166,11 @@ public function workCommand(string $queue = 'batch', int $exitAfter = null, int
159166
'batch' => self::BATCH_QUEUE_NAME,
160167
'live' => self::LIVE_QUEUE_NAME
161168
];
169+
162170
if (!isset($allowedQueues[$queue])) {
163171
$this->output('Invalid queue, should be "live" or "batch"');
164172
}
173+
165174
$queueName = $allowedQueues[$queue];
166175

167176
if ($verbose) {
@@ -181,37 +190,45 @@ public function workCommand(string $queue = 'batch', int $exitAfter = null, int
181190
if ($exitAfter !== null) {
182191
$timeout = max(1, $exitAfter - (time() - $startTime));
183192
}
193+
194+
184195
try {
185196
$message = $this->jobManager->waitAndExecute($queueName, $timeout);
186-
} catch (Exception $exception) {
197+
} catch (\Exception $exception) {
187198
$numberOfJobExecutions++;
188-
$this->outputLine('<error>%s</error>', [$exception->getMessage()]);
189-
if ($verbose && $exception->getPrevious() instanceof \Exception) {
190-
$this->outputLine(' Reason: %s', [$exception->getPrevious()->getMessage()]);
199+
200+
$verbose && $this->outputLine('<error>%s</error>', [$exception->getMessage()]);
201+
202+
if ($exception->getPrevious() instanceof \Exception) {
203+
$verbose && $this->outputLine(' Reason: %s', [$exception->getPrevious()->getMessage()]);
204+
$this->logger->error(sprintf('Indexing job failed: %s. Detailed reason %s', $exception->getMessage(), $exception->getPrevious()->getMessage()), LogEnvironment::fromMethodName(__METHOD__));
205+
} else {
206+
$this->logger->error('Indexing job failed: ' . $exception->getMessage(), LogEnvironment::fromMethodName(__METHOD__));
191207
}
192-
} catch (\Exception $exception) {
193-
$this->outputLine('<error>Unexpected exception during job execution: %s, aborting...</error>', [$exception->getMessage()]);
194-
$this->quit(1);
195208
}
209+
196210
if ($message !== null) {
197211
$numberOfJobExecutions++;
198212
if ($verbose) {
199213
$messagePayload = strlen($message->getPayload()) <= 50 ? $message->getPayload() : substr($message->getPayload(), 0, 50) . '...';
200214
$this->outputLine('<success>Successfully executed job "%s" (%s)</success>', [$message->getIdentifier(), $messagePayload]);
201215
}
202216
}
217+
203218
if ($exitAfter !== null && (time() - $startTime) >= $exitAfter) {
204219
if ($verbose) {
205220
$this->outputLine('Quitting after %d seconds due to <i>--exit-after</i> flag', [time() - $startTime]);
206221
}
207222
$this->quit();
208223
}
224+
209225
if ($limit !== null && $numberOfJobExecutions >= $limit) {
210226
if ($verbose) {
211227
$this->outputLine('Quitting after %d executed job%s due to <i>--limit</i> flag', [$numberOfJobExecutions, $numberOfJobExecutions > 1 ? 's' : '']);
212228
}
213229
$this->quit();
214230
}
231+
215232
} while (true);
216233
}
217234

@@ -232,7 +249,7 @@ public function flushCommand(): void
232249
/**
233250
* Output system report for CLI commands
234251
*/
235-
protected function outputSystemReport()
252+
protected function outputSystemReport(): void
236253
{
237254
$this->outputLine();
238255
$this->outputLine('Memory Usage : %s', [Files::bytesToSizeString(memory_get_peak_usage(true))]);
@@ -291,40 +308,30 @@ protected function indexWorkspace(string $workspaceName, string $indexPostfix):
291308
$this->outputLine();
292309
}
293310

294-
/**
295-
* @param string $indexPostfix
296-
* @return string
297-
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
298-
* @throws ConfigurationException
299-
* @throws \Flowpack\ElasticSearch\Exception
300-
* @throws \Neos\Flow\Http\Exception
301-
*/
302-
protected function createNextIndex(string $indexPostfix): string
303-
{
304-
$this->nodeIndexer->setIndexNamePostfix($indexPostfix);
305-
$this->nodeIndexer->getIndex()->create();
306-
$this->logger->info(sprintf('Index %s created', $this->nodeIndexer->getIndexName()), LogEnvironment::fromMethodName(__METHOD__));
307-
308-
return $this->nodeIndexer->getIndexName();
309-
}
310-
311311
/**
312312
* Update Index Mapping
313313
*
314314
* @param string $indexPostfix
315315
* @return void
316-
* @throws ConfigurationException
317316
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
318317
* @throws \Flowpack\ElasticSearch\Exception
319318
*/
320319
protected function updateMapping(string $indexPostfix): void
321320
{
322-
$nodeTypeMappingCollection = $this->nodeTypeMappingBuilder->buildMappingInformation($this->nodeIndexer->getIndex());
323-
foreach ($nodeTypeMappingCollection as $mapping) {
321+
$combinations = new ArrayCollection($this->contentDimensionCombinator->getAllAllowedCombinations());
322+
$combinations->map(function (array $dimensionValues) use ($indexPostfix) {
323+
$this->nodeIndexer->setDimensions($dimensionValues);
324324
$this->nodeIndexer->setIndexNamePostfix($indexPostfix);
325-
/** @var Mapping $mapping */
326-
$mapping->apply();
327-
}
328-
$this->logger->info(sprintf('Mapping updated for index %s', $this->nodeIndexer->getIndexName()), LogEnvironment::fromMethodName(__METHOD__));
325+
326+
if (!$this->nodeIndexer->getIndex()->exists()) {
327+
$this->nodeIndexer->getIndex()->create();
328+
}
329+
$nodeTypeMappingCollection = $this->nodeTypeMappingBuilder->buildMappingInformation($this->nodeIndexer->getIndex());
330+
foreach ($nodeTypeMappingCollection as $mapping) {
331+
/** @var Mapping $mapping */
332+
$mapping->apply();
333+
}
334+
$this->logger->info(sprintf('Mapping updated for index %s', $this->nodeIndexer->getIndexName()), LogEnvironment::fromMethodName(__METHOD__));
335+
});
329336
}
330337
}

‎Classes/Domain/Repository/NodeDataRepository.php‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use Doctrine\ORM\EntityManagerInterface;
1919
use Doctrine\ORM\Internal\Hydration\IterableResult;
2020
use Doctrine\ORM\QueryBuilder;
21+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\NodeTypeIndexingConfiguration;
2122
use Neos\ContentRepository\Domain\Model\NodeData;
2223
use Neos\Flow\Annotations as Flow;
2324
use Neos\Flow\Persistence\Repository;
@@ -29,6 +30,12 @@ class NodeDataRepository extends Repository
2930
{
3031
public const ENTITY_CLASSNAME = NodeData::class;
3132

33+
/**
34+
* @Flow\Inject
35+
* @var NodeTypeIndexingConfiguration
36+
*/
37+
protected $nodeTypeIndexingConfiguration;
38+
3239
/**
3340
* @Flow\Inject
3441
* @var EntityManagerInterface
@@ -40,15 +47,21 @@ class NodeDataRepository extends Repository
4047
* @param integer $firstResult
4148
* @param integer $maxResults
4249
* @return IterableResult
50+
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
4351
*/
4452
public function findAllBySiteAndWorkspace($workspaceName, $firstResult = 0, $maxResults = 1000): IterableResult
4553
{
4654
/** @var QueryBuilder $queryBuilder */
4755
$queryBuilder = $this->entityManager->createQueryBuilder();
4856

57+
$excludedNodeTypes = array_keys(array_filter($this->nodeTypeIndexingConfiguration->getIndexableConfiguration(), static function($value) {
58+
return !$value;
59+
}));
60+
4961
$queryBuilder->select('n.Persistence_Object_Identifier persistenceObjectIdentifier, n.identifier identifier, n.dimensionValues dimensions, n.nodeType nodeType, n.path path')
5062
->from(NodeData::class, 'n')
51-
->where("n.workspace = :workspace AND n.removed = :removed AND n.movedTo IS NULL")
63+
->where('n.workspace = :workspace AND n.removed = :removed AND n.movedTo IS NULL')
64+
->andWhere($queryBuilder->expr()->notIn('n.nodeType', $excludedNodeTypes))
5265
->setFirstResult((integer)$firstResult)
5366
->setMaxResults((integer)$maxResults)
5467
->setParameters([

‎Classes/IndexingJob.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,12 @@ public function execute(QueueInterface $queue, Message $message): bool
5858

5959
// Skip this iteration if the node can not be fetched from the current context
6060
if (!$currentNode instanceof NodeInterface) {
61-
$this->logger->warning(sprintf('Node %s could not be processed"', $node['identifier']), LogEnvironment::fromMethodName(__METHOD__));
61+
$this->logger->warning(sprintf('Node %s could not be created from node data"', $node['identifier']), LogEnvironment::fromMethodName(__METHOD__));
6262
continue;
6363
}
6464

6565
$this->nodeIndexer->setIndexNamePostfix($this->indexPostfix);
66+
$this->nodeIndexer->setDimensions($node['dimensions']);
6667
$this->nodeIndexer->indexNode($currentNode, $this->targetWorkspaceName);
6768
}
6869

‎Classes/UpdateAliasJob.php‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,21 @@ class UpdateAliasJob implements JobInterface
5959
*/
6060
protected $cleanupIndicesAfterSuccessfulSwitch = true;
6161

62+
/**
63+
* @var array|null
64+
*/
65+
protected $dimensionValues;
6266

6367
/**
6468
* @param string $indexPostfix
69+
* @param array $dimensionValues
6570
* @throws \Exception
6671
*/
67-
public function __construct($indexPostfix)
72+
public function __construct($indexPostfix, array $dimensionValues = [])
6873
{
6974
$this->identifier = Algorithms::generateRandomString(24);
7075
$this->indexPostfix = $indexPostfix;
76+
$this->dimensionValues = $dimensionValues;
7177
}
7278

7379
/**
@@ -82,6 +88,7 @@ public function execute(QueueInterface $queue, Message $message): bool
8288
{
8389
if ($this->shouldIndexBeSwitched($queue)) {
8490
$this->nodeIndexer->setIndexNamePostfix($this->indexPostfix);
91+
$this->nodeIndexer->setDimensions($this->dimensionValues);
8592
$this->nodeIndexer->updateIndexAlias();
8693

8794
if ($this->cleanupIndicesAfterSuccessfulSwitch === true) {

‎composer.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"require": {
77
"neos/flow": "^5.1 || ^6.0",
88
"flowpack/jobqueue-common": "^3.0 || dev-master",
9-
"flowpack/elasticsearch-contentrepositoryadaptor": "^5.0 || ^6.0 || dev-master"
9+
"flowpack/elasticsearch-contentrepositoryadaptor": "^7.0 || dev-master"
1010
},
1111
"autoload": {
1212
"psr-4": {

0 commit comments

Comments
 (0)