Skip to content

Commit 753ce9d

Browse files
ezimuelezimuel
committed
Updated Utility for cleanup YAML tests
1 parent c3164a0 commit 753ce9d

File tree

1 file changed

+209
-34
lines changed

1 file changed

+209
-34
lines changed

‎tests/Elasticsearch/Tests/Utility.php

Lines changed: 209 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,31 @@ class Utility
3030
*/
3131
private static $version;
3232

33+
/**
34+
* @var bool
35+
*/
36+
private static $hasXPack = false;
37+
38+
/**
39+
* @var bool
40+
*/
41+
private static $hasIlm = false;
42+
43+
/**
44+
* @var bool
45+
*/
46+
private static $hasRollups = false;
47+
48+
/**
49+
* @var bool
50+
*/
51+
private static $hasCcr = false;
52+
53+
/**
54+
* @var bool
55+
*/
56+
private static $hasShutdown = false;
57+
3358
/**
3459
* Get the host URL based on ENV variables
3560
*/
@@ -96,15 +121,65 @@ public static function getVersion(Client $client): string
96121
return self::$version;
97122
}
98123

124+
/**
125+
* Read the plugins installed in Elasticsearch using the
126+
* undocumented API GET /_nodes/plugins
127+
*
128+
* @see ESRestTestCase.java:initClient()
129+
*/
130+
private static function readPlugins(Client $client): void
131+
{
132+
$result = $client->transport->performRequest('GET', '/_nodes/plugins');
133+
foreach ($result['nodes'] as $node) {
134+
foreach ($node['modules'] as $module) {
135+
if (substr($module['name'], 0, 6) === 'x-pack') {
136+
self::$hasXPack = true;
137+
}
138+
if ($module['name'] === 'x-pack-ilm') {
139+
self::$hasIlm = true;
140+
}
141+
if ($module['name'] === 'x-pack-rollup') {
142+
self::$hasRollups = true;
143+
}
144+
if ($module['name'] === 'x-pack-ccr') {
145+
self::$hasCcr = true;
146+
}
147+
if ($module['name'] === 'x-pack-shutdown') {
148+
self::$hasShutdown = true;
149+
}
150+
}
151+
}
152+
}
153+
99154
/**
100155
* Clean up the cluster after a test
101156
*
102157
* @see ESRestTestCase.java:cleanUpCluster()
103158
*/
104159
public static function cleanUpCluster(Client $client): void
105160
{
161+
self::readPlugins($client);
162+
163+
self::ensureNoInitializingShards($client);
106164
self::wipeCluster($client);
107165
self::waitForClusterStateUpdatesToFinish($client);
166+
self::checkForUnexpectedlyRecreatedObjects($client);
167+
}
168+
169+
/**
170+
* Waits until all shard initialization is completed.
171+
* This is a handy alternative to ensureGreen as it relates to all shards
172+
* in the cluster and doesn't require to know how many nodes/replica there are.
173+
*
174+
* @see ESRestTestCase.java:ensureNoInitializingShards()
175+
*/
176+
private static function ensureNoInitializingShards(Client $client): void
177+
{
178+
$client->cluster()->health([
179+
'wait_for_no_initializing_shards' => true,
180+
'timeout' => '70s',
181+
'level' => 'shards'
182+
]);
108183
}
109184

110185
/**
@@ -114,55 +189,65 @@ public static function cleanUpCluster(Client $client): void
114189
*/
115190
private static function wipeCluster(Client $client): void
116191
{
117-
if (getenv('TEST_SUITE') === 'platinum') {
192+
if (self::$hasRollups) {
118193
self::wipeRollupJobs($client);
119194
self::waitForPendingRollupTasks($client);
120195
}
121-
if (version_compare(self::getVersion($client), '7.3.99') > 0) {
122-
self::deleteAllSLMPolicies($client);
123-
}
196+
self::deleteAllSLMPolicies($client);
124197

125198
// Clean up searchable snapshots indices before deleting snapshots and repositories
126-
if (getenv('TEST_SUITE') === 'platinum' && version_compare(self::getVersion($client), '7.7.99') > 0) {
199+
if (self::$hasXPack && version_compare(self::getVersion($client), '7.7.99') > 0) {
127200
self::wipeSearchableSnapshotsIndices($client);
128201
}
129202

130203
self::wipeSnapshots($client);
131204
self::wipeDataStreams($client);
132205
self::wipeAllIndices($client);
133206

134-
if (getenv('TEST_SUITE') === 'platinum') {
207+
if (self::$hasXPack) {
135208
self::wipeTemplateForXpack($client);
136209
} else {
137-
// Delete templates
138-
$client->indices()->deleteTemplate([
139-
'name' => '*'
140-
]);
141-
try {
142-
// Delete index template
143-
$client->indices()->deleteIndexTemplate([
144-
'name' => '*'
145-
]);
146-
// Delete component template
147-
$client->cluster()->deleteComponentTemplate([
148-
'name' => '*'
149-
]);
150-
} catch (ElasticsearchException $e) {
151-
// We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
152-
}
210+
self::wipeAllTemplates($client);
153211
}
154212

155213
self::wipeClusterSettings($client);
156214

157-
if (getenv('TEST_SUITE') === 'platinum') {
215+
if (self::$hasIlm) {
158216
self::deleteAllILMPolicies($client);
217+
}
218+
if (self::$hasCcr) {
159219
self::deleteAllAutoFollowPatterns($client);
220+
}
221+
if (self::$hasXPack) {
160222
self::deleteAllTasks($client);
161223
}
162224

163225
self::deleteAllNodeShutdownMetadata($client);
164226
}
165227

228+
/**
229+
* Remove all templates
230+
*/
231+
private static function wipeAllTemplates(Client $client): void
232+
{
233+
// Delete templates
234+
$client->indices()->deleteTemplate([
235+
'name' => '*'
236+
]);
237+
try {
238+
// Delete index template
239+
$client->indices()->deleteIndexTemplate([
240+
'name' => '*'
241+
]);
242+
// Delete component template
243+
$client->cluster()->deleteComponentTemplate([
244+
'name' => '*'
245+
]);
246+
} catch (ElasticsearchException $e) {
247+
// We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
248+
}
249+
}
250+
166251
/**
167252
* Delete all the Roolup Jobs for XPack test suite
168253
*
@@ -295,7 +380,7 @@ private static function deleteAllSLMPolicies(Client $client): void
295380
private static function wipeDataStreams(Client $client): void
296381
{
297382
try {
298-
if (version_compare(self::getVersion($client), '7.8.99') > 0) {
383+
if (self::$hasXPack) {
299384
$client->indices()->deleteDataStream([
300385
'name' => '*',
301386
'expand_wildcards' => 'all'
@@ -304,14 +389,17 @@ private static function wipeDataStreams(Client $client): void
304389
} catch (ElasticsearchException $e) {
305390
// We hit a version of ES that doesn't understand expand_wildcards, try again without it
306391
try {
307-
if (getenv('TEST_SUITE') === 'platinum') {
392+
if (self::$hasXPack) {
308393
$client->indices()->deleteDataStream([
309394
'name' => '*'
310395
]);
311396
}
312-
} catch (ElasticsearchException $e) {
397+
} catch (Exception $e) {
313398
// We hit a version of ES that doesn't serialize DeleteDataStreamAction.Request#wildcardExpressionsOriginallySpecified
314399
// field or that doesn't support data streams so it's safe to ignore
400+
if ($e->getCode() !== '404' && $e->getCode() !== '405') {
401+
throw $e;
402+
}
315403
}
316404
}
317405
}
@@ -483,13 +571,10 @@ private static function isXPackTemplate(string $name): bool
483571
}
484572
switch ($name) {
485573
case ".watches":
486-
case "logstash-index-template":
487-
case ".logstash-management":
488574
case "security_audit_log":
489575
case ".slm-history":
490576
case ".async-search":
491577
case "saml-service-provider":
492-
case "ilm-history":
493578
case "logs":
494579
case "logs-settings":
495580
case "logs-mappings":
@@ -500,13 +585,14 @@ private static function isXPackTemplate(string $name): bool
500585
case "synthetics-settings":
501586
case "synthetics-mappings":
502587
case ".snapshot-blob-cache":
503-
case ".deprecation-indexing-template":
588+
case "ilm-history":
504589
case "logstash-index-template":
505590
case "security-index-template":
506591
case "data-streams-mappings":
507592
return true;
593+
default:
594+
return false;
508595
}
509-
return false;
510596
}
511597

512598
/**
@@ -522,7 +608,15 @@ private static function preserveILMPolicyIds(): array
522608
"watch-history-ilm-policy",
523609
"ml-size-based-ilm-policy",
524610
"logs",
525-
"metrics"
611+
"metrics",
612+
"synthetics",
613+
"7-days-default",
614+
"30-days-default",
615+
"90-days-default",
616+
"180-days-default",
617+
"365-days-default",
618+
".fleet-actions-results-ilm-policy",
619+
".deprecation-indexing-ilm-policy"
526620
];
527621
}
528622

@@ -582,8 +676,8 @@ private static function deleteAllTasks(Client $client): void
582676
*/
583677
private static function deleteAllNodeShutdownMetadata(Client $client)
584678
{
585-
if (getenv('TEST_SUITE') !== 'platinum' || version_compare(self::getVersion($client), '7.15.0') < 0) {
586-
// Node shutdown APIs are only present in xpack from 7.15+
679+
if (!self::$hasShutdown || version_compare(self::getVersion($client), '7.15.0') < 0) {
680+
// Node shutdown APIs are only present in xpack
587681
return;
588682
}
589683
$nodes = $client->shutdown()->getNode();
@@ -629,4 +723,85 @@ private static function waitForClusterStateUpdatesToFinish(Client $client, int $
629723
$stillWaiting = ! empty($result['tasks']);
630724
} while ($stillWaiting && time() < ($start + $timeout));
631725
}
726+
727+
/**
728+
* Returns all the unexpected ilm policies, removing $exclusions from the list
729+
*/
730+
private static function getAllUnexpectedIlmPolicies(Client $client, array $exclusions): array
731+
{
732+
try {
733+
$policies = $client->ilm()->getLifecycle();
734+
} catch (ElasticsearchException $e) {
735+
return [];
736+
}
737+
foreach ($policies as $name => $value) {
738+
if (in_array($name, $exclusions)) {
739+
unset($policies[$name]);
740+
}
741+
}
742+
return $policies;
743+
}
744+
745+
/**
746+
* Returns all the unexpected templates
747+
*/
748+
private static function getAllUnexpectedTemplates(Client $client): array
749+
{
750+
if (!self::$hasXPack) {
751+
return [];
752+
}
753+
$unexpected = [];
754+
// In case of bwc testing, if all nodes are before 7.7.0 then no need
755+
// to attempt to delete component and composable index templates,
756+
// because these were introduced in 7.7.0:
757+
if (version_compare(self::getVersion($client), '7.6.99') > 0) {
758+
$result = $client->indices()->getIndexTemplate();
759+
foreach ($result['index_templates'] as $template) {
760+
if (!self::isXPackTemplate($template['name'])) {
761+
$unexpected[$template['name']] = true;
762+
}
763+
}
764+
$result = $client->cluster()->getComponentTemplate();
765+
foreach ($result['component_templates'] as $template) {
766+
if (!self::isXPackTemplate($template['name'])) {
767+
$unexpected[$template['name']] = true;
768+
}
769+
}
770+
}
771+
$result = $client->indices()->getIndexTemplate();
772+
foreach ($result['index_templates'] as $template) {
773+
if (!self::isXPackTemplate($template['name'])) {
774+
$unexpected[$template['name']] = true;
775+
}
776+
}
777+
return array_keys($unexpected);
778+
}
779+
780+
781+
/**
782+
* This method checks whether ILM policies or templates get recreated after
783+
* they have been deleted. If so, we are probably deleting them unnecessarily,
784+
* potentially causing test performance problems. This could happen for example
785+
* if someone adds a new standard ILM policy but forgets to put it in the
786+
* exclusion list in this test.
787+
*/
788+
private static function checkForUnexpectedlyRecreatedObjects(Client $client): void
789+
{
790+
if (self::$hasIlm) {
791+
$policies = self::getAllUnexpectedIlmPolicies($client, self::preserveILMPolicyIds());
792+
if (count($policies) > 0) {
793+
throw new Exception(sprintf(
794+
"Expected no ILM policies after deletions, but found %s",
795+
implode(',', array_keys($policies))
796+
));
797+
}
798+
}
799+
$templates = self::getAllUnexpectedTemplates($client);
800+
if (count($templates) > 0) {
801+
throw new Exception(sprintf(
802+
"Expected no templates after deletions, but found %s",
803+
implode(',', array_keys($templates))
804+
));
805+
}
806+
}
632807
}

0 commit comments

Comments
 (0)