Skip to content

Commit 3dc56e0

Browse files
picandocodigopicandocodigo
committed
[XPACK] Adds wipe cluster for 8.x
1 parent 4fcafb7 commit 3dc56e0

File tree

4 files changed

+399
-3
lines changed

4 files changed

+399
-3
lines changed

‎api-spec-testing/wipe_cluster_8.rb

Lines changed: 385 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,385 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
require_relative 'logging'
19+
include Elasticsearch::RestAPIYAMLTests::Logging
20+
21+
module Elasticsearch
22+
module RestAPIYAMLTests
23+
module WipeCluster8
24+
PRESERVE_ILM_POLICY_IDS = [
25+
'ilm-history-ilm-policy', 'slm-history-ilm-policy', 'watch-history-ilm-policy',
26+
'ml-size-based-ilm-policy', 'logs', 'metrics', 'synthetics', '7-days-default',
27+
'30-days-default', '90-days-default', '180-days-default', '365-days-default',
28+
'.fleet-actions-results-ilm-policy', '.deprecation-indexing-ilm-policy'
29+
].freeze
30+
31+
PLATINUM_TEMPLATES = [
32+
'.watches', 'logstash-index-template', '.logstash-management',
33+
'security_audit_log', '.slm-history', '.async-search',
34+
'saml-service-provider', 'ilm-history', 'logs', 'logs-settings',
35+
'logs-mappings', 'metrics', 'metrics-settings', 'metrics-mappings',
36+
'synthetics', 'synthetics-settings', 'synthetics-mappings',
37+
'.snapshot-blob-cache', '.deprecation-indexing-template',
38+
'.deprecation-indexing-mappings', '.deprecation-indexing-settings',
39+
'security-index-template', 'data-streams-mappings'
40+
].freeze
41+
42+
# Wipe Cluster, based on PHP's implementation of ESRestTestCase.java:wipeCluster()
43+
# https://github.com/elastic/elasticsearch-php/blob/7.10/tests/Elasticsearch/Tests/Utility.php#L97
44+
def self.run(client)
45+
ensure_no_initializing_shards(client)
46+
wipe_cluster(client)
47+
wait_for_cluster_tasks(client)
48+
check_for_unexpectedly_recreated_objects(client)
49+
end
50+
51+
class << self
52+
private
53+
54+
def wipe_cluster(client)
55+
read_plugins(client)
56+
if @has_rollups
57+
wipe_rollup_jobs(client)
58+
wait_for_pending_rollup_tasks(client)
59+
end
60+
delete_all_slm_policies(client)
61+
wipe_searchable_snapshot_indices(client) if @has_xpack
62+
wipe_snapshots(client)
63+
wipe_datastreams(client)
64+
wipe_all_indices(client)
65+
if platinum?
66+
wipe_templates_for_xpack(client)
67+
else
68+
wipe_all_templates(client)
69+
end
70+
wipe_cluster_settings(client)
71+
if platinum?
72+
clear_ml_jobs(client)
73+
clear_datafeeds(client)
74+
end
75+
delete_all_ilm_policies(client) if @has_ilm
76+
delete_all_follow_patterns(client) if @has_ccr
77+
delete_all_node_shutdown_metadata(client)
78+
end
79+
80+
def ensure_no_initializing_shards(client)
81+
client.cluster.health(wait_for_no_initializing_shards: true, timeout: '70s', level: 'shards')
82+
end
83+
84+
def check_for_unexpectedly_recreated_objects(client)
85+
unexpected_ilm_policies = client.ilm.get_lifecycle
86+
unexpected_ilm_policies.reject! { |k, _| PRESERVE_ILM_POLICY_IDS.include? k }
87+
unless unexpected_ilm_policies.empty?
88+
logger.info(
89+
"Expected no ILM policies after deletions, but found #{unexpected_ilm_policies.keys.join(',')}"
90+
)
91+
end
92+
return unless platinum?
93+
94+
templates = client.indices.get_index_template
95+
unexpected_templates = templates['index_templates'].reject do |t|
96+
# reject platinum templates
97+
PLATINUM_TEMPLATES.include? t['name']
98+
end.map { |t| t['name'] } # only keep the names
99+
legacy_templates = client.indices.get_template
100+
unexpected_templates << legacy_templates.keys.reject { |t| PLATINUM_TEMPLATES.include? t }
101+
102+
unless unexpected_templates.empty?
103+
logger.info(
104+
"Expected no templates after deletions, but found #{unexpected_templates.join(',')}"
105+
)
106+
end
107+
end
108+
109+
def platinum?
110+
ENV['TEST_SUITE'] == 'platinum'
111+
end
112+
113+
def read_plugins(client)
114+
response = client.perform_request('GET', '_nodes/plugins').body
115+
116+
response['nodes'].each do |node|
117+
node[1]['modules'].each do |mod|
118+
@has_xpack = true if mod['name'].include?('x-pack')
119+
@has_ilm = true if mod['name'].include?('x-pack-ilm')
120+
@has_rollups = true if mod['name'].include?('x-pack-rollup')
121+
@has_ccr = true if mod['name'].include?('x-pack-ccr')
122+
@has_shutdown = true if mod['name'].include?('x-pack-shutdown')
123+
end
124+
end
125+
end
126+
127+
def wipe_rollup_jobs(client)
128+
client.rollup.get_jobs(id: '_all')['jobs'].each do |d|
129+
client.rollup.stop_job(id: d['config']['id'], wait_for_completion: true, timeout: '10s', ignore: 404)
130+
client.rollup.delete_job(id: d['config']['id'], ignore: 404)
131+
end
132+
end
133+
134+
def wait_for_pending_rollup_tasks(client)
135+
filter = 'xpack/rollup/job'
136+
loop do
137+
results = client.cat.tasks(detailed: true)
138+
count = 0
139+
140+
time = Time.now.to_i
141+
results.each do |task|
142+
next if task.empty?
143+
144+
logger.debug("Pending task: #{task}")
145+
count += 1 if task.include?(filter)
146+
end
147+
break unless count.positive? && Time.now.to_i < (time + 30)
148+
end
149+
end
150+
151+
def delete_all_slm_policies(client)
152+
policies = client.snapshot_lifecycle_management.get_lifecycle
153+
154+
policies.each do |name, _|
155+
client.snapshot_lifecycle_management.delete_lifecycle(policy_id: name)
156+
end
157+
end
158+
159+
def wipe_searchable_snapshot_indices(client)
160+
indices = client.cluster.state(metric: 'metadata', filter_path: 'metadata.indices.*.settings.index.store.snapshot')
161+
return unless indices.dig('metadata', 'indices')
162+
163+
indices['metadata']['indices'].each do |index|
164+
index_name = if index.is_a?(Array)
165+
index[0]
166+
elsif index.is_a?(Hash)
167+
index.keys.first
168+
end
169+
client.indices.delete(index: index_name, ignore: 404)
170+
end
171+
end
172+
173+
def wipe_snapshots(client)
174+
# Repeatedly delete the snapshots until there aren't any
175+
loop do
176+
repositories = client.snapshot.get_repository(repository: '_all')
177+
break if repositories.empty?
178+
179+
repositories.each_key do |repository|
180+
if repositories[repository]['type'] == 'fs'
181+
response = client.snapshot.get(repository: repository, snapshot: '_all', ignore_unavailable: true)
182+
response['snapshots'].each do |snapshot|
183+
if snapshot['state'] != 'SUCCESS'
184+
logger.debug("Found snapshot that did not succeed #{snapshot}")
185+
end
186+
client.snapshot.delete(repository: repository, snapshot: snapshot['snapshot'], ignore: 404)
187+
end
188+
end
189+
client.snapshot.delete_repository(repository: repository, ignore: 404)
190+
end
191+
end
192+
end
193+
194+
def wipe_datastreams(client)
195+
begin
196+
client.indices.delete_data_stream(name: '*', expand_wildcards: 'all')
197+
rescue StandardError => e
198+
logger.error "Caught exception attempting to delete data streams: #{e}"
199+
client.indices.delete_data_stream(name: '*')
200+
end
201+
end
202+
203+
def wipe_all_indices(client)
204+
client.indices.delete(index: '*,-.ds-ilm-history-*', expand_wildcards: 'open,closed,hidden', ignore: 404)
205+
end
206+
207+
def wipe_templates_for_xpack(client)
208+
templates = client.indices.get_index_template
209+
210+
templates['index_templates'].each do |template|
211+
next if platinum_template? template['name']
212+
213+
begin
214+
client.indices.delete_index_template(name: template['name'], ignore: 404)
215+
end
216+
end
217+
# Delete component template
218+
result = client.cluster.get_component_template
219+
220+
result['component_templates'].each do |template|
221+
next if platinum_template? template['name']
222+
223+
client.cluster.delete_component_template(name: template['name'], ignore: 404)
224+
end
225+
226+
# Always check for legacy templates
227+
templates = client.indices.get_template
228+
templates.each do |name, _|
229+
next if platinum_template? name
230+
231+
begin
232+
client.indices.delete_template(name: name)
233+
rescue StandardError => e
234+
logger.info("Unable to remove index template #{name}")
235+
end
236+
end
237+
end
238+
239+
def wipe_all_templates(client)
240+
client.indices.delete_template(name: '*')
241+
begin
242+
client.indices.delete_index_template(name: '*')
243+
client.cluster.delete_component_template(name: '*')
244+
rescue StandardError => e
245+
logger.info('Using a version of ES that doesn\'t support index templates v2 yet, so it\'s safe to ignore')
246+
end
247+
end
248+
249+
def platinum_template?(template)
250+
platinum_prefixes = ['.monitoring', '.watch', '.triggered-watches', '.data-frame', '.ml-', '.transform', 'data-streams-mappings'].freeze
251+
platinum_prefixes.map { |a| return true if a.include? template }
252+
253+
PLATINUM_TEMPLATES.include? template
254+
end
255+
256+
def wait_for_cluster_tasks(client)
257+
time = Time.now.to_i
258+
count = 0
259+
260+
loop do
261+
results = client.cluster.pending_tasks
262+
results['tasks'].each do |task|
263+
next if task.empty?
264+
265+
logger.debug "Pending cluster task: #{task}"
266+
count += 1
267+
end
268+
break unless count.positive? && Time.now.to_i < (time + 30)
269+
end
270+
end
271+
272+
def delete_all_ilm_policies(client)
273+
policies = client.ilm.get_lifecycle
274+
policies.each do |policy|
275+
client.ilm.delete_lifecycle(policy: policy[0]) unless PRESERVE_ILM_POLICY_IDS.include? policy[0]
276+
end
277+
end
278+
279+
def wipe_cluster_settings(client)
280+
settings = client.cluster.get_settings
281+
new_settings = []
282+
settings.each do |name, value|
283+
next unless !value.empty? && value.is_a?(Array)
284+
285+
new_settings[name] = [] if new_settings[name].empty?
286+
value.each do |key, _v|
287+
new_settings[name]["#{key}.*"] = nil
288+
end
289+
end
290+
client.cluster.put_settings(body: new_settings) unless new_settings.empty?
291+
end
292+
293+
def delete_all_follow_patterns(client)
294+
patterns = client.cross_cluster_replication.get_auto_follow_pattern
295+
296+
patterns['patterns'].each do |pattern|
297+
client.cross_cluster_replication.delete_auto_follow_pattern(name: pattern)
298+
end
299+
end
300+
301+
def create_xpack_rest_user(client)
302+
client.security.put_user(
303+
username: 'x_pack_rest_user',
304+
body: { password: 'x-pack-test-password', roles: ['superuser'] }
305+
)
306+
end
307+
308+
def clear_roles(client)
309+
client.security.get_role.each do |role, _|
310+
begin; client.security.delete_role(name: role); rescue; end
311+
end
312+
end
313+
314+
def clear_users(client)
315+
client.security.get_user.each do |user, _|
316+
begin; client.security.delete_user(username: user); rescue; end
317+
end
318+
end
319+
320+
def clear_privileges(client)
321+
client.security.get_privileges.each do |privilege, _|
322+
begin; client.security.delete_privileges(name: privilege); rescue; end
323+
end
324+
end
325+
326+
def clear_ml_jobs(client)
327+
client.ml.close_job(job_id: '_all', force: true)
328+
client.ml.get_jobs['jobs'].each do |d|
329+
client.ml.delete_job(job_id: d['job_id'])
330+
end
331+
end
332+
333+
def clear_datafeeds(client)
334+
client.ml.stop_datafeed(datafeed_id: '_all', force: true)
335+
client.ml.get_datafeeds['datafeeds'].each do |d|
336+
client.ml.delete_datafeed(datafeed_id: d['datafeed_id'])
337+
end
338+
end
339+
340+
def clear_tasks(client)
341+
tasks = client.tasks.get['nodes'].values.first['tasks'].values.select do |d|
342+
d['cancellable']
343+
end.map do |d|
344+
"#{d['node']}:#{d['id']}"
345+
end
346+
tasks.each { |t| client.tasks.cancel task_id: t }
347+
end
348+
349+
def clear_machine_learning_indices(client)
350+
client.indices.delete(index: '.ml-*', ignore: 404)
351+
end
352+
353+
def clear_index_templates(client)
354+
client.indices.delete_template(name: '*')
355+
templates = client.indices.get_index_template
356+
templates['index_templates'].each do |template|
357+
client.indices.delete_index_template(name: template['name'])
358+
end
359+
end
360+
361+
def clear_transforms(client)
362+
client.transform.get_transform(transform_id: '*')['transforms'].each do |transform|
363+
client.transform.delete_transform(transform_id: transform[:id])
364+
end
365+
end
366+
367+
def clear_ml_filters(client)
368+
filters = client.ml.get_filters['filters']
369+
filters.each do |filter|
370+
client.ml.delete_filter(filter_id: filter['filter_id'])
371+
end
372+
end
373+
374+
def delete_all_node_shutdown_metadata(client)
375+
nodes = client.shutdown.get_node
376+
return unless nodes
377+
378+
nodes['nodes'].each do |node|
379+
client.shutdown.delete_node(node['node_id'])
380+
end
381+
end
382+
end
383+
end
384+
end
385+
end

‎elasticsearch-xpack/spec/rest_yaml_tests_helper.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
require "#{File.expand_path(File.dirname('..'), '..')}/api-spec-testing/test_file"
1919
require "#{File.expand_path(File.dirname('..'), '..')}/api-spec-testing/rspec_matchers"
2020
require "#{File.expand_path(File.dirname('..'), '..')}/api-spec-testing/wipe_cluster"
21+
require "#{File.expand_path(File.dirname('..'), '..')}/api-spec-testing/wipe_cluster_8"
2122
include Elasticsearch::RestAPIYAMLTests
2223

2324
PROJECT_PATH = File.join(File.dirname(__FILE__), '..', '..')

0 commit comments

Comments
 (0)