Skip to content

Commit 3c0f957

Browse files
droberts195droberts195
authored
[7.11] [ML] move the already upgraded model snapshot test (#66471) (#68839)
* [ML] move the already upgraded model snapshot test (#66471) Having the "already upgraded" model snapshot test as a YAML test causes pains whenever there is a version bump. This moves the test to a java internal cluster integration test so that it has access to `Version.CURRENT`. Related to: #66456
1 parent fedd739 commit 3c0f957

File tree

2 files changed

+138
-30
lines changed

2 files changed

+138
-30
lines changed

‎x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobModelSnapshotUpgraderIT.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.xpack.ml.integration;
8+
9+
import static org.hamcrest.Matchers.containsString;
10+
import static org.hamcrest.Matchers.equalTo;
11+
12+
import java.util.ArrayList;
13+
import java.util.Arrays;
14+
import java.util.Collections;
15+
import java.util.HashSet;
16+
import java.util.List;
17+
18+
import org.elasticsearch.ElasticsearchStatusException;
19+
import org.elasticsearch.Version;
20+
import org.elasticsearch.action.support.WriteRequest;
21+
import org.elasticsearch.client.OriginSettingClient;
22+
import org.elasticsearch.cluster.routing.OperationRouting;
23+
import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
24+
import org.elasticsearch.cluster.service.ClusterApplierService;
25+
import org.elasticsearch.cluster.service.ClusterService;
26+
import org.elasticsearch.cluster.service.MasterService;
27+
import org.elasticsearch.common.settings.ClusterSettings;
28+
import org.elasticsearch.common.settings.Settings;
29+
import org.elasticsearch.common.unit.TimeValue;
30+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
31+
import org.elasticsearch.rest.RestStatus;
32+
import org.elasticsearch.threadpool.ThreadPool;
33+
import org.elasticsearch.xpack.core.ClientHelper;
34+
import org.elasticsearch.xpack.core.ml.action.PutJobAction;
35+
import org.elasticsearch.xpack.core.ml.action.UpgradeJobModelSnapshotAction;
36+
import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
37+
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
38+
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
39+
import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
40+
import org.elasticsearch.xpack.core.ml.job.config.Detector;
41+
import org.elasticsearch.xpack.core.ml.job.config.Job;
42+
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
43+
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
44+
import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
45+
import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
46+
import org.elasticsearch.xpack.ml.inference.modelsize.MlModelSizeNamedXContentProvider;
47+
import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
48+
import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
49+
import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
50+
import org.junit.Before;
51+
52+
public class JobModelSnapshotUpgraderIT extends MlSingleNodeTestCase {
53+
54+
private JobResultsPersister jobResultsPersister;
55+
56+
@Before
57+
public void createComponents() throws Exception {
58+
ThreadPool tp = mockThreadPool();
59+
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY,
60+
new HashSet<>(Arrays.asList(InferenceProcessor.MAX_INFERENCE_PROCESSORS,
61+
MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
62+
AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
63+
OperationRouting.USE_ADAPTIVE_REPLICA_SELECTION_SETTING,
64+
ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES,
65+
ClusterService.USER_DEFINED_METADATA,
66+
ClusterApplierService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING)));
67+
ClusterService clusterService = new ClusterService(Settings.EMPTY, clusterSettings, tp);
68+
69+
OriginSettingClient originSettingClient = new OriginSettingClient(client(), ClientHelper.ML_ORIGIN);
70+
ResultsPersisterService resultsPersisterService = new ResultsPersisterService(
71+
tp,
72+
originSettingClient,
73+
clusterService,
74+
Settings.EMPTY
75+
);
76+
AnomalyDetectionAuditor auditor = new AnomalyDetectionAuditor(client(), clusterService);
77+
jobResultsPersister = new JobResultsPersister(originSettingClient, resultsPersisterService, auditor);
78+
waitForMlTemplates();
79+
}
80+
81+
public void testUpgradeAlreadyUpgradedSnapshot() {
82+
String jobId = "job-with-current-snapshot";
83+
84+
createJob(jobId);
85+
ModelSnapshot snapshot = new ModelSnapshot.Builder(jobId).setMinVersion(Version.CURRENT).setSnapshotId("snap_1").build();
86+
indexModelSnapshot(snapshot);
87+
client().admin().indices().prepareRefresh(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)).get();
88+
89+
ElasticsearchStatusException ex = expectThrows(
90+
ElasticsearchStatusException.class,
91+
() -> client().execute(
92+
UpgradeJobModelSnapshotAction.INSTANCE,
93+
new UpgradeJobModelSnapshotAction.Request(jobId, "snap_1", TimeValue.timeValueMinutes(10), true)
94+
).actionGet());
95+
assertThat(ex.status(), equalTo(RestStatus.CONFLICT));
96+
assertThat(
97+
ex.getMessage(),
98+
containsString(
99+
"Cannot upgrade job [job-with-current-snapshot] snapshot [snap_1] as it is already compatible with current version"
100+
)
101+
);
102+
}
103+
104+
private void indexModelSnapshot(ModelSnapshot snapshot) {
105+
jobResultsPersister.persistModelSnapshot(snapshot, WriteRequest.RefreshPolicy.IMMEDIATE, () -> true);
106+
}
107+
108+
private Job.Builder createJob(String jobId) {
109+
Job.Builder builder = new Job.Builder(jobId);
110+
AnalysisConfig.Builder ac = createAnalysisConfig("by_field");
111+
DataDescription.Builder dc = new DataDescription.Builder();
112+
builder.setAnalysisConfig(ac);
113+
builder.setDataDescription(dc);
114+
115+
PutJobAction.Request request = new PutJobAction.Request(builder);
116+
client().execute(PutJobAction.INSTANCE, request).actionGet();
117+
return builder;
118+
}
119+
120+
private AnalysisConfig.Builder createAnalysisConfig(String byFieldName) {
121+
Detector.Builder detector = new Detector.Builder("mean", "field");
122+
detector.setByFieldName(byFieldName);
123+
List<DetectionRule> rules = new ArrayList<>();
124+
125+
detector.setRules(rules);
126+
127+
return new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
128+
}
129+
130+
@Override
131+
public NamedXContentRegistry xContentRegistry() {
132+
List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
133+
namedXContent.addAll(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
134+
namedXContent.addAll(new MlModelSizeNamedXContentProvider().getNamedXContentParsers());
135+
return new NamedXContentRegistry(namedXContent);
136+
}
137+
138+
}

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,6 @@ setup:
1717
}
1818
}
1919
20-
- do:
21-
headers:
22-
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
23-
Content-Type: application/json
24-
index:
25-
index: .ml-anomalies-upgrade-model-snapshot
26-
id: "upgrade-model-snapshot_model_snapshot_snapshot-1"
27-
body: >
28-
{
29-
"job_id" : "upgrade-model-snapshot",
30-
"timestamp": "2016-06-02T00:00:00Z",
31-
"snapshot_id": "snapshot-1",
32-
"snapshot_doc_count": 3,
33-
"min_version": "7.11.0",
34-
"retain": false
35-
}
36-
37-
- do:
38-
headers:
39-
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
40-
indices.refresh:
41-
index: [.ml-anomalies-upgrade-model-snapshot]
42-
4320
---
4421
"Test with unknown job id":
4522
- do:
@@ -54,10 +31,3 @@ setup:
5431
ml.upgrade_job_snapshot:
5532
job_id: "upgrade-model-snapshot"
5633
snapshot_id: "snapshot-9999"
57-
---
58-
"Test with already upgraded snapshot":
59-
- do:
60-
catch: conflict
61-
ml.upgrade_job_snapshot:
62-
job_id: "upgrade-model-snapshot"
63-
snapshot_id: "snapshot-1"

0 commit comments

Comments
 (0)