Skip to content

Commit c40723a

Browse files
valeriy42valeriy42
committed
unit test passes
1 parent c58ac45 commit c40723a

File tree

2 files changed

+79
-28
lines changed

2 files changed

+79
-28
lines changed

‎x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/UnusedStatsRemoverIT.java‎

Lines changed: 60 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.action.index.IndexRequest;
1111
import org.elasticsearch.action.support.PlainActionFuture;
1212
import org.elasticsearch.client.internal.OriginSettingClient;
13+
import org.elasticsearch.common.settings.Settings;
1314
import org.elasticsearch.common.unit.ByteSizeValue;
1415
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
1516
import org.elasticsearch.tasks.TaskId;
@@ -64,8 +65,53 @@ public void createComponents() {
6465

6566
public void testRemoveUnusedStats() throws Exception {
6667

67-
prepareIndex("foo").setId("some-empty-doc").setSource("{}", XContentType.JSON).get();
68+
String modelId = "model-with-stats";
69+
putDFA(modelId);
70+
71+
72+
73+
indexStatDocument(new DataCounts("analytics-with-stats", 1, 1, 1), DataCounts.documentId("analytics-with-stats"));
74+
indexStatDocument(new DataCounts("missing-analytics-with-stats", 1, 1, 1), DataCounts.documentId("missing-analytics-with-stats"));
75+
indexStatDocument(
76+
new InferenceStats(1, 1, 1, 1, TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test", Instant.now()),
77+
InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")
78+
);
79+
indexStatDocument(
80+
new InferenceStats(1, 1, 1, 1, "missing-model", "test", Instant.now()),
81+
InferenceStats.docId("missing-model", "test")
82+
);
83+
indexStatDocument(
84+
new InferenceStats(1, 1, 1, 1, modelId, "test", Instant.now()),
85+
InferenceStats.docId(modelId, "test")
86+
);
87+
client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get();
88+
89+
PlainActionFuture<Boolean> deletionListener = new PlainActionFuture<>();
90+
UnusedStatsRemover statsRemover = new UnusedStatsRemover(client, new TaskId("test", 0L));
91+
statsRemover.remove(10000.0f, deletionListener, () -> false);
92+
deletionListener.actionGet();
93+
94+
client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get();
95+
96+
final String initialStateIndex = MlStatsIndex.TEMPLATE_NAME + "-000001";
97+
98+
// Make sure that stats that should exist still exist
99+
assertTrue(client().prepareGet(initialStateIndex, InferenceStats.docId(modelId, "test")).get().isExists());
100+
assertTrue(
101+
client().prepareGet(
102+
initialStateIndex,
103+
InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")
104+
).get().isExists()
105+
);
106+
assertTrue(client().prepareGet(initialStateIndex, DataCounts.documentId("analytics-with-stats")).get().isExists());
68107

108+
// make sure that unused stats were deleted
109+
assertFalse(client().prepareGet(initialStateIndex, DataCounts.documentId("missing-analytics-with-stats")).get().isExists());
110+
assertFalse(client().prepareGet(initialStateIndex, InferenceStats.docId("missing-model", "test")).get().isExists());
111+
}
112+
113+
private void putDFA(String modelId) {
114+
prepareIndex("foo").setId("some-empty-doc").setSource("{}", XContentType.JSON).get();
69115
PutDataFrameAnalyticsAction.Request request = new PutDataFrameAnalyticsAction.Request(
70116
new DataFrameAnalyticsConfig.Builder().setId("analytics-with-stats")
71117
.setModelMemoryLimit(ByteSizeValue.ofGb(1))
@@ -80,7 +126,7 @@ public void testRemoveUnusedStats() throws Exception {
80126
PutTrainedModelAction.INSTANCE,
81127
new PutTrainedModelAction.Request(
82128
TrainedModelConfig.builder()
83-
.setModelId("model-with-stats")
129+
.setModelId(modelId)
84130
.setInferenceConfig(RegressionConfig.EMPTY_PARAMS)
85131
.setInput(new TrainedModelInput(Arrays.asList("foo", "bar")))
86132
.setParsedDefinition(
@@ -97,21 +143,21 @@ public void testRemoveUnusedStats() throws Exception {
97143
false
98144
)
99145
).actionGet();
146+
}
147+
148+
public void testRemovingUnusedStatsFromReadOnlyIndexShouldFailSilently() throws Exception {
149+
150+
String modelId = "model-with-stats";
151+
putDFA(modelId);
100152

101-
indexStatDocument(new DataCounts("analytics-with-stats", 1, 1, 1), DataCounts.documentId("analytics-with-stats"));
102-
indexStatDocument(new DataCounts("missing-analytics-with-stats", 1, 1, 1), DataCounts.documentId("missing-analytics-with-stats"));
103-
indexStatDocument(
104-
new InferenceStats(1, 1, 1, 1, TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test", Instant.now()),
105-
InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")
106-
);
107153
indexStatDocument(
108154
new InferenceStats(1, 1, 1, 1, "missing-model", "test", Instant.now()),
109155
InferenceStats.docId("missing-model", "test")
110156
);
111-
indexStatDocument(
112-
new InferenceStats(1, 1, 1, 1, "model-with-stats", "test", Instant.now()),
113-
InferenceStats.docId("model-with-stats", "test")
114-
);
157+
158+
// set index to read-only
159+
client().admin().indices().prepareUpdateSettings(MlStatsIndex.indexPattern()).setSettings(Settings.builder().put("index.blocks.write", true)).get();
160+
115161
client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get();
116162

117163
PlainActionFuture<Boolean> deletionListener = new PlainActionFuture<>();
@@ -120,22 +166,9 @@ public void testRemoveUnusedStats() throws Exception {
120166
deletionListener.actionGet();
121167

122168
client().admin().indices().prepareRefresh(MlStatsIndex.indexPattern()).get();
123-
169+
// make sure that unused stats are still there
124170
final String initialStateIndex = MlStatsIndex.TEMPLATE_NAME + "-000001";
125-
126-
// Make sure that stats that should exist still exist
127-
assertTrue(client().prepareGet(initialStateIndex, InferenceStats.docId("model-with-stats", "test")).get().isExists());
128-
assertTrue(
129-
client().prepareGet(
130-
initialStateIndex,
131-
InferenceStats.docId(TrainedModelProvider.MODELS_STORED_AS_RESOURCE.iterator().next(), "test")
132-
).get().isExists()
133-
);
134-
assertTrue(client().prepareGet(initialStateIndex, DataCounts.documentId("analytics-with-stats")).get().isExists());
135-
136-
// make sure that unused stats were deleted
137-
assertFalse(client().prepareGet(initialStateIndex, DataCounts.documentId("missing-analytics-with-stats")).get().isExists());
138-
assertFalse(client().prepareGet(initialStateIndex, InferenceStats.docId("missing-model", "test")).get().isExists());
171+
assertTrue(client().prepareGet(initialStateIndex, InferenceStats.docId("missing-model", "test")).get().isExists());
139172
}
140173

141174
private void indexStatDocument(ToXContentObject object, String docId) throws Exception {

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStatsRemover.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@
99
import org.apache.logging.log4j.LogManager;
1010
import org.apache.logging.log4j.Logger;
1111
import org.elasticsearch.action.ActionListener;
12+
import org.elasticsearch.action.bulk.BulkItemResponse;
1213
import org.elasticsearch.action.support.IndicesOptions;
1314
import org.elasticsearch.client.internal.OriginSettingClient;
15+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1416
import org.elasticsearch.common.Strings;
1517
import org.elasticsearch.index.query.BoolQueryBuilder;
1618
import org.elasticsearch.index.query.QueryBuilder;
1719
import org.elasticsearch.index.query.QueryBuilders;
20+
import org.elasticsearch.index.reindex.BulkByScrollResponse;
1821
import org.elasticsearch.index.reindex.DeleteByQueryAction;
1922
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
2023
import org.elasticsearch.tasks.TaskId;
@@ -101,14 +104,29 @@ private Set<String> getTrainedModelIds() {
101104
return modelIds;
102105
}
103106

107+
private static boolean hasNonReadOnlyBulkFailures(BulkByScrollResponse response) {
108+
for (BulkItemResponse.Failure failure : response.getBulkFailures()) {
109+
if (failure.getMessage().contains(IndexMetadata.INDEX_WRITE_BLOCK.description())) {
110+
LOGGER.debug(
111+
"Ignoring failure to delete orphan stats docs from read-only index [{}]: {}",
112+
failure.getIndex(),
113+
failure.getMessage()
114+
);
115+
} else {
116+
return true;
117+
}
118+
}
119+
return false;
120+
}
121+
104122
private void executeDeleteUnusedStatsDocs(QueryBuilder dbq, float requestsPerSec, ActionListener<Boolean> listener) {
105123
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(MlStatsIndex.indexPattern()).setIndicesOptions(
106124
IndicesOptions.lenientExpandOpen()
107125
).setAbortOnVersionConflict(false).setRequestsPerSecond(requestsPerSec).setTimeout(DEFAULT_MAX_DURATION).setQuery(dbq);
108126
deleteByQueryRequest.setParentTask(parentTaskId);
109127

110128
client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, ActionListener.wrap(response -> {
111-
if (response.getBulkFailures().size() > 0 || response.getSearchFailures().size() > 0) {
129+
if (hasNonReadOnlyBulkFailures(response) || response.getSearchFailures().isEmpty() == false) {
112130
LOGGER.error(
113131
"Some unused stats documents could not be deleted due to failures: {}",
114132
Strings.collectionToCommaDelimitedString(response.getBulkFailures())

0 commit comments

Comments
 (0)