Skip to content

Commit ce569e3

Browse files
jsinovassinjsinovassin
authored
UNOMI-885: fix migration error on rollover alias (#716)
* UNOMI-885: fix migration error on rollover alias * UNOMI-885: change rollover value to have more than one rollover index after migration * UNOMI-885: update test to check rollover indices after migration * UNOMI-885: remove the check on duplicate session as indices are rolloved during migration
1 parent 81989bd commit ce569e3

File tree

12 files changed

+114
-26
lines changed

12 files changed

+114
-26
lines changed

‎.github/workflows/unomi-ci-build-tests.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ jobs:
3737
- name: Integration tests
3838
run: mvn -ntp clean install -Pintegration-tests
3939
- name: Archive code coverage logs
40-
uses: actions/upload-artifact@v3
40+
uses: actions/upload-artifact@v4
4141
if: false # UNOMI-746 Reactivate if necessary
4242
with:
4343
name: unomi-code-coverage-jdk11-${{ github.run_number }}
4444
path: itests/target/site/jacoco
4545
- name: Archive unomi logs
46-
uses: actions/upload-artifact@v3
46+
uses: actions/upload-artifact@v4
4747
if: failure()
4848
with:
4949
name: unomi-log-jdk11-${{ github.run_number }}

‎itests/src/test/java/org/apache/unomi/itests/AllITs.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.unomi.itests;
1919

20-
import org.apache.unomi.itests.migration.Migrate16xTo220IT;
20+
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
2121
import org.apache.unomi.itests.graphql.*;
2222
import org.apache.unomi.itests.migration.MigrationIT;
2323
import org.junit.runner.RunWith;
@@ -31,7 +31,7 @@
3131
*/
3232
@RunWith(Suite.class)
3333
@SuiteClasses({
34-
Migrate16xTo220IT.class,
34+
Migrate16xToCurrentVersionIT.class,
3535
MigrationIT.class,
3636
BasicIT.class,
3737
ConditionEvaluatorIT.class,

‎itests/src/test/java/org/apache/unomi/itests/BaseIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ public Option[] config() {
253253
editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.cluster.name", "contextElasticSearchITests"),
254254
editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.addresses", "localhost:9400"),
255255
editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.taskWaitingPollingInterval", "50"),
256+
editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.rollover.maxDocs", "300"),
256257

257258
systemProperty("org.ops4j.pax.exam.rbc.rmi.port").value("1199"),
258259
systemProperty("org.apache.unomi.hazelcast.group.name").value("cellar"),

itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java renamed to itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,13 @@
3131
import java.io.IOException;
3232
import java.util.*;
3333

34-
public class Migrate16xTo220IT extends BaseIT {
34+
public class Migrate16xToCurrentVersionIT extends BaseIT {
3535

3636
private int eventCount = 0;
3737
private int sessionCount = 0;
3838
private Set<String[]> initialScopes = new HashSet<>();
3939

4040
private static final String SCOPE_NOT_EXIST = "SCOPE_NOT_EXIST";
41-
private static final int NUMBER_DUPLICATE_SESSIONS = 3;
4241
private static final List<String> oldSystemItemsIndices = Arrays.asList("context-actiontype", "context-campaign", "context-campaignevent", "context-goal",
4342
"context-userlist", "context-propertytype", "context-scope", "context-conditiontype", "context-rule", "context-scoring", "context-segment", "context-groovyaction", "context-topic",
4443
"context-patch", "context-jsonschema", "context-importconfig", "context-exportconfig", "context-rulestats");
@@ -100,12 +99,12 @@ public void checkMigratedData() throws Exception {
10099
checkPagePathForEventView();
101100
checkPastEvents();
102101
checkScopeEventHaveBeenUpdated();
102+
countNumberOfSessionIndices();
103103
}
104104

105105
/**
106106
* Checks if at least the new index for events and sessions exists.
107107
* Also checks:
108-
* - duplicated sessions are correctly removed (-3 sessions in final count)
109108
* - persona sessions are now merged in session index due to index reduction in 2_2_0 (+2 sessions in final count)
110109
*/
111110
private void checkEventSessionRollover2_2_0() throws IOException {
@@ -122,7 +121,7 @@ private void checkEventSessionRollover2_2_0() throws IOException {
122121
newSessioncount += countItems(httpClient, sessionIndex, null);
123122
}
124123
Assert.assertEquals(eventCount, newEventcount);
125-
Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS, newSessioncount);
124+
Assert.assertEquals(sessionCount, newSessioncount);
126125
}
127126

128127
private void checkIndexReductions2_2_0() throws IOException {
@@ -339,6 +338,14 @@ private void initCounts(CloseableHttpClient httpClient) {
339338
}
340339
}
341340

341+
private void countNumberOfSessionIndices() {
342+
try {
343+
Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session");
344+
Assert.assertEquals(2, sessionIndices.size());
345+
} catch (IOException e) {
346+
throw new RuntimeException(e);
347+
}
348+
}
342349
private void getScopeFromEvents(CloseableHttpClient httpClient, String eventIndex) throws IOException {
343350
String requestBody = resourceAsString("migration/match_all_login_event_request.json");
344351
JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_search", requestBody, null));

‎tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,23 @@ public static boolean indexExists(CloseableHttpClient httpClient, String esAddre
9797
}
9898
}
9999

100+
public static void configureAlias(CloseableHttpClient httpClient, String esAddress, String alias, String writeIndex, Set<String> readIndices, String configureAliasBody, MigrationContext context) throws IOException {
101+
String readIndicesToAdd = "";
102+
if (!readIndices.isEmpty()) {
103+
readIndicesToAdd = "," + readIndices.stream().map(index -> "{\"add\": {\"index\": \"" + index + "\", \"alias\": \"" + alias + "\", \"is_write_index\": false}}").collect(Collectors.joining(","));
104+
}
105+
if (context != null) {
106+
context.printMessage("Will set " + writeIndex + " as write index for alias " + alias);
107+
context.printMessage("Will set " + readIndices.toString() + " as read indices");
108+
} else {
109+
LOGGER.info("Will set {} as write index for alias {}", writeIndex, alias);
110+
LOGGER.info("Will set {} as read indices", readIndices.toString());
111+
}
112+
String requestBody = configureAliasBody.replace("#writeIndexName", writeIndex).replace("#aliasName", alias).replace("#readIndicesToAdd", readIndicesToAdd);
113+
114+
HttpUtils.executePostRequest(httpClient, esAddress + "/_aliases", requestBody, null);
115+
}
116+
100117
public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException {
101118
try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
102119
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
@@ -300,9 +317,9 @@ public static void waitForYellowStatus(CloseableHttpClient httpClient, String es
300317
* <p>This method sends a request to update documents that match the provided query in the specified index. The update operation is
301318
* performed asynchronously, and the method waits for the task to complete before returning.</p>
302319
*
303-
* @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server
304-
* @param esAddress the address of the Elasticsearch server
305-
* @param indexName the name of the index where documents should be updated
320+
* @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server
321+
* @param esAddress the address of the Elasticsearch server
322+
* @param indexName the name of the index where documents should be updated
306323
* @param requestBody the JSON body containing the query and update instructions for the documents
307324
* @throws Exception if there is an error during the HTTP request or while waiting for the task to finish
308325
*/
@@ -332,23 +349,66 @@ public static void deleteByQuery(CloseableHttpClient httpClient, String esAddres
332349
waitForTaskToFinish(httpClient, esAddress, task.getString("task"), null);
333350
}
334351

352+
private static void printResponseDetail(JSONObject response, MigrationContext migrationContext){
353+
StringBuilder sb = new StringBuilder();
354+
if (response.has("total")) {
355+
sb.append("Total: ").append(response.getInt("total")).append(" ");
356+
}
357+
if (response.has("updated")) {
358+
sb.append("Updated: ").append(response.getInt("updated")).append(" ");
359+
}
360+
if (response.has("created")) {
361+
sb.append("Created: ").append(response.getInt("created")).append(" ");
362+
}
363+
if (response.has("deleted")) {
364+
sb.append("Deleted: ").append(response.getInt("deleted")).append(" ");
365+
}
366+
if (response.has("batches")) {
367+
sb.append("Batches: ").append(response.getInt("batches")).append(" ");
368+
}
369+
if (migrationContext != null) {
370+
migrationContext.printMessage(sb.toString());
371+
} else {
372+
LOGGER.info(sb.toString());
373+
}
374+
}
375+
335376
public static void waitForTaskToFinish(CloseableHttpClient httpClient, String esAddress, String taskId, MigrationContext migrationContext) throws IOException {
336377
while (true) {
337378
final JSONObject status = new JSONObject(
338379
HttpUtils.executeGetRequest(httpClient, esAddress + "/_tasks/" + taskId,
339380
null));
381+
if (status.has("error")) {
382+
final JSONObject error = status.getJSONObject("error");
383+
throw new IOException("Task error: " + error.getString("type") + " - " + error.getString("reason"));
384+
}
340385
if (status.has("completed") && status.getBoolean("completed")) {
341386
if (migrationContext != null) {
342387
migrationContext.printMessage("Task is completed");
343388
} else {
344389
LOGGER.info("Task is completed");
345390
}
391+
if (status.has("response")) {
392+
final JSONObject response = status.getJSONObject("response");
393+
printResponseDetail(response, migrationContext);
394+
if (response.has("failures")) {
395+
final JSONArray failures = response.getJSONArray("failures");
396+
if (!failures.isEmpty()) {
397+
for (int i = 0; i < failures.length(); i++) {
398+
JSONObject failure = failures.getJSONObject(i);
399+
JSONObject cause = failure.getJSONObject("cause");
400+
if (migrationContext != null) {
401+
migrationContext.printMessage("Cause of failure: " + cause.toString());
402+
} else {
403+
LOGGER.error("Cause of failure: {}", cause.toString());
404+
}
405+
}
406+
throw new IOException("Task completed with failures, check previous log for details");
407+
}
408+
}
409+
}
346410
break;
347411
}
348-
if (status.has("error")) {
349-
final JSONObject error = status.getJSONObject("error");
350-
throw new IOException("Task error: " + error.getString("type") + " - " + error.getString("reason"));
351-
}
352412
if (migrationContext != null) {
353413
migrationContext.printMessage("Waiting for Task " + taskId + " to complete");
354414
} else {

‎tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateEventSession.groovy‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ context.performMigrationStep("2.2.0-create-event-index", () -> {
4545
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) {
4646
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json")
4747
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
48+
String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json")
4849

4950
String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias)
5051
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null)
52+
MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverEventAlias, newEventIndex, Collections.emptySet(), configureAliasBody, context)
5153
}
5254
})
5355

@@ -73,9 +75,11 @@ context.performMigrationStep("2.2.0-create-session-index", () -> {
7375
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) {
7476
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json")
7577
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json")
78+
String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json")
7679

7780
String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias)
7881
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newSessionIndex, newIndexSettings, null)
82+
MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverSessionAlias, newSessionIndex, Collections.emptySet(), configureAliasBody, context)
7983
}
8084
})
8185

‎tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-00-cleanPastEventProfileSession.groovy‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
import org.apache.unomi.shell.migration.service.MigrationContext
2-
import org.apache.unomi.shell.migration.utils.HttpUtils
32
import org.apache.unomi.shell.migration.utils.MigrationUtils
4-
import org.osgi.framework.BundleContext
5-
import org.osgi.framework.Bundle
63

74
/*
85
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,7 +22,7 @@ MigrationContext context = migrationContext
2522
String esAddress = context.getConfigString("esAddress")
2623
String indexPrefix = context.getConfigString("indexPrefix")
2724
String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy"
28-
String rolloverEventAlias = indexPrefix + "-session"
25+
String rolloverSessionAlias = indexPrefix + "-session"
2926

3027
context.performMigrationStep("2.5.0-clean-profile-mapping", () -> {
3128
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
@@ -39,10 +36,17 @@ context.performMigrationStep("2.5.0-clean-session-mapping", () -> {
3936
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json")
4037
String cleanPastEventScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.5.0/remove_pastEvents_session.painless")
4138
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json")
42-
String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseSettings, mapping, context, rolloverPolicyName, rolloverEventAlias)
39+
String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseSettings, mapping, context, rolloverPolicyName, rolloverSessionAlias)
4340
Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-")
41+
String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json")
4442

45-
sessionIndices.each { sessionIndex ->
43+
Set<String> sortedSet = new TreeSet<>(sessionIndices)
44+
sortedSet.each { sessionIndex ->
4645
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, newIndexSettings, cleanPastEventScript, context, "2.5.0-clean-session-mapping")
4746
}
47+
SortedSet<String> allExceptLast = Collections.emptySortedSet();
48+
if (sortedSet.size() > 1){
49+
allExceptLast = sortedSet.headSet(sortedSet.last());
50+
}
51+
MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverSessionAlias, sortedSet.last(), allExceptLast, configureAliasBody, context)
4852
})

‎tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import org.apache.unomi.shell.migration.service.MigrationContext
2-
import org.apache.unomi.shell.migration.utils.HttpUtils
32
import org.apache.unomi.shell.migration.utils.MigrationUtils
43

54
/*

‎tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"source": {
3-
"index": "#source"
3+
"index": "#source",
4+
"size": 5000
45
},
56
"dest": {
67
"index": "#dest"

‎tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
},
2424
"aliases": {
2525
"#lifecycleRolloverAlias": {
26-
"is_write_index": true
26+
"is_write_index": false
2727
}
2828
},
2929
"mappings": #mappings
30-
}
30+
}

0 commit comments

Comments
 (0)