Skip to content

Commit 952bf22

Browse files
martijnvgmartijnvg
authored
Conditionally enable logsdb by default (#121049)
Enable logsdb by default if logsdb.prior_logs_usage has not been set to true. Meaning that if no data streams were created matching with the logs-- pattern in 8.x, then logsdb will be enabled by default for data streams matching with logs-*-* pattern. Also removes LogsPatternUsageService as with version 9.0 and beyond, this component is no longer necessary. Followup from #120708 Closes #106489
1 parent 2b66916 commit 952bf22

File tree

14 files changed

+178
-554
lines changed

14 files changed

+178
-554
lines changed

‎build-tools-internal/src/main/resources/changelog-schema.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@
291291
"JVM option",
292292
"Java API",
293293
"Logging",
294+
"Logs",
294295
"Mapping",
295296
"Packaging",
296297
"Painless",

‎docs/changelog/121049.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
pr: 121049
2+
summary: Conditionally enable logsdb by default for data streams matching with logs-*-*
3+
pattern.
4+
area: Logs
5+
type: breaking
6+
issues:
7+
- 106489
8+
breaking:
9+
title: Conditionally enable logsdb by default
10+
area: Logs
11+
details: |-
12+
Logsdb will be enabled by default for data streams matching with logs-*-* pattern.
13+
If upgrading from 8.x to 9.x and data streams matching with log-*-* do exist,
14+
then Logsdb will not be enabled by default.
15+
impact: |-
16+
Logsdb reduce storage footprint in Elasticsearch for logs, but there are side effects
17+
to be taken into account that are described in the Logsdb docs:
18+
https://www.elastic.co/guide/en/elasticsearch/reference/current/logs-data-stream.html#upgrade-to-logsdb-notes
19+
notable: true

‎docs/reference/data-streams/logs.asciidoc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,3 +237,9 @@ The `logsdb` index mode uses the following settings:
237237
* **`index.mapping.ignore_above`**: `8191`
238238

239239
* **`index.mapping.total_fields.ignore_dynamic_beyond_limit`**: `true`
240+
241+
[discrete]
242+
[[upgrade-to-logsdb-notes]]
243+
=== Notes about upgrading to Logsdb
244+
245+
TODO: add notes.

‎docs/reference/rest-api/info.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ Example response:
177177
},
178178
"logsdb": {
179179
"available": true,
180-
"enabled": false
180+
"enabled": true
181181
}
182182
},
183183
"tagline" : "You know, for X"

‎docs/reference/rest-api/usage.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,7 @@ GET /_xpack/usage
514514
},
515515
"logsdb": {
516516
"available": true,
517-
"enabled": false,
517+
"enabled": true,
518518
"indices_count": 0,
519519
"indices_with_synthetic_source": 0,
520520
"num_docs": 0,

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ protected ElasticsearchCluster getUpgradeCluster() {
120120
}""";
121121

122122
public void testLogsIndexing() throws IOException {
123+
assumeTrue(
124+
"otherwise first backing index of logs-apache-production will be in logsdb mode",
125+
getOldClusterTestVersion().before("9.0.0")
126+
);
123127
if (isRunningAgainstOldCluster()) {
124128
assertOK(client().performRequest(putTemplate(client(), "logs-template", STANDARD_TEMPLATE)));
125129
assertOK(client().performRequest(createDataStream("logs-apache-production")));

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsUsageRollingUpgradeIT.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import org.elasticsearch.client.Request;
15+
16+
import java.io.IOException;
17+
import java.time.Instant;
18+
import java.util.Map;
19+
import java.util.concurrent.TimeUnit;
20+
21+
import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.bulkIndex;
22+
import static org.hamcrest.Matchers.hasEntry;
23+
import static org.hamcrest.Matchers.hasKey;
24+
import static org.hamcrest.Matchers.not;
25+
26+
public class LogsUsageRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
27+
28+
public LogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
29+
super(upgradedNodes);
30+
}
31+
32+
public void testUsage() throws Exception {
33+
assumeTrue("logsdb.prior_logs_usage only gets set in 8.x", getOldClusterTestVersion().before("9.0.0"));
34+
String dataStreamName = "logs-mysql-error";
35+
if (isOldCluster()) {
36+
bulkIndex(dataStreamName, 4, 256, Instant.now());
37+
ensureGreen(dataStreamName);
38+
assertBusy(() -> {
39+
var getClusterSettingsResponse = getClusterSettings();
40+
Map<?, ?> persistentSettings = (Map<?, ?>) getClusterSettingsResponse.get("persistent");
41+
assertThat(persistentSettings, hasEntry("logsdb.prior_logs_usage", "true"));
42+
}, 2, TimeUnit.MINUTES);
43+
} else {
44+
String newIndex = rolloverDataStream(dataStreamName);
45+
bulkIndex(dataStreamName, 4, 256, Instant.now());
46+
Map<?, ?> indexResponse = (Map<?, ?>) getIndexSettings(newIndex, true).get(newIndex);
47+
Map<?, ?> settings = (Map<?, ?>) indexResponse.get("settings");
48+
Map<?, ?> defaults = (Map<?, ?>) indexResponse.get("defaults");
49+
assertThat(settings, not(hasKey("index.mode")));
50+
assertThat(defaults, hasEntry("index.mode", "standard"));
51+
}
52+
}
53+
54+
static Map<String, Object> getClusterSettings() throws IOException {
55+
var request = new Request("GET", "/_cluster/settings");
56+
request.addParameter("flat_settings", "true");
57+
request.addParameter("include_defaults", "true");
58+
var response = client().performRequest(request);
59+
assertOK(response);
60+
return entityAsMap(response);
61+
}
62+
63+
static String rolloverDataStream(String dataStreamName) throws IOException {
64+
var request = new Request("POST", "/" + dataStreamName + "/_rollover");
65+
var response = client().performRequest(request);
66+
assertOK(response);
67+
var responseBody = entityAsMap(response);
68+
return (String) responseBody.get("new_index");
69+
}
70+
71+
}

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,18 @@
2222
import java.io.IOException;
2323
import java.io.InputStream;
2424
import java.time.Instant;
25+
import java.util.List;
2526
import java.util.Locale;
2627
import java.util.Map;
2728

2829
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault;
2930
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex;
3031
import static org.elasticsearch.upgrades.TsdbIT.formatInstant;
31-
import static org.hamcrest.Matchers.*;
32+
import static org.hamcrest.Matchers.containsString;
33+
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.greaterThan;
35+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
36+
import static org.hamcrest.Matchers.notNullValue;
3237

3338
public class LogsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
3439

@@ -122,7 +127,8 @@ static void createTemplate(String dataStreamName, String id, String template) th
122127
assertOK(client().performRequest(putIndexTemplateRequest));
123128
}
124129

125-
static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
130+
static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
131+
String firstIndex = null;
126132
for (int i = 0; i < numRequest; i++) {
127133
var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
128134
StringBuilder requestBody = new StringBuilder();
@@ -155,7 +161,11 @@ static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instan
155161
assertOK(response);
156162
var responseBody = entityAsMap(response);
157163
assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false));
164+
if (firstIndex == null) {
165+
firstIndex = (String) ((Map<?, ?>) ((Map<?, ?>) ((List<?>) responseBody.get("items")).get(0)).get("create")).get("_index");
166+
}
158167
}
168+
return firstIndex;
159169
}
160170

161171
void search(String dataStreamName) throws Exception {

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NoLogsUsageRollingUpgradeIT.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import java.time.Instant;
15+
import java.util.Map;
16+
17+
import static org.elasticsearch.upgrades.LogsUsageRollingUpgradeIT.getClusterSettings;
18+
import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.bulkIndex;
19+
import static org.hamcrest.Matchers.hasEntry;
20+
import static org.hamcrest.Matchers.hasKey;
21+
import static org.hamcrest.Matchers.not;
22+
23+
public class NoLogsUsageRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
24+
25+
public NoLogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
26+
super(upgradedNodes);
27+
}
28+
29+
public void testUsage() throws Exception {
30+
String dataStreamName = "logs-mysql-error";
31+
if (isOldCluster()) {
32+
dataStreamName = dataStreamName.replace("logs-", "log-");
33+
bulkIndex(dataStreamName, 4, 256, Instant.now());
34+
ensureGreen(dataStreamName);
35+
} else if (isUpgradedCluster()) {
36+
String newIndex = bulkIndex(dataStreamName, 4, 256, Instant.now());
37+
ensureGreen(dataStreamName);
38+
Map<?, ?> indexResponse = (Map<?, ?>) getIndexSettings(newIndex, true).get(newIndex);
39+
Map<?, ?> settings = (Map<?, ?>) indexResponse.get("settings");
40+
assertThat(settings, hasEntry("index.mode", "logsdb"));
41+
var getClusterSettingsResponse = getClusterSettings();
42+
Map<?, ?> defaults = (Map<?, ?>) getClusterSettingsResponse.get("defaults");
43+
Map<?, ?> persistentSettings = (Map<?, ?>) getClusterSettingsResponse.get("persistent");
44+
assertThat(persistentSettings, not(hasKey("logsdb.prior_logs_usage")));
45+
assertThat(defaults, hasEntry("cluster.logsdb.enabled", "true"));
46+
}
47+
}
48+
49+
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,14 @@ public void setup() throws Exception {
6363

6464
private RestClient client;
6565

66-
public void testLogsSettingsIndexModeDisabled() throws IOException {
66+
public void testLogsSettingsIndexModeEnabledByDefault() throws IOException {
6767
assertOK(createDataStream(client, "logs-custom-dev"));
6868
final String indexMode = (String) getSetting(
6969
client,
7070
getDataStreamBackingIndex(client, "logs-custom-dev", 0),
7171
IndexSettings.MODE.getKey()
7272
);
73-
assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
73+
assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName()));
7474
}
7575

7676
public void testTogglingLogsdb() throws IOException {
@@ -81,29 +81,21 @@ public void testTogglingLogsdb() throws IOException {
8181
getDataStreamBackingIndex(client, "logs-custom-dev", 0),
8282
IndexSettings.MODE.getKey()
8383
);
84-
assertThat(indexModeBefore, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
85-
assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "true"));
84+
assertThat(indexModeBefore, equalTo(IndexMode.LOGSDB.getName()));
85+
assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false"));
8686
final String indexModeAfter = (String) getSetting(
8787
client,
8888
getDataStreamBackingIndex(client, "logs-custom-dev", 0),
8989
IndexSettings.MODE.getKey()
9090
);
91-
assertThat(indexModeAfter, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
91+
assertThat(indexModeAfter, equalTo(IndexMode.LOGSDB.getName()));
9292
assertOK(rolloverDataStream(client, "logs-custom-dev"));
9393
final String indexModeLater = (String) getSetting(
9494
client,
9595
getDataStreamBackingIndex(client, "logs-custom-dev", 1),
9696
IndexSettings.MODE.getKey()
9797
);
98-
assertThat(indexModeLater, equalTo(IndexMode.LOGSDB.getName()));
99-
assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false"));
100-
assertOK(rolloverDataStream(client, "logs-custom-dev"));
101-
final String indexModeFinal = (String) getSetting(
102-
client,
103-
getDataStreamBackingIndex(client, "logs-custom-dev", 2),
104-
IndexSettings.MODE.getKey()
105-
);
106-
assertThat(indexModeFinal, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
98+
assertThat(indexModeLater, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
10799

108100
}
109101

0 commit comments

Comments
 (0)