Skip to content

Commit da1c71f

Browse files
jonathan-buttnerjonathan-buttner
committed
Fixing transport version
2 parents 6be22b5 + 43841a5 commit da1c71f

File tree

8 files changed

+112
-17
lines changed

8 files changed

+112
-17
lines changed

‎docs/changelog/128161.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 128161
2+
summary: Fix system data streams incorrectly showing up in the list of template validation
3+
problems
4+
area: Data streams
5+
type: bug
6+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,10 @@ public static Map<String, Policy> createPluginPolicies(
8181
return pluginPolicies;
8282
}
8383

84+
/**
85+
* @throws PolicyParserException if the supplied policy is formatted incorrectly
86+
* @throws IllegalStateException for any other error parsing the patch, such as nonexistent module names
87+
*/
8488
public static Policy parseEncodedPolicyIfExists(
8589
String encodedPolicy,
8690
String version,
@@ -106,11 +110,8 @@ public static Policy parseEncodedPolicyIfExists(
106110
version
107111
);
108112
}
109-
} catch (Exception ex) {
110-
logger.warn(
111-
Strings.format("Found a policy patch with invalid content. The patch will not be applied. Layer [%s]", layerName),
112-
ex
113-
);
113+
} catch (Exception e) {
114+
throw new IllegalStateException("Unable to parse policy patch for layer [" + layerName + "]", e);
114115
}
115116
}
116117
return null;

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtilsTests.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ public void testNoPatchWithVersionMismatch() {
134134

135135
public void testNoPatchWithValidationError() {
136136

137+
// Nonexistent module names
137138
var policyPatch = """
138139
versions:
139140
- 9.0.0
@@ -149,13 +150,15 @@ public void testNoPatchWithValidationError() {
149150
StandardCharsets.UTF_8
150151
);
151152

152-
var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of());
153-
154-
assertThat(policy, nullValue());
153+
assertThrows(
154+
IllegalStateException.class,
155+
() -> PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of())
156+
);
155157
}
156158

157159
public void testNoPatchWithParsingError() {
158160

161+
// no <version> or <policy> field
159162
var policyPatch = """
160163
entitlement-module-name:
161164
- load_native_libraries
@@ -167,9 +170,10 @@ public void testNoPatchWithParsingError() {
167170
StandardCharsets.UTF_8
168171
);
169172

170-
var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of());
171-
172-
assertThat(policy, nullValue());
173+
assertThrows(
174+
IllegalStateException.class,
175+
() -> PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of())
176+
);
173177
}
174178

175179
public void testMergeScopes() {

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@ static TransportVersion def(int id) {
176176
public static final TransportVersion ESQL_REPORT_SHARD_PARTITIONING_8_19 = def(8_841_0_29);
177177
public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_8_19 = def(8_841_0_30);
178178
public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_31);
179-
public static final TransportVersion ADD_INFERENCE_CUSTOM_MODEL_8_19 = def(8_841_0_32);
179+
public static final TransportVersion V_8_19_FIELD_CAPS_ADD_CLUSTER_ALIAS = def(8_841_0_32);
180+
public static final TransportVersion ADD_INFERENCE_CUSTOM_MODEL_8_19 = def(8_841_0_33);
180181
public static final TransportVersion V_9_0_0 = def(9_000_0_09);
181182
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
182183
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
7070
if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
7171
includeEmptyFields = in.readBoolean();
7272
}
73-
if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_ADD_CLUSTER_ALIAS)) {
73+
if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_ADD_CLUSTER_ALIAS)
74+
|| in.getTransportVersion().isPatchFrom(TransportVersions.V_8_19_FIELD_CAPS_ADD_CLUSTER_ALIAS)) {
7475
clusterAlias = in.readOptionalString();
7576
} else {
7677
clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
@@ -124,7 +125,8 @@ public void writeTo(StreamOutput out) throws IOException {
124125
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
125126
out.writeBoolean(includeEmptyFields);
126127
}
127-
if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_ADD_CLUSTER_ALIAS)) {
128+
if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_ADD_CLUSTER_ALIAS)
129+
|| out.getTransportVersion().isPatchFrom(TransportVersions.V_8_19_FIELD_CAPS_ADD_CLUSTER_ALIAS)) {
128130
out.writeOptionalString(clusterAlias);
129131
}
130132
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -884,7 +884,12 @@ private static void validateDataStreamsStillReferenced(
884884
String templateName,
885885
ComposableIndexTemplate newTemplate
886886
) {
887-
final Set<String> dataStreams = project.dataStreams().keySet();
887+
final Set<String> dataStreams = project.dataStreams()
888+
.entrySet()
889+
.stream()
890+
.filter(entry -> entry.getValue().isSystem() == false)
891+
.map(Map.Entry::getKey)
892+
.collect(Collectors.toSet());
888893

889894
Function<ProjectMetadata, Set<String>> findUnreferencedDataStreams = meta -> {
890895
final Set<String> unreferenced = new HashSet<>();

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.common.settings.IndexScopedSettings;
2323
import org.elasticsearch.common.settings.Settings;
2424
import org.elasticsearch.common.util.concurrent.ThreadContext;
25+
import org.elasticsearch.core.Strings;
2526
import org.elasticsearch.core.TimeValue;
2627
import org.elasticsearch.env.Environment;
2728
import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor;
@@ -1718,6 +1719,76 @@ public void testInvalidNonDataStreamTemplateWithDataStreamOptions() throws Excep
17181719
);
17191720
}
17201721

1722+
public void testSystemDataStreamsIgnoredByValidateIndexTemplateV2() throws Exception {
1723+
/*
1724+
* This test makes sure that system data streams (which do not have named templates) do not appear in the list of data streams
1725+
* without named templates when validateIndexTemplateV2 fails due to another non-system data stream not having a named template.
1726+
*/
1727+
MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
1728+
final String dataStreamTemplateName = "data_stream_template";
1729+
final String indexTemplateName = "index_template";
1730+
final String systemDataStreamName = "system_ds";
1731+
final String ordinaryDataStreamName = "my_ds";
1732+
final String ordinaryDataStreamIndexPattern = "my_ds*";
1733+
ComposableIndexTemplate highPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
1734+
.indexPatterns(List.of(ordinaryDataStreamIndexPattern))
1735+
.priority(275L)
1736+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
1737+
.build();
1738+
ComposableIndexTemplate highPriorityIndexTemplate = ComposableIndexTemplate.builder()
1739+
.indexPatterns(List.of(ordinaryDataStreamIndexPattern))
1740+
.priority(200L)
1741+
.build();
1742+
ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault())
1743+
.dataStreams(
1744+
Map.of(
1745+
systemDataStreamName,
1746+
DataStreamTestHelper.randomInstance(systemDataStreamName, System::currentTimeMillis, randomBoolean(), true),
1747+
ordinaryDataStreamName,
1748+
DataStreamTestHelper.randomInstance(ordinaryDataStreamName, System::currentTimeMillis, randomBoolean(), false)
1749+
),
1750+
Map.of()
1751+
)
1752+
.indexTemplates(Map.of(dataStreamTemplateName, highPriorityDataStreamTemplate, indexTemplateName, highPriorityIndexTemplate))
1753+
.build();
1754+
ComposableIndexTemplate lowPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
1755+
.indexPatterns(List.of(ordinaryDataStreamIndexPattern))
1756+
.priority(1L)
1757+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
1758+
.build();
1759+
/*
1760+
* Here we attempt to change the priority of a template that matches an existing non-system data stream so that it is so low that
1761+
* the data stream matches the index (non data-stream) template instead. We expect an error, but that the error only mentions the
1762+
* non-system data stream.
1763+
*/
1764+
Exception exception = expectThrows(
1765+
Exception.class,
1766+
() -> metadataIndexTemplateService.validateIndexTemplateV2(project, dataStreamTemplateName, lowPriorityDataStreamTemplate)
1767+
);
1768+
assertThat(
1769+
exception.getMessage(),
1770+
containsString(
1771+
Strings.format(
1772+
"composable template [%s] with index patterns [%s], priority [1] would cause data streams [%s] to no longer "
1773+
+ "match a data stream template",
1774+
dataStreamTemplateName,
1775+
ordinaryDataStreamIndexPattern,
1776+
ordinaryDataStreamName
1777+
)
1778+
)
1779+
);
1780+
ComposableIndexTemplate mediumPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
1781+
.indexPatterns(List.of(ordinaryDataStreamIndexPattern))
1782+
.priority(201L)
1783+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
1784+
.build();
1785+
/*
1786+
* We have now corrected the problem -- the priority of the new template is lower than the old data stream template but still higher
1787+
* than the non-data-stream index template. So we expect no validation errors.
1788+
*/
1789+
metadataIndexTemplateService.validateIndexTemplateV2(project, dataStreamTemplateName, mediumPriorityDataStreamTemplate);
1790+
}
1791+
17211792
private ProjectMetadata addComponentTemplate(
17221793
MetadataIndexTemplateService service,
17231794
ProjectMetadata project,

‎test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,11 @@ public static DataStream randomInstance(LongSupplier timeProvider, boolean failu
335335
}
336336

337337
public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore) {
338+
// Some tests don't work well with system data streams, since these data streams require special handling
339+
return randomInstance(dataStreamName, timeProvider, failureStore, false);
340+
}
341+
342+
public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore, boolean system) {
338343
List<Index> indices = randomIndexInstances();
339344
long generation = indices.size() + ESTestCase.randomLongBetween(1, 128);
340345
indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random())));
@@ -360,9 +365,9 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time
360365
generation,
361366
metadata,
362367
randomSettings(),
363-
randomBoolean(),
368+
system ? true : randomBoolean(),
364369
replicated,
365-
false, // Some tests don't work well with system data streams, since these data streams require special handling
370+
system,
366371
timeProvider,
367372
randomBoolean(),
368373
randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass

0 commit comments

Comments
 (0)