Skip to content

Commit f285689

Browse files
elasticmachineelasticmachine
authored
Merge branch 'main' into es-129644-fix
2 parents 1836b8d + 8555c16 commit f285689

File tree

14 files changed

+124
-166
lines changed

14 files changed

+124
-166
lines changed

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.ingest.geoip.stats.CacheStats;
1818
import org.elasticsearch.test.ESTestCase;
1919

20+
import java.nio.file.Path;
2021
import java.util.concurrent.atomic.AtomicInteger;
2122
import java.util.concurrent.atomic.AtomicLong;
2223
import java.util.function.Function;
@@ -140,23 +141,25 @@ public void testPurgeCacheEntriesForDatabase() {
140141
GeoIpCache cache = new GeoIpCache(100);
141142
ProjectId projectId1 = randomUniqueProjectId();
142143
ProjectId projectId2 = randomUniqueProjectId();
143-
String databasePath1 = "path/to/db1";
144-
String databasePath2 = "path/to/db2";
144+
// Turn the path strings into Paths to ensure that we always use the canonical string representation (this string literal does not
145+
// round-trip when converting to a Path and back again on Windows):
146+
Path databasePath1 = PathUtils.get("path/to/db1");
147+
Path databasePath2 = PathUtils.get("path/to/db2");
145148
String ip1 = "127.0.0.1";
146149
String ip2 = "127.0.0.2";
147150

148151
AbstractResponse response = mock(AbstractResponse.class);
149-
cache.putIfAbsent(projectId1, ip1, databasePath1, ip -> response); // cache miss
150-
cache.putIfAbsent(projectId1, ip2, databasePath1, ip -> response); // cache miss
151-
cache.putIfAbsent(projectId2, ip1, databasePath1, ip -> response); // cache miss
152-
cache.putIfAbsent(projectId1, ip1, databasePath2, ip -> response); // cache miss
153-
cache.purgeCacheEntriesForDatabase(projectId1, PathUtils.get(databasePath1));
152+
cache.putIfAbsent(projectId1, ip1, databasePath1.toString(), ip -> response); // cache miss
153+
cache.putIfAbsent(projectId1, ip2, databasePath1.toString(), ip -> response); // cache miss
154+
cache.putIfAbsent(projectId2, ip1, databasePath1.toString(), ip -> response); // cache miss
155+
cache.putIfAbsent(projectId1, ip1, databasePath2.toString(), ip -> response); // cache miss
156+
cache.purgeCacheEntriesForDatabase(projectId1, databasePath1);
154157
// should have purged entries for projectId1 and databasePath1...
155-
assertNull(cache.get(projectId1, ip1, databasePath1));
156-
assertNull(cache.get(projectId1, ip2, databasePath1));
158+
assertNull(cache.get(projectId1, ip1, databasePath1.toString()));
159+
assertNull(cache.get(projectId1, ip2, databasePath1.toString()));
157160
// ...but left the one for projectId2...
158-
assertSame(response, cache.get(projectId2, ip1, databasePath1));
161+
assertSame(response, cache.get(projectId2, ip1, databasePath1.toString()));
159162
// ...and for databasePath2:
160-
assertSame(response, cache.get(projectId1, ip1, databasePath2));
163+
assertSame(response, cache.get(projectId1, ip1, databasePath2.toString()));
161164
}
162165
}

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,8 @@ public void testEmptyBucketName() {
165165
}
166166

167167
private S3Repository createS3Repo(RepositoryMetadata metadata) {
168-
final ProjectId projectId = randomProjectIdOrDefault();
169168
final S3Repository s3Repository = new S3Repository(
170-
projectId,
169+
ProjectId.DEFAULT,
171170
metadata,
172171
NamedXContentRegistry.EMPTY,
173172
new DummyS3Service(
@@ -181,7 +180,7 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
181180
new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
182181
S3RepositoriesMetrics.NOOP
183182
);
184-
assertThat(s3Repository.getProjectId(), equalTo(projectId));
183+
assertThat(s3Repository.getProjectId(), equalTo(ProjectId.DEFAULT));
185184
return s3Repository;
186185
}
187186

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import org.elasticsearch.core.FixForMultiProject;
4141
import org.elasticsearch.core.Nullable;
4242
import org.elasticsearch.core.Tuple;
43-
import org.elasticsearch.gateway.MetadataStateFormat;
4443
import org.elasticsearch.index.Index;
4544
import org.elasticsearch.index.IndexNotFoundException;
4645
import org.elasticsearch.index.IndexVersion;
@@ -50,7 +49,6 @@
5049
import org.elasticsearch.xcontent.NamedObjectNotFoundException;
5150
import org.elasticsearch.xcontent.NamedXContentRegistry;
5251
import org.elasticsearch.xcontent.ToXContent;
53-
import org.elasticsearch.xcontent.XContentBuilder;
5452
import org.elasticsearch.xcontent.XContentParser;
5553

5654
import java.io.IOException;
@@ -2101,30 +2099,6 @@ static <C extends MetadataCustom<C>> void parseCustomObject(
21012099
}
21022100
}
21032101

2104-
private static final ToXContent.Params FORMAT_PARAMS;
2105-
static {
2106-
Map<String, String> params = Maps.newMapWithExpectedSize(2);
2107-
params.put("binary", "true");
2108-
params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
2109-
FORMAT_PARAMS = new ToXContent.MapParams(params);
2110-
}
2111-
2112-
/**
2113-
* State format for {@link Metadata} to write to and load from disk
2114-
*/
2115-
public static final MetadataStateFormat<Metadata> FORMAT = new MetadataStateFormat<>(GLOBAL_STATE_FILE_PREFIX) {
2116-
2117-
@Override
2118-
public void toXContent(XContentBuilder builder, Metadata state) throws IOException {
2119-
ChunkedToXContent.wrapAsToXContent(state).toXContent(builder, FORMAT_PARAMS);
2120-
}
2121-
2122-
@Override
2123-
public Metadata fromXContent(XContentParser parser) throws IOException {
2124-
return Builder.fromXContent(parser);
2125-
}
2126-
};
2127-
21282102
private volatile Metadata.ProjectLookup projectLookup = null;
21292103

21302104
/**

‎server/src/main/java/org/elasticsearch/gateway/MetaStateService.java‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,14 @@
2121
import org.elasticsearch.xcontent.NamedXContentRegistry;
2222

2323
import java.io.IOException;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
2426
import java.util.ArrayList;
2527
import java.util.List;
2628
import java.util.function.Predicate;
2729

30+
import static org.elasticsearch.cluster.metadata.Metadata.GLOBAL_STATE_FILE_PREFIX;
31+
2832
/**
2933
* Handles writing and loading {@link Manifest}, {@link Metadata} and {@link IndexMetadata} as used for cluster state persistence in
3034
* versions prior to {@link Version#V_7_6_0}, used to read this older format during an upgrade from these versions.
@@ -75,20 +79,29 @@ List<IndexMetadata> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
7579
return indexMetadataList;
7680
}
7781

78-
/**
79-
* Loads the global state, *without* index state
80-
*/
81-
Metadata loadGlobalState() throws IOException {
82-
return Metadata.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
83-
}
84-
8582
/**
8683
* Creates empty cluster state file on disk, deleting global metadata and unreferencing all index metadata
8784
* (only used for dangling indices at that point).
8885
*/
8986
public void unreferenceAll() throws IOException {
9087
Manifest.FORMAT.writeAndCleanup(Manifest.empty(), nodeEnv.nodeDataPaths()); // write empty file so that indices become unreferenced
91-
Metadata.FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths());
88+
cleanUpGlobalStateFiles();
89+
}
90+
91+
private void cleanUpGlobalStateFiles() throws IOException {
92+
for (Path location : nodeEnv.nodeDataPaths()) {
93+
logger.trace("cleanupOldFiles: cleaning up {} for global state files", location);
94+
final Path stateLocation = location.resolve(MetadataStateFormat.STATE_DIR_NAME);
95+
try (var paths = Files.list(stateLocation)) {
96+
paths.filter(file -> file.getFileName().toString().startsWith(GLOBAL_STATE_FILE_PREFIX)).forEach(file -> {
97+
try {
98+
Files.deleteIfExists(file);
99+
} catch (IOException e) {
100+
logger.trace("failed to delete global state file: {}", file);
101+
}
102+
});
103+
}
104+
}
92105
}
93106

94107
/**

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ public void testXContentWithIndexGraveyard() throws IOException {
629629
final Metadata originalMeta = Metadata.builder().put(ProjectMetadata.builder(projectId).indexGraveyard(graveyard)).build();
630630
final XContentBuilder builder = JsonXContent.contentBuilder();
631631
builder.startObject();
632-
Metadata.FORMAT.toXContent(builder, originalMeta);
632+
ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
633633
builder.endObject();
634634
try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
635635
final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -647,7 +647,7 @@ public void testXContentClusterUUID() throws IOException {
647647
.build();
648648
final XContentBuilder builder = JsonXContent.contentBuilder();
649649
builder.startObject();
650-
Metadata.FORMAT.toXContent(builder, originalMeta);
650+
ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
651651
builder.endObject();
652652
try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
653653
final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -732,7 +732,7 @@ public void testXContentWithCoordinationMetadata() throws IOException {
732732

733733
final XContentBuilder builder = JsonXContent.contentBuilder();
734734
builder.startObject();
735-
Metadata.FORMAT.toXContent(builder, metadata);
735+
ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, formatParams());
736736
builder.endObject();
737737

738738
try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -3345,6 +3345,10 @@ private static CreateIndexResult createIndices(int numIndices, int numBackingInd
33453345
return new CreateIndexResult(indices, backingIndices, b.build());
33463346
}
33473347

3348+
private static ToXContent.Params formatParams() {
3349+
return new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY));
3350+
}
3351+
33483352
private static class CreateIndexResult {
33493353
final List<Index> indices;
33503354
final List<Index> backingIndices;

‎server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,11 @@ public void testSimpleJsonFromAndTo() throws IOException {
133133

134134
XContentBuilder builder = JsonXContent.contentBuilder();
135135
builder.startObject();
136-
Metadata.FORMAT.toXContent(builder, metadata);
136+
ChunkedToXContent.wrapAsToXContent(metadata)
137+
.toXContent(
138+
builder,
139+
new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY))
140+
);
137141
builder.endObject();
138142

139143
Metadata parsedMetadata;

‎server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,7 @@
99
package org.elasticsearch.gateway;
1010

1111
import org.elasticsearch.cluster.metadata.IndexMetadata;
12-
import org.elasticsearch.cluster.metadata.Metadata;
1312
import org.elasticsearch.common.UUIDs;
14-
import org.elasticsearch.common.settings.Settings;
1513
import org.elasticsearch.env.NodeEnvironment;
1614
import org.elasticsearch.index.Index;
1715
import org.elasticsearch.index.IndexVersion;
@@ -53,20 +51,4 @@ public void testWriteLoadIndex() throws Exception {
5351
public void testLoadMissingIndex() throws Exception {
5452
assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
5553
}
56-
57-
public void testWriteLoadGlobal() throws Exception {
58-
Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
59-
MetaStateWriterUtils.writeGlobalState(env, "test_write", metadata);
60-
assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
61-
}
62-
63-
public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
64-
Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
65-
IndexMetadata index = indexMetadata("test1");
66-
Metadata metadataWithIndex = Metadata.builder(metadata).put(index, true).build();
67-
68-
MetaStateWriterUtils.writeGlobalState(env, "test_write", metadataWithIndex);
69-
assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
70-
assertThat(metaStateService.loadGlobalState().getProject().hasIndex("test1"), equalTo(false));
71-
}
7254
}

‎test/framework/src/main/java/org/elasticsearch/gateway/MetaStateWriterUtils.java‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
import org.apache.logging.log4j.Logger;
1414
import org.elasticsearch.Version;
1515
import org.elasticsearch.cluster.metadata.IndexMetadata;
16-
import org.elasticsearch.cluster.metadata.Manifest;
17-
import org.elasticsearch.cluster.metadata.Metadata;
1816
import org.elasticsearch.env.NodeEnvironment;
1917
import org.elasticsearch.index.Index;
2018

@@ -29,22 +27,6 @@ private MetaStateWriterUtils() {
2927
throw new AssertionError("static methods only");
3028
}
3129

32-
/**
33-
* Writes manifest file (represented by {@link Manifest}) to disk and performs cleanup of old manifest state file if
34-
* the write succeeds or newly created manifest state if the write fails.
35-
*
36-
* @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()}
37-
*/
38-
public static void writeManifestAndCleanup(NodeEnvironment nodeEnv, String reason, Manifest manifest) throws WriteStateException {
39-
logger.trace("[_meta] writing state, reason [{}]", reason);
40-
try {
41-
long generation = Manifest.FORMAT.writeAndCleanup(manifest, nodeEnv.nodeDataPaths());
42-
logger.trace("[_meta] state written (generation: {})", generation);
43-
} catch (WriteStateException ex) {
44-
throw new WriteStateException(ex.isDirty(), "[_meta]: failed to write meta state", ex);
45-
}
46-
}
47-
4830
/**
4931
* Writes the index state.
5032
* <p>
@@ -65,21 +47,4 @@ public static long writeIndex(NodeEnvironment nodeEnv, String reason, IndexMetad
6547
}
6648
}
6749

68-
/**
69-
* Writes the global state, *without* the indices states.
70-
*
71-
* @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return
72-
* false, because new global state file is not yet referenced by manifest file.
73-
*/
74-
static long writeGlobalState(NodeEnvironment nodeEnv, String reason, Metadata metadata) throws WriteStateException {
75-
logger.trace("[_global] writing state, reason [{}]", reason);
76-
try {
77-
long generation = Metadata.FORMAT.write(metadata, nodeEnv.nodeDataPaths());
78-
logger.trace("[_global] state written");
79-
return generation;
80-
} catch (WriteStateException ex) {
81-
throw new WriteStateException(false, "[_global]: failed to write global state", ex);
82-
}
83-
}
84-
8550
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,22 @@ public static CategorizationAnalyzerConfig buildStandardCategorizationAnalyzer(L
210210
.build();
211211
}
212212

213+
/**
214+
* Create a <code>categorization_analyzer</code> that will be used by the ES|QL categorize function.
215+
* The only difference from the DSL analyzer is the tokenizer (standard instead of ml_standard).
216+
* This means the results are slightly different from the categorize text aggregation and the ML job,
217+
* however you can use these tokens for looking up messages in indices generated with the standard
218+
* tokenizer. The latter is considered more important.
219+
*/
220+
public static CategorizationAnalyzerConfig buildStandardEsqlCategorizationAnalyzer() {
221+
222+
return new CategorizationAnalyzerConfig.Builder().addCharFilter("first_line_with_letters")
223+
.setTokenizer("standard")
224+
.addDateWordsTokenFilter()
225+
.addLimitFilter()
226+
.build();
227+
}
228+
213229
private final String analyzer;
214230
private final List<NameOrDefinition> charFilters;
215231
private final NameOrDefinition tokenizer;

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHash.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939

4040
import java.io.IOException;
4141
import java.util.HashMap;
42-
import java.util.List;
4342
import java.util.Map;
4443
import java.util.Objects;
4544

@@ -48,9 +47,8 @@
4847
*/
4948
public class CategorizeBlockHash extends BlockHash {
5049

51-
private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(
52-
List.of()
53-
);
50+
private static final CategorizationAnalyzerConfig ANALYZER_CONFIG = CategorizationAnalyzerConfig
51+
.buildStandardEsqlCategorizationAnalyzer();
5452
private static final int NULL_ORD = 0;
5553

5654
private final int channel;

0 commit comments

Comments
 (0)