Skip to content

Commit 46c47f7

Browse files
original-brownbearoriginal-brownbear
authored
Merge branch 'main' into drier-buckets
2 parents 02b0385 + 9d5df19 commit 46c47f7

File tree

43 files changed

+2506
-59
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+2506
-59
lines changed

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -271,8 +271,7 @@ protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo,
271271
.withProcessInfo(processInfo)
272272
.withServerArgs(args)
273273
.withTempDir(tempDir)
274-
.withJvmOptions(jvmOptions)
275-
.withWorkingDir(args.logsDir());
274+
.withJvmOptions(jvmOptions);
276275
return serverProcessBuilder.start();
277276
}
278277

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.server.cli;
1111

1212
import org.elasticsearch.bootstrap.ServerArgs;
13+
import org.elasticsearch.cli.ExitCodes;
1314
import org.elasticsearch.cli.ProcessInfo;
1415
import org.elasticsearch.cli.Terminal;
1516
import org.elasticsearch.cli.UserException;
@@ -21,6 +22,8 @@
2122
import java.io.IOException;
2223
import java.io.OutputStream;
2324
import java.io.UncheckedIOException;
25+
import java.nio.file.FileAlreadyExistsException;
26+
import java.nio.file.Files;
2427
import java.nio.file.Path;
2528
import java.util.HashMap;
2629
import java.util.List;
@@ -44,7 +47,6 @@ public class ServerProcessBuilder {
4447
private ServerArgs serverArgs;
4548
private ProcessInfo processInfo;
4649
private List<String> jvmOptions;
47-
private Path workingDir;
4850
private Terminal terminal;
4951

5052
// this allows mocking the process building by tests
@@ -84,11 +86,6 @@ public ServerProcessBuilder withJvmOptions(List<String> jvmOptions) {
8486
return this;
8587
}
8688

87-
public ServerProcessBuilder withWorkingDir(Path workingDir) {
88-
this.workingDir = workingDir;
89-
return this;
90-
}
91-
9289
/**
9390
* Specifies the {@link Terminal} to use for reading input and writing output from/to the cli console
9491
*/
@@ -141,6 +138,17 @@ public ServerProcess start() throws UserException {
141138
return start(ProcessBuilder::start);
142139
}
143140

141+
private void ensureWorkingDirExists() throws UserException {
142+
Path workingDir = serverArgs.logsDir();
143+
try {
144+
Files.createDirectories(workingDir);
145+
} catch (FileAlreadyExistsException e) {
146+
throw new UserException(ExitCodes.CONFIG, "Logs dir [" + workingDir + "] exists but is not a directory", e);
147+
} catch (IOException e) {
148+
throw new UserException(ExitCodes.CONFIG, "Unable to create logs dir [" + workingDir + "]", e);
149+
}
150+
}
151+
144152
private static void checkRequiredArgument(Object argument, String argumentName) {
145153
if (argument == null) {
146154
throw new IllegalStateException(
@@ -157,12 +165,14 @@ ServerProcess start(ProcessStarter processStarter) throws UserException {
157165
checkRequiredArgument(jvmOptions, "jvmOptions");
158166
checkRequiredArgument(terminal, "terminal");
159167

168+
ensureWorkingDirExists();
169+
160170
Process jvmProcess = null;
161171
ErrorPumpThread errorPump;
162172

163173
boolean success = false;
164174
try {
165-
jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), workingDir, processStarter);
175+
jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), serverArgs.logsDir(), processStarter);
166176
errorPump = new ErrorPumpThread(terminal, jvmProcess.getErrorStream());
167177
errorPump.start();
168178
sendArgs(serverArgs, jvmProcess.getOutputStream());

‎distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public class ServerProcessTests extends ESTestCase {
6565
protected final Map<String, String> sysprops = new HashMap<>();
6666
protected final Map<String, String> envVars = new HashMap<>();
6767
Path esHomeDir;
68-
Path workingDir;
68+
Path logsDir;
6969
Settings.Builder nodeSettings;
7070
ProcessValidator processValidator;
7171
MainMethod mainCallback;
@@ -94,8 +94,8 @@ public void resetEnv() {
9494
sysprops.put("os.name", "Linux");
9595
sysprops.put("java.home", "javahome");
9696
sysprops.put("es.path.home", esHomeDir.toString());
97+
logsDir = esHomeDir.resolve("logs");
9798
envVars.clear();
98-
workingDir = createTempDir();
9999
nodeSettings = Settings.builder();
100100
processValidator = null;
101101
mainCallback = null;
@@ -207,15 +207,7 @@ ProcessInfo createProcessInfo() {
207207
}
208208

209209
ServerArgs createServerArgs(boolean daemonize, boolean quiet) {
210-
return new ServerArgs(
211-
daemonize,
212-
quiet,
213-
null,
214-
secrets,
215-
nodeSettings.build(),
216-
esHomeDir.resolve("config"),
217-
esHomeDir.resolve("logs")
218-
);
210+
return new ServerArgs(daemonize, quiet, null, secrets, nodeSettings.build(), esHomeDir.resolve("config"), logsDir);
219211
}
220212

221213
ServerProcess startProcess(boolean daemonize, boolean quiet) throws Exception {
@@ -231,8 +223,7 @@ ServerProcess startProcess(boolean daemonize, boolean quiet) throws Exception {
231223
.withProcessInfo(pinfo)
232224
.withServerArgs(createServerArgs(daemonize, quiet))
233225
.withJvmOptions(List.of())
234-
.withTempDir(ServerProcessUtils.setupTempDir(pinfo))
235-
.withWorkingDir(workingDir);
226+
.withTempDir(ServerProcessUtils.setupTempDir(pinfo));
236227
return serverProcessBuilder.start(starter);
237228
}
238229

@@ -241,7 +232,7 @@ public void testProcessBuilder() throws Exception {
241232
assertThat(pb.redirectInput(), equalTo(ProcessBuilder.Redirect.PIPE));
242233
assertThat(pb.redirectOutput(), equalTo(ProcessBuilder.Redirect.INHERIT));
243234
assertThat(pb.redirectError(), equalTo(ProcessBuilder.Redirect.PIPE));
244-
assertThat(String.valueOf(pb.directory()), equalTo(workingDir.toString())); // leave default, which is working directory
235+
assertThat(String.valueOf(pb.directory()), equalTo(esHomeDir.resolve("logs").toString()));
245236
};
246237
mainCallback = (args, stdin, stderr, exitCode) -> {
247238
try (PrintStream err = new PrintStream(stderr, true, StandardCharsets.UTF_8)) {
@@ -315,8 +306,7 @@ public void testCommandLineSysprops() throws Exception {
315306
.withProcessInfo(createProcessInfo())
316307
.withServerArgs(createServerArgs(false, false))
317308
.withJvmOptions(List.of("-Dfoo1=bar", "-Dfoo2=baz"))
318-
.withTempDir(Path.of("."))
319-
.withWorkingDir(workingDir);
309+
.withTempDir(Path.of("."));
320310
serverProcessBuilder.start(starter).waitFor();
321311
}
322312

@@ -433,4 +423,26 @@ public void testProcessDies() throws Exception {
433423
int exitCode = server.waitFor();
434424
assertThat(exitCode, equalTo(-9));
435425
}
426+
427+
public void testLogsDirIsFile() throws Exception {
428+
Files.createFile(logsDir);
429+
var e = expectThrows(UserException.class, this::runForeground);
430+
assertThat(e.getMessage(), containsString("exists but is not a directory"));
431+
}
432+
433+
public void testLogsDirCreateParents() throws Exception {
434+
Path testDir = createTempDir();
435+
logsDir = testDir.resolve("subdir/logs");
436+
processValidator = pb -> assertThat(String.valueOf(pb.directory()), equalTo(logsDir.toString()));
437+
runForeground();
438+
}
439+
440+
public void testLogsCreateFailure() throws Exception {
441+
Path testDir = createTempDir();
442+
Path parentFile = testDir.resolve("exists");
443+
Files.createFile(parentFile);
444+
logsDir = parentFile.resolve("logs");
445+
var e = expectThrows(UserException.class, this::runForeground);
446+
assertThat(e.getMessage(), containsString("Unable to create logs dir"));
447+
}
436448
}

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,9 @@ tests:
414414
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.RepositoryAnalysisFailureIT
415415
method: testFailsOnReadError
416416
issue: https://github.com/elastic/elasticsearch/issues/127029
417+
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
418+
method: test {p0=esql/10_basic/basic with documents_found}
419+
issue: https://github.com/elastic/elasticsearch/issues/127039
417420

418421
# Examples:
419422
#

‎test/framework/build.gradle‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,12 @@ dependencies {
3232

3333
api "org.elasticsearch:mocksocket:${versions.mocksocket}"
3434

35+
testImplementation project(":modules:mapper-extras")
36+
testImplementation project(':x-pack:plugin:core')
3537
testImplementation project(':x-pack:plugin:mapper-unsigned-long')
3638
testImplementation project(':x-pack:plugin:mapper-counted-keyword')
37-
testImplementation project(":modules:mapper-extras")
39+
testImplementation project(':x-pack:plugin:mapper-constant-keyword')
40+
testImplementation project(':x-pack:plugin:wildcard')
3841
}
3942

4043
sourceSets {

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
1313
import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator;
1414
import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
15+
import org.elasticsearch.logsdb.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator;
1516
import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator;
1617
import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator;
1718
import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
@@ -26,6 +27,7 @@
2627
import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
2728
import org.elasticsearch.logsdb.datageneration.fields.leaf.TextFieldDataGenerator;
2829
import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
30+
import org.elasticsearch.logsdb.datageneration.fields.leaf.WildcardFieldDataGenerator;
2931

3032
/**
3133
* Lists all leaf field types that are supported for data generation by default.
@@ -46,7 +48,9 @@ public enum FieldType {
4648
DATE("date"),
4749
GEO_POINT("geo_point"),
4850
TEXT("text"),
49-
IP("ip");
51+
IP("ip"),
52+
CONSTANT_KEYWORD("constant_keyword"),
53+
WILDCARD("wildcard");
5054

5155
private final String name;
5256

@@ -56,7 +60,7 @@ public enum FieldType {
5660

5761
public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
5862
return switch (this) {
59-
case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
63+
case KEYWORD -> new KeywordFieldDataGenerator(dataSource);
6064
case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
6165
case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
6266
case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
@@ -72,6 +76,8 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
7276
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
7377
case TEXT -> new TextFieldDataGenerator(dataSource);
7478
case IP -> new IpFieldDataGenerator(dataSource);
79+
case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator();
80+
case WILDCARD -> new WildcardFieldDataGenerator(dataSource);
7581
};
7682
}
7783

@@ -93,6 +99,8 @@ public static FieldType tryParse(String name) {
9399
case "geo_point" -> FieldType.GEO_POINT;
94100
case "text" -> FieldType.TEXT;
95101
case "ip" -> FieldType.IP;
102+
case "constant_keyword" -> FieldType.CONSTANT_KEYWORD;
103+
case "wildcard" -> FieldType.WILDCARD;
96104
default -> null;
97105
};
98106
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator requ
7878
return null;
7979
}
8080

81+
default DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
82+
return null;
83+
}
84+
8185
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
8286
return null;
8387
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,12 @@ public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) {
126126
}
127127
}
128128

129+
record VersionStringGenerator() implements DataSourceRequest<DataSourceResponse.VersionStringGenerator> {
130+
public DataSourceResponse.VersionStringGenerator accept(DataSourceHandler handler) {
131+
return handler.handle(this);
132+
}
133+
}
134+
129135
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
130136
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
131137
return handler.handle(this);

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ record GeoPointGenerator(Supplier<Object> generator) implements DataSourceRespon
5353

5454
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
5555

56+
record VersionStringGenerator(Supplier<String> generator) implements DataSourceResponse {}
57+
5658
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
5759

5860
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
5151
case GEO_POINT -> geoPointMapping(map);
5252
case TEXT -> textMapping(request, new HashMap<>());
5353
case IP -> ipMapping(map);
54+
case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>());
55+
case WILDCARD -> wildcardMapping(new HashMap<>());
5456
});
5557
}
5658

@@ -225,6 +227,29 @@ private Supplier<Map<String, Object>> ipMapping(Map<String, Object> injected) {
225227
};
226228
}
227229

230+
private Supplier<Map<String, Object>> constantKeywordMapping(Map<String, Object> injected) {
231+
return () -> {
232+
// value is optional and can be set from the first document
233+
// we don't cover this case here
234+
injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
235+
236+
return injected;
237+
};
238+
}
239+
240+
private Supplier<Map<String, Object>> wildcardMapping(Map<String, Object> injected) {
241+
return () -> {
242+
if (ESTestCase.randomDouble() <= 0.2) {
243+
injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
244+
}
245+
if (ESTestCase.randomDouble() <= 0.2) {
246+
injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
247+
}
248+
249+
return injected;
250+
};
251+
}
252+
228253
private static HashMap<String, Object> commonMappingParameters() {
229254
var map = new HashMap<String, Object>();
230255
map.put("store", ESTestCase.randomBoolean());

0 commit comments

Comments
 (0)