Skip to content

Commit 372b76a

Browse files
iveraseiverase
authored
Merge branch 'main' into add-base64-encoded-float32-support
2 parents fe782c4 + ffe6943 commit 372b76a

File tree

30 files changed

+765
-186
lines changed

30 files changed

+765
-186
lines changed

‎distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:7b18967a1329291d113228149b97382406f039171c6537b1afa4067701a6ba9e AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4 AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
104104
# Add entrypoint
105105
################################################################################
106106

107-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:7b18967a1329291d113228149b97382406f039171c6537b1afa4067701a6ba9e
107+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4
108108

109109
RUN <%= retry.loop(package_manager,
110110
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:46fd486e4716d592aa1ad7dad15bb3e6d0aa5b08b2070e9daec9f6b8c8c990a6 AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49 AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
8080
# Add entrypoint
8181
################################################################################
8282

83-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:46fd486e4716d592aa1ad7dad15bb3e6d0aa5b08b2070e9daec9f6b8c8c990a6
83+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49
8484

8585
RUN <%= retry.loop(package_manager,
8686
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎docs/changelog/137055.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 137055
2+
summary: Do not create inference endpoint if ID is used in existing mappings
3+
area: Machine Learning
4+
type: bug
5+
issues:
6+
- 124272

‎docs/changelog/137479.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 137479
2+
summary: "Fix illegal_access_exception: class com.maxmind.db.Decoder from `ip_location`\
3+
\ processor"
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎modules/ingest-geoip/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,4 +99,6 @@ artifacts {
9999
tasks.named("yamlRestCompatTestTransform").configure({ task ->
100100
task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases",
101101
"get databases behavior began returning more results in 8.16")
102+
task.skipTest("ingest_geoip/60_ip_location_databases/Test adding, getting, and removing ip location databases",
103+
"get databases behavior began returning more results in 9.2.1")
102104
})

‎modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,20 @@
2121
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2222
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
2323
import org.junit.Before;
24+
import org.junit.BeforeClass;
2425
import org.junit.ClassRule;
2526
import org.junit.rules.RuleChain;
27+
import org.junit.rules.TemporaryFolder;
2628
import org.junit.rules.TestRule;
2729

30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.util.Objects;
33+
2834
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded;
35+
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.getRootPath;
2936
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline;
37+
import static org.hamcrest.Matchers.is;
3038

3139
@FixForMultiProject(description = "Potentially remove this test after https://elasticco.atlassian.net/browse/ES-12094 is implemented")
3240
public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase {
@@ -35,7 +43,10 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec
3543

3644
private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
3745

46+
public static TemporaryFolder configDir = new TemporaryFolder();
47+
3848
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
49+
.withConfigDir(() -> getRootPath(configDir))
3950
.module("reindex")
4051
.module("ingest-geoip")
4152
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
@@ -51,7 +62,7 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec
5162
.build();
5263

5364
@ClassRule
54-
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
65+
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);
5566

5667
@Override
5768
protected String getTestRestCluster() {
@@ -67,6 +78,19 @@ public static Iterable<Object[]> parameters() throws Exception {
6778
return ESClientYamlSuiteTestCase.createParameters();
6879
}
6980

81+
@BeforeClass
82+
public static void copyExtraDatabase() throws Exception {
83+
Path configPath = getRootPath(configDir);
84+
assertThat(Files.exists(configPath), is(true));
85+
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
86+
Files.createDirectory(ingestGeoipDatabaseDir);
87+
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
88+
Files.copy(
89+
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
90+
ingestGeoipDatabaseDir.resolve("asn.mmdb")
91+
);
92+
}
93+
7094
@Before
7195
public void waitForDatabases() throws Exception {
7296
putGeoipPipeline("pipeline-with-geoip");

‎modules/ingest-geoip/src/main/java/module-info.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919
exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
2020
exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;
2121

22-
exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge;
22+
exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge, com.maxmind.db;
2323
}

‎modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,29 @@
1919
import org.elasticsearch.client.Request;
2020
import org.elasticsearch.common.bytes.BytesReference;
2121
import org.elasticsearch.core.Booleans;
22+
import org.elasticsearch.core.SuppressForbidden;
2223
import org.elasticsearch.test.cluster.ElasticsearchCluster;
2324
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2425
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
2526
import org.elasticsearch.xcontent.XContentBuilder;
2627
import org.elasticsearch.xcontent.json.JsonXContent;
2728
import org.junit.Before;
29+
import org.junit.BeforeClass;
2830
import org.junit.ClassRule;
2931
import org.junit.rules.RuleChain;
32+
import org.junit.rules.TemporaryFolder;
3033
import org.junit.rules.TestRule;
3134

3235
import java.io.IOException;
36+
import java.nio.file.Files;
37+
import java.nio.file.Path;
3338
import java.util.List;
3439
import java.util.Map;
40+
import java.util.Objects;
3541

3642
import static org.hamcrest.Matchers.containsInAnyOrder;
3743
import static org.hamcrest.Matchers.equalTo;
44+
import static org.hamcrest.Matchers.is;
3845
import static org.hamcrest.Matchers.notNullValue;
3946

4047
public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -43,7 +50,10 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
4350

4451
private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
4552

53+
public static TemporaryFolder configDir = new TemporaryFolder();
54+
4655
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
56+
.withConfigDir(() -> getRootPath(configDir))
4757
.module("reindex")
4858
.module("ingest-geoip")
4959
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
@@ -56,7 +66,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
5666
.build();
5767

5868
@ClassRule
59-
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
69+
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);
6070

6171
@Override
6272
protected String getTestRestCluster() {
@@ -72,6 +82,19 @@ public static Iterable<Object[]> parameters() throws Exception {
7282
return ESClientYamlSuiteTestCase.createParameters();
7383
}
7484

85+
@BeforeClass
86+
public static void copyExtraDatabase() throws Exception {
87+
Path configPath = getRootPath(configDir);
88+
assertThat(Files.exists(configPath), is(true));
89+
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
90+
Files.createDirectory(ingestGeoipDatabaseDir);
91+
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
92+
Files.copy(
93+
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
94+
ingestGeoipDatabaseDir.resolve("asn.mmdb")
95+
);
96+
}
97+
7598
@Before
7699
public void waitForDatabases() throws Exception {
77100
putGeoipPipeline("pipeline-with-geoip");
@@ -123,13 +146,23 @@ static void assertDatabasesLoaded() throws Exception {
123146
Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
124147
assertThat(nodes.size(), equalTo(1));
125148
Map<?, ?> node = (Map<?, ?>) nodes.values().iterator().next();
149+
150+
// confirm the downloaded databases are all correct
126151
List<?> databases = ((List<?>) node.get("databases"));
127152
assertThat(databases, notNullValue());
128153
List<String> databaseNames = databases.stream().map(o -> (String) ((Map<?, ?>) o).get("name")).toList();
129154
assertThat(
130155
databaseNames,
131156
containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb")
132157
);
158+
159+
// ensure that the extra config database has been set up, too:
160+
assertThat(node.get("config_databases"), equalTo(List.of("asn.mmdb")));
133161
});
134162
}
163+
164+
@SuppressForbidden(reason = "fixtures use java.io.File based APIs")
165+
public static Path getRootPath(TemporaryFolder folder) {
166+
return folder.getRoot().toPath();
167+
}
135168
}

‎modules/ingest-geoip/src/yamlRestTest/resources/ipinfo/asn_sample.mmdb‎

25 KB
Binary file not shown.

‎modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ teardown:
6969

7070
- do:
7171
ingest.get_geoip_database: {}
72-
- length: { databases: 6 }
72+
- length: { databases: 7 }
7373

7474
- do:
7575
ingest.get_geoip_database:
@@ -95,7 +95,7 @@ teardown:
9595

9696
- do:
9797
ingest.get_geoip_database: {}
98-
- length: { databases: 5 }
98+
- length: { databases: 6 }
9999

100100
- do:
101101
ingest.get_geoip_database:

0 commit comments

Comments
 (0)