Skip to content

Commit 2261883

Browse files
joegallojoegallo
authored
Update geoip fixture files and utility methods (#111913)
1 parent 5510ad9 commit 2261883

15 files changed

+117
-115
lines changed

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,14 @@
1616
import org.elasticsearch.core.TimeValue;
1717
import org.elasticsearch.plugins.Plugin;
1818
import org.elasticsearch.test.ESIntegTestCase;
19-
import org.elasticsearch.test.StreamsUtils;
2019
import org.junit.ClassRule;
2120

22-
import java.io.ByteArrayInputStream;
23-
import java.io.IOException;
24-
import java.io.UncheckedIOException;
25-
import java.nio.file.Files;
2621
import java.nio.file.Path;
2722
import java.util.Collection;
2823
import java.util.List;
2924

25+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
26+
3027
public abstract class AbstractGeoIpIT extends ESIntegTestCase {
3128
private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;
3229

@@ -45,23 +42,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
4542
@Override
4643
protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) {
4744
final Path databasePath = createTempDir();
48-
try {
49-
Files.createDirectories(databasePath);
50-
Files.copy(
51-
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
52-
databasePath.resolve("GeoLite2-City.mmdb")
53-
);
54-
Files.copy(
55-
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
56-
databasePath.resolve("GeoLite2-Country.mmdb")
57-
);
58-
Files.copy(
59-
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
60-
databasePath.resolve("GeoLite2-ASN.mmdb")
61-
);
62-
} catch (final IOException e) {
63-
throw new UncheckedIOException(e);
64-
}
45+
copyDefaultDatabases(databasePath);
6546
return Settings.builder()
6647
.put("ingest.geoip.database_path", databasePath)
6748
.put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false)

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import java.util.zip.GZIPInputStream;
6767

6868
import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
69+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
6970
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
7071
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
7172
import static org.hamcrest.Matchers.anEmptyMap;
@@ -688,12 +689,7 @@ private void setupDatabasesInConfigDirectory() throws Exception {
688689
.forEach(path -> {
689690
try {
690691
Files.createDirectories(path);
691-
Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), path.resolve("GeoLite2-City.mmdb"));
692-
Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), path.resolve("GeoLite2-ASN.mmdb"));
693-
Files.copy(
694-
GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"),
695-
path.resolve("GeoLite2-Country.mmdb")
696-
);
692+
copyDefaultDatabases(path);
697693
} catch (IOException e) {
698694
throw new UncheckedIOException(e);
699695
}

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,8 @@
2222
import org.elasticsearch.watcher.ResourceWatcherService;
2323

2424
import java.io.IOException;
25-
import java.io.InputStream;
2625
import java.nio.file.Files;
2726
import java.nio.file.Path;
28-
import java.nio.file.StandardCopyOption;
2927
import java.util.Arrays;
3028
import java.util.HashMap;
3129
import java.util.List;
@@ -34,7 +32,8 @@
3432
import java.util.concurrent.atomic.AtomicInteger;
3533
import java.util.concurrent.atomic.AtomicReference;
3634

37-
import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
35+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
36+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
3837
import static org.hamcrest.Matchers.equalTo;
3938
import static org.hamcrest.Matchers.greaterThan;
4039
import static org.hamcrest.Matchers.is;
@@ -68,8 +67,8 @@ public void test() throws Exception {
6867
when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
6968
DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService);
7069
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
71-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
72-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
70+
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
71+
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
7372
databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
7473
databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
7574
lazyLoadReaders(databaseNodeService);
@@ -138,18 +137,14 @@ public void test() throws Exception {
138137
assertThat(previous1.current(), equalTo(-1));
139138
});
140139
} else {
141-
Files.copy(
142-
ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
143-
geoIpTmpDir.resolve("GeoLite2-City.mmdb"),
144-
StandardCopyOption.REPLACE_EXISTING
145-
);
140+
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
146141
databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
147142
}
148143
DatabaseReaderLazyLoader previous2 = databaseNodeService.get("GeoLite2-City-Test.mmdb");
149-
InputStream source = ConfigDatabases.class.getResourceAsStream(
150-
i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : "/GeoLite2-City-Test.mmdb"
144+
copyDatabase(
145+
i % 2 == 0 ? "GeoIP2-City-Test.mmdb" : "GeoLite2-City-Test.mmdb",
146+
geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")
151147
);
152-
Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING);
153148
databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
154149

155150
DatabaseReaderLazyLoader current1 = databaseNodeService.get("GeoLite2-City.mmdb");
@@ -194,7 +189,7 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI
194189
throws IOException {
195190
GeoIpCache cache = new GeoIpCache(0);
196191
ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
197-
copyDatabaseFiles(geoIpConfigDir, configDatabases);
192+
copyDefaultDatabases(geoIpConfigDir, configDatabases);
198193
DatabaseNodeService databaseNodeService = new DatabaseNodeService(
199194
geoIpTmpDir,
200195
mock(Client.class),

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,11 @@
2020
import org.junit.After;
2121
import org.junit.Before;
2222

23-
import java.io.IOException;
24-
import java.nio.file.CopyOption;
2523
import java.nio.file.Files;
2624
import java.nio.file.Path;
27-
import java.nio.file.StandardCopyOption;
2825

26+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
27+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
2928
import static org.hamcrest.Matchers.anEmptyMap;
3029
import static org.hamcrest.Matchers.equalTo;
3130
import static org.hamcrest.Matchers.notNullValue;
@@ -62,8 +61,8 @@ public void testLocalDatabasesEmptyConfig() throws Exception {
6261

6362
public void testDatabasesConfigDir() throws Exception {
6463
Path configDir = createTempDir();
65-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
66-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
64+
copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
65+
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
6766

6867
ConfigDatabases configDatabases = new ConfigDatabases(configDir, new GeoIpCache(0));
6968
configDatabases.initialize(resourceWatcherService);
@@ -92,9 +91,9 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
9291
assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country"));
9392
}
9493

95-
CopyOption option = StandardCopyOption.REPLACE_EXISTING;
96-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
97-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option);
94+
copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
95+
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
96+
9897
assertBusy(() -> {
9998
assertThat(configDatabases.getConfigDatabases().size(), equalTo(4));
10099
DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-ASN.mmdb");
@@ -116,7 +115,8 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
116115

117116
public void testDatabasesUpdateExistingConfDatabase() throws Exception {
118117
Path configDir = createTempDir();
119-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
118+
copyDatabase("GeoLite2-City.mmdb", configDir);
119+
120120
GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
121121
ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
122122
configDatabases.initialize(resourceWatcherService);
@@ -131,11 +131,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
131131
assertThat(cache.count(), equalTo(1));
132132
}
133133

134-
Files.copy(
135-
ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
136-
configDir.resolve("GeoLite2-City.mmdb"),
137-
StandardCopyOption.REPLACE_EXISTING
138-
);
134+
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
139135
assertBusy(() -> {
140136
assertThat(configDatabases.getConfigDatabases().size(), equalTo(1));
141137
assertThat(cache.count(), equalTo(0));
@@ -154,11 +150,9 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
154150
});
155151
}
156152

157-
private static Path prepareConfigDir() throws IOException {
153+
private static Path prepareConfigDir() {
158154
Path dir = createTempDir();
159-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb"));
160-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb"));
161-
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb"));
155+
copyDefaultDatabases(dir);
162156
return dir;
163157
}
164158

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383
import java.util.zip.GZIPInputStream;
8484
import java.util.zip.GZIPOutputStream;
8585

86-
import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
86+
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
8787
import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
8888
import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE;
8989
import static org.hamcrest.Matchers.empty;
@@ -117,10 +117,9 @@ public class DatabaseNodeServiceTests extends ESTestCase {
117117
@Before
118118
public void setup() throws IOException {
119119
final Path geoIpConfigDir = createTempDir();
120-
Files.createDirectories(geoIpConfigDir);
121120
GeoIpCache cache = new GeoIpCache(1000);
122121
ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
123-
copyDatabaseFiles(geoIpConfigDir, configDatabases);
122+
copyDefaultDatabases(geoIpConfigDir, configDatabases);
124123

125124
threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName());
126125
Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build();

0 commit comments

Comments
 (0)