From f6fe3246574690f98a9b7a7d021a9575bcb928a0 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 31 Mar 2025 09:55:06 -0500 Subject: [PATCH 1/3] Adding a cleanup method to EnterpriseGeoIpDownloaderIT --- .../geoip/EnterpriseGeoIpDownloaderIT.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index 4cee408d5fab8..e2f2a0c7950d7 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -22,6 +23,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -29,6 +32,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.EnterpriseGeoIpTask; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.plugins.Plugin; @@ -38,12 +42,15 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.xcontent.XContentType; +import org.junit.After; import org.junit.ClassRule; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING; @@ -54,6 +61,8 @@ public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase { private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City"; private static final String IPINFO_DATABASE_TYPE = "asn"; + private static final String MAXMIND_CONFIGURATION = "test-1"; + private static final String IPINFO_CONFIGURATION = "test-2"; @ClassRule public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture( @@ -118,6 +127,7 @@ public void testEnterpriseDownloaderTask() throws Exception { * We know that the databases index has been populated (because we waited around, :wink:), but we don't know for sure that * the databases have been pulled down and made available on all nodes. So we run these ingest-and-check steps in assertBusy blocks. */ + assertBusy(() -> { logger.info("Ingesting a test document"); String documentId = ingestDocument(indexName, geoipPipelineName, sourceField, "89.160.20.128"); @@ -140,6 +150,41 @@ public void testEnterpriseDownloaderTask() throws Exception { }); } + @After + public void cleanup() throws InterruptedException { + /* + * This method cleans up the database configurations that the test created. This allows the test to be run repeatedly. + */ + CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener listener = new LatchedActionListener<>(ActionListener.noop(), latch); + SubscribableListener.newForked( + l -> admin().cluster() + .execute( + DeleteDatabaseConfigurationAction.INSTANCE, + new DeleteDatabaseConfigurationAction.Request( + TimeValue.MAX_VALUE, + TimeValue.timeValueSeconds(10), + MAXMIND_CONFIGURATION + ), + l + ) + ) + .andThen( + l -> admin().cluster() + .execute( + DeleteDatabaseConfigurationAction.INSTANCE, + new DeleteDatabaseConfigurationAction.Request( + TimeValue.MAX_VALUE, + TimeValue.timeValueSeconds(10), + IPINFO_CONFIGURATION + ), + l + ) + ) + .addListener(listener); + latch.await(10, TimeUnit.SECONDS); + } + private void startEnterpriseGeoIpDownloaderTask() { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); persistentTasksService.sendStartRequest( @@ -163,7 +208,7 @@ private void configureMaxmindDatabase(String databaseType) { new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account")) + new DatabaseConfiguration(MAXMIND_CONFIGURATION, databaseType, new DatabaseConfiguration.Maxmind("test_account")) ) ) .actionGet(); @@ -176,7 +221,7 @@ private void configureIpinfoDatabase(String databaseType) { new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo()) + new DatabaseConfiguration(IPINFO_CONFIGURATION, databaseType, new DatabaseConfiguration.Ipinfo()) ) ) .actionGet(); From 39a259dd290808d9e8d5c610255889a5eb0374b6 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 31 Mar 2025 10:05:25 -0500 Subject: [PATCH 2/3] making it easier to read --- .../geoip/EnterpriseGeoIpDownloaderIT.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index e2f2a0c7950d7..2da13618e9fa3 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -157,34 +157,21 @@ public void cleanup() throws InterruptedException { */ CountDownLatch latch = new CountDownLatch(1); LatchedActionListener listener = new LatchedActionListener<>(ActionListener.noop(), latch); - SubscribableListener.newForked( - l -> admin().cluster() - .execute( - DeleteDatabaseConfigurationAction.INSTANCE, - new DeleteDatabaseConfigurationAction.Request( - TimeValue.MAX_VALUE, - TimeValue.timeValueSeconds(10), - MAXMIND_CONFIGURATION - ), - l - ) - ) - .andThen( - l -> admin().cluster() - .execute( - DeleteDatabaseConfigurationAction.INSTANCE, - new DeleteDatabaseConfigurationAction.Request( - TimeValue.MAX_VALUE, - TimeValue.timeValueSeconds(10), - IPINFO_CONFIGURATION - ), - l - ) - ) + SubscribableListener.newForked(l -> deleteDatabaseConfiguration(MAXMIND_CONFIGURATION, l)) + .andThen(l -> deleteDatabaseConfiguration(IPINFO_CONFIGURATION, l)) .addListener(listener); latch.await(10, TimeUnit.SECONDS); } + private void deleteDatabaseConfiguration(String configurationName, ActionListener listener) { + admin().cluster() + .execute( + DeleteDatabaseConfigurationAction.INSTANCE, + new DeleteDatabaseConfigurationAction.Request(TimeValue.MAX_VALUE, TimeValue.timeValueSeconds(10), configurationName), + listener + ); + } + private void startEnterpriseGeoIpDownloaderTask() { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); persistentTasksService.sendStartRequest( From 200a5fe6bc2237ba7cb7a82ffd20c6005b3f5f1c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 31 Mar 2025 10:15:16 -0500 Subject: [PATCH 3/3] removing accidental newline --- .../elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index 2da13618e9fa3..9b2f5400d4f2d 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -127,7 +127,6 @@ public void testEnterpriseDownloaderTask() throws Exception { * We know that the databases index has been populated (because we waited around, :wink:), but we don't know for sure that * the databases have been pulled down and made available on all nodes. So we run these ingest-and-check steps in assertBusy blocks. */ - assertBusy(() -> { logger.info("Ingesting a test document"); String documentId = ingestDocument(indexName, geoipPipelineName, sourceField, "89.160.20.128");