Extend geoip FullClusterRestartIT (#121085)

Author: joegallo

modules/ingest-geoip/qa/full-cluster-restart/build.gradle

@@ -7,8 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-java-rest-test'

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

@@ -14,6 +14,12 @@
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -25,28 +31,46 @@
 import org.junit.rules.TestRule;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
-import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.is;
 
 public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
 
     private static final boolean useFixture = Boolean.getBoolean("geoip_use_service") == false;
 
-    private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
+    private static final GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
 
-    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+    // e.g. use ./gradlew -Dtests.jvm.argline="-Dgeoip_test_with_security=false" ":modules:ingest-geoip:qa:full-cluster-restart:check"
+    // to set this to false, if you so desire
+    private static final boolean useSecurity = Boolean.parseBoolean(System.getProperty("geoip_test_with_security", "true"));
+
+    private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .version(getOldClusterTestVersion())
         .nodes(2)
         .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture)
-        .setting("xpack.security.enabled", "false")
+        .setting("xpack.security.enabled", useSecurity ? "true" : "false")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .build();
 
+    @Override
+    protected Settings restClientSettings() {
+        Settings settings = super.restClientSettings();
+        if (useSecurity) {
+            String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+            settings = Settings.builder().put(settings).put(ThreadContext.PREFIX + ".Authorization", token).build();
+        }
+        return settings;
+    }
+
     @ClassRule
     public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
 
@@ -60,6 +84,9 @@ protected ElasticsearchCluster getUpgradeCluster() {
     }
 
     public void testGeoIpSystemFeaturesMigration() throws Exception {
+        final List<String> maybeSecurityIndex = useSecurity ? List.of(".security-7") : List.of();
+        final List<String> maybeSecurityIndexReindexed = useSecurity ? List.of(".security-7-reindexed-for-10") : List.of();
+
         if (isRunningAgainstOldCluster()) {
             Request enableDownloader = new Request("PUT", "/_cluster/settings");
             enableDownloader.setJsonEntity("""
@@ -86,23 +113,36 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
             assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS);
 
             // the geoip index should be created
-            assertBusy(() -> testCatIndices(".geoip_databases"));
+            assertBusy(() -> testCatIndices(List.of(".geoip_databases"), maybeSecurityIndex));
             assertBusy(() -> testIndexGeoDoc());
+
+            // before the upgrade, Kibana should work
+            assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex));
         } else {
+            // after the upgrade, but before the migration, Kibana should work
+            assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex));
+
+            // migrate the system features and give the cluster a moment to settle
             Request migrateSystemFeatures = new Request("POST", "/_migration/system_features");
             assertOK(client().performRequest(migrateSystemFeatures));
+            ensureHealth(request -> request.addParameter("wait_for_status", "yellow"));
 
-            assertBusy(() -> testCatIndices(".geoip_databases-reindexed-for-10", "my-index-00001"));
+            assertBusy(() -> testCatIndices(List.of(".geoip_databases-reindexed-for-10", "my-index-00001"), maybeSecurityIndexReindexed));
             assertBusy(() -> testIndexGeoDoc());
 
+            // after the migration, Kibana should work
+            if (useSecurity == false) { // BUT IT DOESN'T if security is enabled
+                assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndexReindexed));
+            }
+
             Request disableDownloader = new Request("PUT", "/_cluster/settings");
             disableDownloader.setJsonEntity("""
                 {"persistent": {"ingest.geoip.downloader.enabled": false}}
                 """);
             assertOK(client().performRequest(disableDownloader));
 
             // the geoip index should be deleted
-            assertBusy(() -> testCatIndices("my-index-00001"));
+            assertBusy(() -> testCatIndices(List.of("my-index-00001"), maybeSecurityIndexReindexed));
 
             Request enableDownloader = new Request("PUT", "/_cluster/settings");
             enableDownloader.setJsonEntity("""
@@ -114,7 +154,7 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
             assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS);
 
             // the geoip index should be recreated
-            assertBusy(() -> testCatIndices(".geoip_databases", "my-index-00001"));
+            assertBusy(() -> testCatIndices(List.of(".geoip_databases", "my-index-00001"), maybeSecurityIndexReindexed));
             assertBusy(() -> testIndexGeoDoc());
         }
     }
@@ -146,11 +186,17 @@ private void testDatabasesLoaded() throws IOException {
         }
     }
 
-    private void testCatIndices(String... indexNames) throws IOException {
+    private void testCatIndices(List<String> indexNames, @Nullable List<String> additionalIndexNames) throws IOException {
         Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all");
         String response = EntityUtils.toString(client().performRequest(catIndices).getEntity());
         List<String> indices = List.of(response.trim().split("\\s+"));
-        assertThat(indices, contains(indexNames));
+
+        if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
+            indexNames = new ArrayList<>(indexNames); // recopy into a mutable list
+            indexNames.addAll(additionalIndexNames);
+        }
+
+        assertThat(new HashSet<>(indices), is(new HashSet<>(indexNames)));
     }
 
     private void testIndexGeoDoc() throws IOException {
@@ -165,4 +211,23 @@ private void testIndexGeoDoc() throws IOException {
         assertNull(doc.evaluate("_source.tags"));
         assertEquals("Sweden", doc.evaluate("_source.geo.country_name"));
     }
+
+    private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String> additionalIndexNames) throws IOException {
+        Request getStar = new Request("GET", "*?expand_wildcards=all");
+        getStar.setOptions(
+            RequestOptions.DEFAULT.toBuilder()
+                .addHeader("X-elastic-product-origin", "kibana")
+                .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
+        );
+        Response response = client().performRequest(getStar);
+        assertOK(response);
+
+        if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
+            indexNames = new ArrayList<>(indexNames); // recopy into a mutable list
+            indexNames.addAll(additionalIndexNames);
+        }
+
+        Map<String, Object> map = responseAsMap(response);
+        assertThat(map.keySet(), is(new HashSet<>(indexNames)));
+    }
 }