Adjust SearchableSnapshotsLicenseIntegTests.testShardAllocationOnInvalidLicense (#77757) (#77760)

This tests sometimes fails because it expects the last PostStartTrialRequest
to always "upgrade" the current license that it just nullified to a trial
license; but there is a race in this test with the LicenceService that detects
that no license exists in the cluster state (because the test set it to null)
and self generates a trial license for the cluster too. When the self generation
is processed before the PostStartTrialRequest the latter will return a
TRIAL_ALREADY_ACTIVATED response.

Since the purpose of this test is to verify that the searchable snapshot shards
failed when the license change and came back when the trial license if activated
again, I think we can just adjust the test to accommodate for the 2 types of
responses.

Closes #72329

Author: tlrx

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.DeleteLicenseAction;
@@ -36,7 +35,6 @@
 import org.elasticsearch.license.PostStartTrialResponse;
 import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
 import org.elasticsearch.xpack.searchablesnapshots.action.ClearSearchableSnapshotsCacheAction;
@@ -56,6 +54,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.oneOf;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class SearchableSnapshotsLicenseIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase {
@@ -145,7 +144,6 @@ public void testClearCacheRequiresLicense() throws ExecutionException, Interrupt
         }
     }
 
-    @TestLogging(reason = "https://github.com/elastic/elasticsearch/issues/72329", value = "org.elasticsearch.license:DEBUG")
     public void testShardAllocationOnInvalidLicense() throws Exception {
         // check that shards have been failed as part of invalid license
         assertBusy(
@@ -170,23 +168,19 @@ public void testShardAllocationOnInvalidLicense() throws Exception {
         waitNoPendingTasksOnAll();
         ensureClusterStateConsistency();
 
-        try {
-            PostStartTrialRequest startTrialRequest = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName())
-                .acknowledge(true);
-            PostStartTrialResponse resp = client().execute(PostStartTrialAction.INSTANCE, startTrialRequest).get();
-            assertEquals(PostStartTrialResponse.Status.UPGRADED_TO_TRIAL, resp.getStatus());
-        } catch (AssertionError ae) {
-            try {
-                final ClusterService clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
-                logger.error(
-                    "Failed to start trial license again, cluster state on master node is:\n{}",
-                    Strings.toString(clusterService.state(), false, true)
-                );
-            } catch (Exception e) {
-                ae.addSuppressed(e);
-            }
-            throw ae;
-        }
+        PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true);
+        final PostStartTrialResponse response = client().execute(PostStartTrialAction.INSTANCE, request).get();
+        assertThat(
+            response.getStatus(),
+            oneOf(
+                PostStartTrialResponse.Status.UPGRADED_TO_TRIAL,
+                // The LicenceService automatically generates a license of {@link LicenceService#SELF_GENERATED_LICENSE_TYPE} type
+                // if there is no license found in the cluster state (see {@link LicenceService#registerOrUpdateSelfGeneratedLicense).
+                // Since this test explicitly removes the LicensesMetadata from cluster state it is possible that the self generated
+                // license is created before the PostStartTrialRequest is acked.
+                PostStartTrialResponse.Status.TRIAL_ALREADY_ACTIVATED
+            )
+        );
         // check if cluster goes green again after valid license has been put in place
         ensureGreen(indexName);
     }