[couchbase] Ignore transient primary index creation error. (#4681)

This changeset makes sure that when a primary index is created, a transient
error is ignored since the indexing engine will automatically retry the
statement later.

Transient errors can happen because we just created the bucket and the
indexer might not have caught up to that bucket just yet, but it will
eventually.

In addition, a check has been added afterwards to make sure that the
primary index is online (so, ready for the user to query) before
returning to the caller, improving the overall stability of queries.

Author: daschl

modules/couchbase/src/main/java/org/testcontainers/couchbase/CouchbaseContainer.java

@@ -473,7 +473,30 @@ private void createBuckets() {
                         .add("statement", "CREATE PRIMARY INDEX on `" + bucket.getName() + "`")
                         .build(), true);
 
-                    checkSuccessfulResponse(queryResponse, "Could not create primary index for bucket " + bucket.getName());
+                    try {
+                        checkSuccessfulResponse(queryResponse, "Could not create primary index for bucket " + bucket.getName());
+                    } catch (IllegalStateException ex) {
+                        // potentially ignore the error, the index will be eventually built.
+                        if (!ex.getMessage().contains("Index creation will be retried in background")) {
+                            throw ex;
+                        }
+                    }
+
+                    timePhase(
+                        "createBucket:" + bucket.getName() + ":primaryIndexOnline",
+                        () ->  Unreliables.retryUntilTrue(1, TimeUnit.MINUTES, () -> {
+                            @Cleanup Response stateResponse = doHttpRequest(QUERY_PORT, "/query/service", "POST", new FormBody.Builder()
+                                .add("statement", "SELECT count(*) > 0 AS online FROM system:indexes where keyspace_id = \"" + bucket.getName() + "\" and is_primary = true and state = \"online\"")
+                                .build(), true);
+
+                            String body = stateResponse.body() != null ? stateResponse.body().string() : null;
+                            checkSuccessfulResponse(stateResponse, "Could not poll primary index state for bucket: " + bucket.getName());
+
+                            return Optional.of(MAPPER.readTree(body))
+                                .map(n -> n.at("/results/0/online"))
+                                .map(JsonNode::asBoolean)
+                                .orElse(false);
+                    }));
                 } else {
                     logger().info("Primary index creation for bucket {} ignored, since QUERY service is not present.", bucket.getName());
                 }
@@ -508,7 +531,7 @@ private void checkSuccessfulResponse(final Response response, final String messa
                 }
             }
 
-            throw new IllegalStateException(message + ": " + response.toString() + ", body=" + (body == null ? "<null>" : body));
+            throw new IllegalStateException(message + ": " + response + ", body=" + (body == null ? "<null>" : body));
         }
     }