test: drop old databases from instance before running integration tests. (#1861)

This commit aims to resolve the RESOURCE_EXHAUSTED errors due to
number of databases in an instance reaching the maximum limit.

We drop only those DBs which are created through automated tests,
at least 24 hours ago.

Co-authored-by: Knut Olav Løite <[email protected]>

Author: rajatbhatta

google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTestEnv.java

@@ -19,12 +19,15 @@
 import static com.google.common.base.Preconditions.checkState;
 
 import com.google.api.gax.longrunning.OperationFuture;
+import com.google.api.gax.paging.Page;
+import com.google.cloud.Timestamp;
 import com.google.cloud.spanner.testing.EmulatorSpannerHelper;
 import com.google.cloud.spanner.testing.RemoteSpannerHelper;
 import com.google.common.collect.Iterators;
 import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
 import io.grpc.Status;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.junit.rules.ExternalResource;
@@ -53,6 +56,7 @@ public class IntegrationTestEnv extends ExternalResource {
 
   private TestEnvConfig config;
   private InstanceAdminClient instanceAdminClient;
+  private DatabaseAdminClient databaseAdminClient;
   private boolean isOwnedInstance;
   private RemoteSpannerHelper testHelper;
 
@@ -93,9 +97,12 @@ protected void before() throws Throwable {
     }
     testHelper = createTestHelper(options, instanceId);
     instanceAdminClient = testHelper.getClient().getInstanceAdminClient();
+    databaseAdminClient = testHelper.getClient().getDatabaseAdminClient();
     logger.log(Level.FINE, "Test env endpoint is {0}", options.getHost());
     if (isOwnedInstance) {
       initializeInstance(instanceId);
+    } else {
+      cleanUpOldDatabases(instanceId);
     }
   }
 
@@ -162,6 +169,35 @@ private void initializeInstance(InstanceId instanceId) {
     logger.log(Level.INFO, "Created test instance: {0}", createdInstance.getId());
   }
 
+  private void cleanUpOldDatabases(InstanceId instanceId) {
+    long OLD_DB_THRESHOLD_SECS = TimeUnit.SECONDS.convert(24L, TimeUnit.HOURS);
+    Timestamp currentTimestamp = Timestamp.now();
+    int numDropped = 0;
+    Page<Database> page = databaseAdminClient.listDatabases(instanceId.getInstance());
+    String TEST_DB_REGEX = "(testdb_(.*)_(.*))|(mysample-(.*))";
+
+    logger.log(Level.INFO, "Dropping old test databases from {0}", instanceId.getName());
+    while (page != null) {
+      for (Database db : page.iterateAll()) {
+        try {
+          long timeDiff = db.getCreateTime().getSeconds() - currentTimestamp.getSeconds();
+          // Delete all databases which are more than OLD_DB_THRESHOLD_SECS seconds
+          // old.
+          if ((db.getId().getDatabase().matches(TEST_DB_REGEX))
+              && (timeDiff > OLD_DB_THRESHOLD_SECS)) {
+            logger.log(Level.INFO, "Dropping test database {0}", db.getId());
+            db.drop();
+            ++numDropped;
+          }
+        } catch (SpannerException e) {
+          logger.log(Level.SEVERE, "Failed to drop test database " + db.getId(), e);
+        }
+      }
+      page = page.getNextPage();
+    }
+    logger.log(Level.INFO, "Dropped {0} test database(s)", numDropped);
+  }
+
   private void cleanUpInstance() {
     try {
       if (isOwnedInstance) {