test: move backup tests into slow tests category (#1509)

* test: move backup tests into slow tests

Moves ITDatabaseAdminTests into the ITBackupTest which is only executed
on the nightly build

* test: increase pitr test timeout

Increases PITR update database ddl test timeout to 20 minutes.

Author: thiagotnunes

google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITDatabaseAdminTest.java

@@ -16,50 +16,27 @@
 
 package com.google.cloud.spanner.it;
 
-import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeFalse;
 
-import com.google.api.gax.grpc.GrpcInterceptorProvider;
 import com.google.api.gax.longrunning.OperationFuture;
 import com.google.api.gax.paging.Page;
-import com.google.cloud.Timestamp;
-import com.google.cloud.spanner.Backup;
 import com.google.cloud.spanner.Database;
 import com.google.cloud.spanner.DatabaseAdminClient;
 import com.google.cloud.spanner.ErrorCode;
 import com.google.cloud.spanner.IntegrationTestEnv;
 import com.google.cloud.spanner.Options;
 import com.google.cloud.spanner.ParallelIntegrationTest;
-import com.google.cloud.spanner.Spanner;
 import com.google.cloud.spanner.SpannerException;
-import com.google.cloud.spanner.SpannerOptions;
 import com.google.cloud.spanner.testing.RemoteSpannerHelper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
-import com.google.spanner.admin.database.v1.CreateBackupMetadata;
 import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
-import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata;
 import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata;
-import io.grpc.CallOptions;
-import io.grpc.Channel;
-import io.grpc.ClientCall;
-import io.grpc.ClientInterceptor;
-import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
-import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener;
-import io.grpc.Metadata;
-import io.grpc.MethodDescriptor;
-import io.grpc.Status;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
@@ -72,9 +49,7 @@
 @Category(ParallelIntegrationTest.class)
 @RunWith(JUnit4.class)
 public class ITDatabaseAdminTest {
-  private static final long DATABASE_TIMEOUT_MINUTES = 5;
-  private static final long BACKUP_TIMEOUT_MINUTES = 20;
-  private static final long RESTORE_TIMEOUT_MINUTES = 10;
+  private static final long TIMEOUT_MINUTES = 5;
   @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv();
   private DatabaseAdminClient dbAdminClient;
   private RemoteSpannerHelper testHelper;
@@ -101,7 +76,7 @@ public void databaseOperations() throws Exception {
     String statement1 = "CREATE TABLE T (\n" + "  K STRING(MAX),\n" + ") PRIMARY KEY(K)";
     OperationFuture<Database, CreateDatabaseMetadata> op =
         dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of(statement1));
-    Database db = op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    Database db = op.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
     dbs.add(db);
     assertThat(db.getId().getDatabase()).isEqualTo(dbId);
 
@@ -122,7 +97,7 @@ public void databaseOperations() throws Exception {
     String statement2 = "CREATE TABLE T2 (\n" + "  K2 STRING(MAX),\n" + ") PRIMARY KEY(K2)";
     OperationFuture<?, ?> op2 =
         dbAdminClient.updateDatabaseDdl(instanceId, dbId, ImmutableList.of(statement2), null);
-    op2.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    op2.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
     List<String> statementsInDb = dbAdminClient.getDatabaseDdl(instanceId, dbId);
     assertThat(statementsInDb).containsExactly(statement1, statement2);
 
@@ -143,15 +118,15 @@ public void updateDdlRetry() throws Exception {
     String statement1 = "CREATE TABLE T (\n" + "  K STRING(MAX),\n" + ") PRIMARY KEY(K)";
     OperationFuture<Database, CreateDatabaseMetadata> op =
         dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of(statement1));
-    Database db = op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    Database db = op.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
     dbs.add(db);
     String statement2 = "CREATE TABLE T2 (\n" + "  K2 STRING(MAX),\n" + ") PRIMARY KEY(K2)";
     OperationFuture<Void, UpdateDatabaseDdlMetadata> op1 =
         dbAdminClient.updateDatabaseDdl(instanceId, dbId, ImmutableList.of(statement2), "myop");
     OperationFuture<Void, UpdateDatabaseDdlMetadata> op2 =
         dbAdminClient.updateDatabaseDdl(instanceId, dbId, ImmutableList.of(statement2), "myop");
-    op1.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
-    op2.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    op1.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    op2.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
 
     // Remove the progress list from the metadata before comparing, as there could be small
     // differences between the two in the reported progress depending on exactly when each
@@ -170,7 +145,7 @@ public void databaseOperationsViaEntity() throws Exception {
     String statement1 = "CREATE TABLE T (\n" + "  K STRING(MAX),\n" + ") PRIMARY KEY(K)";
     OperationFuture<Database, CreateDatabaseMetadata> op =
         dbAdminClient.createDatabase(instanceId, dbId, ImmutableList.of(statement1));
-    Database db = op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    Database db = op.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
     dbs.add(db);
     assertThat(db.getId().getDatabase()).isEqualTo(dbId);
 
@@ -179,7 +154,7 @@ public void databaseOperationsViaEntity() throws Exception {
 
     String statement2 = "CREATE TABLE T2 (\n" + "  K2 STRING(MAX),\n" + ") PRIMARY KEY(K2)";
     OperationFuture<?, ?> op2 = db.updateDdl(ImmutableList.of(statement2), null);
-    op2.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    op2.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
     Iterable<String> statementsInDb = db.getDdl();
     assertThat(statementsInDb).containsExactly(statement1, statement2);
     db.drop();
@@ -219,218 +194,4 @@ public void listPagination() throws Exception {
     }
     assertThat(dbIdsGot).containsAtLeastElementsIn(dbIds);
   }
-
-  private static final class InjectErrorInterceptorProvider implements GrpcInterceptorProvider {
-    final AtomicBoolean injectError = new AtomicBoolean(true);
-    final AtomicInteger getOperationCount = new AtomicInteger();
-    final AtomicInteger methodCount = new AtomicInteger();
-    final String methodName;
-
-    private InjectErrorInterceptorProvider(String methodName) {
-      this.methodName = methodName;
-    }
-
-    @Override
-    public List<ClientInterceptor> getInterceptors() {
-      ClientInterceptor interceptor =
-          new ClientInterceptor() {
-            @Override
-            public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
-                MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
-              if (method.getFullMethodName().contains("GetOperation")) {
-                getOperationCount.incrementAndGet();
-              }
-              if (!method.getFullMethodName().contains(methodName)) {
-                return next.newCall(method, callOptions);
-              }
-
-              methodCount.incrementAndGet();
-              final AtomicBoolean errorInjected = new AtomicBoolean();
-              final ClientCall<ReqT, RespT> clientCall = next.newCall(method, callOptions);
-
-              return new SimpleForwardingClientCall<ReqT, RespT>(clientCall) {
-                @Override
-                public void start(Listener<RespT> responseListener, Metadata headers) {
-                  super.start(
-                      new SimpleForwardingClientCallListener<RespT>(responseListener) {
-                        @Override
-                        public void onMessage(RespT message) {
-                          if (injectError.getAndSet(false)) {
-                            errorInjected.set(true);
-                            clientCall.cancel("Cancelling call for injected error", null);
-                          } else {
-                            super.onMessage(message);
-                          }
-                        }
-
-                        @Override
-                        public void onClose(Status status, Metadata metadata) {
-                          if (errorInjected.get()) {
-                            status = Status.UNAVAILABLE.augmentDescription("INJECTED BY TEST");
-                          }
-                          super.onClose(status, metadata);
-                        }
-                      },
-                      headers);
-                }
-              };
-            }
-          };
-      return Collections.singletonList(interceptor);
-    }
-  }
-
-  @Test
-  public void testRetryNonIdempotentRpcsReturningLongRunningOperations() throws Exception {
-    assumeFalse(
-        "Querying long-running operations is not supported on the emulator", isUsingEmulator());
-
-    // RPCs that return a long-running operation such as CreateDatabase, CreateBackup and
-    // RestoreDatabase are non-idempotent and can normally not be automatically retried in case of a
-    // transient failure. The client library will however automatically query the backend to check
-    // whether the corresponding operation was started or not, and if it was, it will pick up the
-    // existing operation. If no operation is found, a new RPC call will be executed to start the
-    // operation.
-
-    List<Database> databases = new ArrayList<>();
-    List<Backup> backups = new ArrayList<>();
-    String initialDatabaseId;
-    Timestamp initialDbCreateTime;
-
-    try {
-      // CreateDatabase
-      InjectErrorInterceptorProvider createDbInterceptor =
-          new InjectErrorInterceptorProvider("CreateDatabase");
-      SpannerOptions options =
-          testHelper.getOptions().toBuilder().setInterceptorProvider(createDbInterceptor).build();
-      try (Spanner spanner = options.getService()) {
-        initialDatabaseId = testHelper.getUniqueDatabaseId();
-        DatabaseAdminClient client = spanner.getDatabaseAdminClient();
-        OperationFuture<Database, CreateDatabaseMetadata> op =
-            client.createDatabase(
-                testHelper.getInstanceId().getInstance(),
-                initialDatabaseId,
-                Collections.emptyList());
-        databases.add(op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES));
-        // Keep track of the original create time of this database, as we will drop this database
-        // later and create another one with the exact same name. That means that the ListOperations
-        // call will return at least two CreateDatabase operations. The retry logic should always
-        // pick the last one.
-        initialDbCreateTime = op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES).getCreateTime();
-        // Assert that the CreateDatabase RPC was called only once, and that the operation tracking
-        // was resumed through a GetOperation call.
-        assertThat(createDbInterceptor.methodCount.get()).isEqualTo(1);
-        assertThat(createDbInterceptor.getOperationCount.get()).isAtLeast(1);
-      }
-
-      // CreateBackup
-      InjectErrorInterceptorProvider createBackupInterceptor =
-          new InjectErrorInterceptorProvider("CreateBackup");
-      options =
-          testHelper
-              .getOptions()
-              .toBuilder()
-              .setInterceptorProvider(createBackupInterceptor)
-              .build();
-      try (Spanner spanner = options.getService()) {
-        String databaseId = databases.get(0).getId().getDatabase();
-        String backupId = String.format("test-bck-%08d", new Random().nextInt(100000000));
-        DatabaseAdminClient client = spanner.getDatabaseAdminClient();
-        OperationFuture<Backup, CreateBackupMetadata> op =
-            client.createBackup(
-                testHelper.getInstanceId().getInstance(),
-                backupId,
-                databaseId,
-                Timestamp.ofTimeSecondsAndNanos(
-                    Timestamp.now().getSeconds() + TimeUnit.SECONDS.convert(7L, TimeUnit.DAYS), 0));
-        backups.add(op.get(BACKUP_TIMEOUT_MINUTES, TimeUnit.MINUTES));
-        // Assert that the CreateBackup RPC was called only once, and that the operation tracking
-        // was resumed through a GetOperation call.
-        assertThat(createDbInterceptor.methodCount.get()).isEqualTo(1);
-        assertThat(createDbInterceptor.getOperationCount.get()).isAtLeast(1);
-      }
-
-      // RestoreBackup
-      int attempts = 0;
-      while (true) {
-        InjectErrorInterceptorProvider restoreBackupInterceptor =
-            new InjectErrorInterceptorProvider("RestoreBackup");
-        options =
-            testHelper
-                .getOptions()
-                .toBuilder()
-                .setInterceptorProvider(restoreBackupInterceptor)
-                .build();
-        try (Spanner spanner = options.getService()) {
-          String backupId = backups.get(0).getId().getBackup();
-          String restoredDbId = testHelper.getUniqueDatabaseId();
-          DatabaseAdminClient client = spanner.getDatabaseAdminClient();
-          OperationFuture<Database, RestoreDatabaseMetadata> op =
-              client.restoreDatabase(
-                  testHelper.getInstanceId().getInstance(),
-                  backupId,
-                  testHelper.getInstanceId().getInstance(),
-                  restoredDbId);
-          databases.add(op.get(RESTORE_TIMEOUT_MINUTES, TimeUnit.MINUTES));
-          // Assert that the RestoreDatabase RPC was called only once, and that the operation
-          // tracking was resumed through a GetOperation call.
-          assertThat(createDbInterceptor.methodCount.get()).isEqualTo(1);
-          assertThat(createDbInterceptor.getOperationCount.get()).isAtLeast(1);
-          break;
-        } catch (ExecutionException e) {
-          if (e.getCause() instanceof SpannerException
-              && ((SpannerException) e.getCause()).getErrorCode() == ErrorCode.FAILED_PRECONDITION
-              && e.getCause()
-                  .getMessage()
-                  .contains("Please retry the operation once the pending restores complete")) {
-            attempts++;
-            if (attempts == 10) {
-              // Still same error after 10 attempts. Ignore.
-              break;
-            }
-            // wait and then retry.
-            Thread.sleep(60_000L);
-          } else {
-            throw e;
-          }
-        }
-      }
-
-      // Create another database with the exact same name as the first database.
-      createDbInterceptor = new InjectErrorInterceptorProvider("CreateDatabase");
-      options =
-          testHelper.getOptions().toBuilder().setInterceptorProvider(createDbInterceptor).build();
-      try (Spanner spanner = options.getService()) {
-        DatabaseAdminClient client = spanner.getDatabaseAdminClient();
-        // First drop the initial database.
-        client.dropDatabase(testHelper.getInstanceId().getInstance(), initialDatabaseId);
-        // Now re-create a database with the exact same name.
-        OperationFuture<Database, CreateDatabaseMetadata> op =
-            client.createDatabase(
-                testHelper.getInstanceId().getInstance(),
-                initialDatabaseId,
-                Collections.emptyList());
-        // Check that the second database was created and has a greater creation time than the
-        // first.
-        Timestamp secondCreationTime =
-            op.get(DATABASE_TIMEOUT_MINUTES, TimeUnit.MINUTES).getCreateTime();
-        // TODO: Change this to greaterThan when the create time of a database is reported back by
-        // the server.
-        assertThat(secondCreationTime).isAtLeast(initialDbCreateTime);
-        // Assert that the CreateDatabase RPC was called only once, and that the operation tracking
-        // was resumed through a GetOperation call.
-        assertThat(createDbInterceptor.methodCount.get()).isEqualTo(1);
-        assertThat(createDbInterceptor.getOperationCount.get()).isAtLeast(1);
-      }
-    } finally {
-      DatabaseAdminClient client = testHelper.getClient().getDatabaseAdminClient();
-      for (Database database : databases) {
-        client.dropDatabase(
-            database.getId().getInstanceId().getInstance(), database.getId().getDatabase());
-      }
-      for (Backup backup : backups) {
-        client.deleteBackup(backup.getInstanceId().getInstance(), backup.getId().getBackup());
-      }
-    }
-  }
 }

google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java

@@ -51,7 +51,7 @@
 @RunWith(JUnit4.class)
 public class ITPitrUpdateDatabaseTest {
 
-  private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(5);
+  private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(20);
   private static final String VERSION_RETENTION_PERIOD = "7d";
 
   @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv();