tests: Fix some flakes (#3246)

Author: amanda-tarafa

pubsub/api/Pubsub.Samples.Tests/CreateTopicWithCloudStorageIngestionTest.cs

@@ -40,9 +40,9 @@ public void CreateTopicWithCloudStorageIngestion()
         DateTimeOffset minimumObjectCreateTime = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero);
         Topic createdTopic = _createTopicWithCloudStorageIngestionSample.CreateTopicWithCloudStorageIngestion(_pubsubFixture.ProjectId, topicId, bucket, inputFormat, textDelimiter, matchGlob, minimumObjectCreateTime);
 
-        // Confirm that the created topic and topic retrieved by ID are equal
+        // Confirm that the created topic and topic retrieved by ID have the same ingestion settings.
         Topic retrievedTopic = _pubsubFixture.GetTopic(topicId);
-        Assert.Equal(createdTopic, retrievedTopic);
+        Assert.Equal(createdTopic.IngestionDataSourceSettings, retrievedTopic.IngestionDataSourceSettings);
 
         // Confirm that all Cloud Storage Ingestion params are equal to expected values
         Assert.Equal(bucket, createdTopic.IngestionDataSourceSettings.CloudStorage.Bucket);

spanner/api/Spanner.Samples.Tests/CreateInstanceWithoutDefaultBackupSchedulesTest.cs

@@ -33,13 +33,11 @@ public async Task TestCreateInstanceAsync()
         CreateInstanceWithoutDefaultBackupSchedulesAsyncSample createInstanceSample =
             new CreateInstanceWithoutDefaultBackupSchedulesAsyncSample();
 
-        var instanceId = SpannerFixture.GenerateId("default-schedule-test-");
+        var instanceId = SpannerFixture.GenerateId(SpannerFixture.BackupInstancePrefix);
+        _spannerFixture.MarkInstanceForDeletion(instanceId);
         Instance instance = await createInstanceSample.CreateInstanceWithoutDefaultBackupSchedulesAsync(
             _spannerFixture.ProjectId, instanceId);
 
         Assert.Equal(Instance.Types.DefaultBackupScheduleType.None, instance.DefaultBackupScheduleType);
-
-        await _spannerFixture.InstanceAdminClient.DeleteInstanceAsync(
-            InstanceName.FromProjectInstance(_spannerFixture.ProjectId, instanceId));
     }
 }

spanner/api/Spanner.Samples.Tests/SpannerFixture.cs

@@ -45,6 +45,11 @@ LastName STRING(1024)
         AlbumTitle STRING(MAX)
         ) PRIMARY KEY (SingerId, AlbumId)";
 
+    public const string BackupInstancePrefix = "default-schedule-test-";
+    public const string InstanceWithProcessingUnitsPrefix = "my-ins-pu-";
+    public const string InstanceWithMultiRegionPrefix = "my-ins-mr-";
+    public const string InstanceWithInstancePartitionPrefix = "my-ins-prt-";
+
     public string ProjectId { get; } = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
     // Allow environment variables to override the default instance and database names.
     public string InstanceId { get; } = Environment.GetEnvironmentVariable("TEST_SPANNER_INSTANCE") ?? "my-instance";
@@ -86,13 +91,15 @@ AlbumTitle STRING(MAX)
         Environment.GetEnvironmentVariable("spanner.test.key.name") ?? "spanner-test-key");
     public CryptoKeyName[] KmsKeyNames { get; private set; }
 
-    public string InstanceIdWithProcessingUnits { get; } = GenerateId("my-ins-pu-");
-    public string InstanceIdWithMultiRegion { get; } = GenerateId("my-ins-mr-");
-    public string InstanceIdWithInstancePartition { get; } = GenerateId("my-ins-prt-");
+    public string InstanceIdWithProcessingUnits { get; } = GenerateId(InstanceWithProcessingUnitsPrefix);
+    public string InstanceIdWithMultiRegion { get; } = GenerateId(InstanceWithMultiRegionPrefix);
+    public string InstanceIdWithInstancePartition { get; } = GenerateId(InstanceWithInstancePartitionPrefix);
     public string InstanceConfigId { get; } = "nam6";
 
     private IList<string> TempDbIds { get; } = new List<string>();
 
+    private IList<string> TempInstanceIds { get; } = new List<string>();
+
     public DatabaseAdminClient DatabaseAdminClient { get; private set; }
     public InstanceAdminClient InstanceAdminClient { get; private set; }
     public SpannerConnection AdminSpannerConnection { get; private set; }
@@ -151,7 +158,9 @@ public async Task DisposeAsync()
                 DeleteInstanceAsync(InstanceIdWithInstancePartition),
                 DeleteBackupAsync(ToBeCancelledBackupId),
                 DeleteBackupAsync(EncryptedBackupId),
-                DeleteBackupAsync(MultiRegionEncryptedRestoreDatabaseId)
+                DeleteBackupAsync(MultiRegionEncryptedRestoreDatabaseId),
+                DeleteStaleInstancesAsync(),
+                DeleteStaleDatabaseAsync(),
             };
             DeleteInstanceConfig(CreateCustomInstanceConfigId);
             DeleteInstanceConfig(UpdateCustomInstanceConfigId);
@@ -162,6 +171,11 @@ public async Task DisposeAsync()
                 cleanupTasks.Add(DeleteDatabaseAsync(id));
             }
 
+            foreach (string id in TempInstanceIds)
+            {
+                cleanupTasks.Add(DeleteInstanceAsync(id));
+            }
+
             await Task.WhenAll(cleanupTasks);
         }
         finally
@@ -422,6 +436,49 @@ private async Task InitializeMultiRegionEncryptedBackupAsync()
         }
     }
 
+    /// <summary>
+    /// Not all tests delete resources appropiately.
+    /// We should fix that, but in the meantime, we can at least clean some stale resources.
+    /// </summary>
+    private async Task DeleteStaleInstancesAsync()
+    {
+        var now = DateTimeOffset.UtcNow;
+        var oneDay = TimeSpan.FromDays(1);
+        await foreach(var instance in InstanceAdminClient.ListInstancesAsync(ProjectName.FromProject(ProjectId)))
+        {
+            if ((instance.InstanceName.InstanceId.StartsWith(BackupInstancePrefix) ||
+                instance.InstanceName.InstanceId.StartsWith(InstanceWithInstancePartitionPrefix) ||
+                instance.InstanceName.InstanceId.StartsWith(InstanceWithMultiRegionPrefix) ||
+                instance.InstanceName.InstanceId.StartsWith(InstanceWithProcessingUnitsPrefix)) &&
+                now - instance.CreateTime.ToDateTimeOffset() > oneDay)
+            {
+                await DeleteInstanceAsync(instance.InstanceName.InstanceId);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Not all tests delete resources appropiately.
+    /// We should fix that, but in the meantime, we can at least clean some stale resources.
+    /// </summary>
+    private async Task DeleteStaleDatabaseAsync()
+    {
+        var now = DateTimeOffset.UtcNow;
+        var oneDay = TimeSpan.FromDays(1);
+        await foreach (var database in DatabaseAdminClient.ListDatabasesAsync(InstanceName.FromProjectInstance(ProjectId, InstanceId)))
+        {
+            if ((database.DatabaseName.DatabaseId.StartsWith("my-db-") ||
+                database.DatabaseName.DatabaseId.StartsWith("my-enc-db-") ||
+                database.DatabaseName.DatabaseId.StartsWith("my-mr-db-") ||
+                database.DatabaseName.DatabaseId.StartsWith("my-restore-db-") ||
+                database.DatabaseName.DatabaseId.StartsWith("temp-db-")) &&
+                now - database.CreateTime.ToDateTimeOffset() > oneDay)
+            {
+                await DeleteDatabaseAsync(database.DatabaseName.DatabaseId);
+            }
+        }
+    }
+
     private async Task DeleteInstanceAsync(string instanceId)
     {
         try
@@ -698,4 +755,6 @@ public async Task CreateTableAsyncPostgres(string createTableStatement)
         using var cmd = PgSpannerConnection.CreateDdlCommand(createTableStatement);
         await cmd.ExecuteNonQueryAsync();
     }
+
+    internal void MarkInstanceForDeletion(string instanceId) => TempInstanceIds.Add(instanceId);
 }

spanner/api/Spanner.Samples.Tests/UpdateInstanceDefaultBackupScheduleTypeTest.cs

@@ -33,7 +33,8 @@ public async Task TestUpdateInstanceAsync()
         CreateInstanceWithoutDefaultBackupSchedulesAsyncSample createInstanceSample =
             new CreateInstanceWithoutDefaultBackupSchedulesAsyncSample();
 
-        var instanceId = SpannerFixture.GenerateId("default-schedule-test-");
+        var instanceId = SpannerFixture.GenerateId(SpannerFixture.BackupInstancePrefix);
+        _spannerFixture.MarkInstanceForDeletion(instanceId);
         Instance instance = await createInstanceSample.CreateInstanceWithoutDefaultBackupSchedulesAsync(
             _spannerFixture.ProjectId, instanceId);
 
@@ -45,8 +46,5 @@ public async Task TestUpdateInstanceAsync()
             _spannerFixture.ProjectId, instanceId);
 
         Assert.Equal(Instance.Types.DefaultBackupScheduleType.Automatic, updatedInstance.DefaultBackupScheduleType);
-        
-        await _spannerFixture.InstanceAdminClient.DeleteInstanceAsync(
-            InstanceName.FromProjectInstance(_spannerFixture.ProjectId, instanceId));
     }
 }

spanner/api/Spanner.Samples.Tests/runTests.ps1

@@ -13,7 +13,7 @@
 # the License.
 import-module -DisableNameChecking ..\..\..\BuildTools.psm1
 
-Set-TestTimeout 7200
+Set-TestTimeout 14400
 
 dotnet restore --force
 dotnet test --no-restore --test-adapter-path:. --logger:junit 2>&1 | %{ "$_" }