multitenant: deflake TestConsumption

This patch adds retries to the `TestConsumption` test for retrieving the
total tenant RU consumption in order to ensure that the delta includes the
bucket flush that was triggered by the test query, instead of background
activity.

Fixes cockroachdb#94286
Fixes cockroachdb#106572

Release note: None

Author: DrewKimball

pkg/ccl/multitenantccl/tenantcostclient/tenant_side_test.go

@@ -873,8 +873,9 @@ func TestConsumption(t *testing.T) {
 	hostServer, _, _ := serverutils.StartServer(t, base.TestServerArgs{DefaultTestTenant: base.TestControlsTenantsExplicitly})
 	defer hostServer.Stopper().Stop(context.Background())
 
+	const targetPeriod = time.Millisecond
 	st := cluster.MakeTestingClusterSettings()
-	tenantcostclient.TargetPeriodSetting.Override(context.Background(), &st.SV, time.Millisecond)
+	tenantcostclient.TargetPeriodSetting.Override(context.Background(), &st.SV, targetPeriod)
 	tenantcostclient.CPUUsageAllowance.Override(context.Background(), &st.SV, 0)
 
 	testProvider := newTestProvider()
@@ -902,21 +903,32 @@ func TestConsumption(t *testing.T) {
 		r.Exec(t, "INSERT INTO t (v) SELECT repeat('1234567890', 1024) FROM generate_series(1, 10) AS g(i)")
 		const expectedBytes = 10 * 10 * 1024
 
-		afterWrite := testProvider.waitForConsumption(t)
-		delta := afterWrite
-		delta.Sub(&beforeWrite)
-		if delta.WriteBatches < 1 || delta.WriteRequests < 2 || delta.WriteBytes < expectedBytes*2 {
-			t.Errorf("usage after write: %s", delta.String())
-		}
+		// Try a few times because background activity can trigger bucket
+		// requests before the test query does.
+		testutils.SucceedsSoon(t, func() error {
+			afterWrite := testProvider.waitForConsumption(t)
+			delta := afterWrite
+			delta.Sub(&beforeWrite)
+			if delta.WriteBatches < 1 || delta.WriteRequests < 2 || delta.WriteBytes < expectedBytes*2 {
+				return errors.Newf("usage after write: %s", delta.String())
+			}
+			return nil
+		})
 
+		beforeRead := testProvider.waitForConsumption(t)
 		r.QueryStr(t, "SELECT min(v) FROM t")
 
-		afterRead := testProvider.waitForConsumption(t)
-		delta = afterRead
-		delta.Sub(&afterWrite)
-		if delta.ReadBatches < 1 || delta.ReadRequests < 1 || delta.ReadBytes < expectedBytes {
-			t.Errorf("usage after read: %s", delta.String())
-		}
+		// Try a few times because background activity can trigger bucket
+		// requests before the test query does.
+		testutils.SucceedsSoon(t, func() error {
+			afterRead := testProvider.waitForConsumption(t)
+			delta := afterRead
+			delta.Sub(&beforeRead)
+			if delta.ReadBatches < 1 || delta.ReadRequests < 1 || delta.ReadBytes < expectedBytes {
+				return errors.Newf("usage after read: %s", delta.String())
+			}
+			return nil
+		})
 		r.Exec(t, "DELETE FROM t WHERE true")
 	}
 	// Make sure some CPU usage is reported.