finatra: fix flaky tests

Problem:
Some of our github actions have been
failing due to the following tests:

In our `InMemoryStatsReceiverUtilityTest`,
by the time we get to the waitFor call,
the metrics have already been updated
too many times.

In `FeatureTestNonInjectionCustomStatsReceiverTest`,
we check that our gauges are nonempty because they
should be populated at startup, but occasionally
the server is not fully started before our check.

Solution:
For `InMemoryStatsReceiverUtilityTest`,
we'll call `waitFor` in its own thread
before adding to stats, counters, gauges

For `FeatureTestNonInjectionCustomStatsReceiverTest`,
we'll wait for the server to complete startup
before running any tests

JIRA Issues: CSL-11274

Differential Revision: https://phabricator.twitter.biz/D766267

Author: joybestourous

inject/inject-core/src/test/scala/com/twitter/inject/tests/InMemoryStatsReceiverUtilityTest.scala

@@ -1,7 +1,8 @@
 package com.twitter.inject.tests
 
 import com.twitter.finagle.stats.InMemoryStatsReceiver
-import com.twitter.inject.{InMemoryStatsReceiverUtility, Test}
+import com.twitter.inject.InMemoryStatsReceiverUtility
+import com.twitter.inject.Test
 import com.twitter.conversions.DurationOps._
 import com.twitter.util.FuturePool
 import org.scalatest.exceptions.TestFailedDueToTimeoutException
@@ -68,32 +69,35 @@ class InMemoryStatsReceiverUtilityTest extends Test {
     var gaugeValue: Float = 0.0f
     val gauge = inMemoryStatsReceiver.addGauge("gauge") { gaugeValue }
 
+    val counterWait = FuturePool.unboundedPool { sr.counters.waitFor("counter", 1L, timeout) }
     // Change metrics every 20 millis
-    val increaseCounter = FuturePool.unboundedPool {
+    FuturePool.unboundedPool {
       for (_ <- 0 to 2) {
         Thread.sleep(20)
         // counter values: 0L, 1L, 2L, 3L
         counter.incr()
       }
     }
-    sr.counters.waitFor("counter", 1L, timeout)
+    await(counterWait)
 
-    val addToHistogram = FuturePool.unboundedPool {
+    val statsWait = FuturePool.unboundedPool { sr.stats.waitFor("stat", Seq(0.0f, 1.0f), timeout) }
+    FuturePool.unboundedPool {
       for (s <- 0 to 2) {
         Thread.sleep(20)
         // histogram values: Seq(0.0f, 1.0f, 2.0f)
         stat.add(s)
       }
     }
-    sr.stats.waitFor("stat", Seq(0.0f, 1.0f), timeout)
+    await(statsWait)
 
-    val increaseGauge = FuturePool.unboundedPool {
+    val gaugeWait = FuturePool.unboundedPool { sr.gauges.waitFor("gauge", 1.0f, timeout) }
+    FuturePool.unboundedPool {
       for (g <- 0 to 2) {
         Thread.sleep(20)
         // gauge values: 0.0f, 1.0f, 2.0f
         gaugeValue = g
       }
     }
-    sr.gauges.waitFor("gauge", 1.0f, timeout)
+    await(gaugeWait)
   }
 }

inject/inject-server/src/test/scala/com/twitter/inject/server/tests/FeatureTestNonInjectionCustomStatsReceiverTest.scala

@@ -1,7 +1,8 @@
 package com.twitter.inject.server.tests
 
 import com.twitter.finagle.http.Status
-import com.twitter.inject.server.{EmbeddedTwitterServer, FeatureTest}
+import com.twitter.inject.server.EmbeddedTwitterServer
+import com.twitter.inject.server.FeatureTest
 
 /** Test a non-inject TwitterServer using a Custom StatsReceiver implementation with the [[FeatureTest]] trait */
 class FeatureTestNonInjectionCustomStatsReceiverTest extends FeatureTest {
@@ -24,6 +25,7 @@ class FeatureTestNonInjectionCustomStatsReceiverTest extends FeatureTest {
    */
   override def beforeAll(): Unit = {
     server.start()
+    super.beforeAll()
   }
 
   test("TestServer#starts up") {