Fix F3 flaky tests (#1023)

* Investigate flaky F3 tests

Attempt to reproduce flaky tests on CI.

* Adjust test progress velocity

* Reduce increment and check granularity in tests

* Disable parallel test runs

* Bump CI timeout in checking that tests are unflaked

* Parallelise race detector test runs

* Re-enable parallel test runs and bump eventual timeout

* Disable paralle tests

* Tests now look stable, remove the CI flaky test run

Author: masih

f3_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/failstore"
 	ds_sync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/host"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@@ -29,8 +30,8 @@ import (
 )
 
 const (
-	eventualCheckInterval = 100 * time.Millisecond
-	eventualCheckTimeout  = time.Minute
+	eventualCheckInterval = 5 * time.Millisecond
+	eventualCheckTimeout  = 2 * time.Minute
 	advanceClockEvery     = 5 * time.Millisecond
 	advanceClockBy        = 100 * time.Millisecond
 )
@@ -44,14 +45,15 @@ func init() {
 }
 
 func TestF3Simple(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).start()
 	env.requireInstanceEventually(5, eventualCheckTimeout, true)
 	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch, eventualCheckTimeout)
 }
 
 func TestF3WithLookback(t *testing.T) {
-	t.Parallel()
+	// Quiet down the logs since the test asserts a scenario that triggers
+	// OhShitStore ERROR level logs.
+	_ = logging.SetLogLevel("f3/ohshitstore", "DPANIC")
 
 	mfst := base
 	mfst.EC.HeadLookback = 20
@@ -93,7 +95,10 @@ func TestF3WithLookback(t *testing.T) {
 }
 
 func TestF3PauseResumeCatchup(t *testing.T) {
-	t.Parallel()
+	// Quiet down the logs since the test asserts a scenario that triggers
+	// OhShitStore ERROR level logs.
+	_ = logging.SetLogLevel("f3/ohshitstore", "DPANIC")
+
 	env := newTestEnvironment(t).withNodes(3).start()
 	env.requireInstanceEventually(1, eventualCheckTimeout, true)
 	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch, eventualCheckTimeout)
@@ -136,7 +141,6 @@ func TestF3PauseResumeCatchup(t *testing.T) {
 }
 
 func TestF3FailRecover(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2)
 
 	// Make it possible to fail a single write for node 0.
@@ -167,7 +171,6 @@ func TestF3FailRecover(t *testing.T) {
 }
 
 func TestF3LateBootstrap(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).start()
 
 	// Wait till we're "caught up".