itest: fix flake with universe sync

Fixes #807.

If we're syncing a node to a universe that is still in the process of
getting proofs pushed to, we sometimes receive leaves with a root that
doesn't match the root we fetched at the beginning of the sync process.
We should re-try the sync, which we'll do in a separate PR.
For now we just want to fix the flake, which we can do by waiting to
start a new node until the existing one has finished syncing.

Author: guggero

itest/assertions.go

@@ -1198,6 +1198,34 @@ func AssertUniverseRootEquality(t *testing.T,
 	))
 }
 
+// AssertUniverseRootEqualityEventually checks that the universe roots returned
+// by two daemons are either equal eventually.
+func AssertUniverseRootEqualityEventually(t *testing.T,
+	clientA, clientB unirpc.UniverseClient) {
+
+	ctxb := context.Background()
+	ctxt, cancel := context.WithTimeout(ctxb, defaultWaitTimeout)
+	defer cancel()
+
+	err := wait.NoError(func() error {
+		rootRequest := &unirpc.AssetRootRequest{}
+		universeRootsAlice, err := clientA.AssetRoots(ctxt, rootRequest)
+		require.NoError(t, err)
+		universeRootsBob, err := clientB.AssetRoots(ctxt, rootRequest)
+		require.NoError(t, err)
+
+		if !AssertUniverseRootsEqual(
+			universeRootsAlice, universeRootsBob,
+		) {
+
+			return fmt.Errorf("roots not equal")
+		}
+
+		return nil
+	}, defaultWaitTimeout)
+	require.NoError(t, err)
+}
+
 // AssertUniverseRoot makes sure the given universe root exists with the given
 // sum, either identified by the asset ID or group key.
 func AssertUniverseRoot(t *testing.T, client unirpc.UniverseClient,

itest/multi_asset_group_test.go

@@ -317,6 +317,10 @@ func testMultiAssetGroupSend(t *harnessTest) {
 		collectibleGroupMembers + 1,
 	})
 
+	AssertUniverseRootEqualityEventually(
+		t.t, t.tapd, t.universeServer.service,
+	)
+
 	// We'll make a second node now that'll be the receiver of all the
 	// assets made above.
 	secondTapd := setupTapdHarness(
@@ -326,6 +330,10 @@ func testMultiAssetGroupSend(t *harnessTest) {
 		require.NoError(t.t, secondTapd.stop(!*noDelete))
 	}()
 
+	AssertUniverseRootEqualityEventually(
+		t.t, secondTapd, t.universeServer.service,
+	)
+
 	// Send 5 of the assets to Bob, and verify that they are received.
 	numUnits := issuableAsset.Asset.Amount
 	assetType := issuableAsset.Asset.AssetType