universe: improve robustness of proof leaf sync

During syncing, the universe root is fetched before the proof leaves,
which are then verified against that root.

Occasionally, the fetched root may become stale before a proof leaf is
verified. In such cases, the proof leaf may belong to a newer version of
the universe tree.

This isn't critical, as the syncer will eventually resolve the mismatch
during subsequent universe root queries.

To improve robustness, this change modifies how proof leaves are
fetched. While queries remain parallel, failed query or verification
attempts no longer prevent other attempts from proceeding. The syncer
now uses whatever proof leaves it can verify with the universe root it
has.

Author: ffranr

universe/syncer.go

@@ -304,7 +304,7 @@ func (s *SimpleSyncer) syncRoot(ctx context.Context, remoteRoot Root,
 
 	// Now that we know where the divergence is, we can fetch the issuance
 	// proofs from the remote party.
-	err = fn.ParSlice(
+	fetchErrs, err := fn.ParSliceErrCollect(
 		ctx, keysToFetch, func(ctx context.Context, key LeafKey) error {
 			newProof, err := diffEngine.FetchProofLeaf(
 				ctx, uniID, key,
@@ -320,8 +320,9 @@ func (s *SimpleSyncer) syncRoot(ctx context.Context, remoteRoot Root,
 			// given.
 			validRoot := leafProof.VerifyRoot(remoteRoot)
 			if !validRoot {
-				return fmt.Errorf("proof for key=%v is "+
-					"invalid", spew.Sdump(key))
+				return fmt.Errorf("proof leaf failed "+
+					"universe root verification "+
+					"(leaf_key=%v)", spew.Sdump(key))
 			}
 
 			// If this is an issuance proof, then we can send
@@ -379,6 +380,13 @@ func (s *SimpleSyncer) syncRoot(ctx context.Context, remoteRoot Root,
 		return err
 	}
 
+	// Report any errors encountered while fetching the leaves.
+	for idx, fetchErr := range fetchErrs {
+		leafKey := keysToFetch[idx]
+		log.Errorf("Error fetching leaf (leaf_key=%x): %v",
+			leafKey.UniverseKey(), fetchErr)
+	}
+
 	// We use an error group to simply the error handling of a goroutine.
 	// This goroutine will handle reading in batches of new leaves to
 	// insert into the DB. We'll fee the output of the goroutines below