universe: optimize syncer call

This commit allows for a much larger (32x) page size than before. This
should allow us to fetch the complete set of roots even faster, given
that we should have 100% cache hits for the syncer calls now.

We also reduce the overall count of requests by 1 in almost all cases by
detecting that the returned page isn't full anymore (instead of reading
until we get an empty page).

Author: guggero

universe/interface.go

@@ -38,7 +38,7 @@ var (
 const (
 	// MaxPageSize is the maximum page size that can be used when querying
 	// for asset roots and leaves.
-	MaxPageSize = 512
+	MaxPageSize = 16384
 )
 
 // IdentifierKey is the compact representation of a universe identifier that can

universe/syncer.go

@@ -525,14 +525,17 @@ func (s *SimpleSyncer) SyncUniverse(ctx context.Context, host ServerAddr,
 // fetchAllRoots fetches all the roots from the remote Universe. This function
 // is used in order to isolate any logic related to the specifics of how we
 // fetch the data from the universe server.
-func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, diffEngine DiffEngine) ([]Root, error) {
+func (s *SimpleSyncer) fetchAllRoots(ctx context.Context,
+	diffEngine DiffEngine) ([]Root, error) {
+
 	offset := int32(0)
 	pageSize := defaultPageSize
 	roots := make([]Root, 0)
 
 	for {
 		log.Debugf("Fetching roots in range: %v to %v", offset,
 			offset+pageSize)
+
 		tempRoots, err := diffEngine.RootNodes(
 			ctx, RootNodesQuery{
 				WithAmountsById: false,
@@ -541,16 +544,17 @@ func (s *SimpleSyncer) fetchAllRoots(ctx context.Context, diffEngine DiffEngine)
 				Limit:           pageSize,
 			},
 		)
-
 		if err != nil {
 			return nil, err
 		}
 
-		if len(tempRoots) == 0 {
+		roots = append(roots, tempRoots...)
+
+		// If we're getting a partial page, then we know we're done.
+		if len(tempRoots) < int(pageSize) {
 			break
 		}
 
-		roots = append(roots, tempRoots...)
 		offset += pageSize
 	}