bboltcachestorage: delete links outside of cursor

The `emptyBranchWithParents` method could accidentally leave link
entries that shouldn't exist. When finding these links, deleting during
the iteration can sometimes cause the cursor to jump entries that should
be deleted.

This changes the code path to delete the links outside of the iteration
to avoid this.

This is caused by a long-standing bug in bolt that can't be fixed
easily. See etcd-io/bbolt#611 for details.

Signed-off-by: Jonathan A. Sternberg <[email protected]>

Author: jsternberg

solver/bboltcachestorage/storage.go

@@ -267,19 +267,28 @@ func (s *Store) emptyBranchWithParents(tx *bolt.Tx, id []byte) error {
 	if backlinks := tx.Bucket([]byte(backlinksBucket)).Bucket(id); backlinks != nil {
 		if err := backlinks.ForEach(func(k, v []byte) error {
 			if subLinks := tx.Bucket([]byte(linksBucket)).Bucket(k); subLinks != nil {
+				// Perform deletion outside of the iteration.
+				// https://github.com/etcd-io/bbolt/pull/611
+				var toDelete []string
 				if err := subLinks.ForEach(func(k, v []byte) error {
 					parts := bytes.Split(k, []byte("@"))
 					if len(parts) != 2 {
 						return errors.Errorf("invalid key %s", k)
 					}
 					if bytes.Equal(id, parts[1]) {
-						return subLinks.Delete(k)
+						toDelete = append(toDelete, string(k))
 					}
 					return nil
 				}); err != nil {
 					return err
 				}
 
+				for _, k := range toDelete {
+					if err := subLinks.Delete([]byte(k)); err != nil {
+						return err
+					}
+				}
+
 				if isEmptyBucket(subLinks) {
 					if subResult := tx.Bucket([]byte(resultBucket)).Bucket(k); isEmptyBucket(subResult) {
 						if err := tx.Bucket([]byte(linksBucket)).DeleteBucket(k); err != nil {