feat: resend batch msgs if some failed (#112)

* feat: send msg again, just removing the failed msg in batch by parsing the failed msg

* chore: add #112 to cl

* chore: add godoc to reliablySendMsgsResendingOnMsgErr

* chore: address pr comment set regex compiled as vars

* chore: add max retries to resend batch

* fix: babylon retry send reliably msg

* chore: remove withespace

* test: add e2e test testing the retry logic

* fix: add new param to test manager

* fix: e2e test

* fix: e2e test delegations retry

* fix: test

* test: 2 invalid delegations

* revert e2e test.

* lint fix

* lint fix

* adds e2e for sending batch

* lint fix

* fix e2e

* lint fix

---------

Co-authored-by: Vlad <[email protected]>
Co-authored-by: Lazar <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 * [#134](https://github.com/babylonlabs-io/covenant-emulator/pull/134) chore: add golangci lint rules
 * [#141](https://github.com/babylonlabs-io/covenant-emulator/pull/141) chore: bumps go1.25
+* [#112](https://github.com/babylonlabs-io/covenant-emulator/pull/112) feat: resend batch msgs if some failed
 
 ## v0.16.0-rc.1
 

clientcontroller/babylon.go

@@ -2,10 +2,16 @@ package clientcontroller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
+	"regexp"
+	"strconv"
+	"strings"
 	"time"
 
+	sdkErrors "cosmossdk.io/errors"
+
 	sdkmath "cosmossdk.io/math"
 	"github.com/btcsuite/btcd/btcec/v2"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
@@ -30,19 +36,23 @@ import (
 var (
 	_                  ClientController = &BabylonController{}
 	MaxPaginationLimit                  = uint64(1000)
+	messageIndexRegex                   = regexp.MustCompile(`message index:\s*(\d+)`)
 )
 
 type BabylonController struct {
 	bbnClient *bbnclient.Client
 	cfg       *config.BBNConfig
 	btcParams *chaincfg.Params
 	logger    *zap.Logger
+
+	MaxRetiresBatchRemovingMsgs uint64
 }
 
 func NewBabylonController(
 	cfg *config.BBNConfig,
 	btcParams *chaincfg.Params,
 	logger *zap.Logger,
+	maxRetiresBatchRemovingMsgs uint64,
 ) (*BabylonController, error) {
 	bbnConfig := config.BBNConfigToBabylonConfig(cfg)
 
@@ -63,6 +73,7 @@ func NewBabylonController(
 		cfg,
 		btcParams,
 		logger,
+		maxRetiresBatchRemovingMsgs,
 	}, nil
 }
 
@@ -154,7 +165,7 @@ func (bc *BabylonController) reliablySendMsgs(msgs []sdk.Msg) (*babylonclient.Re
 	return bc.bbnClient.ReliablySendMsgs(
 		context.Background(),
 		msgs,
-		expectedErrors,
+		nil,
 		unrecoverableErrors,
 	)
 }
@@ -182,16 +193,92 @@ func (bc *BabylonController) SubmitCovenantSigs(covSigs []*types.CovenantSigs) (
 
 		msgs = append(msgs, msg)
 	}
-	res, err := bc.reliablySendMsgs(msgs)
-	if err != nil {
-		return nil, err
+
+	return bc.reliablySendMsgsResendingOnMsgErr(msgs)
+}
+
+// reliablySendMsgsResendingOnMsgErr sends the msgs to the chain, if some msg fails to execute
+// and contains 'message index: %d', it will remove that msg from the batch and send again
+// if there is no more message available, returns the last error.
+func (bc *BabylonController) reliablySendMsgsResendingOnMsgErr(msgs []sdk.Msg) (*types.TxResponse, error) {
+	var err error
+
+	maxRetries := BatchRetries(msgs, bc.MaxRetiresBatchRemovingMsgs)
+	for i := uint64(0); i < maxRetries; i++ {
+		res, errSendMsg := bc.reliablySendMsgs(msgs)
+		if errSendMsg != nil {
+			// concatenate the errors, to throw out if needed
+			err = errors.Join(err, errSendMsg)
+
+			if strings.Contains(errSendMsg.Error(), "message index: ") {
+				// remove the failed msg from the batch and send again
+				failedIndex, found := FailedMessageIndex(errSendMsg)
+				if !found {
+					return nil, errSendMsg
+				}
+
+				msgs = RemoveMsgAtIndex(msgs, failedIndex)
+
+				continue
+			}
+
+			return nil, errSendMsg
+		}
+
+		if res == nil { // expected error happened
+			return &types.TxResponse{}, nil
+		}
+
+		return &types.TxResponse{TxHash: res.TxHash, Events: res.Events}, nil
 	}
 
-	if res == nil {
+	if err != nil && errorContained(err, expectedErrors) {
 		return &types.TxResponse{}, nil
 	}
 
-	return &types.TxResponse{TxHash: res.TxHash, Events: res.Events}, nil
+	return nil, fmt.Errorf("failed to send batch of msgs: %w", err)
+}
+
+// BatchRetries returns the max number of retries it should execute based on the
+// amount of messages in the batch
+func BatchRetries(msgs []sdk.Msg, maxRetiresBatchRemovingMsgs uint64) uint64 {
+	maxRetriesByMsgLen := uint64(len(msgs))
+
+	if maxRetiresBatchRemovingMsgs == 0 {
+		return maxRetriesByMsgLen
+	}
+
+	if maxRetiresBatchRemovingMsgs > maxRetriesByMsgLen {
+		return maxRetriesByMsgLen
+	}
+
+	return maxRetiresBatchRemovingMsgs
+}
+
+// RemoveMsgAtIndex removes any msg inside the slice, based on the index is given
+// if the index is out of bounds, it just returns the slice of msgs.
+func RemoveMsgAtIndex(msgs []sdk.Msg, index int) []sdk.Msg {
+	if index < 0 || index >= len(msgs) {
+		return msgs
+	}
+
+	return append(msgs[:index], msgs[index+1:]...)
+}
+
+// FailedMessageIndex finds the message index which failed in a error which contains
+// the substring 'message index: %d'.
+// ex.:  rpc error: code = Unknown desc = failed to execute message; message index: 1: the covenant signature is already submitted
+func FailedMessageIndex(err error) (int, bool) {
+	matches := messageIndexRegex.FindStringSubmatch(err.Error())
+
+	if len(matches) > 1 {
+		index, errAtoi := strconv.Atoi(matches[1])
+		if errAtoi == nil {
+			return index, true
+		}
+	}
+
+	return 0, false
 }
 
 func (bc *BabylonController) QueryPendingDelegations(limit uint64, filter FilterFn) ([]*types.Delegation, error) {
@@ -589,3 +676,13 @@ func (bc *BabylonController) QueryBtcLightClientTip() (*btclctypes.BTCHeaderInfo
 
 	return res.Header, nil
 }
+
+func errorContained(err error, errList []*sdkErrors.Error) bool {
+	for _, e := range errList {
+		if strings.Contains(err.Error(), e.Error()) {
+			return true
+		}
+	}
+
+	return false
+}