fix(solana): call ClearSignatures in SetRoot on "account in use" error (#374)

This PR enhances the Solana `SetRoot` implementation so as to handle
"Account already in use" errors, which we got several times when
executing changesets in mainnet. In theory, these errors are happening
because Solana's SetRoot does not execute atomically -- instead, it
requires at least 4 instructions:

1. initialize signatures
2. append signatures (at least once)
3. finalize signatures
4. set root

If the first instruction completes successfully but any of the others
fails, it'll leave the signatures PDA in a initialized state but without
the root properly set. This prevents retries at the CI level from
working as the program will then fail, stating the the pda "account is
already in use".

The workaround in this PR is to identify this error and issue a "clear
signatures" instruction -- which should reset the signatures pda -- and
then retry the SetRoot instructions as usual.

The PR also bumps the chainlink-ccip version so as to include a patch
from Terry Tata to a "send transaction" helper that should help deal
with frequent RPC errors found in mainnet.

[DX-768](https://smartcontract-it.atlassian.net/browse/DX-768)

[DX-768]:
https://smartcontract-it.atlassian.net/browse/DX-768?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ

Author: gustavogama-cll

.changeset/swift-cougars-tickle.md

@@ -0,0 +1,5 @@
+---
+"@smartcontractkit/mcms": patch
+---
+
+fix: call ClearSignatures in Solana's SetRoot upon receiving an "account in use" error

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52
 	github.com/smartcontractkit/chain-selectors v1.0.48
 	github.com/smartcontractkit/chainlink-aptos v0.0.0-20250414155853-651b4e583ee9
-	github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250422205932-c33527859fd6
+	github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250512155317-4615e5659ce4
 	github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7
 	github.com/spf13/cast v1.7.1
 	github.com/stretchr/testify v1.10.0
@@ -189,6 +189,7 @@ require (
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/smartcontractkit/chainlink-ccip v0.0.0-20250320090719-315440f5b0a7 // indirect
 	github.com/smartcontractkit/chainlink-common v0.6.1-0.20250329081313-84ec641e0758 // indirect
 	github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect
 	github.com/smartcontractkit/libocr v0.0.0-20250220133800-f3b940c4f298 // indirect

go.sum

@@ -599,8 +599,12 @@ github.com/smartcontractkit/chain-selectors v1.0.48 h1:wa03tlcGj08qZfv1p+LXZIimp
 github.com/smartcontractkit/chain-selectors v1.0.48/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
 github.com/smartcontractkit/chainlink-aptos v0.0.0-20250414155853-651b4e583ee9 h1:lw8RZ8IR4UX1M7djAB3IuMtcAqFX4Z4bzQczClfb8bs=
 github.com/smartcontractkit/chainlink-aptos v0.0.0-20250414155853-651b4e583ee9/go.mod h1:Sq/ddMOYch6ZuAnW2k5u9V4+TlGKFzuHQnTM8JXEU+g=
+github.com/smartcontractkit/chainlink-ccip v0.0.0-20250320090719-315440f5b0a7 h1:/VKrPJRo4y58whyBRhc9Fszu2eTNn0LNISaS0pjhTpk=
+github.com/smartcontractkit/chainlink-ccip v0.0.0-20250320090719-315440f5b0a7/go.mod h1:AhqYIeGF2k94J+/gzRx5dQttlgUdZid2N6E4HlHVIVA=
 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250422205932-c33527859fd6 h1:EWpKT2h/jyqCcblfmtHuY5oN2k8k2Mrmi5KqX+hc2lo=
 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250422205932-c33527859fd6/go.mod h1:k3/Z6AvwurPUlfuDFEonRbkkiTSgNSrtVNhJEWNlUZA=
+github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250512155317-4615e5659ce4 h1:ZopJiCUcshj79A3tjh0f5cncXEjlfp7QOPWFppb4gxM=
+github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20250512155317-4615e5659ce4/go.mod h1:k3/Z6AvwurPUlfuDFEonRbkkiTSgNSrtVNhJEWNlUZA=
 github.com/smartcontractkit/chainlink-common v0.6.1-0.20250329081313-84ec641e0758 h1:SyaVoJtYZ54dO4HyUcfWsuvC0hRsjk+Lyy/k++WQc7Y=
 github.com/smartcontractkit/chainlink-common v0.6.1-0.20250329081313-84ec641e0758/go.mod h1:ASXpANdCfcKd+LF3Vhz37q4rmJ/XYQKEQ3La1k7idp0=
 github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 h1:E7k5Sym9WnMOc4X40lLnQb6BMosxi8DfUBU9pBJjHOQ=

sdk/solana/executor.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"regexp"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/gagliardetto/solana-go"
@@ -19,6 +20,8 @@ import (
 
 var _ sdk.Executor = (*Executor)(nil)
 
+const maxPreloadSignaturesAttempts = 3
+
 // Executor is an Executor implementation for Solana chains, allowing for the execution of
 // operations on the MCMS contract
 type Executor struct {
@@ -161,7 +164,7 @@ func (e *Executor) SetRoot(
 		return types.TransactionResult{}, err
 	}
 
-	err = e.preloadSignatures(ctx, pdaSeed, root, validUntil, sortedSignatures, rootSignaturesPDA)
+	err = e.preloadSignatures(ctx, pdaSeed, root, validUntil, sortedSignatures, rootSignaturesPDA, 0)
 	if err != nil {
 		return types.TransactionResult{}, err
 	}
@@ -200,11 +203,16 @@ func (e *Executor) preloadSignatures(
 	validUntil uint32,
 	sortedSignatures []types.Signature,
 	signaturesPDA solana.PublicKey,
+	attempt int,
 ) error {
 	initSignaturesInstruction := mcm.NewInitSignaturesInstruction(mcmName, root, validUntil,
 		uint8(len(sortedSignatures)), signaturesPDA, e.auth.PublicKey(), solana.SystemProgramID) //nolint:gosec
 	_, _, err := e.sendAndConfirm(ctx, e.client, e.auth, initSignaturesInstruction, rpc.CommitmentConfirmed)
 	if err != nil {
+		if isAccountAlreadyInUseError(err) {
+			return e.retryPreloadSignatures(ctx, mcmName, root, validUntil, sortedSignatures, signaturesPDA, attempt)
+		}
+
 		return fmt.Errorf("unable to initialize signatures: %w", err)
 	}
 
@@ -229,6 +237,30 @@ func (e *Executor) preloadSignatures(
 	return nil
 }
 
+// retryPreloadSignatures clears the signatures pda and then calls preloadSignatures
+func (e *Executor) retryPreloadSignatures(
+	ctx context.Context,
+	mcmName [32]byte,
+	root [32]byte,
+	validUntil uint32,
+	sortedSignatures []types.Signature,
+	signaturesPDA solana.PublicKey,
+	attempt int,
+) error {
+	if attempt >= maxPreloadSignaturesAttempts {
+		return fmt.Errorf("maximum attempts to retry preload signatures reached (%d); aborting", attempt)
+	}
+
+	clearSignaturesInstruction := mcm.NewClearSignaturesInstruction(mcmName, root, validUntil, signaturesPDA,
+		e.auth.PublicKey())
+	_, _, err := e.sendAndConfirm(ctx, e.client, e.auth, clearSignaturesInstruction, rpc.CommitmentConfirmed)
+	if err != nil {
+		return fmt.Errorf("unable to clear signatures: %w", err)
+	}
+
+	return e.preloadSignatures(ctx, mcmName, root, validUntil, sortedSignatures, signaturesPDA, attempt+1)
+}
+
 // solanaMetadata returns the root metadata input for the MCM program
 func (e *Executor) solanaMetadata(metadata types.ChainMetadata, configPDA [32]byte) mcm.RootMetadataInput {
 	return mcm.RootMetadataInput{
@@ -264,3 +296,9 @@ func solanaSignatures(signatures []types.Signature) []mcm.Signature {
 
 	return solanaSignatures
 }
+
+var accountAlreadyInUsePattern = regexp.MustCompile(`Allocate: account Address.*already in use`)
+
+func isAccountAlreadyInUseError(err error) bool {
+	return accountAlreadyInUsePattern.MatchString(err.Error())
+}

sdk/solana/executor_test.go

@@ -99,7 +99,6 @@ func TestExecutor_ExecuteOperation(t *testing.T) {
 				},
 			},
 			mockSetup: func(m *mocks.JSONRPCClient) {
-
 			},
 			want:    "",
 			wantErr: errors.New("invalid contract ID provided"),
@@ -155,7 +154,6 @@ func TestExecutor_ExecuteOperation(t *testing.T) {
 				},
 			},
 			mockSetup: func(m *mocks.JSONRPCClient) {
-
 			},
 			want:    "",
 			wantErr: errors.New("invalid contract ID provided"),
@@ -216,7 +214,7 @@ func TestExecutor_SetRoot(t *testing.T) {
 		wantErr    string
 	}{
 		{
-			name:       "success",
+			name:       "success - direct",
 			metadata:   defaultMetadata,
 			proof:      defaultProof,
 			root:       defaultRoot,
@@ -238,6 +236,42 @@ func TestExecutor_SetRoot(t *testing.T) {
 			},
 			want: "oaV9FKKPDVneUANQ9hJqEuhgwfUgbxucUC4TmzpgGJhuSxBueapWc9HJ4cJQMqT2PPQX6rhTbKnXkebsaravnLo",
 		},
+		{
+			name:       "success - after account already in use error",
+			metadata:   defaultMetadata,
+			proof:      defaultProof,
+			root:       defaultRoot,
+			validUntil: defaultValidUntil,
+			signatures: defaultSignatures,
+			setup: func(t *testing.T, executor *Executor, mockJSONRPCClient *mocks.JSONRPCClient) {
+				t.Helper()
+
+				accountAlreadyInUseError := errors.New(`
+					(string) (len=4) "logs": ([]interface {}) (len=7 cap=8) {
+					(string) (len=63) "Program 5vNJx78mz7KVMjhuipyr9jKBKcMrKYGdjGkgE4LUmjKk invoke [1]",
+					(string) (len=40) "Program log: Instruction: InitSignatures",
+					(string) (len=51) "Program 11111111111111111111111111111111 invoke [2]",
+					(string) (len=110) "Allocate: account Address { address: DKvwoHhcRMZwsedUvJqmsNuat52WdcYQD7agqMjJybbF, base: None } already in use",
+					(string) (len=74) "Program 11111111111111111111111111111111 failed: custom program error: 0x0",
+					(string) (len=90) "Program 5vNJx78mz7KVMjhuipyr9jKBKcMrKYGdjGkgE4LUmjKk consumed 7202 of 200000 compute units",
+					(string) (len=86) "Program 5vNJx78mz7KVMjhuipyr9jKBKcMrKYGdjGkgE4LUmjKk failed: custom program error: 0x0"`)
+
+				// 6 transactions: init-signatures, clear-signatures, init-signatures append-signatures, finalize-signatures, set-root
+				mockSolanaTransaction(t, mockJSONRPCClient, 80, 70,
+					"AxzwxQ2DLR4zEFxEPGaafR4z3MY4CP1CAdSs1ZZhArtgS3G4F9oYSy3Nx1HyA1Macb4bYEi4jU6F1CL4SRrZz1v", nil, accountAlreadyInUseError)
+				mockSolanaTransaction(t, mockJSONRPCClient, 81, 71,
+					"5qm3BUCF1DswRm4r32mipWZa5NrbHYgPst8BJXr1BysNaqfEz4kVGnGzCx3vLWJoWi2FRszzgLUxfcmAfLzzHw9n", nil, nil)
+				mockSolanaTransaction(t, mockJSONRPCClient, 82, 72,
+					"KCAkcwG8LG3cS3bUemdR1grv6EoyqfgDJN3BJfN58azEDkpRFa9S66RDFvtNWHga9htimSnfNkGoWVLLx7AJKrs", nil, nil)
+				mockSolanaTransaction(t, mockJSONRPCClient, 83, 63,
+					"4GWCdxQAsAmbqyaB1SCHpmnyRWmBwBY5S6hUpSMz2gENErgt8zqmsXnz9dbXCchucZqAf7ZdctzTNtUUTjD8rMcv", nil, nil)
+				mockSolanaTransaction(t, mockJSONRPCClient, 84, 64,
+					"npfXXzfJzSkB6QcwS36P9dzc61itiDnLX9yGVyzaooftiSFs1A53JWHGQq6F9MzjWxKD8bRLPKWuGseUxHK56s3", nil, nil)
+				mockSolanaTransaction(t, mockJSONRPCClient, 85, 65,
+					"2W1d6qQAVPjssJgm6WMeLxwatKZ9TUUZcvNHdFaguvLxrCFqPdpuikiz9YdfkXG5eLojQNjrW6L6W2sFRWEujER", nil, nil)
+			},
+			want: "2W1d6qQAVPjssJgm6WMeLxwatKZ9TUUZcvNHdFaguvLxrCFqPdpuikiz9YdfkXG5eLojQNjrW6L6W2sFRWEujER",
+		},
 		{
 			name:       "failure: invalid address",
 			metadata:   types.ChainMetadata{StartingOpCount: 100, MCMAddress: "invalid-mcm-address"},
@@ -377,3 +411,42 @@ func newTestExecutor(t *testing.T, auth solana.PrivateKey, chainSelector types.C
 
 	return NewExecutor(encoder, client, auth), mockJSONRPCClient
 }
+
+func Test_isAccountAlreadyInUseError(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "account already in use error",
+			err: errors.New(`
+				(string) (len=51) "Program 11111111111111111111111111111111 invoke [2]",
+				(string) (len=110) "Allocate: account Address { address: DKvwoHhcRMZwsedUvJqmsNuat52WdcYQD7agqMjJybbF, base: None } already in use",
+				(string) (len=74) "Program 11111111111111111111111111111111 failed: custom program error: 0x0",
+				(string) (len=90) "Program 5vNJx78mz7KVMjhuipyr9jKBKcMrKYGdjGkgE4LUmjKk consumed 7202 of 200000 compute units",
+			`),
+			want: true,
+		},
+		{
+			name: "other error",
+			err:  errors.New(`other error`),
+			want: false,
+		},
+		{
+			name: "other similar but not quite the same error",
+			err:  errors.New(`Allocate: already in use`),
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := isAccountAlreadyInUseError(tt.err)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}