feat: add IsOperationDone and IsChainDone to timelock executable (#337)

In order to support concurrent timelock executions between the
chainlink-deployments CI and the legacy timelock-worker service, the GHA
running on the CI need to be able to check if an operation and/or the
operations of a chain have been marked as "done" by the timelock worker.

This PR adds two methods to the TimelockExecutable type, implementing
these two checks. The implementation was adapted from the existing
methods: `IsOperationReady` and `IsChainReady`.

JIRA: [DPA-1613](https://smartcontract-it.atlassian.net/browse/DPA-1613)

[DPA-1613]:
https://smartcontract-it.atlassian.net/browse/DPA-1613?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ

Author: gustavogama-cll

.changeset/tall-tigers-wave.md

@@ -0,0 +1,5 @@
+---
+"@smartcontractkit/mcms": minor
+---
+
+feat: add IsOperationDone and IsChainDone to TimelockExecutable

errors.go

@@ -19,6 +19,16 @@ func (e *OperationNotReadyError) Error() string {
 	return fmt.Sprintf("operation %d is not ready", e.OpIndex)
 }
 
+// OperationNotDoneError is returned when an operation is not yet done.
+type OperationNotDoneError struct {
+	OpIndex int
+}
+
+// Error implements the error interface.
+func (e *OperationNotDoneError) Error() string {
+	return fmt.Sprintf("operation %d is not done", e.OpIndex)
+}
+
 // InvalidProposalKindError is returned when an invalid proposal kind is provided.
 type InvalidProposalKindError struct {
 	ProvidedKind types.ProposalKind

timelock_executable.go

@@ -131,6 +131,43 @@ func (t *TimelockExecutable) IsOperationReady(ctx context.Context, idx int) erro
 	return nil
 }
 
+// IsChainDone checks if the chain is done executing
+func (t *TimelockExecutable) IsChainDone(ctx context.Context, chainSelector types.ChainSelector) error {
+	// Check readiness for each global operation in the proposal
+	for globalIndex, op := range t.proposal.Operations {
+		if op.ChainSelector == chainSelector {
+			err := t.IsOperationDone(ctx, globalIndex)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (t *TimelockExecutable) IsOperationDone(ctx context.Context, idx int) error {
+	op := t.proposal.Operations[idx]
+
+	cs := op.ChainSelector
+	timelock := t.proposal.TimelockAddresses[cs]
+
+	operationID, err := t.GetOpID(ctx, idx, op, cs)
+	if err != nil {
+		return fmt.Errorf("unable to get operation ID: %w", err)
+	}
+
+	isDone, err := t.executors[cs].IsOperationDone(ctx, timelock, operationID)
+	if err != nil {
+		return err
+	}
+	if !isDone {
+		return &OperationNotDoneError{OpIndex: idx}
+	}
+
+	return nil
+}
+
 type Option func(*executeOptions)
 
 type executeOptions struct {

timelock_executable_test.go

@@ -573,27 +573,44 @@ func scheduleAndExecuteGrantRolesProposal(t *testing.T, ctx context.Context, tar
 	require.NoError(t, sim.Backend.AdjustTime(5*time.Second))
 	sim.Backend.Commit() // Note < 1.14 geth needs a commit after adjusting time.
 
-	// Check that the operation is now ready
+	opIdx := 0
+
+	// IsReady
 	err = tExecutable.IsReady(ctx)
 	require.NoError(t, err)
-
-	// Check IsChainReady function succeeds
 	for chainSelector := range proposal.ChainMetadata {
 		err = tExecutable.IsChainReady(ctx, chainSelector)
 		require.NoError(t, err)
 	}
 
+	// !IsDone
+	err = tExecutable.IsOperationDone(ctx, opIdx)
+	require.Error(t, err)
+	for chainSelector := range proposal.ChainMetadata {
+		err = tExecutable.IsChainDone(ctx, chainSelector)
+		require.Error(t, err)
+	}
+
 	// Execute the proposal
-	idx := 0
-	_, err = tExecutable.Execute(ctx, idx)
+	_, err = tExecutable.Execute(ctx, opIdx)
 	require.NoError(t, err)
 	sim.Backend.Commit()
-	opID, err := tExecutable.GetOpID(ctx, idx, proposal.Operations[idx], proposal.Operations[idx].ChainSelector)
-	require.NoError(t, err)
-	// Check that the operation is done
-	isOperationDone, err := timelockC.IsOperationDone(&bind.CallOpts{}, opID)
+
+	// IsDone
+	err = tExecutable.IsOperationDone(ctx, opIdx)
 	require.NoError(t, err)
-	require.True(t, isOperationDone)
+	for chainSelector := range proposal.ChainMetadata {
+		err = tExecutable.IsChainDone(ctx, chainSelector)
+		require.NoError(t, err)
+	}
+
+	// !IsReady
+	err = tExecutable.IsOperationReady(ctx, opIdx)
+	require.Error(t, err)
+	for chainSelector := range proposal.ChainMetadata {
+		err = tExecutable.IsChainReady(ctx, chainSelector)
+		require.Error(t, err)
+	}
 
 	// Check the state of the timelock contract
 	for _, role := range targetRoles {