feat(cadence-matching): Add drain observer for SD executor client (#7751)

<!-- 1-2 line summary of WHAT changed technically:
- Always link the relevant projects GitHub issue, unless it is a minor
bugfix
- Good: "Modified FailoverDomain mapper to allow null ActiveClusterName
#320"
- Bad: "added nil check" -->
**What changed?**
This PR adds DrainObserver field to resource.Params to inject a
DrainSignalObserver into the matching service at initialization.
We wire the drain observer through matching.Service -> handler.NewEngine
-> setupExecutor -> executorclient.Params, enabling the executor
client's heartbeat loop to react to infrastructure drain/undrain
signals.

<!-- Your goal is to provide all the required context for a future
maintainer
to understand the reasons for making this change (see
https://cbea.ms/git-commit/#why-not-how).
How did this work previously (and what was wrong with it)? What has
changed, and why did you solve it
this way?
- Good: "Active-active domains have independent cluster attributes per
region. Previously,
modifying cluster attributes required spedifying the default
ActiveClusterName which
updates the global domain default. This prevents operators from updating
regional
configurations without affecting the primary cluster designation. This
change allows
  attribute updates to be independent of active cluster selection."
- Bad: "Improves domain handling" -->
**Why?**
The shard-distributor executor client already supports drain-aware
behavior via the DrainObserver field on executorclient.Params. However,
there was no way to wire a DrainSignalObserver into the matching
service's executor client because resource.Params didn't have it.
PR follows the same pattern used by ShardDistributorMatchingConfig
initialization.

<!-- Include specific test commands and setup. Please include the exact
commands such that
another maintainer or contributor can reproduce the test steps taken. 
- e.g Unit test commands with exact invocation
`go test -v ./common/types/mapper/proto -run TestFailoverDomainRequest`
- For integration tests include setup steps and test commands
Example: "Started local server with `./cadence start`, then ran `make
test_e2e`"
- For local simulation testing include setup steps for the server and
how you ran the tests
- Good: Full commands that reviewers can copy-paste to verify
- Bad: "Tested locally" or "Added tests" -->
**How did you test it?**
Unit tests with `go test -v ./service/matching/handler/`

<!-- If there are risks that the release engineer should know about
document them here.
For example:
- Has an API/IDL been modified? Is it backwards/forwards compatible? If
not, what are the repecussions?
- Has a schema change been introduced? Is it possible to roll back?
- Has a feature flag been re-used for a new purpose? 
- Is there a potential performance concern? Is the change modifying core
task processing logic?
- If truly N/A, you can mark it as such -->
**Potential risks**
Since drainObserver field is optional there wouldn't be any changes in
cadence-matching's behaviour.

<!-- If this PR completes a user facing feature or changes functionality
add release notes here.
Your release notes should allow a user and the release engineer to
understand the changes with little context.
Always ensure that the description contains a link to the relevant
GitHub issue. -->
**Release notes**
N/A

<!-- Consider whether this change requires documentation updates in the
Cadence-Docs repo
- If yes: mention what needs updating (or link to docs PR in
cadence-docs repo)
- If in doubt, add a note about potential doc needs
- Only mark N/A if you're certain no docs are affected -->
**Documentation Changes**
N/A

---

## Reviewer Validation

**PR Description Quality** (check these before reviewing code):

- [ ] **"What changed"** provides a clear 1-2 line summary
  - [ ] Project Issue is linked
- [ ] **"Why"** explains the full motivation with sufficient context
- [ ] **Testing is documented:**
- [ ] Unit test commands are included (with exact `go test` invocation)
- [ ] Integration test setup/commands included (if integration tests
were run)
  - [ ] Canary testing details included (if canary was mentioned)
- [ ] **Potential risks** section is thoughtfully filled out (or
legitimately N/A)
- [ ] **Release notes** included if this completes a user-facing feature
- [ ] **Documentation** needs are addressed (or noted if uncertain)

Signed-off-by: Gaziza Yestemirova <gaziza@uber.com>

Author: gazi-yestemirova

common/resource/params.go

@@ -101,5 +101,11 @@ type (
 
 		// ShardDistributorMatchingConfig is the config for shard distributor executor client in matching service
 		ShardDistributorMatchingConfig clientcommon.Config
+
+		// DrainObserver is an optional observer that signals when this instance is
+		// drained from service discovery.
+		// It is used by shard-distributor executor clients to
+		// gracefully stop processing during drains.
+		DrainObserver clientcommon.DrainSignalObserver
 	}
 )

service/matching/handler/engine.go

@@ -111,6 +111,7 @@ type (
 		timeSource                     clock.TimeSource
 		failoverNotificationVersion    int64
 		ShardDistributorMatchingConfig clientcommon.Config
+		drainObserver                  clientcommon.DrainSignalObserver
 	}
 )
 
@@ -140,6 +141,7 @@ func NewEngine(
 	timeSource clock.TimeSource,
 	shardDistributorClient executorclient.Client,
 	ShardDistributorMatchingConfig clientcommon.Config,
+	drainObserver clientcommon.DrainSignalObserver,
 ) Engine {
 	e := &matchingEngineImpl{
 		taskListRegistry:               tasklist.NewTaskListRegistry(metricsClient),
@@ -161,6 +163,7 @@ func NewEngine(
 		isolationState:                 isolationState,
 		timeSource:                     timeSource,
 		ShardDistributorMatchingConfig: ShardDistributorMatchingConfig,
+		drainObserver:                  drainObserver,
 	}
 
 	e.setupExecutor(shardDistributorClient)
@@ -215,6 +218,7 @@ func (e *matchingEngineImpl) setupExecutor(shardDistributorExecutorClient execut
 			"grpc":     fmt.Sprintf("%d", e.config.RPCConfig.GRPCPort),
 			"hostIP":   hostIP.String(),
 		},
+		DrainObserver: e.drainObserver,
 	}
 	executor, err := executorclient.NewExecutor[tasklist.ShardProcessor](params)
 	if err != nil {

service/matching/handler/engine_integration_test.go

@@ -194,6 +194,7 @@ func (s *matchingEngineSuite) newMatchingEngine(
 		s.mockTimeSource,
 		s.mockShardExecutorClient,
 		defaultSDExecutorConfig(),
+		nil,
 	).(*matchingEngineImpl)
 }
 

service/matching/handler/membership_test.go

@@ -139,6 +139,7 @@ func TestGetTaskListManager_OwnerShip(t *testing.T) {
 				mockTimeSource,
 				mockShardDistributorExecutorClient,
 				defaultSDExecutorConfig(),
+				nil,
 			).(*matchingEngineImpl)
 
 			resolverMock.EXPECT().Lookup(gomock.Any(), gomock.Any()).Return(

service/matching/service.go

@@ -45,6 +45,7 @@ type Service struct {
 	stopC                          chan struct{}
 	config                         *config.Config
 	ShardDistributorMatchingConfig clientcommon.Config
+	drainObserver                  clientcommon.DrainSignalObserver
 }
 
 // NewService builds a new cadence-matching service
@@ -84,6 +85,7 @@ func NewService(
 		config:                         serviceConfig,
 		stopC:                          make(chan struct{}),
 		ShardDistributorMatchingConfig: params.ShardDistributorMatchingConfig,
+		drainObserver:                  params.DrainObserver,
 	}, nil
 }
 
@@ -111,6 +113,7 @@ func (s *Service) Start() {
 		s.GetTimeSource(),
 		s.GetShardDistributorExecutorClient(),
 		s.ShardDistributorMatchingConfig,
+		s.drainObserver,
 	)
 
 	s.handler = handler.NewHandler(engine, s.config, s.GetDomainCache(), s.GetMetricsClient(), s.GetLogger(), s.GetThrottledLogger())