Prune two workflow update related metrics in favor of logs w/ namespace (#9269)

## What changed?
Remove `invalid_state_transition_workflow_update_message` and
`workflow_update_registry_size_limited` metrics. Add the `namespace` tag
to the logs/softasserts in the `instrumentation` methods the metrics are
used for.

## Why?
The logs will give us the same information and the namespace without
cardinality concerns. These metrics are not expected to fire frequently.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

## Potential risks
Minimal, metrics changes only.

Author: spkane31

common/metrics/metric_defs.go

@@ -882,35 +882,34 @@ var (
 		"activity_schedule_to_close_latency",
 		WithDescription("Duration of activity execution from scheduled time to terminal state. Includes retries and backoffs."),
 	)
-	ActivitySuccess                                      = NewCounterDef("activity_success", WithDescription("Number of activities that succeeded (doesn't include retries)."))
-	ActivityFail                                         = NewCounterDef("activity_fail", WithDescription("Number of activities that failed and won't be retried anymore."))
-	ActivityTaskFail                                     = NewCounterDef("activity_task_fail", WithDescription("Number of activity task failures (includes retries)."))
-	ActivityCancel                                       = NewCounterDef("activity_cancel", WithDescription("Number of activities that are cancelled."))
-	ActivityTerminate                                    = NewCounterDef("activity_terminate", WithDescription("Number of activities that are terminated."))
-	ActivityTaskTimeout                                  = NewCounterDef("activity_task_timeout", WithDescription("Number of activity task timeouts (including retries)."))
-	ActivityTimeout                                      = NewCounterDef("activity_timeout", WithDescription("Number of terminal activity timeouts."))
-	ActivityPayloadSize                                  = NewCounterDef("activity_payload_size", WithDescription("Size of activity payloads in bytes."))
-	AckLevelUpdateCounter                                = NewCounterDef("ack_level_update")
-	AckLevelUpdateFailedCounter                          = NewCounterDef("ack_level_update_failed")
-	CommandCounter                                       = NewCounterDef("command")
-	MessageTypeRequestWorkflowExecutionUpdateCounter     = NewCounterDef("request_workflow_update_message")
-	MessageTypeAcceptWorkflowExecutionUpdateCounter      = NewCounterDef("accept_workflow_update_message")
-	MessageTypeRespondWorkflowExecutionUpdateCounter     = NewCounterDef("respond_workflow_update_message")
-	MessageTypeRejectWorkflowExecutionUpdateCounter      = NewCounterDef("reject_workflow_update_message")
-	InvalidStateTransitionWorkflowExecutionUpdateCounter = NewCounterDef("invalid_state_transition_workflow_update_message")
-	WorkflowExecutionUpdateRegistrySize                  = NewBytesHistogramDef("workflow_update_registry_size")
-	WorkflowExecutionUpdateRegistrySizeLimited           = NewCounterDef("workflow_update_registry_size_limited")
-	WorkflowExecutionUpdateRequestRateLimited            = NewCounterDef("workflow_update_request_rate_limited")
-	WorkflowExecutionUpdateTooMany                       = NewCounterDef("workflow_update_request_too_many")
-	WorkflowExecutionUpdateAborted                       = NewCounterDef("workflow_update_aborted")
-	WorkflowExecutionUpdateSentToWorker                  = NewCounterDef("workflow_update_sent_to_worker")
-	WorkflowExecutionUpdateSentToWorkerAgain             = NewCounterDef("workflow_update_sent_to_worker_again")
-	WorkflowExecutionUpdateWaitStageAccepted             = NewCounterDef("workflow_update_wait_stage_accepted")
-	WorkflowExecutionUpdateWaitStageCompleted            = NewCounterDef("workflow_update_wait_stage_completed")
-	WorkflowExecutionUpdateClientTimeout                 = NewCounterDef("workflow_update_client_timeout")
-	WorkflowExecutionUpdateServerTimeout                 = NewCounterDef("workflow_update_server_timeout")
-	SpeculativeWorkflowTaskCommits                       = NewCounterDef("speculative_workflow_task_commits")
-	SpeculativeWorkflowTaskRollbacks                     = NewCounterDef("speculative_workflow_task_rollbacks")
+	ActivitySuccess                                  = NewCounterDef("activity_success", WithDescription("Number of activities that succeeded (doesn't include retries)."))
+	ActivityFail                                     = NewCounterDef("activity_fail", WithDescription("Number of activities that failed and won't be retried anymore."))
+	ActivityTaskFail                                 = NewCounterDef("activity_task_fail", WithDescription("Number of activity task failures (includes retries)."))
+	ActivityCancel                                   = NewCounterDef("activity_cancel", WithDescription("Number of activities that are cancelled."))
+	ActivityTerminate                                = NewCounterDef("activity_terminate", WithDescription("Number of activities that are terminated."))
+	ActivityTaskTimeout                              = NewCounterDef("activity_task_timeout", WithDescription("Number of activity task timeouts (including retries)."))
+	ActivityTimeout                                  = NewCounterDef("activity_timeout", WithDescription("Number of terminal activity timeouts."))
+	ActivityPayloadSize                              = NewCounterDef("activity_payload_size", WithDescription("Size of activity payloads in bytes."))
+	AckLevelUpdateCounter                            = NewCounterDef("ack_level_update")
+	AckLevelUpdateFailedCounter                      = NewCounterDef("ack_level_update_failed")
+	CommandCounter                                   = NewCounterDef("command")
+	MessageTypeRequestWorkflowExecutionUpdateCounter = NewCounterDef("request_workflow_update_message")
+	MessageTypeAcceptWorkflowExecutionUpdateCounter  = NewCounterDef("accept_workflow_update_message")
+	MessageTypeRespondWorkflowExecutionUpdateCounter = NewCounterDef("respond_workflow_update_message")
+	MessageTypeRejectWorkflowExecutionUpdateCounter  = NewCounterDef("reject_workflow_update_message")
+	WorkflowExecutionUpdateRegistrySize              = NewBytesHistogramDef("workflow_update_registry_size")
+	WorkflowExecutionUpdateRegistrySizeLimited       = NewCounterDef("workflow_update_registry_size_limited")
+	WorkflowExecutionUpdateRequestRateLimited        = NewCounterDef("workflow_update_request_rate_limited")
+	WorkflowExecutionUpdateTooMany                   = NewCounterDef("workflow_update_request_too_many")
+	WorkflowExecutionUpdateAborted                   = NewCounterDef("workflow_update_aborted")
+	WorkflowExecutionUpdateSentToWorker              = NewCounterDef("workflow_update_sent_to_worker")
+	WorkflowExecutionUpdateSentToWorkerAgain         = NewCounterDef("workflow_update_sent_to_worker_again")
+	WorkflowExecutionUpdateWaitStageAccepted         = NewCounterDef("workflow_update_wait_stage_accepted")
+	WorkflowExecutionUpdateWaitStageCompleted        = NewCounterDef("workflow_update_wait_stage_completed")
+	WorkflowExecutionUpdateClientTimeout             = NewCounterDef("workflow_update_client_timeout")
+	WorkflowExecutionUpdateServerTimeout             = NewCounterDef("workflow_update_server_timeout")
+	SpeculativeWorkflowTaskCommits                   = NewCounterDef("speculative_workflow_task_commits")
+	SpeculativeWorkflowTaskRollbacks                 = NewCounterDef("speculative_workflow_task_rollbacks")
 
 	ActivityEagerExecutionCounter = NewCounterDef("activity_eager_execution")
 	// WorkflowEagerExecutionCounter is emitted any time eager workflow start is requested.

service/history/workflow/context.go

@@ -968,6 +968,7 @@ func (c *ContextImpl) UpdateRegistry(ctx context.Context) update.Registry {
 
 		c.updateRegistry = update.NewRegistry(
 			c.MutableState,
+			update.WithNamespace(nsName),
 			update.WithLogger(c.logger),
 			update.WithMetrics(c.metricsHandler),
 			update.WithTracerProvider(trace.SpanFromContext(ctx).TracerProvider()),

service/history/workflow/update/registry.go

@@ -136,6 +136,13 @@ func WithTotalLimitSuggestCAN(f func() float64) Option {
 	}
 }
 
+// WithNamespace sets the namespace name to be used in Registry metrics and logs.
+func WithNamespace(ns string) Option {
+	return func(r *registry) {
+		r.instrumentation.namespace = ns
+	}
+}
+
 // WithLogger sets the log.Logger to be used by Registry and its Updates.
 func WithLogger(l log.Logger) Option {
 	return func(r *registry) {

service/history/workflow/update/util.go

@@ -14,9 +14,10 @@ import (
 
 type (
 	instrumentation struct {
-		log     log.Logger
-		metrics metrics.Handler
-		tracer  trace.Tracer
+		log       log.Logger
+		metrics   metrics.Handler
+		tracer    trace.Tracer
+		namespace string
 	}
 )
 
@@ -50,11 +51,12 @@ func (i *instrumentation) countRateLimited() {
 
 func (i *instrumentation) countRegistrySizeLimited(updateCount, registrySize, payloadSize int) {
 	i.oneOf(metrics.WorkflowExecutionUpdateRegistrySizeLimited.Name())
-	// TODO: remove log once limit is enforced everywhere
 	i.log.Warn("update registry size limit reached",
 		tag.Int("registry-size", registrySize),
 		tag.Int("payload-size", payloadSize),
-		tag.Int("update-count", updateCount))
+		tag.Int("update-count", updateCount),
+		tag.String("namespace", i.namespace),
+	)
 }
 
 func (i *instrumentation) countTooMany() {
@@ -79,14 +81,14 @@ func (i *instrumentation) countSentAgain() {
 }
 
 func (i *instrumentation) invalidStateTransition(updateID string, msg proto.Message, state state) {
-	i.oneOf(metrics.InvalidStateTransitionWorkflowExecutionUpdateCounter.Name())
 	softassert.Fail(
 		i.log,
 		"invalid state transition attempted",
 		tag.ComponentWorkflowUpdate,
 		tag.String("update-id", updateID),
 		tag.String("message", fmt.Sprintf("%T", msg)),
 		tag.Stringer("state", state),
+		tag.String("namespace", i.namespace),
 	)
 }