sql: add metrics counting SQL-level automatic retries

Add metrics counting the number of automatic retries of transactions and
statements within conn executor.

Informs: #145377

Release note (sql change): Add new metrics `sql.txn.auto_retry.count`
and `sql.statements.auto_retry.count` which count the number of
automatic retries of SQL transactions and statements, respectively,
within the database.

These metrics differ from the related `txn.restarts.*` metrics in that
the latter count retryable errors emitted by the KV layer of the
database, which must be retried, and the former count auto-retry actions
taken by the SQL layer of the database in response to some of those
retryable errors.

Author: michae2

docs/generated/metrics/metrics.yaml

@@ -8823,6 +8823,22 @@ layers:
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
+    - name: sql.statements.auto_retry.count
+      exported_name: sql_statements_auto_retry_count
+      description: Number of SQL statement automatic retries
+      y_axis_label: SQL Statements
+      type: COUNTER
+      unit: COUNT
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
+    - name: sql.statements.auto_retry.count.internal
+      exported_name: sql_statements_auto_retry_count_internal
+      description: Number of SQL statement automatic retries (internal queries)
+      y_axis_label: SQL Internal Statements
+      type: COUNTER
+      unit: COUNT
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
     - name: sql.stats.activity.update.latency
       exported_name: sql_stats_activity_update_latency
       description: The latency of updates made by the SQL activity updater job. Includes failed update attempts
@@ -8991,6 +9007,22 @@ layers:
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
+    - name: sql.txn.auto_retry.count
+      exported_name: sql_txn_auto_retry_count
+      description: Number of SQL transaction automatic retries
+      y_axis_label: SQL Transactions
+      type: COUNTER
+      unit: COUNT
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
+    - name: sql.txn.auto_retry.count.internal
+      exported_name: sql_txn_auto_retry_count_internal
+      description: Number of SQL transaction automatic retries (internal queries)
+      y_axis_label: SQL Internal Statements
+      type: COUNTER
+      unit: COUNT
+      aggregation: AVG
+      derivative: NON_NEGATIVE_DERIVATIVE
     - name: sql.txn.begin.started.count
       exported_name: sql_txn_begin_started_count
       description: Number of SQL transaction BEGIN statements started

pkg/sql/conn_executor.go

@@ -616,6 +616,8 @@ func makeMetrics(internal bool, sv *settings.Values) Metrics {
 			TransactionTimeoutCount:           metric.NewCounter(getMetricMeta(MetaTransactionTimeout, internal)),
 			FullTableOrIndexScanCount:         aggmetric.NewSQLCounter(getMetricMeta(MetaFullTableOrIndexScan, internal)),
 			FullTableOrIndexScanRejectedCount: metric.NewCounter(getMetricMeta(MetaFullTableOrIndexScanRejected, internal)),
+			TxnRetryCount:                     metric.NewCounter(getMetricMeta(MetaTxnRetry, internal)),
+			StatementRetryCount:               metric.NewCounter(getMetricMeta(MetaStatementRetry, internal)),
 		},
 		StartedStatementCounters:  makeStartedStatementCounters(internal),
 		ExecutedStatementCounters: makeExecutedStatementCounters(internal),

pkg/sql/conn_executor_exec.go

@@ -2811,6 +2811,7 @@ func (ex *connExecutor) dispatchReadCommittedStmtToExecutionEngine(
 			ppInfo.dispatchReadCommittedStmtToExecutionEngine.autoRetryStmtReason = p.autoRetryStmtReason
 			ppInfo.dispatchReadCommittedStmtToExecutionEngine.autoRetryStmtCounter = p.autoRetryStmtCounter
 		}
+		ex.metrics.EngineMetrics.StatementRetryCount.Inc(1)
 	}
 	return nil
 }
@@ -4398,6 +4399,7 @@ func (ex *connExecutor) recordTransactionFinish(
 		ex.sessionData().Database, ex.sessionData().ApplicationName)
 	ex.metrics.EngineMetrics.SQLTxnLatency.RecordValue(elapsedTime.Nanoseconds(),
 		ex.sessionData().Database, ex.sessionData().ApplicationName)
+	ex.metrics.EngineMetrics.TxnRetryCount.Inc(int64(ex.state.mu.autoRetryCounter))
 
 	ex.txnIDCacheWriter.Record(contentionpb.ResolvedTxnID{
 		TxnID:            ev.txnID,

pkg/sql/exec_util.go

@@ -1348,6 +1348,18 @@ var (
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
+	MetaTxnRetry = metric.Metadata{
+		Name:        "sql.txn.auto_retry.count",
+		Help:        "Number of SQL transaction automatic retries",
+		Measurement: "SQL Transactions",
+		Unit:        metric.Unit_COUNT,
+	}
+	MetaStatementRetry = metric.Metadata{
+		Name:        "sql.statements.auto_retry.count",
+		Help:        "Number of SQL statement automatic retries",
+		Measurement: "SQL Statements",
+		Unit:        metric.Unit_COUNT,
+	}
 )
 
 func getMetricMeta(meta metric.Metadata, internal bool) metric.Metadata {

pkg/sql/executor_statement_metrics.go

@@ -75,6 +75,14 @@ type EngineMetrics struct {
 	// FullTableOrIndexScanRejectedCount counts the number of queries that were
 	// rejected because of the `disallow_full_table_scans` guardrail.
 	FullTableOrIndexScanRejectedCount *metric.Counter
+
+	// TxnRetryCount counts the number of automatic transaction retries that
+	// have occurred.
+	TxnRetryCount *metric.Counter
+
+	// StatementRetryCount counts the number of automatic statement retries that
+	// have occurred under READ COMMITTED isolation.
+	StatementRetryCount *metric.Counter
 }
 
 // EngineMetrics implements the metric.Struct interface.