MCP server: Add usage metrics (#711)

- Instrument every MCP tool with the new `shared.TrackToolExecution`
helper so each handler automatically emits `tools.<tool_name>.count_*`
and `duration_ms` metrics, marking success via `tracker.MarkSuccess()`
just before returning.

- Tag metrics with human-readable organization names by introducing a
cached org-name resolver (pkg/tools/internal/shared/org_resolver.go)
wired through `tools.ConfigureMetrics(provider)` (called once during
server bootstrap) and backed by the organization gRPC client + stub
support.

- Document the workflow in `AGENTS.md` so future tools follow the same
pattern, and add targeted unit tests for both the metrics helper and
resolver caching logic.

Author: DamjanBecirovic

mcp_server/AGENTS.md

@@ -17,3 +17,9 @@ Follow Conventional Commits (e.g., `feat(auth):`, `fix(front):`, `docs:`) and ke
 
 ## Security & Configuration Tips
 Surface dependency issues early with `make check.ex.deps`, `make check.go.deps`, and `make check.docker`. Store secrets in local `.env` files; never commit credentials. Runtime configuration reads internal gRPC endpoints from `INTERNAL_API_URL_PLUMBER`, `INTERNAL_API_URL_JOB`, `INTERNAL_API_URL_LOGHUB`, and `INTERNAL_API_URL_LOGHUB2`, falling back to legacy `MCP_*` variables. Export `DOCKER_BUILDKIT=1` to mirror CI Docker builds.
+
+## MCP Tool Metrics Quickstart
+- Shared instrumentation lives in `pkg/tools/internal/shared/metrics.go`. Inside every MCP tool handler, create a tracker with `tracker := shared.TrackToolExecution(ctx, "<tool_name>", orgID)`, `defer tracker.Cleanup()`, and call `tracker.MarkSuccess()` right before you return a successful result.
+- Organization tags resolve via `pkg/tools/internal/shared/org_resolver.go`. The resolver is configured once through `tools.ConfigureMetrics(provider)` during server bootstrap, so new tools only need to supply the org ID (or `""` when not applicable).
+- For org-agnostic tools (e.g., `organizations_list`), pass an empty org ID so we still emit `count_*` and `duration_ms` metrics without tags.
+- Following this pattern ensures every tool automatically publishes `tools.<tool_name>.count_total|count_passed|count_failed` and `tools.<tool_name>.duration_ms` metrics, with human-readable org tags whenever available, keeping dashboards consistent without extra boilerplate.

mcp_server/cmd/mcp_server/main.go

@@ -19,22 +19,31 @@ import (
 
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/internalapi"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/logging"
+	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/jobs"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/organizations"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/pipelines"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/projects"
 	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/workflows"
+	"github.com/semaphoreio/semaphore/mcp_server/pkg/watchman"
 	support "github.com/semaphoreio/semaphore/mcp_server/test/support"
 )
 
 var (
-	versionFlag = flag.Bool("version", false, "print the server version and exit")
-	nameFlag    = flag.String("name", "semaphore-mcp-server", "implementation name advertised to MCP clients")
-	httpAddr    = flag.String("http", ":3001", "address to serve the streamable MCP transport")
-	version     = "0.1.0"
+	versionFlag      = flag.Bool("version", false, "print the server version and exit")
+	nameFlag         = flag.String("name", "semaphore-mcp-server", "implementation name advertised to MCP clients")
+	httpAddr         = flag.String("http", ":3001", "address to serve the streamable MCP transport")
+	version          = "0.1.0"
+	metricsNamespace = os.Getenv("METRICS_NAMESPACE")
+)
+
+const (
+	metricService = "mcp-server"
 )
 
 func main() {
+	watchman.Configure(fmt.Sprintf("%s.%s", metricService, metricsNamespace))
+
 	flag.Parse()
 
 	if *versionFlag {
@@ -91,6 +100,10 @@ func main() {
 		}()
 	}
 
+	// Configure organization name resolver for metrics tagging.
+	// This must be called once before registering tools that emit metrics.
+	tools.ConfigureMetrics(provider)
+
 	organizations.Register(srv, provider)
 	projects.Register(srv, provider)
 	workflows.Register(srv, provider)

mcp_server/pkg/tools/config.go

@@ -0,0 +1,16 @@
+package tools
+
+import (
+	"github.com/semaphoreio/semaphore/mcp_server/pkg/internalapi"
+	"github.com/semaphoreio/semaphore/mcp_server/pkg/tools/internal/shared"
+)
+
+// ConfigureMetrics initializes global metrics configuration for all tools.
+// This should be called once during server initialization before registering any tools.
+//
+// It configures the organization name resolver used for metrics tagging,
+// allowing metrics to be tagged with human-readable organization names
+// instead of UUIDs.
+func ConfigureMetrics(provider internalapi.Provider) {
+	shared.ConfigureDefaultOrgResolver(provider)
+}

mcp_server/pkg/tools/internal/shared/metrics.go

@@ -0,0 +1,190 @@
+package shared
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	watchman "github.com/renderedtext/go-watchman"
+
+	"github.com/semaphoreio/semaphore/mcp_server/pkg/logging"
+)
+
+var (
+	watchmanBenchmarkWithTags = watchman.BenchmarkWithTags
+	watchmanIncrementWithTags = watchman.IncrementWithTags
+)
+
+// ToolMetrics emits Watchman metrics for a specific tool invocation.
+type ToolMetrics struct {
+	base string
+	tags []string
+}
+
+// NewToolMetrics prepares a metrics emitter scoped to a tool and optional organization ID.
+func NewToolMetrics(ctx context.Context, toolName, orgID string) *ToolMetrics {
+	resolver := getOrgNameResolver()
+	return newToolMetricsWithResolver(ctx, toolName, orgID, resolver)
+}
+
+func newToolMetricsWithResolver(ctx context.Context, toolName, orgID string, resolver OrgNameResolver) *ToolMetrics {
+	name := strings.TrimSpace(toolName)
+	if name == "" {
+		return nil
+	}
+
+	base := "tools." + name
+	tags := make([]string, 0, 1)
+
+	if tag := resolveOrgTag(ctx, orgID, resolver); tag != "" {
+		tags = append(tags, tag)
+	}
+
+	return &ToolMetrics{
+		base: base,
+		tags: tags,
+	}
+}
+
+// IncrementTotal bumps the total execution counter.
+func (tm *ToolMetrics) IncrementTotal() {
+	tm.increment("count_total")
+}
+
+// IncrementSuccess bumps the successful execution counter.
+func (tm *ToolMetrics) IncrementSuccess() {
+	tm.increment("count_passed")
+}
+
+// IncrementFailure bumps the failed execution counter.
+func (tm *ToolMetrics) IncrementFailure() {
+	tm.increment("count_failed")
+}
+
+// TrackDuration submits the elapsed duration since start.
+func (tm *ToolMetrics) TrackDuration(start time.Time) {
+	if tm == nil {
+		return
+	}
+
+	name := tm.metricName("duration_ms")
+	if err := watchmanBenchmarkWithTags(start, name, tm.tags); err != nil {
+		logMetricError(name, err)
+	}
+}
+
+func (tm *ToolMetrics) increment(suffix string) {
+	if tm == nil {
+		return
+	}
+	name := tm.metricName(suffix)
+	if err := watchmanIncrementWithTags(name, tm.tags); err != nil {
+		logMetricError(name, err)
+	}
+}
+
+func (tm *ToolMetrics) metricName(suffix string) string {
+	if tm == nil {
+		return suffix
+	}
+	if suffix == "" {
+		return tm.base
+	}
+	return tm.base + "." + suffix
+}
+
+func resolveOrgTag(ctx context.Context, orgID string, resolver OrgNameResolver) string {
+	orgID = strings.TrimSpace(orgID)
+	if orgID == "" {
+		return ""
+	}
+
+	value := orgID
+	if resolver != nil {
+		if name, err := resolver.Resolve(ctx, orgID); err == nil {
+			name = strings.TrimSpace(name)
+			if name != "" {
+				value = name
+			}
+		} else {
+			logging.ForComponent("metrics").
+				WithError(err).
+				WithField("orgId", orgID).
+				Debug("failed to resolve organization name for metrics")
+		}
+	}
+
+	return sanitizeMetricTag("org_" + value)
+}
+
+func sanitizeMetricTag(value string) string {
+	value = strings.TrimSpace(strings.ToLower(value))
+	if value == "" {
+		return ""
+	}
+	value = strings.ReplaceAll(value, " ", "_")
+	return value
+}
+
+func logMetricError(metric string, err error) {
+	if err == nil {
+		return
+	}
+	logging.ForComponent("metrics").
+		WithError(err).
+		WithField("metric", metric).
+		Debug("failed to submit Watchman metric")
+}
+
+// ToolExecutionTracker helps track tool execution metrics with a consistent pattern.
+// It provides methods to mark success and automatically handles cleanup via defer.
+type ToolExecutionTracker struct {
+	metrics *ToolMetrics
+	start   time.Time
+	success *bool
+}
+
+// TrackToolExecution creates a new tracker for monitoring tool execution metrics.
+// It automatically increments the total counter and sets up cleanup logic.
+//
+// Usage:
+//
+//	tracker := shared.TrackToolExecution(ctx, toolName, orgID)
+//	defer tracker.Cleanup()
+//	// ... tool logic ...
+//	tracker.MarkSuccess() // Call before successful return
+func TrackToolExecution(ctx context.Context, toolName, orgID string) *ToolExecutionTracker {
+	metrics := NewToolMetrics(ctx, toolName, orgID)
+	if metrics != nil {
+		metrics.IncrementTotal()
+	}
+
+	success := false
+	return &ToolExecutionTracker{
+		metrics: metrics,
+		start:   time.Now(),
+		success: &success,
+	}
+}
+
+// MarkSuccess marks the tool execution as successful.
+// This should be called just before returning a successful result.
+func (t *ToolExecutionTracker) MarkSuccess() {
+	if t != nil && t.success != nil {
+		*t.success = true
+	}
+}
+
+// Cleanup emits duration and success/failure metrics.
+// This should be called via defer immediately after creating the tracker.
+func (t *ToolExecutionTracker) Cleanup() {
+	if t == nil || t.metrics == nil {
+		return
+	}
+	t.metrics.TrackDuration(t.start)
+	if t.success != nil && *t.success {
+		t.metrics.IncrementSuccess()
+	} else {
+		t.metrics.IncrementFailure()
+	}
+}