fix: Improve telemetry allocations (#2185)

This PR: 
- moves OTEL counter initialization to a single place
- removes startTime and endTime non-counter metrics
- introduces duration metric
- removes client-level cardinality from the metrics, preventing memory accumulation

Author: murarustefaan

scheduler/metrics/duration.go

@@ -0,0 +1,38 @@
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+type durationMeasurement struct {
+	startTime time.Time
+	started   bool
+	duration  time.Duration
+	sem       sync.Mutex
+}
+
+func (dm *durationMeasurement) Start(start time.Time) {
+	// If we have already started, don't start again. This can happen for relational tables that are resolved multiple times (per parent resource)
+	dm.sem.Lock()
+	defer dm.sem.Unlock()
+	if dm.started {
+		return
+	}
+
+	dm.started = true
+	dm.startTime = start
+}
+
+// End calculates, updates and returns the delta duration for updating OTEL counters.
+func (dm *durationMeasurement) End(end time.Time) time.Duration {
+	var delta time.Duration
+	newDuration := end.Sub(dm.startTime)
+
+	dm.sem.Lock()
+	defer dm.sem.Unlock()
+
+	delta = newDuration - dm.duration
+	dm.duration = newDuration
+	return delta
+}