feat: improve metrics (#549)

Author: masontikhonov

venona/VERSION

@@ -1 +1 @@
-2.0.2
+2.0.3

venona/pkg/agent/agent.go

@@ -262,14 +262,19 @@ func (a *Agent) reportTaskStatus(ctx context.Context, taskDef task.Task, err err
 }
 
 func (a *Agent) getTasks(ctx context.Context) (task.Tasks, []*workflow.Workflow) {
+	metrics.IncGetTasksRequests()
 	tasks := a.pullTasks(ctx)
 	return a.splitTasks(tasks)
 }
 
 func (a *Agent) pullTasks(ctx context.Context) task.Tasks {
 	start := time.Now()
 	tasks, err := a.cf.Tasks(ctx)
-	metrics.ObserveGetTasks(start)
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+	metrics.ObserveGetTasks(start, status)
 
 	if err != nil {
 		a.log.Error("Failed pulling tasks", "error", err)

venona/pkg/metrics/metrics.go

@@ -56,11 +56,16 @@ var (
 		Name:      "queue_size",
 		Help:      "Current number of waiting tasks",
 	})
-	getTasksDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+	getTasksDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: runnerNamespace,
 		Name:      "get_tasks_duration_sec",
 		Help:      "How long each GetTasks request takes (seconds)",
-		Buckets:   []float64{0.25, 0.5, 1, 2, 3, 6},
+		Buckets:   []float64{0.25, 0.5, 1, 2, 3, 6, 12, 30, 60},
+	}, []string{"status"})
+	getTasksRequests = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: runnerNamespace,
+		Name:      "get_tasks_requests",
+		Help:      "Number of GetTasks requests",
 	})
 	handlingTimeSinceCreation = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: runnerNamespace,
@@ -136,10 +141,15 @@ func IncWorkflowRetries(podName string) {
 	wfTaskRetries.With(labels).Inc()
 }
 
-func ObserveGetTasks(start time.Time) {
+func ObserveGetTasks(start time.Time, status string) {
 	end := time.Now()
 	diff := end.Sub(start)
-	getTasksDuration.Observe(diff.Seconds())
+	labels := prometheus.Labels{"status": status}
+	getTasksDuration.With(labels).Observe(diff.Seconds())
+}
+
+func IncGetTasksRequests() {
+	getTasksRequests.Inc()
 }
 
 func ObserveAgentTaskMetrics(agentType string, sinceCreation, inRunner, processed time.Duration) {

venonactl/VERSION

@@ -1 +1 @@
-2.0.2
+2.0.3