Add metrics and keep app alive (#50)

* - Add metrics for sucessful and failed queries
- Add keepAlive flag to not kill the app when the query fail

* - Add metric names prefix
- Remove duplicated matric calls
- Add extra label to update duration metic

Author: guerremdq

main.go

@@ -26,13 +26,29 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	counterSources = flagx.StringArray{}
-	gaugeSources   = flagx.StringArray{}
-	project        = flag.String("project", "", "GCP project name.")
-	refresh        = flag.Duration("refresh", 5*time.Minute, "Interval between updating metrics.")
+	gaugeSources = flagx.StringArray{}
+	project      = flag.String("project", "", "GCP project name.")
+	refresh      = flag.Duration("refresh", 5*time.Minute, "Interval between updating metrics.")
+	keepAlive    = flag.Bool("keepAlive", false, "Keep the process alive even if query fails to execute.")
+
+	successFilesCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "bqx_success_files_executed_total",
+		Help: "The total number of successfully executed files",
+	}, []string{"filename"})
+
+	failedFilesCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "bqx_failed_files_executed_total",
+		Help: "The total number of failed executed files",
+	}, []string{"filename"})
+	updateDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "bqx_query_runtime_duration_seconds",
+		Help:    "Duration taken for updating files",
+		Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600},
+	}, []string{"filename", "status"})
 )
 
 func init() {
@@ -69,12 +85,13 @@ func fileToQuery(filename string, vars map[string]string) string {
 	return q
 }
 
-func reloadRegisterUpdate(client *bigquery.Client, files []setup.File, vars map[string]string) {
+func reloadRegisterUpdate(client *bigquery.Client, files []setup.File, vars map[string]string, keepAlive bool) {
 	var wg sync.WaitGroup
 	for i := range files {
 		wg.Add(1)
 		go func(f *setup.File) {
 			modified, err := f.IsModified()
+			start := time.Now()
 			if modified && err == nil {
 				c := sql.NewCollector(
 					newRunner(client), prometheus.GaugeValue,
@@ -85,14 +102,21 @@ func reloadRegisterUpdate(client *bigquery.Client, files []setup.File, vars map[
 				// uses the same name but changes the metrics reported. Because
 				// this cannot be recovered, we use rtx.Must to exit and allow
 				// the runtime environment to restart.
-				rtx.Must(f.Register(c), "Failed to register collector: aborting")
+				err = f.Register(c)
+				if !keepAlive {
+					rtx.Must(f.Register(c), "Failed to register collector: aborting")
+				}
 			} else {
-				start := time.Now()
 				err = f.Update()
 				log.Println("Updating:", fileToMetric(f.Name), time.Since(start))
 			}
 			if err != nil {
+				failedFilesCounter.WithLabelValues(fileToMetric(f.Name)).Inc()
+				updateDuration.WithLabelValues(fileToMetric(f.Name), "failed").Observe(time.Since(start).Seconds())
 				log.Println("Error:", f.Name, err)
+			} else {
+				successFilesCounter.WithLabelValues(fileToMetric(f.Name)).Inc()
+				updateDuration.WithLabelValues(fileToMetric(f.Name), "success").Observe(time.Since(start).Seconds())
 			}
 			wg.Done()
 		}(&files[i])
@@ -125,7 +149,7 @@ func main() {
 	}
 
 	for mainCtx.Err() == nil {
-		reloadRegisterUpdate(client, files, vars)
+		reloadRegisterUpdate(client, files, vars, *keepAlive)
 		sleepUntilNext(*refresh)
 	}
 }