added counter-query parameter

.gitignore

@@ -1 +1,32 @@
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
 /prometheus-bigquery-exporter
+config.yaml
+/queries
+credentials.json

README.md

@@ -4,6 +4,9 @@
 
 An exporter for converting BigQuery results into Prometheus metrics.
 
+This Fork adds a config file for multiple queries and Counter type metrics
+
+
 ## Limitations: No historical values
 
 Prometheus collects the *current* status of a system as reported by an exporter.

configuration/example_config.yml

@@ -0,0 +1,9 @@
+project: "skyita-da-logging-noprod"
+gauge-queries:
+  - query:
+    file: "/queries/metric1_name.sql"
+    #refresh "*/5 * * * *" #every 5 minutes by default
+counter-queries:
+  - query:
+    file: "/queries/metric2_name.sql"
+    #refresh "30 9 * * *" #every day at 9:30

configuration/test/test_1.yml

@@ -0,0 +1,13 @@
+project: "abc"
+gauge-queries:
+  - query: "abc"
+    file: "/queries/metric1_name.sql"
+  - query: "def"
+    file: "/queries/metric2_name.sql"
+counter-queries:
+  - query: "abc_2"
+    file: "/queries/metric3_name.sql"
+  - query: "def_2"
+    file: "/queries/metric4_name.sql"
+  - query: "ghi_2"
+    file: "/queries/metric5_name.sql"

configuration/test/test_2.yml

@@ -0,0 +1,9 @@
+project: "abc"
+gauge-queries:
+counter-queries:
+  - query: "abc_2"
+    file: "/queries/metric3_name.sql"
+  - query: "def_2"
+    file: "/queries/metric4_name.sql"
+  - query: "ghi_2"
+    file: "/queries/metric5_name.sql"

configuration/test/test_3.yml

@@ -0,0 +1,7 @@
+project: "abc"
+gauge-queries:
+  - query: "abc"
+    file: "/queries/metric1_name.sql"
+  - query: "def"
+    file: "/queries/metric2_name.sql"
+counter-queries:

go.mod

@@ -41,3 +41,4 @@ require (
 	gopkg.in/yaml.v2 v2.3.0 // indirect
 	honnef.co/go/tools v0.0.1-2019.2.3 // indirect
 )
+

go.sum

@@ -103,8 +103,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/m-lab/go v1.4.0 h1:Au2Vt15+H8oOd3xYZFfW3gK86GRciKlfGJs/FzsJwK4=
 github.com/m-lab/go v1.4.0/go.mod h1:f22d1CtoFIho8yt0wPNYo0Lx5h8YfgRW4+1pzQTeQRw=
@@ -309,6 +311,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/config/config.go

@@ -0,0 +1,111 @@
+package config
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/m-lab/go/logx"
+	"github.com/spf13/afero"
+	"gopkg.in/yaml.v2"
+)
+
+var fs = afero.NewOsFs()
+
+type Config struct {
+	Name    string      `yaml:"-"`
+	stat    os.FileInfo `yaml:"-"`
+	Gauge   []Query     `yaml:"gauge-queries"`
+	Counter []Query     `yaml:"counter-queries"`
+}
+
+type Query struct {
+	Query string `yaml:"query"`
+	File  string `yaml:"file"`
+}
+
+func (cfg *Config) GetGaugeFiles() []string {
+
+	var paths []string
+	for _, path := range cfg.Gauge {
+
+		paths = append(paths, path.File)
+	}
+	return paths
+}
+
+func (cfg *Config) IsModified() (bool, error) {
+
+	var err error
+	if cfg.stat == nil {
+		cfg.stat, err = fs.Stat(cfg.Name)
+		logx.Debug.Println("IsModified:stat1:", cfg.Name, err)
+		// Return true on the first successful Stat(), or the error otherwise.
+		return err == nil, err
+	}
+	curr, err := fs.Stat(cfg.Name)
+	if err != nil {
+		log.Printf("Failed to stat %q: %v", cfg.Name, err)
+		return false, err
+	}
+	logx.Debug.Println("IsModified:stat2:", cfg.Name, curr.ModTime(), cfg.stat.ModTime(),
+		curr.ModTime().After(cfg.stat.ModTime()))
+	modified := curr.ModTime().After(cfg.stat.ModTime())
+	if modified {
+		// Update the stat cache to the latest version.
+		cfg.stat = curr
+	}
+	return modified, nil
+}
+
+func (cfg *Config) GetCounterFiles() []string {
+
+	var paths []string
+	for _, path := range cfg.Counter {
+
+		paths = append(paths, path.File)
+	}
+	return paths
+}
+
+func ReadConfigFile(path string) (*Config, error) {
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("something wrong during file opening: %s", err.Error())
+	}
+
+	defer f.Close()
+
+	var cfg Config
+	decoder := yaml.NewDecoder(f)
+	err = decoder.Decode(&cfg)
+	if err != nil {
+		return nil, fmt.Errorf("something wrong during configuration unmarshalling: %s", err.Error())
+	}
+
+	err = validate(&cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.Name = path
+	cfg.stat, err = fs.Stat(cfg.Name)
+	if err != nil {
+		return nil, fmt.Errorf("something wrong during file stat extraction: %s", err.Error())
+	}
+
+	return &cfg, nil
+}
+
+func validate(cfg *Config) error {
+
+	if len(cfg.Counter) == 0 {
+		return fmt.Errorf("no Counter parameters available")
+	}
+
+	if len(cfg.Gauge) == 0 {
+		return fmt.Errorf("no Gauge parameters available")
+	}
+	return nil
+}

internal/config/config_test.go

@@ -0,0 +1,64 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadConfigFile__good(t *testing.T) {
+
+	result, err := ReadConfigFile("../../configuration/test/test_1.yml")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(result.Gauge))
+
+	gaugeQuery0 := result.Gauge[0]
+	assert.Equal(t, "abc", gaugeQuery0.Query)
+	assert.Equal(t, "/queries/metric1_name.sql", gaugeQuery0.File)
+
+	gaugeQuery1 := result.Gauge[1]
+	assert.Equal(t, "def", gaugeQuery1.Query)
+	assert.Equal(t, "/queries/metric2_name.sql", gaugeQuery1.File)
+
+	assert.Equal(t, 3, len(result.Counter))
+
+	counterQuery0 := result.Counter[0]
+	assert.Equal(t, "abc_2", counterQuery0.Query)
+	assert.Equal(t, "/queries/metric3_name.sql", counterQuery0.File)
+
+	counterQuery1 := result.Counter[1]
+	assert.Equal(t, "def_2", counterQuery1.Query)
+	assert.Equal(t, "/queries/metric4_name.sql", counterQuery1.File)
+
+	counterQuery2 := result.Counter[2]
+	assert.Equal(t, "ghi_2", counterQuery2.Query)
+	assert.Equal(t, "/queries/metric5_name.sql", counterQuery2.File)
+}
+
+func TestReadConfigFile__empty_file(t *testing.T) {
+
+	_, err := ReadConfigFile("../../configuration/test/test_0.yml")
+	assert.NotNil(t, err)
+	assert.Equal(t, "something wrong during configuration unmarshalling: EOF", err.Error())
+}
+
+func TestReadConfigFile__no_counter_parameters(t *testing.T) {
+
+	_, err := ReadConfigFile("../../configuration/test/test_3.yml")
+	assert.NotNil(t, err)
+	assert.Equal(t, "no Counter parameters available", err.Error())
+}
+
+func TestReadConfigFile__no_gauge_parameters(t *testing.T) {
+
+	_, err := ReadConfigFile("../../configuration/test/test_2.yml")
+	assert.NotNil(t, err)
+	assert.Equal(t, "no Gauge parameters available", err.Error())
+}
+
+func TestReadConfigFile__wrong_path(t *testing.T) {
+
+	_, err := ReadConfigFile("../../not/existing/path")
+	assert.NotNil(t, err)
+	assert.Equal(t, "something wrong during file opening: open ../../not/existing/path: no such file or directory", err.Error())
+}

internal/setup/setup.go

@@ -71,6 +71,16 @@ func (f *File) Register(c *sql.Collector) error {
 	return c.RegisterErr
 }
 
+func (f *File) Unregister() error {
+
+	unregitered := prometheus.Unregister(f.c)
+	if !unregitered {
+		return fmt.Errorf("something wrong during unregistering of sql collector of file: %s", f.Name)
+	}
+	logx.Debug.Println("Unregister:", f.Name, f.c.RegisterErr)
+	return nil
+}
+
 // Update runs the collector query again.
 func (f *File) Update() error {
 	if f.c != nil {