diff --git a/.gitignore b/.gitignore index ec4ce3a..e92416c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,32 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + /prometheus-bigquery-exporter +config.yaml +/queries +credentials.json \ No newline at end of file diff --git a/README.md b/README.md index bc764ed..908a292 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ An exporter for converting BigQuery results into Prometheus metrics. +This Fork adds a config file for multiple queries and Counter type metrics + + ## Limitations: No historical values Prometheus collects the *current* status of a system as reported by an exporter. diff --git a/configuration/example_config.yml b/configuration/example_config.yml new file mode 100644 index 0000000..6f93cd8 --- /dev/null +++ b/configuration/example_config.yml @@ -0,0 +1,9 @@ +project: "skyita-da-logging-noprod" +gauge-queries: + - query: + file: "/queries/metric1_name.sql" + #refresh "*/5 * * * *" #every 5 minutes by default +counter-queries: + - query: + file: "/queries/metric2_name.sql" + #refresh "30 9 * * *" #every day at 9:30 \ No newline at end of file diff --git a/configuration/test/test_0.yml b/configuration/test/test_0.yml new file mode 100644 index 0000000..e69de29 diff --git a/configuration/test/test_1.yml b/configuration/test/test_1.yml new file mode 100644 index 0000000..a0914de --- /dev/null +++ b/configuration/test/test_1.yml @@ -0,0 +1,13 @@ +project: "abc" +gauge-queries: + - query: "abc" + file: "/queries/metric1_name.sql" + - query: "def" + file: "/queries/metric2_name.sql" +counter-queries: + - query: "abc_2" + file: "/queries/metric3_name.sql" + - query: "def_2" + file: "/queries/metric4_name.sql" + - query: "ghi_2" + file: "/queries/metric5_name.sql" \ No newline at end of file diff --git a/configuration/test/test_2.yml b/configuration/test/test_2.yml new file mode 100644 index 0000000..3c20fea --- /dev/null +++ b/configuration/test/test_2.yml @@ -0,0 +1,9 @@ +project: "abc" +gauge-queries: +counter-queries: + - query: "abc_2" + file: "/queries/metric3_name.sql" + - query: "def_2" + file: "/queries/metric4_name.sql" + - query: "ghi_2" + file: "/queries/metric5_name.sql" \ No newline at end of file diff --git a/configuration/test/test_3.yml b/configuration/test/test_3.yml new file mode 100644 index 0000000..1f28aeb --- /dev/null +++ b/configuration/test/test_3.yml @@ -0,0 +1,7 @@ +project: "abc" +gauge-queries: + - query: "abc" + file: "/queries/metric1_name.sql" + - query: "def" + file: "/queries/metric2_name.sql" +counter-queries: \ No newline at end of file diff --git a/go.mod b/go.mod index d3d57f4..061a346 100644 --- a/go.mod +++ b/go.mod @@ -41,3 +41,4 @@ require ( gopkg.in/yaml.v2 v2.3.0 // indirect honnef.co/go/tools v0.0.1-2019.2.3 // indirect ) + diff --git a/go.sum b/go.sum index 0638aab..ad89ce5 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/m-lab/go v1.4.0 h1:Au2Vt15+H8oOd3xYZFfW3gK86GRciKlfGJs/FzsJwK4= github.com/m-lab/go v1.4.0/go.mod h1:f22d1CtoFIho8yt0wPNYo0Lx5h8YfgRW4+1pzQTeQRw= @@ -309,6 +311,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..d360dcd --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,111 @@ +package config + +import ( + "fmt" + "log" + "os" + + "github.com/m-lab/go/logx" + "github.com/spf13/afero" + "gopkg.in/yaml.v2" +) + +var fs = afero.NewOsFs() + +type Config struct { + Name string `yaml:"-"` + stat os.FileInfo `yaml:"-"` + Gauge []Query `yaml:"gauge-queries"` + Counter []Query `yaml:"counter-queries"` +} + +type Query struct { + Query string `yaml:"query"` + File string `yaml:"file"` +} + +func (cfg *Config) GetGaugeFiles() []string { + + var paths []string + for _, path := range cfg.Gauge { + + paths = append(paths, path.File) + } + return paths +} + +func (cfg *Config) IsModified() (bool, error) { + + var err error + if cfg.stat == nil { + cfg.stat, err = fs.Stat(cfg.Name) + logx.Debug.Println("IsModified:stat1:", cfg.Name, err) + // Return true on the first successful Stat(), or the error otherwise. + return err == nil, err + } + curr, err := fs.Stat(cfg.Name) + if err != nil { + log.Printf("Failed to stat %q: %v", cfg.Name, err) + return false, err + } + logx.Debug.Println("IsModified:stat2:", cfg.Name, curr.ModTime(), cfg.stat.ModTime(), + curr.ModTime().After(cfg.stat.ModTime())) + modified := curr.ModTime().After(cfg.stat.ModTime()) + if modified { + // Update the stat cache to the latest version. + cfg.stat = curr + } + return modified, nil +} + +func (cfg *Config) GetCounterFiles() []string { + + var paths []string + for _, path := range cfg.Counter { + + paths = append(paths, path.File) + } + return paths +} + +func ReadConfigFile(path string) (*Config, error) { + + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("something wrong during file opening: %s", err.Error()) + } + + defer f.Close() + + var cfg Config + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&cfg) + if err != nil { + return nil, fmt.Errorf("something wrong during configuration unmarshalling: %s", err.Error()) + } + + err = validate(&cfg) + if err != nil { + return nil, err + } + + cfg.Name = path + cfg.stat, err = fs.Stat(cfg.Name) + if err != nil { + return nil, fmt.Errorf("something wrong during file stat extraction: %s", err.Error()) + } + + return &cfg, nil +} + +func validate(cfg *Config) error { + + if len(cfg.Counter) == 0 { + return fmt.Errorf("no Counter parameters available") + } + + if len(cfg.Gauge) == 0 { + return fmt.Errorf("no Gauge parameters available") + } + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..f87ddef --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,64 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadConfigFile__good(t *testing.T) { + + result, err := ReadConfigFile("../../configuration/test/test_1.yml") + assert.Nil(t, err) + assert.Equal(t, 2, len(result.Gauge)) + + gaugeQuery0 := result.Gauge[0] + assert.Equal(t, "abc", gaugeQuery0.Query) + assert.Equal(t, "/queries/metric1_name.sql", gaugeQuery0.File) + + gaugeQuery1 := result.Gauge[1] + assert.Equal(t, "def", gaugeQuery1.Query) + assert.Equal(t, "/queries/metric2_name.sql", gaugeQuery1.File) + + assert.Equal(t, 3, len(result.Counter)) + + counterQuery0 := result.Counter[0] + assert.Equal(t, "abc_2", counterQuery0.Query) + assert.Equal(t, "/queries/metric3_name.sql", counterQuery0.File) + + counterQuery1 := result.Counter[1] + assert.Equal(t, "def_2", counterQuery1.Query) + assert.Equal(t, "/queries/metric4_name.sql", counterQuery1.File) + + counterQuery2 := result.Counter[2] + assert.Equal(t, "ghi_2", counterQuery2.Query) + assert.Equal(t, "/queries/metric5_name.sql", counterQuery2.File) +} + +func TestReadConfigFile__empty_file(t *testing.T) { + + _, err := ReadConfigFile("../../configuration/test/test_0.yml") + assert.NotNil(t, err) + assert.Equal(t, "something wrong during configuration unmarshalling: EOF", err.Error()) +} + +func TestReadConfigFile__no_counter_parameters(t *testing.T) { + + _, err := ReadConfigFile("../../configuration/test/test_3.yml") + assert.NotNil(t, err) + assert.Equal(t, "no Counter parameters available", err.Error()) +} + +func TestReadConfigFile__no_gauge_parameters(t *testing.T) { + + _, err := ReadConfigFile("../../configuration/test/test_2.yml") + assert.NotNil(t, err) + assert.Equal(t, "no Gauge parameters available", err.Error()) +} + +func TestReadConfigFile__wrong_path(t *testing.T) { + + _, err := ReadConfigFile("../../not/existing/path") + assert.NotNil(t, err) + assert.Equal(t, "something wrong during file opening: open ../../not/existing/path: no such file or directory", err.Error()) +} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index cdec817..d2368a4 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -71,6 +71,16 @@ func (f *File) Register(c *sql.Collector) error { return c.RegisterErr } +func (f *File) Unregister() error { + + unregitered := prometheus.Unregister(f.c) + if !unregitered { + return fmt.Errorf("something wrong during unregistering of sql collector of file: %s", f.Name) + } + logx.Debug.Println("Unregister:", f.Name, f.c.RegisterErr) + return nil +} + // Update runs the collector query again. func (f *File) Update() error { if f.c != nil { diff --git a/main.go b/main.go index 116a0b7..f9eb998 100644 --- a/main.go +++ b/main.go @@ -10,14 +10,17 @@ import ( "fmt" "io/ioutil" "log" + "os" "path/filepath" "strings" "sync" "time" "github.com/m-lab/go/flagx" + "github.com/m-lab/go/logx" "github.com/m-lab/go/prometheusx" "github.com/m-lab/go/rtx" + "github.com/m-lab/prometheus-bigquery-exporter/internal/config" "github.com/m-lab/prometheus-bigquery-exporter/internal/setup" "github.com/m-lab/prometheus-bigquery-exporter/query" "github.com/m-lab/prometheus-bigquery-exporter/sql" @@ -29,16 +32,17 @@ import ( ) var ( - counterSources = flagx.StringArray{} - gaugeSources = flagx.StringArray{} - project = flag.String("project", "", "GCP project name.") - refresh = flag.Duration("refresh", 5*time.Minute, "Interval between updating metrics.") + //counterSources = flagx.StringArray{} + //gaugeSources = flagx.StringArray{} + project = flag.String("project", "", "GCP project name.") + configFile = flag.String("config", "config.yaml", "Configuration file name") + refresh = flag.Duration("refresh", 5*time.Minute, "Interval between updating metrics.") ) func init() { - // TODO: support counter queries. - // flag.Var(&counterSources, "counter-query", "Name of file containing a counter query.") - flag.Var(&gaugeSources, "gauge-query", "Name of file containing a gauge query.") + + //flag.Var(&counterSources, "counter-query", "Name of file containing a counter query.") + //flag.Var(&gaugeSources, "gauge-query", "Name of file containing a gauge query.") // Port registered at https://github.com/prometheus/prometheus/wiki/Default-port-allocations *prometheusx.ListenAddress = ":9348" @@ -69,9 +73,9 @@ func fileToQuery(filename string, vars map[string]string) string { return q } -func reloadRegisterUpdate(client *bigquery.Client, files []setup.File, vars map[string]string) { +func reloadRegisterUpdate(client *bigquery.Client, GaugeFiles []setup.File, CounterFiles []setup.File, vars map[string]string) { var wg sync.WaitGroup - for i := range files { + for i := range GaugeFiles { wg.Add(1) go func(f *setup.File) { modified, err := f.IsModified() @@ -95,7 +99,29 @@ func reloadRegisterUpdate(client *bigquery.Client, files []setup.File, vars map[ log.Println("Error:", f.Name, err) } wg.Done() - }(&files[i]) + }(&GaugeFiles[i]) + } + wg.Wait() + for i := range CounterFiles { + wg.Add(1) + go func(f *setup.File) { + modified, err := f.IsModified() + if modified && err == nil { + c := sql.NewCollector( + newRunner(client), prometheus.CounterValue, + fileToMetric(f.Name), fileToQuery(f.Name, vars)) + + log.Println("Registering:", fileToMetric(f.Name)) + err = f.Register(c) + } else { + log.Println("Updating:", fileToMetric(f.Name)) + err = f.Update() + } + if err != nil { + log.Println("Error:", f.Name, err) + } + wg.Done() + }(&CounterFiles[i]) } wg.Wait() } @@ -106,16 +132,22 @@ var newRunner = func(client *bigquery.Client) sql.QueryRunner { } func main() { + flag.Parse() rtx.Must(flagx.ArgsFromEnv(flag.CommandLine), "Could not get args from env") + if configFile == nil { + fmt.Printf("Undefined config file path") + os.Exit(1) + } + + cfg := initConfig(*configFile) + srv := prometheusx.MustServeMetrics() defer srv.Shutdown(mainCtx) - files := make([]setup.File, len(gaugeSources)) - for i := range files { - files[i].Name = gaugeSources[i] - } + GaugeFiles := toFiles(cfg.GetGaugeFiles()) + CounterFiles := toFiles(cfg.GetCounterFiles()) client, err := bigquery.NewClient(mainCtx, *project) rtx.Must(err, "Failed to allocate a new bigquery.Client") @@ -125,7 +157,61 @@ func main() { } for mainCtx.Err() == nil { - reloadRegisterUpdate(client, files, vars) + + isModified, err := cfg.IsModified() + if err != nil { + logx.Debug.Fatalf("Something wrong during configuration reload: %s", err.Error()) + os.Exit(1) + } + if isModified { + log.Println("Main configuration file change detected, reloading") + logx.Debug.Printf("Start reload configuration") + + unregisterCollectors(GaugeFiles) + logx.Debug.Printf("Unregistered old Gauge sql collectors") + + unregisterCollectors(CounterFiles) + logx.Debug.Printf("Unregistered old Counter sql collectors") + + cfg = initConfig(*configFile) + GaugeFiles = toFiles(cfg.GetGaugeFiles()) + CounterFiles = toFiles(cfg.GetCounterFiles()) + log.Println("Configuration reload completed successfully") + logx.Debug.Printf("%+v", cfg) + } + + reloadRegisterUpdate(client, GaugeFiles, CounterFiles, vars) sleepUntilNext(*refresh) } } + +func unregisterCollectors(files []setup.File) { + + for _, file := range files { + err := file.Unregister() + if err != nil { + logx.Debug.Fatalf("Something wrong during unregistration: %s", err.Error()) + os.Exit(1) + } + } +} + +func toFiles(paths []string) []setup.File { + files := make([]setup.File, len(paths)) + for i := range paths { + files[i].Name = paths[i] + } + return files +} + +func initConfig(configPath string) *config.Config { + + cfg, err := config.ReadConfigFile(configPath) + if err != nil { + logx.Debug.Fatalf("%s", err.Error()) + os.Exit(1) + } + + logx.Debug.Printf("Configuration unmarshalled successfully: %+v", cfg) + return cfg +} diff --git a/main_test.go b/main_test.go index 05606d0..bb8870f 100644 --- a/main_test.go +++ b/main_test.go @@ -61,7 +61,11 @@ func Test_main(t *testing.T) { // Set the refresh period to a very small delay. *refresh = time.Second - gaugeSources.Set(tmp.Name()) + + //TODO aggiornare i test dopo il completamento della logica + //gaugeSources.Set(tmp.Name()) + //counterSources.Set(tmp.Name()) + // Reset mainCtx to timeout after a second. mainCtx, mainCancel = context.WithTimeout(mainCtx, time.Second) @@ -70,7 +74,7 @@ func Test_main(t *testing.T) { main() // Verify that the fakeRunner was called twice. - if f.updated != 2 { - t.Errorf("main() failed to update; got %d, want 2", f.updated) + if f.updated != 4 { + t.Errorf("main() failed to update; got %d, want 4", f.updated) } }