Loki loading (#80)

* vendors loki
Signed-off-by: Owen Diehl <[email protected]>

* loki backend support
Signed-off-by: Owen Diehl <[email protected]>

* updates changelog
Signed-off-by: Owen Diehl <[email protected]>

* updates to yaml v3
Signed-off-by: Owen Diehl <[email protected]>

Author: owen-d

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## unreleased / master
 
+* [FEATURE] Added loki backend support for the rules commands, configurable with `--backend=loki` (defaults to cortex).
+
 ## v0.2.4
 
 * [BUGFIX] Fix alertmanager registration subcommand and path for the configuration API #72

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/snappy v0.0.1
 	github.com/google/go-cmp v0.4.0
+	github.com/grafana/loki v1.6.2-0.20200831213751-f905e77259f2
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
@@ -23,7 +24,6 @@ require (
 	github.com/stretchr/testify v1.5.1
 	google.golang.org/api v0.29.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/yaml.v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 	gotest.tools v2.2.0+incompatible
 )
@@ -42,3 +42,6 @@ replace github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915
 
 // Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86
 replace github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
+
+// Same as Cortex, we can't upgrade to grpc 1.30.0 until go.etcd.io/etcd will support it.
+replace google.golang.org/grpc => google.golang.org/grpc v1.29.1

go.sum

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 const alertmanagerAPIPath = "/api/v1/alerts"

pkg/client/alerts.go

@@ -8,7 +8,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // CreateRuleGroup creates a new rule group

pkg/client/rules.go

@@ -16,7 +16,7 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	chunkTool "github.com/grafana/cortex-tools/pkg/chunk"
 	"github.com/grafana/cortex-tools/pkg/chunk/filter"
@@ -68,7 +68,7 @@ func (cfg *SchemaConfig) Load() error {
 	}
 
 	decoder := yaml.NewDecoder(f)
-	decoder.SetStrict(true)
+	decoder.KnownFields(true)
 	return decoder.Decode(&cfg)
 }
 

pkg/commands/chunks.go

@@ -4,7 +4,7 @@ import (
 	"os"
 
 	"gopkg.in/alecthomas/kingpin.v2"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/cortex-tools/pkg/chunk/migrate"
 	"github.com/grafana/cortex-tools/pkg/chunk/migrate/reader"
@@ -30,7 +30,7 @@ func (c *migrateChunksCommandOptions) run(k *kingpin.ParseContext) error {
 	}
 
 	decoder := yaml.NewDecoder(f)
-	decoder.SetStrict(true)
+	decoder.KnownFields(true)
 
 	if err := decoder.Decode(&c.Config); err != nil {
 		return err

pkg/commands/migrate.go

@@ -35,6 +35,8 @@ var (
 		Name:      "last_rule_load_success_timestamp_seconds",
 		Help:      "The timestamp of the last successful rule load.",
 	})
+
+	backends = []string{rules.CortexBackend, rules.LokiBackend} // list of supported backend types
 )
 
 // RuleCommand configures and executes rule related cortex operations
@@ -43,6 +45,9 @@ type RuleCommand struct {
 
 	cli *client.CortexClient
 
+	// Backend type (cortex | loki)
+	Backend string
+
 	// Get Rule Groups Configs
 	Namespace string
 	RuleGroup string
@@ -73,6 +78,7 @@ type RuleCommand struct {
 func (r *RuleCommand) Register(app *kingpin.Application) {
 	rulesCmd := app.Command("rules", "View & edit rules stored in cortex.").PreAction(r.setup)
 	rulesCmd.Flag("key", "Api key to use when contacting cortex, alternatively set $CORTEX_API_KEY.").Default("").Envar("CORTEX_API_KEY").StringVar(&r.ClientConfig.Key)
+	rulesCmd.Flag("backend", "Backend type to interact with: <cortex|loki>").Default("cortex").EnumVar(&r.Backend, backends...)
 
 	// Register rule commands
 	listCmd := rulesCmd.
@@ -323,7 +329,7 @@ func (r *RuleCommand) deleteRuleGroup(k *kingpin.ParseContext) error {
 }
 
 func (r *RuleCommand) loadRules(k *kingpin.ParseContext) error {
-	nss, err := rules.ParseFiles(r.RuleFilesList)
+	nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "load operation unsuccessful, unable to parse rules files")
 	}
@@ -372,7 +378,7 @@ func (r *RuleCommand) diffRules(k *kingpin.ParseContext) error {
 		return errors.Wrap(err, "diff operation unsuccessful, unable to load rules files")
 	}
 
-	nss, err := rules.ParseFiles(r.RuleFilesList)
+	nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "diff operation unsuccessful, unable to parse rules files")
 	}
@@ -426,7 +432,7 @@ func (r *RuleCommand) syncRules(k *kingpin.ParseContext) error {
 		return errors.Wrap(err, "sync operation unsuccessful, unable to load rules files")
 	}
 
-	nss, err := rules.ParseFiles(r.RuleFilesList)
+	nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "sync operation unsuccessful, unable to parse rules files")
 	}
@@ -527,7 +533,7 @@ func (r *RuleCommand) prepare(k *kingpin.ParseContext) error {
 		return errors.Wrap(err, "prepare operation unsuccessful, unable to load rules files")
 	}
 
-	namespaces, err := rules.ParseFiles(r.RuleFilesList)
+	namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "prepare operation unsuccessful, unable to parse rules files")
 	}
@@ -559,7 +565,7 @@ func (r *RuleCommand) lint(k *kingpin.ParseContext) error {
 		return errors.Wrap(err, "prepare operation unsuccessful, unable to load rules files")
 	}
 
-	namespaces, err := rules.ParseFiles(r.RuleFilesList)
+	namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "prepare operation unsuccessful, unable to parse rules files")
 	}
@@ -593,7 +599,7 @@ func (r *RuleCommand) checkRecordingRuleNames(k *kingpin.ParseContext) error {
 		return errors.Wrap(err, "check operation unsuccessful, unable to load rules files")
 	}
 
-	namespaces, err := rules.ParseFiles(r.RuleFilesList)
+	namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList)
 	if err != nil {
 		return errors.Wrap(err, "check operation unsuccessful, unable to parse rules files")
 	}

pkg/commands/rules.go

@@ -8,7 +8,7 @@ import (
 	"github.com/alecthomas/chroma/quick"
 	"github.com/mitchellh/colorstring"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/cortex-tools/pkg/rules"
 )

pkg/printer/printer.go

@@ -9,7 +9,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/mitchellh/colorstring"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 )
 
 var (