Adding ability to filter for dashboards (#23)

Author: mariusoe

README.md

@@ -7,6 +7,10 @@ A possible use case is: push Grafana dashboards from one Grafana instance to a G
 
 As an example this is useful to stage dashboards from "dev" to "prod" environments.
 
+The special thing is that the synchronization of dashboards is based on tags, which can be created by the users themselves. Thus, users can determine when a dashboard is ready for synchronization, e.g. so that it is synchronized from a "dev" to a "prod" environment.
+
+If a dashboard is imported to Grafana but a dashboard with the same name or ID already exists there, it will be overwritten. For security reasons, dashboards **are not deleted** by the application. If a dashboard is obsolete, it must be deleted manually by the user.
+
 ## Usage
 
 The application can be used as follows:
@@ -55,10 +59,11 @@ See the following configuration for available configuration options:
          enable: true
          # the branch to use for exporting dashboards
          git-branch: "push-branch"
-         # only dashboards with match this pattern will be considered in the sync process
+         # only dashboards with match this pattern will be considered in the sync process.
+         # this value is a WHITELIST in case it is not empty!!!
          filter: ""
          # the tag to determine which dashboards should be exported
-         tag-pattern: "agent"
+         tag-pattern: "sync"
          # whether the sync-tag should be kept during exporting
          push-tags: true
 
@@ -68,7 +73,8 @@ See the following configuration for available configuration options:
          enable: true
          # the branch to use for importing dashboards
          git-branch: "pull-branch"
-         # only dashboards with match this pattern will be considered in the sync process
+         # only dashboards with match this pattern will be considered in the sync process.
+         # this value is a WHITELIST in case it is not empty!!!
          filter: ""
 
 ## Development

configuration.yml

@@ -2,21 +2,27 @@
 # example configuration
 ##########################
 
+- job-name:      "example-job"
   # API token to interact with the specified Grafana instance
+  grafana-token: "eyJrIjoiSEp4dzhGdVBxMUhBdm5Db..."
   # Base URL of the Grafana instance
   grafana-url:   "http://localhost:3000"
   # SSH-URL of the Git repository to use
-  private-key-file:   "<PRIVATE_KEY_FILE>"
+  git-repository-url: "<GIT_REPOSITORY_URL>"
   # Private key to use for authentication against the Git repository
+  private-key-file:   "<PRIVATE_SSH_KEY>"
 
   # push (export) related configurations
   push-configuration:
     # whether to export dashboards
     enable: true
     # the branch to use for exporting dashboards
-    # only dashboards with match this pattern will be considered in the sync process
+    git-branch: "push-branch"
+    # only dashboards with match this pattern will be considered in the sync process.
+    # this value is a WHITELIST in case it is not empty!
     filter: ""
     # the tag to determine which dashboards should be exported
+    tag-pattern: "sync"
     # whether the sync-tag should be kept during exporting
     push-tags: true
 
@@ -25,5 +31,7 @@
     # whether to import dashboards
     enable: true
     # the branch to use for importing dashboards
-    # only dashboards with match this pattern will be considered in the sync process
+    git-branch: "pull-branch"
+    # only dashboards with match this pattern will be considered in the sync process.
+    # this value is a WHITELIST in case it is not empty!
     filter: ""

pkg/internal/synchronizer.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 
+	sdk "github.com/NovatecConsulting/grafana-api-go-sdk"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -81,14 +83,32 @@ func (s *Synchronization) Synchronize(dryRun bool) error {
 
 // Pushs dashboards from the configured Grafana into Git.
 func (s *Synchronization) pushDashboards(dryRun bool) error {
+	configuration := s.options.PushConfiguration
+
 	log.WithFields(log.Fields{
-		"target-branch": s.options.PushConfiguration.GitBranch,
-		"filter":        s.options.PushConfiguration.Filter,
-		"tag-pattern":   s.options.PushConfiguration.TagPattern,
-		"push-tags":     s.options.PushConfiguration.PushTags,
+		"job":           s.options.JobName,
+		"target-branch": configuration.GitBranch,
+		"filter":        configuration.Filter,
+		"tag-pattern":   configuration.TagPattern,
+		"push-tags":     configuration.PushTags,
 	}).Info("Starting dashboard synchroization (export) into the Git repository.")
 
-	dashboardTag := s.options.PushConfiguration.TagPattern
+	// initializing the dashboard filter
+	var regexFilter *regexp.Regexp
+	var err error
+	if configuration.Filter != "" {
+		regexFilter, err = regexp.Compile(configuration.Filter)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"job":    s.options.JobName,
+				"filter": configuration.Filter,
+			}).Fatal("Invalid filter pattern for the push configuration. Skipping exportation of dashboard.")
+			return err
+		}
+	}
+
+	dashboardTag := configuration.TagPattern
 
 	resultBoards, err := s.grafanaApi.SearchDashboardsWithTag(dashboardTag)
 
@@ -100,7 +120,7 @@ func (s *Synchronization) pushDashboards(dryRun bool) error {
 		log.WithField("amount", len(resultBoards)).Info("Successfully fetched dashboards.")
 
 		// clone repo from specific branch
-		repository, err := s.gitApi.CloneRepo(s.options.PushConfiguration.GitBranch)
+		repository, err := s.gitApi.CloneRepo(configuration.GitBranch)
 		if err != nil {
 			log.WithField("error", err).Fatal("Error while cloning repository.")
 			return err
@@ -110,8 +130,25 @@ func (s *Synchronization) pushDashboards(dryRun bool) error {
 			// get dashboard Object and Properties
 			dashboard, boardProperties := s.grafanaApi.GetDashboardObjectByUID(board.UID)
 
+			// synchronize only dashboards matching the filter
+			if regexFilter != nil {
+				folderAndTitle := boardProperties.FolderTitle + "/" + dashboard.Title
+				if regexFilter.FindStringIndex(folderAndTitle) == nil {
+					log.WithFields(log.Fields{
+						"dashboard-path": folderAndTitle,
+						"filter":         configuration.Filter,
+					}).Info("Skipping export because dashboard does not match the specified filter pattern.")
+					continue
+				}
+			}
+
 			// delete Tag from dashboard Object
-			dashboardWithDeletedTag := s.grafanaApi.DeleteTagFromDashboardObjectByID(dashboard, dashboardTag)
+			var dashboardWithDeletedTag sdk.Board
+			if configuration.PushTags {
+				dashboardWithDeletedTag = dashboard
+			} else {
+				dashboardWithDeletedTag = s.grafanaApi.DeleteTagFromDashboardObjectByID(dashboard, dashboardTag)
+			}
 
 			// get folder name and id, required for update processes and git folder structure
 			folderId := boardProperties.FolderID
@@ -142,21 +179,39 @@ func (s *Synchronization) pushDashboards(dryRun bool) error {
 
 		log.Info("Successfully pushed dashboards to the remote Git repository.")
 	} else {
-		log.WithField("tag-pattern", s.options.PushConfiguration.TagPattern).Info("No dashboards found using the configured tag pattern.")
+		log.WithField("tag-pattern", configuration.TagPattern).Info("No dashboards found using the configured tag pattern.")
 	}
 
 	return nil
 }
 
 // Pulling dashboards from the configured Git and importing them into Grafana.
 func (s *Synchronization) pullDashboards(dryRun bool) error {
+	configuration := s.options.PullConfiguration
+
 	log.WithFields(log.Fields{
-		"target-branch": s.options.PullConfiguration.GitBranch,
-		"filter":        s.options.PullConfiguration.Filter,
+		"job":           s.options.JobName,
+		"target-branch": configuration.GitBranch,
+		"filter":        configuration.Filter,
 	}).Info("Starting dashboard synchroization (import) from the Git repository.")
 
+	// initializing the dashboard filter
+	var regexFilter *regexp.Regexp
+	var err error
+	if configuration.Filter != "" {
+		regexFilter, err = regexp.Compile(configuration.Filter)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"job":    s.options.JobName,
+				"filter": configuration.Filter,
+			}).Fatal("Invalid filter pattern for the pull configuration. Skipping importation of dashboard.")
+			return err
+		}
+	}
+
 	// clone and fetch the configured repository
-	repository, err := s.gitApi.CloneRepo(s.options.PullConfiguration.GitBranch)
+	repository, err := s.gitApi.CloneRepo(configuration.GitBranch)
 	if err != nil {
 		log.WithField("error", err).Fatal("Error while cloning repository.")
 		return err
@@ -210,6 +265,18 @@ func (s *Synchronization) pullDashboards(dryRun bool) error {
 				}).Fatal("Failed to unmarshal dashboard.")
 			}
 
+			// synchronize only dashboards matching the filter
+			if regexFilter != nil {
+				folderAndTitle := folderName + "/" + dashboard.Title
+				if regexFilter.FindStringIndex(folderAndTitle) == nil {
+					log.WithFields(log.Fields{
+						"dashboard-path": folderAndTitle,
+						"filter":         configuration.Filter,
+					}).Info("Skipping import because dashboard does not match the specified filter pattern.")
+					continue
+				}
+			}
+
 			//gitDashboardExtended := getDashboardObjectFromRawDashboard(gitRawDashboard)
 			grafanaDashboard, _ := s.grafanaApi.GetDashboardObjectByUID(dashboard.UID)