Change app to read configuration and support both labels and descriptions in json format

Author: Foivos Filippopoulos

README.md

@@ -6,20 +6,45 @@ Snapshots (only the ones created by the service) are deleted when they become ol
 ## Usage
 ```
 Usage of /gcp-disk-snapshotter:
-  -interval_secs int
-        Interval between snapshots in seconds. Defaults to 43200s = 12h (default 43200)
-  -labels string
-        (Required) Comma separated list of disk labels in format <name>:<value>
+ -conf_file string
+        (Required) Path of the configuration file tha contains the targets based on label or description
   -log_level string
         Log Level, defaults to INFO (default "info")
   -project string
         (Required) GCP Project to use
-  -retention_hours int
-        Retention Duration in hours. Defaults to 720h = 1 month (default 720)
   -snap_prefix string
         Prefix for created snapshots
   -watch_interval int
         Interval between watch cycles in seconds. Defaults to 60s (default 60)
   -zones string
         (Required) Comma separated list of zones where projects disks may live
 ```
+
+#Configuration File
+
+Example Configuration File:
+
+```
+{
+  "Descriptions": [
+    {
+      "retentionPeriodHours" : 2,
+      "intervalSeconds" : 100,
+      "description": {
+        "key": "kubernetes.io/created-for/pvc/name",
+        "value": "some-app-pd-pvc"
+      }
+    }
+  ],
+  "Labels": [
+    {
+      "retentionPeriodHours" : 2,
+      "intervalSeconds" : 100,
+      "label": {
+        "key": "name",
+        "value": "some-app-name"
+      }
+    }
+  ]
+}
+```

main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
+	"io/ioutil"
 	"os"
 	"strings"
 
@@ -13,14 +15,12 @@ import (
 
 var (
 	// flags
-	flagProject         = flag.String("project", "", "(Required) GCP Project to use")
-	flagZones           = flag.String("zones", "", "(Required) Comma separated list of zones where projects disks may live")
-	flagLabels          = flag.String("labels", "", "(Required) Comma separated list of disk labels in format <name>:<value>")
-	flagSnapPrefix      = flag.String("snap_prefix", "", "Prefix for created snapshots")
-	flagWatchInterval   = flag.Int("watch_interval", 60, "Interval between watch cycles in seconds. Defaults to 60s")
-	flagRetentionHours  = flag.Int("retention_hours", 720, "Retention Duration in hours. Defaults to 720h = 1 month")
-	flagIntervalSeconds = flag.Int("interval_secs", 43200, "Interval between snapshots in seconds. Defaults to 43200s = 12h")
-	flagLogLevel        = flag.String("log_level", "info", "Log Level, defaults to INFO")
+	flagProject       = flag.String("project", "", "(Required) GCP Project to use")
+	flagZones         = flag.String("zones", "", "(Required) Comma separated list of zones where projects disks may live")
+	flagConfFile      = flag.String("conf_file", "", "(Required) Path of the configuration file tha contains the targets based on label or description")
+	flagSnapPrefix    = flag.String("snap_prefix", "", "Prefix for created snapshots")
+	flagWatchInterval = flag.Int("watch_interval", 60, "Interval between watch cycles in seconds. Defaults to 60s")
+	flagLogLevel      = flag.String("log_level", "info", "Log Level, defaults to INFO")
 )
 
 func usage() {
@@ -55,41 +55,51 @@ func main() {
 	}
 	zones := strings.Split(*flagZones, ",")
 
-	if *flagLabels == "" {
+	if *flagConfFile == "" {
 		usage()
 	}
 
-	labels := &models.LabelList{}
-
-	for _, label := range strings.Split(*flagLabels, ",") {
-
-		l := strings.Split(label, ":")
-		if len(l) < 2 {
-			usage()
-		}
-
-		labels.AddLabel(l[0], l[1])
-	}
-
 	snapPrefix := *flagSnapPrefix
 	watchInterval := *flagWatchInterval
-	retentionHours := *flagRetentionHours
-	intervalSecs := *flagIntervalSeconds
 	logLevel := *flagLogLevel
 
 	// Init logging
 	initLogging(logLevel)
 
+	// Load config
+	snapshotConfigs := loadSnapshotConfig(*flagConfFile)
+
+	log.Debug("Reading Configuration")
+	for _, d := range snapshotConfigs.Descriptions {
+		log.Debug("description: ", d.Description.Key, " ", d.Description.Value)
+	}
+	for _, l := range snapshotConfigs.Labels {
+		log.Debug("label: ", l.Label.Key, " ", l.Label.Value)
+	}
 	// Create a snapshotter
-	gsc := snapshot.CreateGCPSnapClient(project, snapPrefix, zones, *labels)
+	gsc := snapshot.CreateGCPSnapClient(project, snapPrefix, zones)
 
 	// Start watching
 	watcher := &watch.Watcher{
-		GSC:            gsc,
-		WatchInterval:  watchInterval,
-		RetentionHours: retentionHours,
-		IntervalSecs:   intervalSecs,
+		GSC:           gsc,
+		WatchInterval: watchInterval,
 	}
-	watcher.Watch()
+	watcher.Watch(snapshotConfigs)
 
 }
+
+func loadSnapshotConfig(snapshotConfigFile string) *models.SnapshotConfigs {
+	confFile, err := os.Open(snapshotConfigFile)
+	if err != nil {
+		log.Fatal("Error while opening volume snapshot config file: ", err)
+	}
+	fileContent, err := ioutil.ReadAll(confFile)
+	if err != nil {
+		log.Fatal("Error while reading volume snapshot config file: ", err)
+	}
+	snapshotConfigs := &models.SnapshotConfigs{}
+	if err = json.Unmarshal(fileContent, snapshotConfigs); err != nil {
+		log.Fatal("Error unmarshalling snapshots config file: ", err)
+	}
+	return snapshotConfigs
+}

models/labels.go

@@ -0,0 +1,28 @@
+package models
+
+type Label struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+type LabelSnapshotConfig struct {
+	Label                *Label `json:"label"`
+	IntervalSeconds      int64  `json:"intervalSeconds"`
+	RetentionPeriodHours int64  `json:"retentionPeriodHours"`
+}
+
+type Description struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+type DescriptionSnapshotConfig struct {
+	Description          *Description `json:"description"`
+	IntervalSeconds      int64        `json:"intervalSeconds"`
+	RetentionPeriodHours int64        `json:"retentionPeriodHours"`
+}
+
+type SnapshotConfigs struct {
+	Descriptions []*DescriptionSnapshotConfig `json:"Descriptions"`
+	Labels       []*LabelSnapshotConfig       `json:"Labels"`
+}

models/types.go

@@ -2,6 +2,7 @@ package snapshot
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	log "github.com/sirupsen/logrus"
 	"net/http"
@@ -25,13 +26,13 @@ var googleClient *http.Client
 type GCPSnapClient struct {
 	Project        string
 	Zones          []string
-	Labels         models.LabelList
 	SnapPrefix     string
 	ComputeService compute.Service
 }
 
 type GCPSnapClientInterface interface {
-	GetDiskList() ([]compute.Disk, error)
+	GetDisksFromLabel(label *models.Label) ([]compute.Disk, error)
+	GetDisksFromDescription(label *models.Description) ([]compute.Disk, error)
 	ListSnapshots(diskSelfLink string) ([]compute.Snapshot, error)
 	ListClientCreatedSnapshots(diskSelfLink string) ([]compute.Snapshot, error)
 	CreateSnapshot(diskName, zone string) (string, error)
@@ -41,7 +42,7 @@ type GCPSnapClientInterface interface {
 }
 
 // Basic Init function for the snapshotter
-func CreateGCPSnapClient(project, snapPrefix string, zones []string, labels models.LabelList) *GCPSnapClient {
+func CreateGCPSnapClient(project, snapPrefix string, zones []string) *GCPSnapClient {
 
 	ctx := context.Background()
 	googleClient, err := google.DefaultClient(ctx, compute.ComputeScope)
@@ -57,7 +58,6 @@ func CreateGCPSnapClient(project, snapPrefix string, zones []string, labels mode
 	return &GCPSnapClient{
 		Project:        project,
 		Zones:          zones,
-		Labels:         labels,
 		SnapPrefix:     snapPrefix,
 		ComputeService: *computeService,
 	}
@@ -74,7 +74,7 @@ func formatLinkString(in string) string {
 }
 
 // GetDiskList: Returns a list of disks that contain one of the given labels
-func (gsc *GCPSnapClient) GetDiskList() ([]compute.Disk, error) {
+func (gsc *GCPSnapClient) GetDisksFromLabel(label *models.Label) ([]compute.Disk, error) {
 
 	disks := []compute.Disk{}
 
@@ -85,12 +85,36 @@ func (gsc *GCPSnapClient) GetDiskList() ([]compute.Disk, error) {
 		}
 
 		for _, disk := range resp.Items {
-			for _, label := range gsc.Labels.Items {
-				if val, ok := disk.Labels[label.Name]; ok {
-					if label.Value == val {
-						disks = append(disks, *disk)
-						break
-					}
+			if val, ok := disk.Labels[label.Key]; ok {
+				if label.Value == val {
+					disks = append(disks, *disk)
+				}
+			}
+		}
+	}
+
+	return disks, nil
+}
+
+func (gsc *GCPSnapClient) GetDisksFromDescription(desc *models.Description) ([]compute.Disk, error) {
+
+	disks := []compute.Disk{}
+
+	for _, zone := range gsc.Zones {
+		resp, err := gsc.ComputeService.Disks.List(gsc.Project, zone).Do()
+		if err != nil {
+			return disks, errors.Wrap(err, "error listing disks")
+		}
+
+		for _, disk := range resp.Items {
+			var dObj map[string]string
+			if err = json.Unmarshal([]byte(disk.Description), &dObj); err != nil {
+				log.Debug("Skipping: error unmarshalling disk description to map: ", err)
+				continue
+			}
+			if val, ok := dObj[desc.Key]; ok {
+				if desc.Value == val {
+					disks = append(disks, *disk)
 				}
 			}
 		}
@@ -163,7 +187,7 @@ func (gsc *GCPSnapClient) CreateSnapshot(diskName, zone string) (string, error)
 	// Name must match regex '(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)'
 	snapshot := &compute.Snapshot{
 		Description: fmt.Sprintf("Snapshot of %s", diskName),
-		Name:        fmt.Sprintf("%s%s-snapshot-%s", gsc.SnapPrefix, diskName, time.Now().Format("20060102150405")),
+		Name:        fmt.Sprintf("%s%s-%s", gsc.SnapPrefix, diskName, time.Now().Format("20060102150405")),
 		Labels:      snapLabels,
 	}