trial version

Author: Alex8Efremov

Makefile

@@ -21,11 +21,11 @@ all: down up
 
 test:
 	go build -o app/main cmd/main.go
-	docker compose exec -u postgres postgres /app/main --project test --backup_type psql --backup_cron "*/1 * * * *"
+	docker compose exec -u postgres postgres /app/main --project test --backup_type psql --backup_cron "*/1 * * * *" --delete_cron "*/1 * * * *" --delete_retain 5
 
 tc:
 	go build -o app/main cmd/main.go
-	docker compose exec -u postgres postgres /app/main --project test --backup_type psql --config_file /var/lib/postgre
+	docker compose exec -u postgres postgres /app/main --project test --backup_type psql
 
 tm:
 	go build -o app/main cmd/main.go
@@ -35,9 +35,7 @@ tmy:
 	docker compose exec -u mysql mysql /app/main --project test --backup_type mysql --backup_cron "*/1 * * * *"
 tmd:
 	go build -o app/main cmd/main.go
-	docker compose exec -u mongodb mongo wal-g backup-list
-	docker compose exec -u mongodb mongo wal-g delete retain FULL 5 --confirm
-	docker compose exec -u mongodb mongo /app/main --project test --backup_type mongodb --backup_cron "*/1 * * * *"
+	docker compose exec -u mongodb mongo /app/main --project test --backup_type mongodb --backup_cron "*/1 * * * *" --delete_cron "*/1 * * * *" --delete_retain 5
 
 
 

README.md

@@ -1,8 +1,32 @@
-# Вся логика по созданию бекапа в create.go
+# Бекапилка не несет ответственности за то в каком виде конфиг файлы.
+Установка wal-g, clickhouse-backup осуществляется отдельно
+
+Пример команды для запуска 
+```
+main --project projectName --backup_type <psql|mysql|mariadb|mongodb|clickhouse> --cron "* * * * *" 
+Support only for postgresql --delete_cron "*/1 * * * *" --delete_retain 5
+```
+
+# Вся логика по созданию бекапа в create/create.go
+Удаление старых бекапов поддерживается долько для postgresql
+Проверка проходит на стадии задания переменных в config.go
+Также здесь создаю `config.DeleteEnable` которая в main.go определяет состояние удаления старых бекапов
 
 # Переменные создаются в config/config.go
 
 # Метрики exporter/collector.go
-clickhouse need create table
 
-wal-g psql retein
+# Крон запускает выполнение тасок cron/cron.go
+
+# Локальная проверка
+В build/docker лежат Докер файлы и конфиги для запуска и теста
+Бекап предпологает наличие конфиг файлов в дирах по умолчанию (кроме mysql там установил home dir /var/lib/mysql руками)
+Если нужно изменить место конфига, добовляем --config флаг.
+Команды запускаются от имени пользователя бекапилки.
+Обязательные поля для заполнения (в конфигах бекапилок)
+- "AWS_ACCESS_KEY_ID": "",
+- "AWS_SECRET_ACCESS_KEY": "",
+- "AWS_REGION": "eu-central-1",
+- "AWS_ENDPOINT": "",
+- "WALE_S3_PREFIX": "",
+

app/main

@@ -18,19 +18,8 @@ import (
 	"github.com/efremov-it/backup_exporter/pkg/cron"
 )
 
-// c := NewCron()
-// if err := c.AddJob("First", "* * * * *", func(ctx context.Context) {
-// }); err != nil {
-//  panic(err)
-// }
-// if err := c.AddJob("Second", "* * * * *", func(ctx context.Context) {
-// }); err != nil {
-//  panic(err)
-// }
-
 func main() {
 	config, err := config.ParseFlags()
-
 	if err != nil {
 		log.Fatal("Error parsing flags:\n", err)
 	}
@@ -43,10 +32,16 @@ func main() {
 	prometheus.MustRegister(collectorVars)
 	backupService := create.NewBackupService(config, collectorVars)
 
-	if err := c.AddJob("Backup", config.CronTime, backupService.Run); err != nil {
+	// create backup
+	if err := c.AddJob("Backup", config.CronTime, backupService.Create); err != nil {
 		panic(err)
 	}
-	// create.NewBackupService(config, collectorVars).Run(ctx)
+	// retain Backup for postgresql.
+	if config.DeleteEnable {
+		if err := c.AddJob("Backup-Retain", config.DeleteCronTime, backupService.Retain); err != nil {
+			panic(err)
+		}
+	}
 	// Create a Prometheus registry
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
@@ -63,7 +58,6 @@ func main() {
 	}); err != nil {
 		panic(err)
 	}
-
 	if err := c.Run(ctx); err != nil {
 		panic(err)
 	}

cmd/main.go

@@ -9,15 +9,22 @@ import (
 )
 
 type CollectorMetrics struct {
-	mutex          sync.RWMutex
-	result         Result
-	config         *config.Config
-	BackupStatus   *prometheus.Desc
-	BackupDuration *prometheus.Desc
-	BackupNextTime *prometheus.Desc
+	mutex                sync.RWMutex
+	resultCreate         ResultCreate
+	resultRetain         ResultRetain
+	config               *config.Config
+	BackupCreateStatus   *prometheus.Desc
+	BackupCreateDuration *prometheus.Desc
+	BackupRetainStatus   *prometheus.Desc
+	BackupRetainDuration *prometheus.Desc
 }
 
-type Result struct {
+type ResultCreate struct {
+	BackupStartTime time.Time
+	BackupDuration  float64
+	BackupStatus    int
+}
+type ResultRetain struct {
 	BackupStartTime time.Time
 	BackupDuration  float64
 	BackupStatus    int
@@ -26,16 +33,20 @@ type Result struct {
 func NewCollector(config *config.Config) *CollectorMetrics {
 	labelNames := []string{"backup_type", "project_name", "instance_name"}
 	return &CollectorMetrics{
-		BackupStatus: prometheus.V2.NewDesc("backup_status",
+		BackupCreateStatus: prometheus.V2.NewDesc("backup_create_status",
 			"Show backup status",
 			prometheus.UnconstrainedLabels(labelNames),
 			prometheus.Labels(nil)),
-		BackupDuration: prometheus.V2.NewDesc("backup_duration",
+		BackupCreateDuration: prometheus.V2.NewDesc("backup_create_duration",
 			"Show backup duration",
 			prometheus.UnconstrainedLabels(labelNames),
 			prometheus.Labels(nil)),
-		BackupNextTime: prometheus.V2.NewDesc("backup_next_time",
-			"Show when the next backup will be created",
+		BackupRetainStatus: prometheus.V2.NewDesc("backup_retain_status",
+			"Show backup retain status (only for postgresql)",
+			prometheus.UnconstrainedLabels(labelNames),
+			prometheus.Labels(nil)),
+		BackupRetainDuration: prometheus.V2.NewDesc("backup_retain_duration",
+			"Show backup retain duration (only for postgresql)",
 			prometheus.UnconstrainedLabels(labelNames),
 			prometheus.Labels(nil)),
 		config: config,
@@ -44,9 +55,10 @@ func NewCollector(config *config.Config) *CollectorMetrics {
 
 func (collector *CollectorMetrics) Describe(ch chan<- *prometheus.Desc) {
 	ds := []*prometheus.Desc{
-		collector.BackupStatus,
-		collector.BackupDuration,
-		collector.BackupNextTime,
+		collector.BackupCreateStatus,
+		collector.BackupCreateDuration,
+		collector.BackupRetainStatus,
+		collector.BackupRetainDuration,
 	}
 
 	for _, d := range ds {
@@ -58,23 +70,42 @@ func (c *CollectorMetrics) Collect(ch chan<- prometheus.Metric) {
 	c.mutex.RLock()
 	defer c.mutex.RUnlock()
 
-	ch <- prometheus.NewMetricWithTimestamp(c.result.BackupStartTime, prometheus.MustNewConstMetric(
-		c.BackupDuration,
+	ch <- prometheus.NewMetricWithTimestamp(c.resultCreate.BackupStartTime, prometheus.MustNewConstMetric(
+		c.BackupCreateDuration,
+		prometheus.GaugeValue,
+		c.resultCreate.BackupDuration,
+		c.config.BackupType, c.config.ProjectName, c.config.Host,
+	))
+	ch <- prometheus.NewMetricWithTimestamp(c.resultCreate.BackupStartTime, prometheus.MustNewConstMetric(
+		c.BackupCreateStatus,
 		prometheus.GaugeValue,
-		c.result.BackupDuration,
+		float64(c.resultCreate.BackupStatus),
 		c.config.BackupType, c.config.ProjectName, c.config.Host,
 	))
-	ch <- prometheus.NewMetricWithTimestamp(c.result.BackupStartTime, prometheus.MustNewConstMetric(
-		c.BackupStatus,
+	ch <- prometheus.NewMetricWithTimestamp(c.resultRetain.BackupStartTime, prometheus.MustNewConstMetric(
+		c.BackupRetainDuration,
 		prometheus.GaugeValue,
-		float64(c.result.BackupStatus),
+		c.resultRetain.BackupDuration,
 		c.config.BackupType, c.config.ProjectName, c.config.Host,
 	))
+	ch <- prometheus.NewMetricWithTimestamp(c.resultRetain.BackupStartTime, prometheus.MustNewConstMetric(
+		c.BackupRetainStatus,
+		prometheus.GaugeValue,
+		float64(c.resultRetain.BackupStatus),
+		c.config.BackupType, c.config.ProjectName, c.config.Host,
+	))
+}
+
+func (c *CollectorMetrics) SetResultCreate(result ResultCreate) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	c.resultCreate = result
 }
 
-func (c *CollectorMetrics) SetResult(result Result) {
+func (c *CollectorMetrics) SetResultRetain(result ResultRetain) {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 
-	c.result = result
+	c.resultRetain = result
 }

pkg/collector/collector.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 )
 
 type Config struct {
@@ -14,6 +15,7 @@ type Config struct {
 	ConfigFile     string
 	BackupStorage  string
 	CronTime       string
+	DeleteEnable   bool
 	DeleteRetain   string
 	DeleteCronTime string
 }
@@ -37,39 +39,48 @@ func ParseFlags() (*Config, error) {
 	flag.StringVar(&config.ConfigFile, "config_file", "", "Path to config file (default is different for each backup tools)")
 	flag.StringVar(&config.BackupStorage, "backup_storage", "", "When uploading backups to storage, the user should pass the Postgres data directory as an argument (default: $PGDATA)")
 	flag.StringVar(&config.CronTime, "backup_cron", "0 2 * * *", "How often you should create your backup. Format crontab --backup_cron \"* * * * *\" (default: \"0 2 * * *\" At 02:00)")
-	flag.StringVar(&config.DeleteRetain, "delete_retain", "", "when set keep 5(or) full backups and all deltas of them (default: not set. example 5)")
+	flag.StringVar(&config.DeleteRetain, "delete_retain", "", "when set keep 5(or) full backups and all deltas of them (default: not set. example 5 int type)")
 	flag.StringVar(&config.DeleteCronTime, "delete_cron", "0 3* * 6", "How often you should retain your backups (default: \"0 3* * 6\" At 03:00 on Saturday)")
 
 	flag.Parse()
 
-	errUsage := fmt.Errorf("usage: main --project projectName --backup_type <psql|mysql|mariadb|mongodb|clickhouse> --cron \"* * * * *\"")
-
-	fmt.Println("DeleteRetain \n DeleteCronTime", config.DeleteRetain, config.DeleteCronTime)
-	// Set default values
+	errUsage := fmt.Errorf("usage: main --project projectName --backup_type <psql|mysql|mariadb|mongodb|clickhouse> --cron \"* * * * *\"\nSupport only for postgresql (--delete_cron \"*/1 * * * *\" --delete_retain 5)")
 
+	// Set up default values
 	if config.InstanceName == "" {
 		hostname, err := getHostname()
 		if err != nil {
 			return nil, err
 		}
 		config.InstanceName = hostname
 	}
-
 	if config.BackupStorage == "" && config.BackupType == "psql" {
 		pgdata := os.Getenv("PGDATA")
 		if pgdata == "" {
 			return nil, fmt.Errorf("error: $PGDATA environment variable is not set")
 		}
 		config.BackupStorage = pgdata
 	}
-
+	// enable backup Retain
+	config.DeleteEnable = false
+	if config.BackupType == "psql" {
+		if config.DeleteRetain != "" {
+			// if delete_retain contain string
+			_, err := strconv.Atoi(config.DeleteRetain)
+			if err != nil {
+				return nil, errUsage
+			}
+			config.DeleteEnable = true
+		} else {
+			fmt.Print("For enable deleting old backups you need to use flags:\n --delete_cron \"*/1 * * * *\" --delete_retain 5\n")
+		}
+	}
 	checkBackupType := true
 	for _, db := range []string{"psql", "mysql", "mariadb", "mongodb", "clickhouse"} {
 		if db == config.BackupType {
 			checkBackupType = false
 		}
 	}
-
 	// Validate required flags
 	if config.ProjectName == "" || checkBackupType {
 		return nil, errUsage

pkg/config/config.go

@@ -24,7 +24,7 @@ type BackupService struct {
 	metrics *collector.CollectorMetrics
 }
 
-func (bs *BackupService) Run(ctx context.Context) {
+func (bs *BackupService) Create(ctx context.Context) {
 	var command string
 	var use_flags []string
 	// set backup command and arguments
@@ -47,15 +47,15 @@ func (bs *BackupService) Run(ctx context.Context) {
 	// create backup
 	n := 0
 	for n < 3 {
-		result, err := backup(command, use_flags...)
+		t, d, s, err := backup(command, use_flags...)
 		if err != nil {
-			bs.metrics.SetResult(result)
+			bs.metrics.SetResultCreate(collector.ResultCreate{t, d, s})
 			fmt.Fprint(os.Stdout, []any{"Backup failed\nStart backup one more time\n"}...)
 			time.Sleep(5 * time.Second)
 		} else {
 			n = 3
 			print("Backup created\n")
-			bs.metrics.SetResult(result)
+			bs.metrics.SetResultCreate(collector.ResultCreate{t, d, s})
 			return
 		}
 		n++
@@ -64,42 +64,43 @@ func (bs *BackupService) Run(ctx context.Context) {
 
 func (bs *BackupService) Retain(ctx context.Context) {
 	var use_flags []string
-	// for psql we need to use BackupStorage
 	if bs.config.BackupType == "psql" {
 		if bs.config.DeleteRetain != "" {
 			use_flags = append(use_flags, "delete", "retain", "FULL", bs.config.DeleteRetain, "--confirm")
 		}
 		use_flags = append(use_flags, bs.config.BackupStorage)
+	} else {
+		print("Retain backup support only for postgresql\n")
 	}
 
 	if bs.config.ConfigFile != "" {
 		use_flags = append(use_flags, "--config", bs.config.ConfigFile)
 	}
 
 	// Retain backup
-	result, err := backup("wal-g", use_flags...)
+	t, d, s, err := backup("wal-g", use_flags...)
 	if err != nil {
-		bs.metrics.SetResult(result)
+		bs.metrics.SetResultRetain(collector.ResultRetain{t, d, s})
 		fmt.Fprint(os.Stdout, []any{"Retain backup failed\n"}...)
 	} else {
 		print("Retain backup finished\n")
-		bs.metrics.SetResult(result)
+		bs.metrics.SetResultRetain(collector.ResultRetain{t, d, s})
 		return
 	}
 }
 
-func backup(command string, args ...string) (result collector.Result, err error) {
-	result.BackupStartTime = time.Now()
+func backup(command string, args ...string) (startTime time.Time, duration float64, status int, err error) {
+	startTime = time.Now()
 	cmd := exec.Command(command, args...)
 	output, err := cmd.CombinedOutput()
 
 	defer func() {
-		result.BackupDuration = float64(time.Since(result.BackupStartTime).Seconds())
+		duration = float64(time.Since(startTime).Seconds())
 		if err != nil {
-			result.BackupStatus = 1
+			status = 1
 			log.Printf("Error:\n", err, string(output))
 		} else {
-			result.BackupStatus = 0
+			status = 0
 			log.Printf("Command Output: %s\n", string(output))
 		}
 	}()