Skip to content

Commit ced4a48

Browse files
deitchdeitch
authored
Merge pull request #277 from databacker/retention
enable retention
2 parents 613f323 + 8a36813 commit ced4a48

File tree

23 files changed

+959
-175
lines changed

23 files changed

+959
-175
lines changed

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Back up mysql databases to... anywhere!
44

55
## Overview
66

7-
mysql-backup is a simple way to do MySQL database backups and restores.
7+
mysql-backup is a simple way to do MySQL database backups and restores, as well as manage your backups.
88

99
It has the following features:
1010

@@ -14,6 +14,7 @@ It has the following features:
1414
* connect to any container running on the same system
1515
* select how often to run a dump
1616
* select when to start the first dump, whether time of day or relative to container start time
17+
* prune backups older than a specific time period or quantity
1718

1819
Please see [CONTRIBUTORS.md](./CONTRIBUTORS.md) for a list of contributors.
1920

‎cmd/common_test.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,25 @@ func newMockExecs() *mockExecs {
1717
return m
1818
}
1919

20-
func (m *mockExecs) timerDump(opts core.DumpOptions, timerOpts core.TimerOptions) error {
21-
args := m.Called(opts, timerOpts)
20+
func (m *mockExecs) dump(opts core.DumpOptions) error {
21+
args := m.Called(opts)
2222
return args.Error(0)
2323
}
2424

2525
func (m *mockExecs) restore(target storage.Storage, targetFile string, dbconn database.Connection, databasesMap map[string]string, compressor compression.Compressor) error {
2626
args := m.Called(target, targetFile, dbconn, databasesMap, compressor)
2727
return args.Error(0)
2828
}
29+
30+
func (m *mockExecs) prune(opts core.PruneOptions) error {
31+
args := m.Called(opts)
32+
return args.Error(0)
33+
}
34+
func (m *mockExecs) timer(timerOpts core.TimerOptions, cmd func() error) error {
35+
args := m.Called(timerOpts)
36+
err := args.Error(0)
37+
if err != nil {
38+
return err
39+
}
40+
return cmd()
41+
}

‎cmd/dump.go‎

Lines changed: 60 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ const (
2020
defaultMaxAllowedPacket = 4194304
2121
)
2222

23-
func dumpCmd(execs execs) (*cobra.Command, error) {
23+
func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
24+
if cmdConfig == nil {
25+
return nil, fmt.Errorf("cmdConfig is nil")
26+
}
2427
var v *viper.Viper
2528
var cmd = &cobra.Command{
2629
Use: "dump",
@@ -43,18 +46,18 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
4346
)
4447
if len(targetURLs) > 0 {
4548
for _, t := range targetURLs {
46-
store, err := storage.ParseURL(t, creds)
49+
store, err := storage.ParseURL(t, cmdConfig.creds)
4750
if err != nil {
4851
return fmt.Errorf("invalid target url: %v", err)
4952
}
5053
targets = append(targets, store)
5154
}
5255
} else {
5356
// try the config file
54-
if configuration != nil {
57+
if cmdConfig.configuration != nil {
5558
// parse the target objects, then the ones listed for the backup
56-
targetStructures := configuration.Targets
57-
dumpTargets := configuration.Dump.Targets
59+
targetStructures := cmdConfig.configuration.Targets
60+
dumpTargets := cmdConfig.configuration.Dump.Targets
5861
for _, t := range dumpTargets {
5962
var store storage.Storage
6063
if target, ok := targetStructures[t]; !ok {
@@ -73,49 +76,49 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
7376
return fmt.Errorf("no targets specified")
7477
}
7578
safechars := v.GetBool("safechars")
76-
if !v.IsSet("safechars") && configuration != nil {
77-
safechars = configuration.Dump.Safechars
79+
if !v.IsSet("safechars") && cmdConfig.configuration != nil {
80+
safechars = cmdConfig.configuration.Dump.Safechars
7881
}
7982
include := v.GetStringSlice("include")
80-
if len(include) == 0 && configuration != nil {
81-
include = configuration.Dump.Include
83+
if len(include) == 0 && cmdConfig.configuration != nil {
84+
include = cmdConfig.configuration.Dump.Include
8285
}
8386
// make this slice nil if it's empty, so it is consistent; used mainly for test consistency
8487
if len(include) == 0 {
8588
include = nil
8689
}
8790
exclude := v.GetStringSlice("exclude")
88-
if len(exclude) == 0 && configuration != nil {
89-
exclude = configuration.Dump.Exclude
91+
if len(exclude) == 0 && cmdConfig.configuration != nil {
92+
exclude = cmdConfig.configuration.Dump.Exclude
9093
}
9194
// make this slice nil if it's empty, so it is consistent; used mainly for test consistency
9295
if len(exclude) == 0 {
9396
exclude = nil
9497
}
9598
preBackupScripts := v.GetString("pre-backup-scripts")
96-
if preBackupScripts == "" && configuration != nil {
97-
preBackupScripts = configuration.Dump.Scripts.PreBackup
99+
if preBackupScripts == "" && cmdConfig.configuration != nil {
100+
preBackupScripts = cmdConfig.configuration.Dump.Scripts.PreBackup
98101
}
99102
noDatabaseName := v.GetBool("no-database-name")
100-
if !v.IsSet("no-database-name") && configuration != nil {
101-
noDatabaseName = configuration.Dump.NoDatabaseName
103+
if !v.IsSet("no-database-name") && cmdConfig.configuration != nil {
104+
noDatabaseName = cmdConfig.configuration.Dump.NoDatabaseName
102105
}
103106
compact := v.GetBool("compact")
104-
if !v.IsSet("compact") && configuration != nil {
105-
compact = configuration.Dump.Compact
107+
if !v.IsSet("compact") && cmdConfig.configuration != nil {
108+
compact = cmdConfig.configuration.Dump.Compact
106109
}
107110
maxAllowedPacket := v.GetInt("max-allowed-packet")
108-
if !v.IsSet("max-allowed-packet") && configuration != nil && configuration.Dump.MaxAllowedPacket != 0 {
109-
maxAllowedPacket = configuration.Dump.MaxAllowedPacket
111+
if !v.IsSet("max-allowed-packet") && cmdConfig.configuration != nil && cmdConfig.configuration.Dump.MaxAllowedPacket != 0 {
112+
maxAllowedPacket = cmdConfig.configuration.Dump.MaxAllowedPacket
110113
}
111114

112115
// compression algorithm: check config, then CLI/env var overrides
113116
var (
114117
compressionAlgo string
115118
compressor compression.Compressor
116119
)
117-
if configuration != nil {
118-
compressionAlgo = configuration.Dump.Compression
120+
if cmdConfig.configuration != nil {
121+
compressionAlgo = cmdConfig.configuration.Dump.Compression
119122
}
120123
compressionVar := v.GetString("compression")
121124
if compressionVar != "" {
@@ -131,7 +134,7 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
131134
Targets: targets,
132135
Safechars: safechars,
133136
DBNames: include,
134-
DBConn: dbconn,
137+
DBConn: cmdConfig.dbconn,
135138
Compressor: compressor,
136139
Exclude: exclude,
137140
PreBackupScripts: preBackupScripts,
@@ -141,35 +144,56 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
141144
MaxAllowedPacket: maxAllowedPacket,
142145
}
143146

147+
// retention, if enabled
148+
retention := v.GetString("retention")
149+
if retention == "" && cmdConfig.configuration != nil {
150+
retention = cmdConfig.configuration.Prune.Retention
151+
}
152+
144153
// timer options
145154
once := v.GetBool("once")
146-
if !v.IsSet("once") && configuration != nil {
147-
once = configuration.Dump.Schedule.Once
155+
if !v.IsSet("once") && cmdConfig.configuration != nil {
156+
once = cmdConfig.configuration.Dump.Schedule.Once
148157
}
149158
cron := v.GetString("cron")
150-
if cron == "" && configuration != nil {
151-
cron = configuration.Dump.Schedule.Cron
159+
if cron == "" && cmdConfig.configuration != nil {
160+
cron = cmdConfig.configuration.Dump.Schedule.Cron
152161
}
153162
begin := v.GetString("begin")
154-
if begin == "" && configuration != nil {
155-
begin = configuration.Dump.Schedule.Begin
163+
if begin == "" && cmdConfig.configuration != nil {
164+
begin = cmdConfig.configuration.Dump.Schedule.Begin
156165
}
157166
frequency := v.GetInt("frequency")
158-
if frequency == 0 && configuration != nil {
159-
frequency = configuration.Dump.Schedule.Frequency
167+
if frequency == 0 && cmdConfig.configuration != nil {
168+
frequency = cmdConfig.configuration.Dump.Schedule.Frequency
160169
}
161170
timerOpts := core.TimerOptions{
162171
Once: once,
163172
Cron: cron,
164173
Begin: begin,
165174
Frequency: frequency,
166175
}
167-
dump := core.TimerDump
176+
dump := core.Dump
177+
prune := core.Prune
178+
timer := core.TimerCommand
168179
if execs != nil {
169-
dump = execs.timerDump
180+
dump = execs.dump
181+
prune = execs.prune
182+
timer = execs.timer
170183
}
171-
if err := dump(dumpOpts, timerOpts); err != nil {
172-
return err
184+
if err := timer(timerOpts, func() error {
185+
err := dump(dumpOpts)
186+
if err != nil {
187+
return fmt.Errorf("error running dump: %w", err)
188+
}
189+
if retention != "" {
190+
if err := prune(core.PruneOptions{Targets: targets, Retention: retention}); err != nil {
191+
return fmt.Errorf("error running prune: %w", err)
192+
}
193+
}
194+
return nil
195+
}); err != nil {
196+
return fmt.Errorf("error running command: %w", err)
173197
}
174198
log.Info("Backup complete")
175199
return nil
@@ -232,6 +256,8 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S
232256
cmd.MarkFlagsMutuallyExclusive("once", "frequency")
233257
cmd.MarkFlagsMutuallyExclusive("cron", "begin")
234258
cmd.MarkFlagsMutuallyExclusive("cron", "frequency")
259+
// retention
260+
flags.String("retention", "", "Retention period for backups. Optional. If not specified, no pruning will be done. Can be number of backups or time-based. For time-based, the format is: 1d, 1w, 1m, 1y for days, weeks, months, years, respectively. For number-based, the format is: 1c, 2c, 3c, etc. for the count of backups to keep.")
235261

236262
return cmd, nil
237263
}

‎cmd/dump_test.go‎

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,70 +25,97 @@ func TestDumpCmd(t *testing.T) {
2525
wantErr bool
2626
expectedDumpOptions core.DumpOptions
2727
expectedTimerOptions core.TimerOptions
28+
expectedPruneOptions *core.PruneOptions
2829
}{
29-
{"missing server and target options", []string{""}, "", true, core.DumpOptions{}, core.TimerOptions{}},
30-
{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: database.Connection{Host: "abc"}}, core.TimerOptions{}},
30+
// invalid ones
31+
{"missing server and target options", []string{""}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
32+
{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: database.Connection{Host: "abc"}}, core.TimerOptions{}, nil},
33+
34+
// file URL
3135
{"file URL", []string{"--server", "abc", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{
3236
Targets: []storage.Storage{file.New(*fileTargetURL)},
3337
MaxAllowedPacket: defaultMaxAllowedPacket,
3438
Compressor: &compression.GzipCompressor{},
3539
DBConn: database.Connection{Host: "abc"},
36-
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}},
40+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
41+
{"file URL with prune", []string{"--server", "abc", "--target", "file:///foo/bar", "--retention", "1h"}, "", false, core.DumpOptions{
42+
Targets: []storage.Storage{file.New(*fileTargetURL)},
43+
MaxAllowedPacket: defaultMaxAllowedPacket,
44+
Compressor: &compression.GzipCompressor{},
45+
DBConn: database.Connection{Host: "abc"},
46+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
47+
48+
// config file
49+
{"config file", []string{"--config-file", "testdata/config.yml"}, "", false, core.DumpOptions{
50+
Targets: []storage.Storage{file.New(*fileTargetURL)},
51+
MaxAllowedPacket: defaultMaxAllowedPacket,
52+
Compressor: &compression.GzipCompressor{},
53+
DBConn: database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
54+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
55+
56+
// timer options
3757
{"once flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--once"}, "", false, core.DumpOptions{
3858
Targets: []storage.Storage{file.New(*fileTargetURL)},
3959
MaxAllowedPacket: defaultMaxAllowedPacket,
4060
Compressor: &compression.GzipCompressor{},
4161
DBConn: database.Connection{Host: "abc"},
42-
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Once: true}},
62+
}, core.TimerOptions{Once: true, Frequency: defaultFrequency, Begin: defaultBegin}, nil},
4363
{"cron flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *"}, "", false, core.DumpOptions{
4464
Targets: []storage.Storage{file.New(*fileTargetURL)},
4565
MaxAllowedPacket: defaultMaxAllowedPacket,
4666
Compressor: &compression.GzipCompressor{},
4767
DBConn: database.Connection{Host: "abc"},
48-
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}},
68+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}, nil},
4969
{"begin flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--begin", "1234"}, "", false, core.DumpOptions{
5070
Targets: []storage.Storage{file.New(*fileTargetURL)},
5171
MaxAllowedPacket: defaultMaxAllowedPacket,
5272
Compressor: &compression.GzipCompressor{},
5373
DBConn: database.Connection{Host: "abc"},
54-
}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}},
74+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}, nil},
5575
{"frequency flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--frequency", "10"}, "", false, core.DumpOptions{
5676
Targets: []storage.Storage{file.New(*fileTargetURL)},
5777
MaxAllowedPacket: defaultMaxAllowedPacket,
5878
Compressor: &compression.GzipCompressor{},
5979
DBConn: database.Connection{Host: "abc"},
60-
}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}},
61-
{"config file", []string{"--config-file", "testdata/config.yml"}, "", false, core.DumpOptions{
62-
Targets: []storage.Storage{file.New(*fileTargetURL)},
63-
MaxAllowedPacket: defaultMaxAllowedPacket,
64-
Compressor: &compression.GzipCompressor{},
65-
DBConn: database.Connection{Host: "abc", Port: 3306, User: "user", Pass: "xxxx"},
66-
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}},
67-
{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
68-
{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
69-
{"incompatible flags: once/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
70-
{"incompatible flags: cron/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
71-
{"incompatible flags: cron/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
80+
}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}, nil},
81+
{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
82+
{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
83+
{"incompatible flags: once/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
84+
{"incompatible flags: cron/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
85+
{"incompatible flags: cron/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--frequency", "10"}, "", true, core.DumpOptions{
86+
DBConn: database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
87+
}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
7288
}
7389

7490
for _, tt := range tests {
7591
t.Run(tt.name, func(t *testing.T) {
7692
m := newMockExecs()
77-
m.On("timerDump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
93+
m.On("dump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
7894
diff := deep.Equal(dumpOpts, tt.expectedDumpOptions)
7995
if diff == nil {
8096
return true
8197
}
8298
t.Errorf("dumpOpts compare failed: %v", diff)
8399
return false
84-
}), mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
100+
})).Return(nil)
101+
m.On("timer", mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
85102
diff := deep.Equal(timerOpts, tt.expectedTimerOptions)
86103
if diff == nil {
87104
return true
88105
}
89106
t.Errorf("timerOpts compare failed: %v", diff)
90107
return false
91108
})).Return(nil)
109+
if tt.expectedPruneOptions != nil {
110+
m.On("prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
111+
diff := deep.Equal(pruneOpts, *tt.expectedPruneOptions)
112+
if diff == nil {
113+
return true
114+
}
115+
t.Errorf("pruneOpts compare failed: %v", diff)
116+
return false
117+
})).Return(nil)
118+
}
92119

93120
cmd, err := rootCmd(m)
94121
if err != nil {

0 commit comments

Comments
 (0)