Skip to content

Commit 67e7288

Browse files
michalmiddletonMichal Middleton
michalmiddleton
and
Michal Middleton
authored
Add support for zstd compression (#249)
Co-authored-by: Michal Middleton <jafa81@gmail.com>
1 parent 1765b06 commit 67e7288

File tree

6 files changed

+195
-74
lines changed

6 files changed

+195
-74
lines changed

‎README.md‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,22 @@ You can populate below template according to your requirements and use it as you
148148

149149
# BACKUP_CRON_EXPRESSION="0 2 * * *"
150150

151-
# The name of the backup file including the `.tar.gz` extension.
151+
# The compression algorithm used in conjunction with tar.
152+
# Valid options are: "gz" (Gzip) and "zst" (Zstd).
153+
# Note that the selection affects the file extension.
154+
155+
# BACKUP_COMPRESSION="gz"
156+
157+
# The name of the backup file including the extension.
152158
# Format verbs will be replaced as in `strftime`. Omitting them
153159
# will result in the same filename for every backup run, which means previous
154-
# versions will be overwritten on subsequent runs. The default results
155-
# in filenames like `backup-2021-08-29T04-00-00.tar.gz`.
160+
# versions will be overwritten on subsequent runs.
161+
# Extension can be defined literally or via "{{ .Extension }}" template,
162+
# in which case it will become either "tar.gz" or "tar.zst" (depending
163+
# on your BACKUP_COMPRESSION setting).
164+
# The default results in filenames like: `backup-2021-08-29T04-00-00.tar.gz`.
156165

157-
# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.tar.gz"
166+
# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
158167

159168
# Setting BACKUP_FILENAME_EXPAND to true allows for environment variable
160169
# placeholders in BACKUP_FILENAME, BACKUP_LATEST_SYMLINK and in

‎cmd/backup/archive.go‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ import (
1515
"path"
1616
"path/filepath"
1717
"strings"
18+
19+
"github.com/klauspost/compress/zstd"
1820
)
1921

20-
func createArchive(files []string, inputFilePath, outputFilePath string) error {
22+
func createArchive(files []string, inputFilePath, outputFilePath string, compression string) error {
2123
inputFilePath = stripTrailingSlashes(inputFilePath)
2224
inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
2325
if err != nil {
@@ -27,7 +29,7 @@ func createArchive(files []string, inputFilePath, outputFilePath string) error {
2729
return fmt.Errorf("createArchive: error creating output file path: %w", err)
2830
}
2931

30-
if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
32+
if err := compress(files, outputFilePath, filepath.Dir(inputFilePath), compression); err != nil {
3133
return fmt.Errorf("createArchive: error creating archive: %w", err)
3234
}
3335

@@ -51,18 +53,30 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error)
5153
return inputFilePath, outputFilePath, err
5254
}
5355

54-
func compress(paths []string, outFilePath, subPath string) error {
56+
func compress(paths []string, outFilePath, subPath string, algo string) error {
5557
file, err := os.Create(outFilePath)
58+
var compressWriter io.WriteCloser
5659
if err != nil {
5760
return fmt.Errorf("compress: error creating out file: %w", err)
5861
}
5962

6063
prefix := path.Dir(outFilePath)
61-
gzipWriter := gzip.NewWriter(file)
62-
tarWriter := tar.NewWriter(gzipWriter)
64+
switch algo {
65+
case "gz":
66+
compressWriter = gzip.NewWriter(file)
67+
case "zst":
68+
compressWriter, err = zstd.NewWriter(file)
69+
if err != nil {
70+
return fmt.Errorf("compress: zstd error: %w", err)
71+
}
72+
default:
73+
return fmt.Errorf("compress: unsupported compression algorithm: %s", algo)
74+
}
75+
76+
tarWriter := tar.NewWriter(compressWriter)
6377

6478
for _, p := range paths {
65-
if err := writeTarGz(p, tarWriter, prefix); err != nil {
79+
if err := writeTarball(p, tarWriter, prefix); err != nil {
6680
return fmt.Errorf("compress: error writing %s to archive: %w", p, err)
6781
}
6882
}
@@ -72,9 +86,9 @@ func compress(paths []string, outFilePath, subPath string) error {
7286
return fmt.Errorf("compress: error closing tar writer: %w", err)
7387
}
7488

75-
err = gzipWriter.Close()
89+
err = compressWriter.Close()
7690
if err != nil {
77-
return fmt.Errorf("compress: error closing gzip writer: %w", err)
91+
return fmt.Errorf("compress: error closing compression writer: %w", err)
7892
}
7993

8094
err = file.Close()
@@ -85,10 +99,10 @@ func compress(paths []string, outFilePath, subPath string) error {
8599
return nil
86100
}
87101

88-
func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
102+
func writeTarball(path string, tarWriter *tar.Writer, prefix string) error {
89103
fileInfo, err := os.Lstat(path)
90104
if err != nil {
91-
return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
105+
return fmt.Errorf("writeTarball: error getting file infor for %s: %w", path, err)
92106
}
93107

94108
if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
@@ -99,19 +113,19 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
99113
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
100114
var err error
101115
if link, err = os.Readlink(path); err != nil {
102-
return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
116+
return fmt.Errorf("writeTarball: error resolving symlink %s: %w", path, err)
103117
}
104118
}
105119

106120
header, err := tar.FileInfoHeader(fileInfo, link)
107121
if err != nil {
108-
return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
122+
return fmt.Errorf("writeTarball: error getting file info header: %w", err)
109123
}
110124
header.Name = strings.TrimPrefix(path, prefix)
111125

112126
err = tarWriter.WriteHeader(header)
113127
if err != nil {
114-
return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
128+
return fmt.Errorf("writeTarball: error writing file info header: %w", err)
115129
}
116130

117131
if !fileInfo.Mode().IsRegular() {
@@ -120,13 +134,13 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
120134

121135
file, err := os.Open(path)
122136
if err != nil {
123-
return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
137+
return fmt.Errorf("writeTarball: error opening %s: %w", path, err)
124138
}
125139
defer file.Close()
126140

127141
_, err = io.Copy(tarWriter, file)
128142
if err != nil {
129-
return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
143+
return fmt.Errorf("writeTarball: error copying %s to tar writer: %w", path, err)
130144
}
131145

132146
return nil

‎cmd/backup/config.go‎

Lines changed: 70 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -16,59 +16,60 @@ import (
1616
// Config holds all configuration values that are expected to be set
1717
// by users.
1818
type Config struct {
19-
AwsS3BucketName string `split_words:"true"`
20-
AwsS3Path string `split_words:"true"`
21-
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
22-
AwsEndpointProto string `split_words:"true" default:"https"`
23-
AwsEndpointInsecure bool `split_words:"true"`
24-
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
25-
AwsStorageClass string `split_words:"true"`
26-
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
27-
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
28-
AwsSecretAccessKey string `split_words:"true"`
29-
AwsSecretAccessKeyFile string `split_words:"true"`
30-
AwsIamRoleEndpoint string `split_words:"true"`
31-
AwsPartSize int64 `split_words:"true"`
32-
BackupSources string `split_words:"true" default:"/backup"`
33-
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
34-
BackupFilenameExpand bool `split_words:"true"`
35-
BackupLatestSymlink string `split_words:"true"`
36-
BackupArchive string `split_words:"true" default:"/archive"`
37-
BackupRetentionDays int32 `split_words:"true" default:"-1"`
38-
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
39-
BackupPruningPrefix string `split_words:"true"`
40-
BackupStopContainerLabel string `split_words:"true" default:"true"`
41-
BackupFromSnapshot bool `split_words:"true"`
42-
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
43-
GpgPassphrase string `split_words:"true"`
44-
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
45-
NotificationLevel string `split_words:"true" default:"error"`
46-
EmailNotificationRecipient string `split_words:"true"`
47-
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
48-
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
49-
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
50-
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
51-
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
52-
WebdavUrl string `split_words:"true"`
53-
WebdavUrlInsecure bool `split_words:"true"`
54-
WebdavPath string `split_words:"true" default:"/"`
55-
WebdavUsername string `split_words:"true"`
56-
WebdavPassword string `split_words:"true"`
57-
SSHHostName string `split_words:"true"`
58-
SSHPort string `split_words:"true" default:"22"`
59-
SSHUser string `split_words:"true"`
60-
SSHPassword string `split_words:"true"`
61-
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
62-
SSHIdentityPassphrase string `split_words:"true"`
63-
SSHRemotePath string `split_words:"true"`
64-
ExecLabel string `split_words:"true"`
65-
ExecForwardOutput bool `split_words:"true"`
66-
LockTimeout time.Duration `split_words:"true" default:"60m"`
67-
AzureStorageAccountName string `split_words:"true"`
68-
AzureStoragePrimaryAccountKey string `split_words:"true"`
69-
AzureStorageContainerName string `split_words:"true"`
70-
AzureStoragePath string `split_words:"true"`
71-
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
19+
AwsS3BucketName string `split_words:"true"`
20+
AwsS3Path string `split_words:"true"`
21+
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
22+
AwsEndpointProto string `split_words:"true" default:"https"`
23+
AwsEndpointInsecure bool `split_words:"true"`
24+
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
25+
AwsStorageClass string `split_words:"true"`
26+
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
27+
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
28+
AwsSecretAccessKey string `split_words:"true"`
29+
AwsSecretAccessKeyFile string `split_words:"true"`
30+
AwsIamRoleEndpoint string `split_words:"true"`
31+
AwsPartSize int64 `split_words:"true"`
32+
BackupCompression CompressionType `split_words:"true" default:"gz"`
33+
BackupSources string `split_words:"true" default:"/backup"`
34+
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
35+
BackupFilenameExpand bool `split_words:"true"`
36+
BackupLatestSymlink string `split_words:"true"`
37+
BackupArchive string `split_words:"true" default:"/archive"`
38+
BackupRetentionDays int32 `split_words:"true" default:"-1"`
39+
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
40+
BackupPruningPrefix string `split_words:"true"`
41+
BackupStopContainerLabel string `split_words:"true" default:"true"`
42+
BackupFromSnapshot bool `split_words:"true"`
43+
BackupExcludeRegexp RegexpDecoder `split_words:"true"`
44+
GpgPassphrase string `split_words:"true"`
45+
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
46+
NotificationLevel string `split_words:"true" default:"error"`
47+
EmailNotificationRecipient string `split_words:"true"`
48+
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
49+
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
50+
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
51+
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
52+
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
53+
WebdavUrl string `split_words:"true"`
54+
WebdavUrlInsecure bool `split_words:"true"`
55+
WebdavPath string `split_words:"true" default:"/"`
56+
WebdavUsername string `split_words:"true"`
57+
WebdavPassword string `split_words:"true"`
58+
SSHHostName string `split_words:"true"`
59+
SSHPort string `split_words:"true" default:"22"`
60+
SSHUser string `split_words:"true"`
61+
SSHPassword string `split_words:"true"`
62+
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
63+
SSHIdentityPassphrase string `split_words:"true"`
64+
SSHRemotePath string `split_words:"true"`
65+
ExecLabel string `split_words:"true"`
66+
ExecForwardOutput bool `split_words:"true"`
67+
LockTimeout time.Duration `split_words:"true" default:"60m"`
68+
AzureStorageAccountName string `split_words:"true"`
69+
AzureStoragePrimaryAccountKey string `split_words:"true"`
70+
AzureStorageContainerName string `split_words:"true"`
71+
AzureStoragePath string `split_words:"true"`
72+
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
7273
}
7374

7475
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
@@ -82,6 +83,22 @@ func (c *Config) resolveSecret(envVar string, secretPath string) (string, error)
8283
return string(data), nil
8384
}
8485

86+
type CompressionType string
87+
88+
func (c *CompressionType) Decode(v string) error {
89+
switch v {
90+
case "gz", "zst":
91+
*c = CompressionType(v)
92+
return nil
93+
default:
94+
return fmt.Errorf("config: error decoding compression type %s", v)
95+
}
96+
}
97+
98+
func (c *CompressionType) String() string {
99+
return string(*c)
100+
}
101+
85102
type CertDecoder struct {
86103
Cert *x509.Certificate
87104
}

‎cmd/backup/script.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package main
55

66
import (
7+
"bytes"
78
"context"
89
"errors"
910
"fmt"
@@ -89,6 +90,20 @@ func newScript() (*script, error) {
8990
}
9091

9192
s.file = path.Join("/tmp", s.c.BackupFilename)
93+
94+
tmplFileName, tErr := template.New("extension").Parse(s.file)
95+
if tErr != nil {
96+
return nil, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
97+
}
98+
99+
var bf bytes.Buffer
100+
if tErr := tmplFileName.Execute(&bf, map[string]string{
101+
"Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression),
102+
}); tErr != nil {
103+
return nil, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
104+
}
105+
s.file = bf.String()
106+
92107
if s.c.BackupFilenameExpand {
93108
s.file = os.ExpandEnv(s.file)
94109
s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
@@ -454,7 +469,7 @@ func (s *script) createArchive() error {
454469
return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
455470
}
456471

457-
if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
472+
if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String()); err != nil {
458473
return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
459474
}
460475

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ require (
1010
github.com/docker/docker v24.0.5+incompatible
1111
github.com/gofrs/flock v0.8.1
1212
github.com/kelseyhightower/envconfig v1.4.0
13+
github.com/klauspost/compress v1.16.7
1314
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
1415
github.com/minio/minio-go/v7 v7.0.61
1516
github.com/otiai10/copy v1.11.0
@@ -33,7 +34,6 @@ require (
3334
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
3435
github.com/google/uuid v1.3.0 // indirect
3536
github.com/json-iterator/go v1.1.12 // indirect
36-
github.com/klauspost/compress v1.16.7 // indirect
3737
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
3838
github.com/kr/fs v0.1.0 // indirect
3939
github.com/kylelemons/godebug v1.1.0 // indirect

0 commit comments

Comments
 (0)