Skip to content

Commit 6848fd4

Browse files
saolofmnencia
saolof
and
mnencia
authored
feat: add lz4, xz, and zstd compression (#82)
This adds support for the three compression options that were added as compressors in Barman 3.12, and which should hopefully improve RTO for compressed object store backups. Signed-off-by: Olof Salberger <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 1730bc9 commit 6848fd4

File tree

6 files changed

+124
-15
lines changed

6 files changed

+124
-15
lines changed

‎pkg/api/config.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,17 @@ const (
5555
// CompressionTypeBzip2 means bzip2 compression is performed
5656
CompressionTypeBzip2 = CompressionType("bzip2")
5757

58+
// CompressionTypeLz4 means lz4 compression is performed
59+
CompressionTypeLz4 = CompressionType("lz4")
60+
5861
// CompressionTypeSnappy means snappy compression is performed
5962
CompressionTypeSnappy = CompressionType("snappy")
63+
64+
// CompressionTypeXz means xz compression is performed
65+
CompressionTypeXz = CompressionType("xz")
66+
67+
// CompressionTypeZstd means zstd compression is performed
68+
CompressionTypeZstd = CompressionType("zstd")
6069
)
6170

6271
// BarmanCredentials an object containing the potential credentials for each cloud provider
@@ -204,8 +213,9 @@ type BarmanObjectStoreConfiguration struct {
204213
// WAL stream
205214
type WalBackupConfiguration struct {
206215
// Compress a WAL file before sending it to the object store. Available
207-
// options are empty string (no compression, default), `gzip`, `bzip2` or `snappy`.
208-
// +kubebuilder:validation:Enum=gzip;bzip2;snappy
216+
// options are empty string (no compression, default), `gzip`, `bzip2`,
217+
// `lz4`, `snappy`, `xz`, and `zstd`.
218+
// +kubebuilder:validation:Enum=bzip2;gzip;lz4;snappy;xz;zstd
209219
// +optional
210220
Compression CompressionType `json:"compression,omitempty"`
211221

@@ -264,8 +274,8 @@ type WalBackupConfiguration struct {
264274
type DataBackupConfiguration struct {
265275
// Compress a backup file (a tar file per tablespace) while streaming it
266276
// to the object store. Available options are empty string (no
267-
// compression, default), `gzip`, `bzip2` or `snappy`.
268-
// +kubebuilder:validation:Enum=gzip;bzip2;snappy
277+
// compression, default), `gzip`, `bzip2`, `lz4`, `snappy`, `xz`, and `zstd`.
278+
// +kubebuilder:validation:Enum=bzip2;gzip;lz4;snappy;xz;zstd
269279
// +optional
270280
Compression CompressionType `json:"compression,omitempty"`
271281

‎pkg/archiver/command.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,9 @@ func (archiver *WALArchiver) BarmanCloudWalArchiveOptions(
125125

126126
var options []string
127127
if configuration.Wal != nil {
128-
if configuration.Wal.Compression == barmanApi.CompressionTypeSnappy && !capabilities.HasSnappy {
129-
return nil, fmt.Errorf("snappy compression is not supported in Barman %v", capabilities.Version)
128+
if !capabilities.HasCompression(configuration.Wal.Compression) {
129+
return nil, fmt.Errorf("%v compression is not supported in Barman %v",
130+
configuration.Wal.Compression, capabilities.Version)
130131
}
131132
if len(configuration.Wal.Compression) != 0 {
132133
options = append(

‎pkg/backup/backup.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ func (b *Command) GetDataConfiguration(
5959
return options, nil
6060
}
6161

62-
if b.configuration.Data.Compression == barmanApi.CompressionTypeSnappy && !b.capabilities.HasSnappy {
63-
return nil, fmt.Errorf("snappy compression is not supported in Barman %v", b.capabilities.Version)
62+
if !b.capabilities.HasCompression(b.configuration.Data.Compression) {
63+
return nil, fmt.Errorf("%v compression is not supported in Barman %v",
64+
b.configuration.Data.Compression, b.capabilities.Version)
6465
}
6566

6667
if len(b.configuration.Data.Compression) != 0 {

‎pkg/capabilities/detect.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import (
2424

2525
"github.com/blang/semver"
2626
"github.com/cloudnative-pg/machinery/pkg/log"
27+
28+
barmanApi "github.com/cloudnative-pg/barman-cloud/pkg/api"
2729
)
2830

2931
// capabilities stores the current Barman capabilities
@@ -34,6 +36,12 @@ var capabilities *Capabilities
3436
func detect(version *semver.Version) *Capabilities {
3537
contextLogger := log.FromContext(context.Background())
3638
newCapabilities := new(Capabilities)
39+
newCapabilities.supportedCompressions = []barmanApi.CompressionType{
40+
barmanApi.CompressionTypeNone,
41+
barmanApi.CompressionTypeBzip2,
42+
barmanApi.CompressionTypeGzip,
43+
}
44+
3745
if version == nil {
3846
contextLogger.Info("Missing Barman Cloud installation in the operand image")
3947
return newCapabilities
@@ -42,6 +50,15 @@ func detect(version *semver.Version) *Capabilities {
4250
newCapabilities.Version = version
4351

4452
switch {
53+
case version.GE(semver.Version{Major: 3, Minor: 12}):
54+
// lz4, xz, and zstd compression support, added in barman 3.12
55+
newCapabilities.supportedCompressions = append(
56+
newCapabilities.supportedCompressions,
57+
barmanApi.CompressionTypeLz4,
58+
barmanApi.CompressionTypeXz,
59+
barmanApi.CompressionTypeZstd,
60+
)
61+
fallthrough
4562
case version.GE(semver.Version{Major: 3, Minor: 4}):
4663
// The --name flag was added to Barman in version 3.3 but we also require the
4764
// barman-cloud-backup-show command which was not added until Barman version 3.4
@@ -57,7 +74,10 @@ func detect(version *semver.Version) *Capabilities {
5774
// Barman-cloud-check-wal-archive, added in Barman >= 2.18
5875
newCapabilities.HasCheckWalArchive = true
5976
// Snappy compression support, added in Barman >= 2.18
60-
newCapabilities.HasSnappy = true
77+
newCapabilities.supportedCompressions = append(
78+
newCapabilities.supportedCompressions,
79+
barmanApi.CompressionTypeSnappy,
80+
)
6181
// error codes for wal-restore command added in Barman >= 2.18
6282
newCapabilities.HasErrorCodesForWALRestore = true
6383
// azure-identity credential of type managed-identity added in Barman >= 2.18

‎pkg/capabilities/detect_test.go‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,42 @@ package capabilities
1919
import (
2020
"github.com/blang/semver"
2121

22+
barmanApi "github.com/cloudnative-pg/barman-cloud/pkg/api"
23+
2224
. "github.com/onsi/ginkgo/v2"
2325
. "github.com/onsi/gomega"
2426
)
2527

2628
var _ = Describe("detect capabilities", func() {
27-
It("ensures that all capabilities are true for the 3.4 version", func() {
29+
It("ensures that all capabilities are true for the 3.12 version", func() {
30+
version, err := semver.ParseTolerant("3.12.0")
31+
Expect(err).ToNot(HaveOccurred())
32+
capabilities := detect(&version)
33+
Expect(capabilities).To(Equal(&Capabilities{
34+
Version: &version,
35+
hasName: true,
36+
HasAzure: true,
37+
HasS3: true,
38+
HasGoogle: true,
39+
HasRetentionPolicy: true,
40+
HasTags: true,
41+
HasCheckWalArchive: true,
42+
HasErrorCodesForWALRestore: true,
43+
HasErrorCodesForRestore: true,
44+
HasAzureManagedIdentity: true,
45+
supportedCompressions: []barmanApi.CompressionType{
46+
barmanApi.CompressionTypeNone,
47+
barmanApi.CompressionTypeBzip2,
48+
barmanApi.CompressionTypeGzip,
49+
barmanApi.CompressionTypeLz4,
50+
barmanApi.CompressionTypeXz,
51+
barmanApi.CompressionTypeZstd,
52+
barmanApi.CompressionTypeSnappy,
53+
},
54+
}))
55+
})
56+
57+
It("ensures that barman versions 3.4 have no additional compression capabilities", func() {
2858
version, err := semver.ParseTolerant("3.4.0")
2959
Expect(err).ToNot(HaveOccurred())
3060
capabilities := detect(&version)
@@ -37,10 +67,15 @@ var _ = Describe("detect capabilities", func() {
3767
HasRetentionPolicy: true,
3868
HasTags: true,
3969
HasCheckWalArchive: true,
40-
HasSnappy: true,
4170
HasErrorCodesForWALRestore: true,
4271
HasErrorCodesForRestore: true,
4372
HasAzureManagedIdentity: true,
73+
supportedCompressions: []barmanApi.CompressionType{
74+
barmanApi.CompressionTypeNone,
75+
barmanApi.CompressionTypeBzip2,
76+
barmanApi.CompressionTypeGzip,
77+
barmanApi.CompressionTypeSnappy,
78+
},
4479
}))
4580
})
4681

@@ -56,10 +91,15 @@ var _ = Describe("detect capabilities", func() {
5691
HasRetentionPolicy: true,
5792
HasTags: true,
5893
HasCheckWalArchive: true,
59-
HasSnappy: true,
6094
HasErrorCodesForWALRestore: true,
6195
HasErrorCodesForRestore: true,
6296
HasAzureManagedIdentity: true,
97+
supportedCompressions: []barmanApi.CompressionType{
98+
barmanApi.CompressionTypeNone,
99+
barmanApi.CompressionTypeBzip2,
100+
barmanApi.CompressionTypeGzip,
101+
barmanApi.CompressionTypeSnappy,
102+
},
63103
}))
64104
})
65105

@@ -74,14 +114,19 @@ var _ = Describe("detect capabilities", func() {
74114
HasRetentionPolicy: true,
75115
HasTags: true,
76116
HasCheckWalArchive: true,
77-
HasSnappy: true,
78117
HasErrorCodesForWALRestore: true,
79118
HasErrorCodesForRestore: true,
80119
HasAzureManagedIdentity: true,
120+
supportedCompressions: []barmanApi.CompressionType{
121+
barmanApi.CompressionTypeNone,
122+
barmanApi.CompressionTypeBzip2,
123+
barmanApi.CompressionTypeGzip,
124+
barmanApi.CompressionTypeSnappy,
125+
},
81126
}))
82127
})
83128

84-
It("ensures that barmans versions below 2.18.0 only return the expected capabilities", func() {
129+
It("ensures that barman versions below 2.18.0 only return the expected capabilities", func() {
85130
version, err := semver.ParseTolerant("2.17.0")
86131
Expect(err).ToNot(HaveOccurred())
87132
capabilities := detect(&version)
@@ -90,6 +135,11 @@ var _ = Describe("detect capabilities", func() {
90135
HasAzure: true,
91136
HasS3: true,
92137
HasRetentionPolicy: true,
138+
supportedCompressions: []barmanApi.CompressionType{
139+
barmanApi.CompressionTypeNone,
140+
barmanApi.CompressionTypeBzip2,
141+
barmanApi.CompressionTypeGzip,
142+
},
93143
}), "unexpected capabilities set to true")
94144
})
95145

@@ -101,6 +151,11 @@ var _ = Describe("detect capabilities", func() {
101151
Version: &version,
102152
HasAzure: true,
103153
HasS3: true,
154+
supportedCompressions: []barmanApi.CompressionType{
155+
barmanApi.CompressionTypeNone,
156+
barmanApi.CompressionTypeBzip2,
157+
barmanApi.CompressionTypeGzip,
158+
},
104159
}))
105160
})
106161

@@ -110,6 +165,11 @@ var _ = Describe("detect capabilities", func() {
110165
capabilities := detect(&version)
111166
Expect(capabilities).To(Equal(&Capabilities{
112167
Version: &version,
168+
supportedCompressions: []barmanApi.CompressionType{
169+
barmanApi.CompressionTypeNone,
170+
barmanApi.CompressionTypeBzip2,
171+
barmanApi.CompressionTypeGzip,
172+
},
113173
}))
114174
})
115175
})

‎pkg/capabilities/type.go‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package capabilities
1919

2020
import (
2121
"github.com/blang/semver"
22+
23+
barmanApi "github.com/cloudnative-pg/barman-cloud/pkg/api"
2224
)
2325

2426
// Capabilities collects a set of boolean values that shows the possible capabilities of Barman and the version
@@ -32,10 +34,10 @@ type Capabilities struct {
3234
HasRetentionPolicy bool
3335
HasTags bool
3436
HasCheckWalArchive bool
35-
HasSnappy bool
3637
HasErrorCodesForWALRestore bool
3738
HasErrorCodesForRestore bool
3839
HasAzureManagedIdentity bool
40+
supportedCompressions []barmanApi.CompressionType
3941
}
4042

4143
// LegacyExecutor allows this code to know
@@ -52,3 +54,18 @@ func (c *Capabilities) ShouldExecuteBackupWithName(exec LegacyExecutor) bool {
5254

5355
return c.hasName
5456
}
57+
58+
// HasCompression returns true if the given compression is supported by Barman
59+
func (c *Capabilities) HasCompression(compression barmanApi.CompressionType) bool {
60+
if c == nil {
61+
return false
62+
}
63+
64+
for _, item := range c.supportedCompressions {
65+
if item == compression {
66+
return true
67+
}
68+
}
69+
70+
return false
71+
}

0 commit comments

Comments
 (0)