Skip to content

Commit 9fa1c0b

Browse files
armruleonardocefcanovai
authored
feat(spike): backup method (#20)
Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Signed-off-by: Francesco Canovai <[email protected]> Co-authored-by: Leonardo Cecchi <[email protected]> Co-authored-by: Francesco Canovai <[email protected]>
1 parent ed2626a commit 9fa1c0b

File tree

9 files changed

+169
-20
lines changed

9 files changed

+169
-20
lines changed

docs/examples/backup-example.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: postgresql.cnpg.io/v1
2+
kind: Backup
3+
metadata:
4+
name: backup-example
5+
spec:
6+
method: plugin
7+
8+
cluster:
9+
name: cluster-example
10+
11+
pluginConfiguration:
12+
name: barman-cloud.cloudnative-pg.io

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ require (
77
github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241001084914-829808376542
88
github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248
99
github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6
10-
github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f
10+
github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392
1111
github.com/onsi/ginkgo/v2 v2.20.2
1212
github.com/onsi/gomega v1.34.2
1313
github.com/spf13/cobra v1.8.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248 h1:eUGzb7YNj
2222
github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248/go.mod h1:K9/4eAT3rh2bKIWyujoN8BIPRXa4d1Ls+eBY8PE8y6w=
2323
github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6 h1:C4CU5fBTYTiJBPDqcgHpXSc5IvRTy+5KTaFZzdKHfAQ=
2424
github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6/go.mod h1:mHEVy/Guae+rij1qlgwHg+lyFKDX48qjTL4lAqE7OJs=
25-
github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f h1:RgPmQJkuSu3eTdfd4T2K95RYQi57LHB2+Jfsu/faKOM=
26-
github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f/go.mod h1:bWp1Es5zlxElg4Z/c5f0RKOkDcyNvDHdYIvNcPQU4WM=
25+
github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392 h1:DHaSe0PoLnIQFWIpRqB9RiBlNzbdLuVbiCtc9tN+FL0=
26+
github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392/go.mod h1:bWp1Es5zlxElg4Z/c5f0RKOkDcyNvDHdYIvNcPQU4WM=
2727
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
2828
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2929
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/cnpgi/instance/backup.go

Lines changed: 112 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,46 @@ package instance
22

33
import (
44
"context"
5+
"fmt"
6+
"os"
7+
"strconv"
8+
"time"
59

10+
barmanBackup "github.com/cloudnative-pg/barman-cloud/pkg/backup"
11+
barmanCapabilities "github.com/cloudnative-pg/barman-cloud/pkg/capabilities"
12+
barmanCredentials "github.com/cloudnative-pg/barman-cloud/pkg/credentials"
13+
"github.com/cloudnative-pg/cloudnative-pg/pkg/postgres"
614
"github.com/cloudnative-pg/cnpg-i/pkg/backup"
15+
"github.com/cloudnative-pg/machinery/pkg/fileutils"
16+
"github.com/cloudnative-pg/machinery/pkg/log"
17+
pgTime "github.com/cloudnative-pg/machinery/pkg/postgres/time"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
"sigs.k8s.io/controller-runtime/pkg/client"
20+
21+
barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1"
22+
"github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata"
723
)
824

925
// BackupServiceImplementation is the implementation
1026
// of the Backup CNPG capability
1127
type BackupServiceImplementation struct {
28+
BarmanObjectKey client.ObjectKey
29+
ClusterObjectKey client.ObjectKey
30+
Client client.Client
31+
InstanceName string
1232
backup.UnimplementedBackupServer
1333
}
1434

35+
// This is an implementation of the barman executor
36+
// that always instruct the barman library to use the
37+
// "--name" option for backups. We don't support old
38+
// Barman versions that do not implement that option.
39+
type barmanCloudExecutor struct{}
40+
41+
func (barmanCloudExecutor) ShouldForceLegacyBackup() bool {
42+
return false
43+
}
44+
1545
// GetCapabilities implements the BackupService interface
1646
func (b BackupServiceImplementation) GetCapabilities(
1747
_ context.Context, _ *backup.BackupCapabilitiesRequest,
@@ -30,7 +60,86 @@ func (b BackupServiceImplementation) GetCapabilities(
3060
}
3161

3262
// Backup implements the Backup interface
33-
func (b BackupServiceImplementation) Backup(_ context.Context, _ *backup.BackupRequest) (*backup.BackupResult, error) {
34-
// TODO implement me
35-
panic("implement me")
63+
func (b BackupServiceImplementation) Backup(
64+
ctx context.Context,
65+
_ *backup.BackupRequest,
66+
) (*backup.BackupResult, error) {
67+
contextLogger := log.FromContext(ctx)
68+
69+
var objectStore barmancloudv1.ObjectStore
70+
if err := b.Client.Get(ctx, b.BarmanObjectKey, &objectStore); err != nil {
71+
return nil, err
72+
}
73+
74+
if err := fileutils.EnsureDirectoryExists(postgres.BackupTemporaryDirectory); err != nil {
75+
contextLogger.Error(err, "Cannot create backup temporary directory", "err", err)
76+
return nil, err
77+
}
78+
79+
capabilities, err := barmanCapabilities.CurrentCapabilities()
80+
if err != nil {
81+
return nil, err
82+
}
83+
backupCmd := barmanBackup.NewBackupCommand(
84+
&objectStore.Spec.Configuration,
85+
capabilities,
86+
)
87+
88+
// We need to connect to PostgreSQL and to do that we need
89+
// PGHOST (and the like) to be available
90+
osEnvironment := os.Environ()
91+
caBundleEnvironment := getRestoreCABundleEnv(&objectStore.Spec.Configuration)
92+
env, err := barmanCredentials.EnvSetBackupCloudCredentials(
93+
ctx,
94+
b.Client,
95+
objectStore.Namespace,
96+
&objectStore.Spec.Configuration,
97+
mergeEnv(osEnvironment, caBundleEnvironment))
98+
if err != nil {
99+
return nil, err
100+
}
101+
102+
backupName := fmt.Sprintf("backup-%v", pgTime.ToCompactISO8601(time.Now()))
103+
104+
if err = backupCmd.Take(
105+
ctx,
106+
backupName,
107+
b.InstanceName,
108+
env,
109+
barmanCloudExecutor{},
110+
postgres.BackupTemporaryDirectory,
111+
); err != nil {
112+
return nil, err
113+
}
114+
115+
executedBackupInfo, err := backupCmd.GetExecutedBackupInfo(
116+
ctx,
117+
backupName,
118+
b.InstanceName,
119+
barmanCloudExecutor{},
120+
env)
121+
if err != nil {
122+
return nil, err
123+
}
124+
125+
return &backup.BackupResult{
126+
BackupId: executedBackupInfo.ID,
127+
BackupName: executedBackupInfo.BackupName,
128+
StartedAt: metav1.Time{Time: executedBackupInfo.BeginTime}.Unix(),
129+
StoppedAt: metav1.Time{Time: executedBackupInfo.EndTime}.Unix(),
130+
BeginWal: executedBackupInfo.BeginWal,
131+
EndWal: executedBackupInfo.EndWal,
132+
BeginLsn: executedBackupInfo.BeginLSN,
133+
EndLsn: executedBackupInfo.EndLSN,
134+
BackupLabelFile: nil,
135+
TablespaceMapFile: nil,
136+
InstanceId: b.InstanceName,
137+
Online: true,
138+
Metadata: map[string]string{
139+
"timeline": strconv.Itoa(executedBackupInfo.TimeLine),
140+
"version": metadata.Data.Version,
141+
"name": metadata.Data.Name,
142+
"displayName": metadata.Data.DisplayName,
143+
},
144+
}, nil
36145
}

internal/cnpgi/instance/start.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,12 @@ func (c *CNPGI) Start(ctx context.Context) error {
3535
PGDataPath: c.PGDataPath,
3636
PGWALPath: c.PGWALPath,
3737
})
38-
backup.RegisterBackupServer(server, BackupServiceImplementation{})
38+
backup.RegisterBackupServer(server, BackupServiceImplementation{
39+
Client: c.Client,
40+
BarmanObjectKey: c.BarmanObjectKey,
41+
ClusterObjectKey: c.ClusterObjectKey,
42+
InstanceName: c.InstanceName,
43+
})
3944
return nil
4045
}
4146

internal/cnpgi/instance/wal.go

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,8 @@ func (w WALServiceImplementation) Restore(
124124
contextLogger := log.FromContext(ctx)
125125
startTime := time.Now()
126126

127-
var cluster *cnpgv1.Cluster
128-
129-
if err := w.Client.Get(ctx, w.ClusterObjectKey, cluster); err != nil {
127+
var cluster cnpgv1.Cluster
128+
if err := w.Client.Get(ctx, w.ClusterObjectKey, &cluster); err != nil {
130129
return nil, err
131130
}
132131

@@ -152,7 +151,7 @@ func (w WALServiceImplementation) Restore(
152151
if err != nil {
153152
return nil, fmt.Errorf("while getting recover credentials: %w", err)
154153
}
155-
mergeEnv(env, credentialsEnv)
154+
env = mergeEnv(env, credentialsEnv)
156155

157156
options, err := barmanCommand.CloudWalRestoreOptions(ctx, barmanConfiguration, objectStore.Name)
158157
if err != nil {
@@ -178,7 +177,7 @@ func (w WALServiceImplementation) Restore(
178177
}
179178

180179
// We skip this step if streaming connection is not available
181-
if isStreamingAvailable(cluster, w.InstanceName) {
180+
if isStreamingAvailable(&cluster, w.InstanceName) {
182181
if err := checkEndOfWALStreamFlag(walRestorer); err != nil {
183182
return nil, err
184183
}
@@ -213,7 +212,7 @@ func (w WALServiceImplementation) Restore(
213212

214213
// We skip this step if streaming connection is not available
215214
endOfWALStream := isEndOfWALStream(walStatus)
216-
if isStreamingAvailable(cluster, w.InstanceName) && endOfWALStream {
215+
if isStreamingAvailable(&cluster, w.InstanceName) && endOfWALStream {
217216
contextLogger.Info(
218217
"Set end-of-wal-stream flag as one of the WAL files to be prefetched was not found")
219218

@@ -262,18 +261,29 @@ func (w WALServiceImplementation) SetFirstRequired(
262261
}
263262

264263
// mergeEnv merges all the values inside incomingEnv into env.
265-
func mergeEnv(env []string, incomingEnv []string) {
264+
func mergeEnv(env []string, incomingEnv []string) []string {
265+
result := make([]string, len(env), len(env)+len(incomingEnv))
266+
copy(result, env)
267+
266268
for _, incomingItem := range incomingEnv {
267269
incomingKV := strings.SplitAfterN(incomingItem, "=", 2)
268270
if len(incomingKV) != 2 {
269271
continue
270272
}
271-
for idx, item := range env {
273+
274+
found := false
275+
for idx, item := range result {
272276
if strings.HasPrefix(item, incomingKV[0]) {
273-
env[idx] = incomingItem
277+
result[idx] = incomingItem
278+
found = true
274279
}
275280
}
281+
if !found {
282+
result = append(result, incomingItem)
283+
}
276284
}
285+
286+
return result
277287
}
278288

279289
// TODO: refactor.

scripts/cleanup.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
5+
cd "$(dirname "$0")/.." || exit
6+
7+
kubectl delete clusters --all
8+
kubectl delete backups --all
9+
kubectl exec -ti mc -- mc rm -r --force minio/backups

scripts/run.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,19 @@ if [ -f .env ]; then
88
source .env
99
fi
1010

11+
12+
MYTMPDIR="$(mktemp -d)"
13+
trap 'rm -rf -- "$MYTMPDIR"' EXIT
14+
1115
current_context=$(kubectl config view --raw -o json | jq -r '."current-context"' | sed "s/kind-//")
1216
operator_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/operator)
1317
instance_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local KO_DEFAULTBASEIMAGE="ghcr.io/cloudnative-pg/postgresql:17.0" ko build -BP ./cmd/instance)
1418

19+
# Now we deploy the plugin inside the `cnpg-system` workspace
1520
(
16-
cd kubernetes;
21+
cp -r kubernetes config "$MYTMPDIR"
22+
cd "$MYTMPDIR/kubernetes"
1723
kustomize edit set image "plugin-barman-cloud=$operator_image"
1824
kustomize edit set secret plugin-barman-cloud "--from-literal=SIDECAR_IMAGE=$instance_image"
25+
kubectl apply -k .
1926
)
20-
21-
# Now we deploy the plugin inside the `cnpg-system` workspace
22-
kubectl apply -k kubernetes/

0 commit comments

Comments
 (0)