Skip to content

Commit 4217566

Browse files
pooknullhors
pooknull
and
hors
authored
K8SPS-254: prohibit to run several restores in parallel (#382)
* K8SPS-254: prohibit to run several restores in parallel https://jira.percona.com/browse/K8SPS-254 * small improvement --------- Co-authored-by: Viacheslav Sarzhan <[email protected]>
1 parent 40224f3 commit 4217566

File tree

2 files changed

+208
-32
lines changed

2 files changed

+208
-32
lines changed

‎pkg/controller/psrestore/controller.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package psrestore
1919
import (
2020
"context"
2121
"fmt"
22+
"sync"
2223
"time"
2324

2425
appsv1 "k8s.io/api/apps/v1"
@@ -51,6 +52,8 @@ type PerconaServerMySQLRestoreReconciler struct {
5152
client.Client
5253
Scheme *runtime.Scheme
5354
ServerVersion *platform.ServerVersion
55+
56+
sm sync.Map
5457
}
5558

5659
var ErrWaitingTermination error = errors.New("waiting for MySQL pods to terminate")
@@ -162,6 +165,32 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req
162165
return ctrl.Result{}, nil
163166
}
164167

168+
restoreList := &apiv1alpha1.PerconaServerMySQLRestoreList{}
169+
if err := r.List(ctx, restoreList, &client.ListOptions{Namespace: cr.Namespace}); err != nil {
170+
return ctrl.Result{}, errors.Wrap(err, "get restore jobs list")
171+
}
172+
for _, restore := range restoreList.Items {
173+
if restore.Spec.ClusterName != cr.Spec.ClusterName || restore.Name == cr.Name {
174+
continue
175+
}
176+
177+
switch restore.Status.State {
178+
case apiv1alpha1.RestoreSucceeded, apiv1alpha1.RestoreFailed, apiv1alpha1.RestoreError, apiv1alpha1.RestoreNew:
179+
default:
180+
status.State = apiv1alpha1.RestoreError
181+
status.StateDesc = fmt.Sprintf("PerconaServerMySQLRestore %s is already running", restore.Name)
182+
log.Info("PerconaServerMySQLRestore is already running", "restore", restore.Name)
183+
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
184+
}
185+
}
186+
// The above code is to prevent multiple restores from running at the same time. It works only if restore job is created.
187+
// But if multiple restores are created at the same time, then the above code will not work, because there are no restore jobs yet.
188+
// Therefore, we need to use sync.Map to prevent multiple restores from creating restore jobs at the same time.
189+
if _, ok := r.sm.LoadOrStore(cr.Spec.ClusterName, 1); ok {
190+
return ctrl.Result{RequeueAfter: time.Second * 5}, nil
191+
}
192+
defer r.sm.Delete(cr.Spec.ClusterName)
193+
165194
log.Info("Pausing cluster", "cluster", cluster.Name)
166195
if err := r.pauseCluster(ctx, cluster); err != nil {
167196
if errors.Is(err, ErrWaitingTermination) {
@@ -308,6 +337,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req
308337
if err := r.unpauseCluster(ctx, cluster); err != nil {
309338
return ctrl.Result{}, errors.Wrap(err, "unpause cluster")
310339
}
340+
log.Info("PerconaServerMySQLRestore is finished", "restore", cr.Name, "cluster", cluster.Name)
311341
}
312342

313343
return ctrl.Result{}, nil

‎pkg/controller/psrestore/controller_test.go

Lines changed: 178 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/util/yaml"
1616
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
1717
controllerruntime "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
1819
"sigs.k8s.io/controller-runtime/pkg/client/fake"
1920

2021
apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1"
@@ -37,8 +38,7 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
3738
name string
3839
cr *apiv1alpha1.PerconaServerMySQLRestore
3940
cluster *apiv1alpha1.PerconaServerMySQL
40-
backup *apiv1alpha1.PerconaServerMySQLBackup
41-
secret *corev1.Secret
41+
objects []client.Object
4242
stateDesc string
4343
shouldSucceed bool
4444
}{
@@ -112,14 +112,16 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
112112
{
113113
name: "without backup storage in cluster",
114114
cr: cr,
115-
backup: &apiv1alpha1.PerconaServerMySQLBackup{
116-
ObjectMeta: metav1.ObjectMeta{
117-
Name: backupName,
118-
Namespace: namespace,
119-
},
120-
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
121-
ClusterName: clusterName,
122-
StorageName: storageName,
115+
objects: []client.Object{
116+
&apiv1alpha1.PerconaServerMySQLBackup{
117+
ObjectMeta: metav1.ObjectMeta{
118+
Name: backupName,
119+
Namespace: namespace,
120+
},
121+
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
122+
ClusterName: clusterName,
123+
StorageName: storageName,
124+
},
123125
},
124126
},
125127
cluster: &apiv1alpha1.PerconaServerMySQL{
@@ -138,14 +140,16 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
138140
{
139141
name: "without secret",
140142
cr: cr,
141-
backup: &apiv1alpha1.PerconaServerMySQLBackup{
142-
ObjectMeta: metav1.ObjectMeta{
143-
Name: backupName,
144-
Namespace: namespace,
145-
},
146-
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
147-
ClusterName: clusterName,
148-
StorageName: storageName,
143+
objects: []client.Object{
144+
&apiv1alpha1.PerconaServerMySQLBackup{
145+
ObjectMeta: metav1.ObjectMeta{
146+
Name: backupName,
147+
Namespace: namespace,
148+
},
149+
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
150+
ClusterName: clusterName,
151+
StorageName: storageName,
152+
},
149153
},
150154
},
151155
cluster: &apiv1alpha1.PerconaServerMySQL{
@@ -172,14 +176,77 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
172176
{
173177
name: "should succeed",
174178
cr: cr,
175-
backup: &apiv1alpha1.PerconaServerMySQLBackup{
179+
objects: []client.Object{
180+
&apiv1alpha1.PerconaServerMySQLBackup{
181+
ObjectMeta: metav1.ObjectMeta{
182+
Name: backupName,
183+
Namespace: namespace,
184+
},
185+
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
186+
ClusterName: clusterName,
187+
StorageName: storageName,
188+
},
189+
},
190+
&corev1.Secret{
191+
ObjectMeta: metav1.ObjectMeta{
192+
Name: "aws-secret",
193+
Namespace: namespace,
194+
},
195+
},
196+
},
197+
cluster: &apiv1alpha1.PerconaServerMySQL{
176198
ObjectMeta: metav1.ObjectMeta{
177-
Name: backupName,
199+
Name: clusterName,
178200
Namespace: namespace,
179201
},
180-
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
181-
ClusterName: clusterName,
182-
StorageName: storageName,
202+
Spec: apiv1alpha1.PerconaServerMySQLSpec{
203+
Backup: &apiv1alpha1.BackupSpec{
204+
Storages: map[string]*apiv1alpha1.BackupStorageSpec{
205+
storageName: {
206+
S3: &apiv1alpha1.BackupStorageS3Spec{
207+
CredentialsSecret: "aws-secret",
208+
},
209+
Type: apiv1alpha1.BackupStorageS3,
210+
},
211+
},
212+
InitImage: "operator-image",
213+
},
214+
},
215+
},
216+
stateDesc: "",
217+
shouldSucceed: true,
218+
},
219+
{
220+
name: "with running restore",
221+
cr: cr,
222+
objects: []client.Object{
223+
&apiv1alpha1.PerconaServerMySQLBackup{
224+
ObjectMeta: metav1.ObjectMeta{
225+
Name: backupName,
226+
Namespace: namespace,
227+
},
228+
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
229+
ClusterName: clusterName,
230+
StorageName: storageName,
231+
},
232+
},
233+
&corev1.Secret{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: "aws-secret",
236+
Namespace: namespace,
237+
},
238+
},
239+
&apiv1alpha1.PerconaServerMySQLRestore{
240+
ObjectMeta: metav1.ObjectMeta{
241+
Name: "running-restore",
242+
Namespace: namespace,
243+
},
244+
Spec: apiv1alpha1.PerconaServerMySQLRestoreSpec{
245+
ClusterName: clusterName,
246+
},
247+
Status: apiv1alpha1.PerconaServerMySQLRestoreStatus{
248+
State: apiv1alpha1.RestoreRunning,
249+
},
183250
},
184251
},
185252
cluster: &apiv1alpha1.PerconaServerMySQL{
@@ -201,11 +268,96 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
201268
},
202269
},
203270
},
204-
secret: &corev1.Secret{
271+
stateDesc: "PerconaServerMySQLRestore running-restore is already running",
272+
shouldSucceed: false,
273+
},
274+
{
275+
name: "with new, failed, errored and succeeded restore",
276+
cr: cr,
277+
objects: []client.Object{
278+
&apiv1alpha1.PerconaServerMySQLBackup{
279+
ObjectMeta: metav1.ObjectMeta{
280+
Name: backupName,
281+
Namespace: namespace,
282+
},
283+
Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{
284+
ClusterName: clusterName,
285+
StorageName: storageName,
286+
},
287+
},
288+
&corev1.Secret{
289+
ObjectMeta: metav1.ObjectMeta{
290+
Name: "aws-secret",
291+
Namespace: namespace,
292+
},
293+
},
294+
&apiv1alpha1.PerconaServerMySQLRestore{
295+
ObjectMeta: metav1.ObjectMeta{
296+
Name: "new-restore",
297+
Namespace: namespace,
298+
},
299+
Spec: apiv1alpha1.PerconaServerMySQLRestoreSpec{
300+
ClusterName: clusterName,
301+
},
302+
Status: apiv1alpha1.PerconaServerMySQLRestoreStatus{
303+
State: apiv1alpha1.RestoreNew,
304+
},
305+
},
306+
&apiv1alpha1.PerconaServerMySQLRestore{
307+
ObjectMeta: metav1.ObjectMeta{
308+
Name: "failed-restore",
309+
Namespace: namespace,
310+
},
311+
Spec: apiv1alpha1.PerconaServerMySQLRestoreSpec{
312+
ClusterName: clusterName,
313+
},
314+
Status: apiv1alpha1.PerconaServerMySQLRestoreStatus{
315+
State: apiv1alpha1.RestoreFailed,
316+
},
317+
},
318+
&apiv1alpha1.PerconaServerMySQLRestore{
319+
ObjectMeta: metav1.ObjectMeta{
320+
Name: "succeeded-restore",
321+
Namespace: namespace,
322+
},
323+
Spec: apiv1alpha1.PerconaServerMySQLRestoreSpec{
324+
ClusterName: clusterName,
325+
},
326+
Status: apiv1alpha1.PerconaServerMySQLRestoreStatus{
327+
State: apiv1alpha1.RestoreSucceeded,
328+
},
329+
},
330+
&apiv1alpha1.PerconaServerMySQLRestore{
331+
ObjectMeta: metav1.ObjectMeta{
332+
Name: "error-restore",
333+
Namespace: namespace,
334+
},
335+
Spec: apiv1alpha1.PerconaServerMySQLRestoreSpec{
336+
ClusterName: clusterName,
337+
},
338+
Status: apiv1alpha1.PerconaServerMySQLRestoreStatus{
339+
State: apiv1alpha1.RestoreError,
340+
},
341+
},
342+
},
343+
cluster: &apiv1alpha1.PerconaServerMySQL{
205344
ObjectMeta: metav1.ObjectMeta{
206-
Name: "aws-secret",
345+
Name: clusterName,
207346
Namespace: namespace,
208347
},
348+
Spec: apiv1alpha1.PerconaServerMySQLSpec{
349+
Backup: &apiv1alpha1.BackupSpec{
350+
Storages: map[string]*apiv1alpha1.BackupStorageSpec{
351+
storageName: {
352+
S3: &apiv1alpha1.BackupStorageS3Spec{
353+
CredentialsSecret: "aws-secret",
354+
},
355+
Type: apiv1alpha1.BackupStorageS3,
356+
},
357+
},
358+
InitImage: "operator-image",
359+
},
360+
},
209361
},
210362
stateDesc: "",
211363
shouldSucceed: true,
@@ -224,7 +376,7 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
224376

225377
for _, tt := range tests {
226378
t.Run(tt.name, func(t *testing.T) {
227-
cb := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.cr).WithStatusSubresource(tt.cr)
379+
cb := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.cr).WithStatusSubresource(tt.cr).WithObjects(tt.objects...)
228380
if tt.cluster != nil {
229381
cb.WithObjects(tt.cluster, &appsv1.StatefulSet{
230382
ObjectMeta: metav1.ObjectMeta{
@@ -233,12 +385,6 @@ func TestRestoreStatusErrStateDesc(t *testing.T) {
233385
},
234386
})
235387
}
236-
if tt.backup != nil {
237-
cb.WithObjects(tt.backup)
238-
}
239-
if tt.secret != nil {
240-
cb.WithObjects(tt.secret)
241-
}
242388

243389
r := PerconaServerMySQLRestoreReconciler{
244390
Client: cb.Build(),

0 commit comments

Comments
 (0)