Skip to content

Commit b8dbdf3

Browse files
egegunesegegunes
committed
fix duplicate backup commands
1 parent 0b2fc17 commit b8dbdf3

File tree

11 files changed

+92
-57
lines changed

11 files changed

+92
-57
lines changed

‎e2e-tests/functions‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,10 @@ wait_backup_agent() {
229229
set_debug
230230
}
231231

232+
print_operator_logs() {
233+
kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
234+
}
235+
232236
wait_backup() {
233237
local backup_name=$1
234238
local target_state=${2:-"ready"}
@@ -243,15 +247,9 @@ wait_backup() {
243247
let retry+=1
244248
current_status=$(kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.state}')
245249
if [[ $retry -ge 600 || ${current_status} == 'error' ]]; then
246-
kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
247-
| grep -v 'level=info' \
248-
| grep -v 'level=debug' \
249-
| grep -v 'Getting tasks for pod' \
250-
| grep -v 'Getting pods from source' \
251-
| tail -200
252-
kubectl_bin get psmdb-backup
250+
print_operator_logs | tail -n 200
251+
kubectl_bin describe psmdb-backup/${backup_name}
253252
echo "Backup object psmdb-backup/${backup_name} is in ${current_status} state."
254-
echo something went wrong with operator or kubernetes cluster
255253
exit 1
256254
fi
257255
done

‎e2e-tests/pitr/conf/some-name-rs0.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ spec:
2222
enabled: true
2323
oplogSpanMin: 2
2424
oplogOnly: false
25-
oplogSpanMin: 10
2625
compressionType: gzip
2726
compressionLevel: 6
2827
replsets:

‎pkg/controller/perconaservermongodb/pbm.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ func (r *ReconcilePerconaServerMongoDB) resyncPBMIfNeeded(ctx context.Context, c
536536
}
537537

538538
l := r.lockers.LoadOrCreate(cr.NamespacedName().String())
539-
if !l.resyncMutex.TryLock() {
539+
if !l.ResyncMutex.TryLock() {
540540
return nil
541541
}
542542

@@ -554,7 +554,7 @@ func (r *ReconcilePerconaServerMongoDB) resyncPBMIfNeeded(ctx context.Context, c
554554
// running in separate goroutine to not block reconciliation
555555
// until all resync operations finished
556556
go func() {
557-
defer l.resyncMutex.Unlock()
557+
defer l.ResyncMutex.Unlock()
558558

559559
pbm, err := backup.NewPBM(ctx, r.client, cr)
560560
if err != nil {

‎pkg/controller/perconaservermongodb/psmdb_controller.go‎

Lines changed: 10 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"sort"
1010
"strings"
1111
"sync"
12-
"sync/atomic"
1312
"time"
1413

1514
v "github.com/hashicorp/go-version"
@@ -43,6 +42,7 @@ import (
4342
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/secret"
4443
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls"
4544
"github.com/percona/percona-server-mongodb-operator/pkg/util"
45+
"github.com/percona/percona-server-mongodb-operator/pkg/util/lockstore"
4646
"github.com/percona/percona-server-mongodb-operator/pkg/version"
4747
)
4848

@@ -94,7 +94,7 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
9494
serverVersion: sv,
9595
reconcileIn: time.Second * 5,
9696
crons: NewCronRegistry(),
97-
lockers: newLockStore(),
97+
lockers: lockstore.New(),
9898
newPBM: backup.NewPBM,
9999
restConfig: mgr.GetConfig(),
100100
newCertManagerCtrlFunc: tls.NewCertManagerController,
@@ -191,40 +191,9 @@ type ReconcilePerconaServerMongoDB struct {
191191

192192
initImage string
193193

194-
lockers lockStore
194+
lockers *lockstore.LockStore
195195
}
196196

197-
type lockStore struct {
198-
store *sync.Map
199-
}
200-
201-
func newLockStore() lockStore {
202-
return lockStore{
203-
store: new(sync.Map),
204-
}
205-
}
206-
207-
func (l lockStore) LoadOrCreate(key string) lock {
208-
val, _ := l.store.LoadOrStore(key, lock{
209-
statusMutex: new(sync.Mutex),
210-
resyncMutex: new(sync.Mutex),
211-
updateSync: new(int32),
212-
})
213-
214-
return val.(lock)
215-
}
216-
217-
type lock struct {
218-
statusMutex *sync.Mutex
219-
resyncMutex *sync.Mutex
220-
updateSync *int32
221-
}
222-
223-
const (
224-
updateDone = 0
225-
updateWait = 1
226-
)
227-
228197
// Reconcile reads that state of the cluster for a PerconaServerMongoDB object and makes changes based on the state read
229198
// and what is in the PerconaServerMongoDB.Spec
230199
// Note:
@@ -239,16 +208,19 @@ func (r *ReconcilePerconaServerMongoDB) Reconcile(ctx context.Context, request r
239208

240209
// As operator can handle a few clusters
241210
// lock should be created per cluster to not lock cron jobs of other clusters
242-
l := r.lockers.LoadOrCreate(request.NamespacedName.String())
211+
lock := r.lockers.LoadOrCreate(request.NamespacedName.String())
243212

244213
// PerconaServerMongoDB object is also accessed and changed by a version service's cron job (that runs concurrently)
245-
l.statusMutex.Lock()
246-
defer l.statusMutex.Unlock()
214+
lock.ReconcileMutex.Lock()
215+
defer lock.ReconcileMutex.Unlock()
247216
// we have to be sure the reconcile loop will be run at least once
248217
// in-between any version service jobs (hence any two vs jobs shouldn't be run sequentially).
249218
// the version service job sets the state to `updateWait` and the next job can be run only
250219
// after the state was dropped to`updateDone` again
251-
defer atomic.StoreInt32(l.updateSync, updateDone)
220+
defer lock.SetToUpdateDone()
221+
222+
log.V(1).Info("Reconciling")
223+
defer log.V(1).Info("Reconcile finished")
252224

253225
// Fetch the PerconaServerMongoDB instance
254226
cr := &api.PerconaServerMongoDB{}

‎pkg/controller/perconaservermongodb/status_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
1717
fakeBackup "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/fake"
1818
faketls "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls/fake"
19+
"github.com/percona/percona-server-mongodb-operator/pkg/util/lockstore"
1920
"github.com/percona/percona-server-mongodb-operator/pkg/version"
2021
)
2122

@@ -40,7 +41,7 @@ func buildFakeClient(objs ...client.Object) *ReconcilePerconaServerMongoDB {
4041
return &ReconcilePerconaServerMongoDB{
4142
client: cl,
4243
scheme: s,
43-
lockers: newLockStore(),
44+
lockers: lockstore.New(),
4445
newPBM: fakeBackup.NewPBM,
4546
newCertManagerCtrlFunc: faketls.NewCertManagerController,
4647
}

‎pkg/controller/perconaservermongodb/suite_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
2222
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup"
2323
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls"
24+
"github.com/percona/percona-server-mongodb-operator/pkg/util/lockstore"
2425
"github.com/percona/percona-server-mongodb-operator/pkg/version"
2526
)
2627

@@ -81,7 +82,7 @@ func reconciler() *ReconcilePerconaServerMongoDB {
8182
client: k8sClient,
8283
scheme: k8sClient.Scheme(),
8384
crons: NewCronRegistry(),
84-
lockers: newLockStore(),
85+
lockers: lockstore.New(),
8586
clientcmd: cli,
8687
restConfig: cfg,
8788
newPBM: backup.NewPBM,

‎pkg/controller/perconaservermongodb/version.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"math/rand"
77
"os"
88
"strings"
9-
"sync/atomic"
109

1110
v "github.com/hashicorp/go-version"
1211
"github.com/pkg/errors"
@@ -86,13 +85,13 @@ func (r *ReconcilePerconaServerMongoDB) scheduleEnsureVersion(ctx context.Contex
8685
Namespace: cr.Namespace,
8786
}
8887

89-
l := r.lockers.LoadOrCreate(nn.String())
88+
lock := r.lockers.LoadOrCreate(nn.String())
9089

9190
id, err := r.crons.AddFuncWithSeconds(cr.Spec.UpgradeOptions.Schedule, func() {
92-
l.statusMutex.Lock()
93-
defer l.statusMutex.Unlock()
91+
lock.ReconcileMutex.Lock()
92+
defer lock.ReconcileMutex.Unlock()
9493

95-
if !atomic.CompareAndSwapInt32(l.updateSync, updateDone, updateWait) {
94+
if !lock.SwapToUpdateWait() {
9695
return
9796
}
9897

‎pkg/controller/perconaservermongodbbackup/backup.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424

2525
const (
2626
// pbmStartingDeadline is timeout after which continuous starting state is considered as error
27-
pbmStartingDeadline = time.Duration(120) * time.Second
27+
pbmStartingDeadline = time.Duration(600) * time.Second // TODO: we should make this configurable
2828
pbmStartingDeadlineErrMsg = "starting deadline exceeded"
2929
)
3030

‎pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"github.com/percona/percona-server-mongodb-operator/pkg/naming"
3535
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
3636
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup"
37+
"github.com/percona/percona-server-mongodb-operator/pkg/util/lockstore"
3738
"github.com/percona/percona-server-mongodb-operator/pkg/version"
3839
)
3940

@@ -64,6 +65,7 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
6465
scheme: mgr.GetScheme(),
6566
newPBMFunc: backup.NewPBM,
6667
clientcmd: cli,
68+
lockers: lockstore.New(),
6769
}, nil
6870
}
6971

@@ -94,6 +96,8 @@ type ReconcilePerconaServerMongoDBBackup struct {
9496
scheme *runtime.Scheme
9597
clientcmd *clientcmd.Client
9698

99+
lockers *lockstore.LockStore
100+
97101
newPBMFunc backup.NewPBMFunc
98102
}
99103

@@ -105,6 +109,11 @@ type ReconcilePerconaServerMongoDBBackup struct {
105109
func (r *ReconcilePerconaServerMongoDBBackup) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
106110
log := logf.FromContext(ctx)
107111

112+
lock := r.lockers.LoadOrCreate(request.NamespacedName.String())
113+
114+
lock.ReconcileMutex.Lock()
115+
defer lock.ReconcileMutex.Unlock()
116+
108117
log.V(1).Info("Reconciling")
109118
defer log.V(1).Info("Reconcile finished")
110119

@@ -262,6 +271,7 @@ func (r *ReconcilePerconaServerMongoDBBackup) reconcile(
262271

263272
switch status.State {
264273
case psmdbv1.BackupStateNew, psmdbv1.BackupStateWaiting:
274+
log.Info("Sending backup", "status.State", status.State, "cr.Status.State", cr.Status.State)
265275
return bcp.Start(ctx, r.client, cluster, cr)
266276
}
267277

‎pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
3131
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup"
3232
"github.com/percona/percona-server-mongodb-operator/pkg/util"
33+
"github.com/percona/percona-server-mongodb-operator/pkg/util/lockstore"
3334
"github.com/percona/percona-server-mongodb-operator/pkg/version"
3435
)
3536

@@ -56,6 +57,7 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
5657
scheme: mgr.GetScheme(),
5758
clientcmd: cli,
5859
newPBMFunc: backup.NewPBM,
60+
lockers: lockstore.New(),
5961
}, nil
6062
}
6163

@@ -85,6 +87,7 @@ type ReconcilePerconaServerMongoDBRestore struct {
8587
scheme *runtime.Scheme
8688
clientcmd *clientcmd.Client
8789

90+
lockers *lockstore.LockStore
8891
newPBMFunc backup.NewPBMFunc
8992
}
9093

@@ -95,6 +98,13 @@ type ReconcilePerconaServerMongoDBRestore struct {
9598
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
9699
func (r *ReconcilePerconaServerMongoDBRestore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
97100
log := logf.FromContext(ctx)
101+
lock := r.lockers.LoadOrCreate(request.NamespacedName.String())
102+
103+
lock.ReconcileMutex.Lock()
104+
defer lock.ReconcileMutex.Unlock()
105+
106+
log.V(1).Info("Reconciling")
107+
defer log.V(1).Info("Reconcile finished")
98108

99109
rr := reconcile.Result{
100110
RequeueAfter: time.Second * 5,

0 commit comments

Comments
 (0)