Skip to content

Commit e2b7e97

Browse files
pooknullpooknull
committed
K8SPSMDB-1211: handle FULL CLUSTER CRASH error during the restore
https://perconadev.atlassian.net/browse/K8SPSMDB-1211
1 parent b009df6 commit e2b7e97

28 files changed

+594
-442
lines changed

‎pkg/controller/common/common.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package common
2+
3+
import (
4+
"context"
5+
6+
"github.com/pkg/errors"
7+
corev1 "k8s.io/api/core/v1"
8+
"k8s.io/apimachinery/pkg/runtime"
9+
"sigs.k8s.io/controller-runtime/pkg/client"
10+
11+
api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
12+
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
13+
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup"
14+
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo"
15+
)
16+
17+
func New(client client.Client, scheme *runtime.Scheme, newPBMFunc backup.NewPBMFunc, mongoClientProvider psmdb.MongoClientProvider) CommonReconciler {
18+
return CommonReconciler{
19+
client: client,
20+
scheme: scheme,
21+
newPBMFunc: newPBMFunc,
22+
mongoClientProvider: mongoClientProvider,
23+
}
24+
}
25+
26+
type CommonReconciler struct {
27+
client client.Client
28+
scheme *runtime.Scheme
29+
newPBMFunc backup.NewPBMFunc
30+
mongoClientProvider psmdb.MongoClientProvider
31+
}
32+
33+
func (r *CommonReconciler) Client() client.Client {
34+
return r.client
35+
}
36+
37+
func (r *CommonReconciler) Scheme() *runtime.Scheme {
38+
return r.scheme
39+
}
40+
41+
func (r *CommonReconciler) NewPBM(ctx context.Context, cluster *api.PerconaServerMongoDB) (backup.PBM, error) {
42+
return r.newPBMFunc(ctx, r.client, cluster)
43+
}
44+
45+
func (r *CommonReconciler) NewPBMFunc() backup.NewPBMFunc {
46+
return r.newPBMFunc
47+
}
48+
49+
func (r *CommonReconciler) getMongoClientProvider() psmdb.MongoClientProvider {
50+
if r.mongoClientProvider == nil {
51+
return psmdb.NewProvider(r.client)
52+
}
53+
return r.mongoClientProvider
54+
}
55+
56+
func (r *CommonReconciler) MongoClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, rs *api.ReplsetSpec, role api.SystemUserRole) (mongo.Client, error) {
57+
return r.getMongoClientProvider().Mongo(ctx, cr, rs, role)
58+
}
59+
60+
func (r *CommonReconciler) MongosClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, role api.SystemUserRole) (mongo.Client, error) {
61+
return r.getMongoClientProvider().Mongos(ctx, cr, role)
62+
}
63+
64+
func (r *CommonReconciler) StandaloneClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, rs *api.ReplsetSpec, role api.SystemUserRole, pod corev1.Pod) (mongo.Client, error) {
65+
host, err := psmdb.MongoHost(ctx, r.client, cr, cr.Spec.ClusterServiceDNSMode, rs, rs.Expose.Enabled, pod)
66+
if err != nil {
67+
return nil, errors.Wrap(err, "failed to get mongo host")
68+
}
69+
return r.getMongoClientProvider().Standalone(ctx, cr, role, host, cr.TLSEnabled())
70+
}
71+
72+
/*
73+
// ReconcilePerconaServerMongoDB reconciles a PerconaServerMongoDB object
74+
type ReconcilePerconaServerMongoDB struct {
75+
// This client, initialized using mgr.Client() above, is a split client
76+
// that reads objects from the cache and writes to the apiserver
77+
client client.Client
78+
scheme *runtime.Scheme
79+
restConfig *rest.Config
80+
81+
crons CronRegistry
82+
clientcmd *clientcmd.Client
83+
serverVersion *version.ServerVersion
84+
reconcileIn time.Duration
85+
mongoClientProvider psmdb.MongoClientProvider
86+
87+
newCertManagerCtrlFunc tls.NewCertManagerControllerFunc
88+
89+
newPBM backup.NewPBMFunc
90+
91+
initImage string
92+
93+
lockers lockStore
94+
}
95+
96+
// ReconcilePerconaServerMongoDBRestore reconciles a PerconaServerMongoDBRestore object
97+
type ReconcilePerconaServerMongoDBRestore struct {
98+
// This client, initialized using mgr.Client() above, is a split client
99+
// that reads objects from the cache and writes to the apiserver
100+
client client.Client
101+
scheme *runtime.Scheme
102+
clientcmd *clientcmd.Client
103+
104+
newPBMFunc backup.NewPBMFunc
105+
}
106+
// ReconcilePerconaServerMongoDBBackup reconciles a PerconaServerMongoDBBackup object
107+
type ReconcilePerconaServerMongoDBBackup struct {
108+
// This client, initialized using mgr.Client() above, is a split client
109+
// that reads objects from the cache and writes to the apiserver
110+
client client.Client
111+
scheme *runtime.Scheme
112+
clientcmd *clientcmd.Client
113+
114+
newPBMFunc backup.NewPBMFunc
115+
}
116+
*/

‎pkg/controller/perconaservermongodb/backup.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateBackupTask(ctx context.Con
6060
if err != nil {
6161
return errors.Wrap(err, "can't create job")
6262
}
63-
err = setControllerReference(cr, &cjob, r.scheme)
63+
err = setControllerReference(cr, &cjob, r.Scheme())
6464
if err != nil {
6565
return errors.Wrapf(err, "set owner reference for backup task %s", cjob.Name)
6666
}

‎pkg/controller/perconaservermongodb/balancer.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ import (
44
"context"
55
"time"
66

7-
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
87
"github.com/pkg/errors"
98
corev1 "k8s.io/api/core/v1"
109
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
1110
"k8s.io/apimachinery/pkg/types"
1211
logf "sigs.k8s.io/controller-runtime/pkg/log"
1312

1413
api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
14+
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
1515
)
1616

1717
func (r *ReconcilePerconaServerMongoDB) enableBalancerIfNeeded(ctx context.Context, cr *api.PerconaServerMongoDB) error {
@@ -85,7 +85,7 @@ func (r *ReconcilePerconaServerMongoDB) enableBalancerIfNeeded(ctx context.Conte
8585
}
8686
}
8787

88-
mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
88+
mongosSession, err := r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
8989
if err != nil {
9090
return errors.Wrap(err, "failed to get mongos connection")
9191
}
@@ -141,7 +141,7 @@ func (r *ReconcilePerconaServerMongoDB) disableBalancer(ctx context.Context, cr
141141
return errors.Wrapf(err, "get mongos statefulset %s", msSts.Name)
142142
}
143143

144-
mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
144+
mongosSession, err := r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
145145
if err != nil {
146146
return errors.Wrap(err, "failed to get mongos connection")
147147
}

‎pkg/controller/perconaservermongodb/connections.go

Lines changed: 0 additions & 73 deletions
This file was deleted.

‎pkg/controller/perconaservermongodb/connections_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2020

2121
api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
22+
"github.com/percona/percona-server-mongodb-operator/pkg/controller/common"
2223
"github.com/percona/percona-server-mongodb-operator/pkg/naming"
2324
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
2425
"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo"
@@ -158,7 +159,7 @@ func TestConnectionLeaks(t *testing.T) {
158159
connectionCount := new(int)
159160

160161
r := buildFakeClient(obj...)
161-
r.mongoClientProvider = &fakeMongoClientProvider{pods: rsPods, cr: cr, connectionCount: connectionCount}
162+
r.CommonReconciler = common.New(r.Client(), r.Scheme(), r.NewPBMFunc(), &fakeMongoClientProvider{pods: rsPods, cr: cr, connectionCount: connectionCount})
162163
r.serverVersion = &version.ServerVersion{Platform: version.PlatformKubernetes}
163164
r.crons = NewCronRegistry()
164165

‎pkg/controller/perconaservermongodb/custom_users.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCustomUsers(ctx context.Context
3535
var err error
3636
var mongoCli mongo.Client
3737
if cr.Spec.Sharding.Enabled {
38-
mongoCli, err = r.mongosClientWithRole(ctx, cr, api.RoleUserAdmin)
38+
mongoCli, err = r.MongosClientWithRole(ctx, cr, api.RoleUserAdmin)
3939
} else {
40-
mongoCli, err = r.mongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleUserAdmin)
40+
mongoCli, err = r.MongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleUserAdmin)
4141
}
4242
if err != nil {
4343
return errors.Wrap(err, "failed to get mongo client")
@@ -310,7 +310,8 @@ func updatePass(
310310
user *api.User,
311311
userInfo *mongo.User,
312312
secret *corev1.Secret,
313-
annotationKey, passKey string) error {
313+
annotationKey, passKey string,
314+
) error {
314315
log := logf.FromContext(ctx)
315316

316317
if userInfo == nil || user.IsExternalDB() {
@@ -395,7 +396,8 @@ func createUser(
395396
mongoCli mongo.Client,
396397
user *api.User,
397398
secret *corev1.Secret,
398-
annotationKey, passKey string) error {
399+
annotationKey, passKey string,
400+
) error {
399401
log := logf.FromContext(ctx)
400402

401403
roles := make([]mongo.Role, 0)

‎pkg/controller/perconaservermongodb/fcv.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
)
1313

1414
func (r *ReconcilePerconaServerMongoDB) getFCV(ctx context.Context, cr *api.PerconaServerMongoDB) (string, error) {
15-
c, err := r.mongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleClusterAdmin)
15+
c, err := r.MongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleClusterAdmin)
1616
if err != nil {
1717
return "", errors.Wrap(err, "failed to get connection")
1818
}
@@ -40,9 +40,9 @@ func (r *ReconcilePerconaServerMongoDB) setFCV(ctx context.Context, cr *api.Perc
4040
var connErr error
4141

4242
if cr.Spec.Sharding.Enabled {
43-
cli, connErr = r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
43+
cli, connErr = r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
4444
} else {
45-
cli, connErr = r.mongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleClusterAdmin)
45+
cli, connErr = r.MongoClientWithRole(ctx, cr, cr.Spec.Replsets[0], api.RoleClusterAdmin)
4646
}
4747

4848
if connErr != nil {

‎pkg/controller/perconaservermongodb/finalizers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (r *ReconcilePerconaServerMongoDB) checkFinalizers(ctx context.Context, cr
8585
}
8686

8787
func (r *ReconcilePerconaServerMongoDB) deleteAllPITRChunks(ctx context.Context, cr *api.PerconaServerMongoDB) error {
88-
pbmc, err := r.newPBM(ctx, r.client, cr)
88+
pbmc, err := r.NewPBM(ctx, cr)
8989
if err != nil {
9090
return errors.Wrap(err, "new pbm")
9191
}

‎pkg/controller/perconaservermongodb/mgo.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCluster(ctx context.Context, cr
9595
}
9696
}
9797

98-
cli, err := r.mongoClientWithRole(ctx, cr, replset, api.RoleClusterAdmin)
98+
cli, err := r.MongoClientWithRole(ctx, cr, replset, api.RoleClusterAdmin)
9999
if err != nil {
100100
if cr.Spec.Unmanaged {
101101
return api.AppStateInit, nil, nil
@@ -193,7 +193,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCluster(ctx context.Context, cr
193193
replset.ClusterRole == api.ClusterRoleShardSvr &&
194194
len(mongosPods) > 0 && cr.Spec.Sharding.Mongos.Size > 0 {
195195

196-
mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
196+
mongosSession, err := r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
197197
if err != nil {
198198
return api.AppStateError, nil, errors.Wrap(err, "failed to get mongos connection")
199199
}
@@ -571,7 +571,7 @@ func (r *ReconcilePerconaServerMongoDB) removeRSFromShard(ctx context.Context, c
571571
return nil
572572
}
573573

574-
cli, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
574+
cli, err := r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
575575
if err != nil {
576576
return errors.Errorf("failed to get mongos connection: %v", err)
577577
}
@@ -621,7 +621,7 @@ func (r *ReconcilePerconaServerMongoDB) handleRsAddToShard(ctx context.Context,
621621
return errors.Wrapf(err, "get rsPod %s host", rspod.Name)
622622
}
623623

624-
cli, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
624+
cli, err := r.MongosClientWithRole(ctx, cr, api.RoleClusterAdmin)
625625
if err != nil {
626626
return errors.Wrap(err, "failed to get mongos client")
627627
}
@@ -724,7 +724,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c
724724
time.Sleep(time.Second * 5)
725725

726726
log.Info("creating user admin", "replset", replsetName, "pod", pod.Name, "user", api.RoleUserAdmin)
727-
userAdmin, err := getInternalCredentials(ctx, r.client, cr, api.RoleUserAdmin)
727+
userAdmin, err := psmdb.GetCredentials(ctx, r.client, cr, api.RoleUserAdmin)
728728
if err != nil {
729729
return nil, nil, errors.Wrap(err, "failed to get userAdmin credentials")
730730
}
@@ -757,7 +757,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplicaSetNoPrimary(ctx context.Co
757757
}
758758

759759
log.Info("Connecting to pod", "pod", pod.Name, "user", api.RoleClusterAdmin)
760-
cli, err := r.standaloneClientWithRole(ctx, cr, replset, api.RoleClusterAdmin, pod)
760+
cli, err := r.StandaloneClientWithRole(ctx, cr, replset, api.RoleClusterAdmin, pod)
761761
if err != nil {
762762
return errors.Wrap(err, "get standalone mongo client")
763763
}
@@ -922,7 +922,7 @@ func compareRoles(x []mongo.Role, y []mongo.Role) bool {
922922
func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemUsers(ctx context.Context, cr *api.PerconaServerMongoDB, replset *api.ReplsetSpec) error {
923923
log := logf.FromContext(ctx)
924924

925-
cli, err := r.mongoClientWithRole(ctx, cr, replset, api.RoleUserAdmin)
925+
cli, err := r.MongoClientWithRole(ctx, cr, replset, api.RoleUserAdmin)
926926
if err != nil {
927927
return errors.Wrap(err, "failed to get mongo client")
928928
}
@@ -1013,7 +1013,7 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemUsers(ctx context.Co
10131013
}
10141014

10151015
for _, role := range users {
1016-
creds, err := getInternalCredentials(ctx, r.client, cr, role)
1016+
creds, err := psmdb.GetCredentials(ctx, r.client, cr, role)
10171017
if err != nil {
10181018
log.Error(err, "failed to get credentials", "role", role)
10191019
continue

0 commit comments

Comments
 (0)