Skip to content

Commit 354f7b1

Browse files
AMeceaAMecea
committed
Nodes that are not present in orc set to unknown.
1 parent 4ca37ce commit 354f7b1

File tree

6 files changed

+84
-24
lines changed

6 files changed

+84
-24
lines changed

‎pkg/apis/mysql/v1alpha1/cluster.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ func (c *MysqlCluster) GetBackupCandidate() string {
259259
return node.Name
260260
}
261261
}
262-
glog.Warning("No healthy slave node found so returns the master node: %s.", c.GetPodHostname(0))
262+
glog.Warningf("No healthy slave node found so returns the master node: %s.", c.GetPodHostname(0))
263263
return c.GetPodHostname(0)
264264
}
265265

‎pkg/backupfactory/backupfactory.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func (f *bFactory) Sync(ctx context.Context) error {
8888
}
8989

9090
func (f *bFactory) getJobName() string {
91-
return fmt.Sprintf("%s-%s-backup", f.backup.Name, f.backup.Spec.ClusterName)
91+
return fmt.Sprintf("%s-backupjob", f.backup.Name)
9292
}
9393

9494
func (f *bFactory) ensurePodSpec(in core.PodSpec) core.PodSpec {

‎pkg/controller/clustercontroller/backups.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func (c *Controller) registerClusterInBackupCron(cluster *api.MysqlCluster) erro
6060

6161
schedule, err := cron.Parse(cluster.Spec.BackupSchedule)
6262
if err != nil {
63-
return fmt.Errorf("fail to parse schedule: %s", err)
63+
return fmt.Errorf("failed to parse schedule: %s", err)
6464
}
6565

6666
lockJobRegister.Lock()

‎pkg/mysqlcluster/orc_reconciliation.go‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -72,23 +72,18 @@ func (f *cFactory) SyncOrchestratorStatus(ctx context.Context) error {
7272
}
7373

7474
func (f *cFactory) updateStatusFromOrc(insts []orc.Instance) {
75-
for i := 0; i < int(f.cluster.Spec.Replicas); i++ {
76-
host := f.cluster.GetPodHostname(i)
77-
// select instance from orchestrator
78-
var node *orc.Instance
79-
for _, inst := range insts {
80-
if inst.Key.Hostname == host {
81-
node = &inst
82-
break
75+
updatedNodes := []string{}
76+
for _, node := range insts {
77+
host := node.Key.Hostname
78+
updatedNodes = append(updatedNodes, host)
79+
80+
if !node.IsUpToDate {
81+
if !node.IsLastCheckValid {
82+
f.updateNodeCondition(host, api.NodeConditionLagged, core.ConditionUnknown)
83+
f.updateNodeCondition(host, api.NodeConditionReplicating, core.ConditionUnknown)
84+
f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionUnknown)
8385
}
84-
}
85-
86-
if node == nil {
87-
f.updateNodeCondition(host, api.NodeConditionLagged, core.ConditionUnknown)
88-
f.updateNodeCondition(host, api.NodeConditionReplicating, core.ConditionUnknown)
89-
f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionUnknown)
90-
91-
return
86+
continue
9287
}
9388

9489
maxSlaveLatency := defaultMaxSlaveLatency
@@ -116,6 +111,8 @@ func (f *cFactory) updateStatusFromOrc(insts []orc.Instance) {
116111
f.updateNodeCondition(host, api.NodeConditionMaster, core.ConditionFalse)
117112
}
118113
}
114+
115+
f.removeNodeConditionNotIn(updatedNodes)
119116
}
120117

121118
func (f *cFactory) updateStatusForRecoveries(recoveries []orc.TopologyRecovery) {
@@ -230,3 +227,20 @@ func (f *cFactory) updateNodeCondition(host string, cType api.NodeConditionType,
230227
}
231228
}
232229
}
230+
231+
func (f *cFactory) removeNodeConditionNotIn(hosts []string) {
232+
for _, ns := range f.cluster.Status.Nodes {
233+
updated := false
234+
for _, h := range hosts {
235+
if h == ns.Name {
236+
updated = true
237+
}
238+
}
239+
240+
if !updated {
241+
f.updateNodeCondition(ns.Name, api.NodeConditionLagged, core.ConditionUnknown)
242+
f.updateNodeCondition(ns.Name, api.NodeConditionReplicating, core.ConditionUnknown)
243+
f.updateNodeCondition(ns.Name, api.NodeConditionMaster, core.ConditionUnknown)
244+
}
245+
}
246+
}

‎pkg/mysqlcluster/orc_reconciliation_test.go‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ var _ = Describe("Mysql cluster reconcilation", func() {
8383

8484
It("should update status", func() {
8585
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
86-
true, -1, false)
86+
true, -1, false, true)
8787
orcClient.AddRecoveries("asd.default", 1, true)
8888
factory.createPod("asd-mysql-0")
8989

@@ -104,7 +104,7 @@ var _ = Describe("Mysql cluster reconcilation", func() {
104104

105105
It("should have pending recoveries", func() {
106106
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
107-
true, -1, false)
107+
true, -1, false, true)
108108
orcClient.AddRecoveries("asd.default", 11, false)
109109
Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
110110
Expect(getCCond(
@@ -114,7 +114,7 @@ var _ = Describe("Mysql cluster reconcilation", func() {
114114

115115
It("should have pending recoveries but cluster not ready enough", func() {
116116
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
117-
true, -1, false)
117+
true, -1, false, true)
118118
orcClient.AddRecoveries("asd.default", 111, false)
119119
cluster.UpdateStatusCondition(api.ClusterConditionReady, core.ConditionTrue, "", "")
120120
Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
@@ -126,7 +126,7 @@ var _ = Describe("Mysql cluster reconcilation", func() {
126126

127127
It("should have pending recoveries that will be recovered", func() {
128128
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
129-
true, -1, false)
129+
true, -1, false, true)
130130
orcClient.AddRecoveries("asd.default", 112, false)
131131
min20, _ := time.ParseDuration("-20m")
132132
cluster.Status.Conditions = []api.ClusterCondition{
@@ -148,6 +148,31 @@ var _ = Describe("Mysql cluster reconcilation", func() {
148148
Expect(event).To(ContainSubstring("RecoveryAcked"))
149149
})
150150

151+
It("node not uptodate in orc", func() {
152+
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
153+
true, -1, false, false)
154+
Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
155+
156+
Expect(cluster.Status.Nodes[0].GetCondition(api.NodeConditionMaster).Status).To(
157+
Equal(core.ConditionUnknown))
158+
})
159+
160+
It("node not in orc", func() {
161+
orcClient.AddInstance("asd.default", cluster.GetPodHostname(0),
162+
true, -1, false, true)
163+
Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
164+
165+
Expect(cluster.Status.Nodes[0].GetCondition(api.NodeConditionMaster).Status).To(
166+
Equal(core.ConditionTrue))
167+
168+
orcClient.RemoveInstance("asd.default", cluster.GetPodHostname(0))
169+
Ω(factory.SyncOrchestratorStatus(ctx)).Should(Succeed())
170+
171+
Expect(cluster.Status.Nodes[0].GetCondition(api.NodeConditionMaster).Status).To(
172+
Equal(core.ConditionUnknown))
173+
174+
})
175+
151176
})
152177
})
153178
})

‎pkg/util/orchestrator/fake/client.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func New() *FakeOrc {
3434
return &FakeOrc{}
3535
}
3636

37-
func (o *FakeOrc) AddInstance(cluster string, host string, master bool, sls int64, slaveR bool) {
37+
func (o *FakeOrc) AddInstance(cluster, host string, master bool, sls int64, slaveR, upToDate bool) {
3838
valid := true
3939
if sls < 0 {
4040
valid = false
@@ -52,6 +52,8 @@ func (o *FakeOrc) AddInstance(cluster string, host string, master bool, sls int6
5252
ClusterName: cluster,
5353
Slave_SQL_Running: slaveR,
5454
Slave_IO_Running: slaveR,
55+
IsUpToDate: upToDate,
56+
IsLastCheckValid: upToDate,
5557
}
5658
if o.Clusters == nil {
5759
o.Clusters = make(map[string][]Instance)
@@ -63,6 +65,25 @@ func (o *FakeOrc) AddInstance(cluster string, host string, master bool, sls int6
6365
o.Clusters[cluster] = []Instance{inst}
6466
}
6567

68+
func (o *FakeOrc) RemoveInstance(cluster, host string) {
69+
instances, ok := o.Clusters[cluster]
70+
if !ok {
71+
return
72+
}
73+
index := -1
74+
for i, inst := range instances {
75+
if inst.Key.Hostname == host {
76+
index = i
77+
}
78+
}
79+
80+
if index == -1 {
81+
return
82+
}
83+
84+
o.Clusters[cluster] = append(instances[:index], instances[index+1:]...)
85+
}
86+
6687
func (o *FakeOrc) AddRecoveries(cluster string, id int64, ack bool) {
6788
tr := TopologyRecovery{
6889
Id: id,

0 commit comments

Comments
 (0)