Skip to content

Commit 339e333

Browse files
3pointer3pointer
authored
tici: ensure config rolling update applies and use DNS flash.service_addr when TiCI enabled (#6744)
1 parent f256c18 commit 339e333

File tree

4 files changed

+90
-13
lines changed

4 files changed

+90
-13
lines changed

‎pkg/manager/member/tici_member_manager.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,8 @@ func (m *ticiMemberManager) syncTiCIMetaStatefulSet(tc *v1alpha1.TidbCluster) er
225225
return err
226226
}
227227

228+
prepareTiCIRollingUpgrade(newSts, oldSts)
229+
228230
return mngerutils.UpdateStatefulSetWithPrecheck(m.deps, tc, "FailedUpdateTiCIMetaSTS", newSts, oldSts)
229231
}
230232

@@ -270,9 +272,24 @@ func (m *ticiMemberManager) syncTiCIWorkerStatefulSet(tc *v1alpha1.TidbCluster)
270272
return err
271273
}
272274

275+
prepareTiCIRollingUpgrade(newSts, oldSts)
276+
273277
return mngerutils.UpdateStatefulSetWithPrecheck(m.deps, tc, "FailedUpdateTiCIWorkerSTS", newSts, oldSts)
274278
}
275279

280+
func prepareTiCIRollingUpgrade(newSet, oldSet *appsv1.StatefulSet) {
281+
if newSet == nil || oldSet == nil {
282+
return
283+
}
284+
if newSet.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
285+
return
286+
}
287+
needsUpgrade := !templateEqual(newSet, oldSet) || oldSet.Status.CurrentRevision != oldSet.Status.UpdateRevision
288+
if needsUpgrade {
289+
mngerutils.SetUpgradePartition(newSet, 0)
290+
}
291+
}
292+
276293
func (m *ticiMemberManager) syncTiCIMetaStatus(tc *v1alpha1.TidbCluster, sts *appsv1.StatefulSet) error {
277294
if sts == nil {
278295
tc.Status.TiCIMeta.StatefulSet = &appsv1.StatefulSetStatus{}

‎pkg/manager/member/tici_member_manager_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@
1414
package member
1515

1616
import (
17+
"encoding/json"
1718
"strings"
1819
"testing"
1920

2021
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
2122
tcconfig "github.com/pingcap/tidb-operator/pkg/apis/util/config"
23+
24+
appsv1 "k8s.io/api/apps/v1"
25+
corev1 "k8s.io/api/core/v1"
2226
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2327
)
2428

@@ -106,6 +110,46 @@ func TestAppendTiCICustomConfig(t *testing.T) {
106110
}
107111
}
108112

113+
func TestPrepareTiCIRollingUpgrade(t *testing.T) {
114+
partition := int32(1)
115+
oldSet := &appsv1.StatefulSet{
116+
ObjectMeta: metav1.ObjectMeta{
117+
Name: "tc-tici-meta",
118+
Namespace: "default",
119+
Annotations: map[string]string{},
120+
},
121+
Spec: appsv1.StatefulSetSpec{
122+
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
123+
Type: appsv1.RollingUpdateStatefulSetStrategyType,
124+
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{Partition: &partition},
125+
},
126+
Template: corev1.PodTemplateSpec{
127+
Spec: corev1.PodSpec{
128+
Containers: []corev1.Container{{Name: "tici-meta", Image: "tici:v1"}},
129+
},
130+
},
131+
},
132+
Status: appsv1.StatefulSetStatus{
133+
CurrentRevision: "tc-tici-meta-old",
134+
UpdateRevision: "tc-tici-meta-new",
135+
},
136+
}
137+
specData, err := json.Marshal(appsv1.StatefulSetSpec{Template: oldSet.Spec.Template})
138+
if err != nil {
139+
t.Fatalf("marshal spec failed: %v", err)
140+
}
141+
oldSet.Annotations[LastAppliedConfigAnnotation] = string(specData)
142+
143+
newSet := oldSet.DeepCopy()
144+
prepareTiCIRollingUpgrade(newSet, oldSet)
145+
if newSet.Spec.UpdateStrategy.RollingUpdate == nil || newSet.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
146+
t.Fatalf("rolling update partition should be set")
147+
}
148+
if got := *newSet.Spec.UpdateStrategy.RollingUpdate.Partition; got != 0 {
149+
t.Fatalf("expected partition 0 when TiCI has pending upgrade, got: %d", got)
150+
}
151+
}
152+
109153
func newTidbClusterForTiCIConfig() *v1alpha1.TidbCluster {
110154
return &v1alpha1.TidbCluster{
111155
ObjectMeta: metav1.ObjectMeta{

‎pkg/manager/member/tiflash_util.go‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper
161161
listenHost = listenHostForIPv6
162162
}
163163
version := tc.TiFlashVersion()
164+
ticiEnabled := tc.Spec.TiCI != nil && tc.Spec.TiCI.S3 != nil
164165

165166
// common
166167
{
@@ -207,7 +208,12 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper
207208
}
208209
common.SetIfNil("flash.tidb_status_addr", tidbStatusAddr)
209210
if !mountCMInTiflashContainer {
210-
common.SetIfNil("flash.service_addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort))
211+
flashServiceAddr := fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort)
212+
if ticiEnabled {
213+
flashServiceAddr = fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name),
214+
controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashFlashPort)
215+
}
216+
common.SetIfNil("flash.service_addr", flashServiceAddr)
211217
}
212218
common.SetIfNil("flash.flash_cluster.log", defaultClusterLog)
213219
common.SetIfNil("flash.proxy.addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashProxyPort))
@@ -314,12 +320,13 @@ func getTiFlashConfig(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper {
314320
noLocalPD := tc.WithoutLocalPD()
315321
acrossK8s := tc.AcrossK8s()
316322
noLocalTiDB := tc.WithoutLocalTiDB()
323+
ticiEnabled := tc.Spec.TiCI != nil && tc.Spec.TiCI.S3 != nil
317324
listenHost := listenHostForIPv4
318325
if tc.Spec.PreferIPv6 {
319326
listenHost = listenHostForIPv6
320327
}
321328

322-
setTiFlashConfigDefault(config, ref, tc.Name, tc.Namespace, tc.Spec.ClusterDomain, listenHost, noLocalPD, noLocalTiDB, acrossK8s)
329+
setTiFlashConfigDefault(config, ref, tc.Name, tc.Namespace, tc.Spec.ClusterDomain, listenHost, noLocalPD, noLocalTiDB, acrossK8s, ticiEnabled)
323330

324331
// Note the config of tiflash use "_" by convention, others(proxy) use "-".
325332
if tc.IsTLSClusterEnabled() {
@@ -361,12 +368,12 @@ func setTiFlashLogConfigDefault(config *v1alpha1.TiFlashConfigWraper) {
361368

362369
// setTiFlashConfigDefault sets default configs for TiFlash
363370
func setTiFlashConfigDefault(config *v1alpha1.TiFlashConfigWraper, ref *v1alpha1.TidbClusterRef,
364-
clusterName, ns, clusterDomain, listenHost string, noLocalPD bool, noLocalTiDB bool, acrossK8s bool,
371+
clusterName, ns, clusterDomain, listenHost string, noLocalPD bool, noLocalTiDB bool, acrossK8s bool, ticiEnabled bool,
365372
) {
366373
if config.Common == nil {
367374
config.Common = v1alpha1.NewTiFlashCommonConfig()
368375
}
369-
setTiFlashCommonConfigDefault(config.Common, ref, clusterName, ns, clusterDomain, listenHost, noLocalPD, noLocalTiDB, acrossK8s)
376+
setTiFlashCommonConfigDefault(config.Common, ref, clusterName, ns, clusterDomain, listenHost, noLocalPD, noLocalTiDB, acrossK8s, ticiEnabled)
370377

371378
if config.Proxy == nil {
372379
config.Proxy = v1alpha1.NewTiFlashProxyConfig()
@@ -383,7 +390,7 @@ func setTiFlashProxyConfigDefault(config *v1alpha1.TiFlashProxyConfigWraper, clu
383390
controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort))
384391
}
385392

386-
func setTiFlashCommonConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, ref *v1alpha1.TidbClusterRef, clusterName, ns, clusterDomain, listenHost string, noLocalPD bool, noLocalTiDB bool, acrossK8s bool) {
393+
func setTiFlashCommonConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, ref *v1alpha1.TidbClusterRef, clusterName, ns, clusterDomain, listenHost string, noLocalPD bool, noLocalTiDB bool, acrossK8s bool, ticiEnabled bool) {
387394
config.SetIfNil("tmp_path", "/data0/tmp")
388395
config.SetIfNil("display_name", "TiFlash")
389396
config.SetIfNil("default_profile", "default")
@@ -396,7 +403,7 @@ func setTiFlashCommonConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, r
396403
config.SetIfNil("https_port", int64(v1alpha1.DefaultTiFlashHttpPort))
397404
config.SetIfNil("http_port", int64(v1alpha1.DefaultTiFlashHttpPort))
398405
config.SetIfNil("interserver_http_port", int64(v1alpha1.DefaultTiFlashInternalPort))
399-
setTiFlashFlashConfigDefault(config, ref, clusterName, ns, clusterDomain, listenHost, noLocalTiDB, acrossK8s)
406+
setTiFlashFlashConfigDefault(config, ref, clusterName, ns, clusterDomain, listenHost, noLocalTiDB, acrossK8s, ticiEnabled)
400407
setTiFlashLoggerConfigDefault(config)
401408
setTiFlashApplicationConfigDefault(config)
402409

@@ -431,7 +438,7 @@ func setTiFlashCommonConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, r
431438
config.SetIfNil("profiles.default.use_uncompressed_cache", int64(0))
432439
}
433440

434-
func setTiFlashFlashConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, ref *v1alpha1.TidbClusterRef, clusterName, ns, clusterDomain, listenHost string, noLocalTiDB, acrossK8s bool) {
441+
func setTiFlashFlashConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, ref *v1alpha1.TidbClusterRef, clusterName, ns, clusterDomain, listenHost string, noLocalTiDB, acrossK8s bool, ticiEnabled bool) {
435442
tidbStatusAddr := fmt.Sprintf("%s.%s.svc:%d", controller.TiDBMemberName(clusterName), ns, v1alpha1.DefaultTiDBStatusPort)
436443
if noLocalTiDB {
437444
// TODO: support first cluster without TiDB when deploy cluster across mutli Kubernete clusters
@@ -449,7 +456,16 @@ func setTiFlashFlashConfigDefault(config *v1alpha1.TiFlashCommonConfigWraper, re
449456
}
450457

451458
config.SetIfNil("flash.tidb_status_addr", tidbStatusAddr)
452-
config.SetIfNil("flash.service_addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort))
459+
flashServiceAddr := fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort)
460+
if ticiEnabled {
461+
flashServiceAddr = fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d",
462+
controller.TiFlashMemberName(clusterName),
463+
controller.TiFlashPeerMemberName(clusterName),
464+
ns,
465+
controller.FormatClusterDomain(clusterDomain),
466+
v1alpha1.DefaultTiFlashFlashPort)
467+
}
468+
config.SetIfNil("flash.service_addr", flashServiceAddr)
453469
config.SetIfNil("flash.overlap_threshold", 0.6)
454470
config.SetIfNil("flash.compact_log_min_period", int64(200))
455471

‎pkg/manager/member/tiflash_util_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,7 @@ func TestSetTiFlashConfigDefault(t *testing.T) {
506506
for _, test := range tests {
507507
t.Run(test.name, func(t *testing.T) {
508508
// g := NewGomegaWithT(t)
509-
setTiFlashConfigDefault(test.config, nil, "test", "test", "", "0.0.0.0", false, false, false)
509+
setTiFlashConfigDefault(test.config, nil, "test", "test", "", "0.0.0.0", false, false, false, false)
510510
// g.Expect(test.config).To(Equal(test.expect))
511511
if diff := cmp.Diff(*test.expect, *test.config); diff != "" {
512512
t.Fatalf("unexpected configuration (-want, +got): %s", diff)
@@ -1971,7 +1971,7 @@ func TestGetTiFlashConfigV2FlashServiceAddr(t *testing.T) {
19711971
expectVal: "0.0.0.0:3930",
19721972
},
19731973
{
1974-
name: "tici cluster keeps config flash service listen addr",
1974+
name: "tici cluster uses headless flash service addr",
19751975
setTC: func(tc *v1alpha1.TidbCluster) {
19761976
tc.Spec.TiCI = &v1alpha1.TiCISpec{
19771977
S3: &v1alpha1.TiCIS3Spec{
@@ -1980,7 +1980,7 @@ func TestGetTiFlashConfigV2FlashServiceAddr(t *testing.T) {
19801980
},
19811981
}
19821982
},
1983-
expectVal: "0.0.0.0:3930",
1983+
expectVal: "test-tiflash-POD_NUM.test-tiflash-peer.default.svc:3930",
19841984
},
19851985
}
19861986

@@ -2019,7 +2019,7 @@ func TestGetTiFlashConfigFlashServiceAddr(t *testing.T) {
20192019
expectVal: "0.0.0.0:3930",
20202020
},
20212021
{
2022-
name: "tici cluster keeps config flash service listen addr",
2022+
name: "tici cluster uses headless flash service addr",
20232023
setTC: func(tc *v1alpha1.TidbCluster) {
20242024
tc.Spec.TiCI = &v1alpha1.TiCISpec{
20252025
S3: &v1alpha1.TiCIS3Spec{
@@ -2028,7 +2028,7 @@ func TestGetTiFlashConfigFlashServiceAddr(t *testing.T) {
20282028
},
20292029
}
20302030
},
2031-
expectVal: "0.0.0.0:3930",
2031+
expectVal: "test-tiflash-POD_NUM.test-tiflash-peer.default.svc:3930",
20322032
},
20332033
}
20342034

0 commit comments

Comments
 (0)