From 7197624e3119238973d74f9d0026c9886a3af43f Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Thu, 6 Jun 2024 15:43:56 +0200 Subject: [PATCH 1/5] Support extra flags for mysqld_exporter Signed-off-by: Sergey Stankevich --- .../crds/planetscale.com_vitessclusters.yaml | 8 + .../crds/planetscale.com_vitesskeyspaces.yaml | 8 + deploy/crds/planetscale.com_vitessshards.yaml | 4 + docs/api.md | 16 ++ docs/api/index.html | 16 ++ pkg/apis/planetscale/v2/vitessshard_types.go | 8 + .../planetscale/v2/zz_generated.deepcopy.go | 7 + pkg/operator/vitess/flags.go | 39 +++++ pkg/operator/vitess/flags_test.go | 165 ++++++++++++++++++ pkg/operator/vttablet/flags.go | 12 ++ pkg/operator/vttablet/lazy_values.go | 2 + pkg/operator/vttablet/pod.go | 23 ++- 12 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 pkg/operator/vitess/flags_test.go diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml index 043ea904..11ddaf8a 100644 --- a/deploy/crds/planetscale.com_vitessclusters.yaml +++ b/deploy/crds/planetscale.com_vitessclusters.yaml @@ -2012,6 +2012,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -2453,6 +2457,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: diff --git a/deploy/crds/planetscale.com_vitesskeyspaces.yaml b/deploy/crds/planetscale.com_vitesskeyspaces.yaml index 6fbf5424..4bd43977 100644 --- a/deploy/crds/planetscale.com_vitesskeyspaces.yaml +++ b/deploy/crds/planetscale.com_vitesskeyspaces.yaml @@ -553,6 +553,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -994,6 +998,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: diff --git a/deploy/crds/planetscale.com_vitessshards.yaml b/deploy/crds/planetscale.com_vitessshards.yaml index 2e676b06..b5f708b6 100644 --- a/deploy/crds/planetscale.com_vitessshards.yaml +++ b/deploy/crds/planetscale.com_vitessshards.yaml @@ -536,6 +536,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: diff --git a/docs/api.md b/docs/api.md index a063db8f..456827e1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1526,6 +1526,22 @@ EtcdLockserverStatus +extraFlags
+ +map[string]string + + + +

ExtraFlags can optionally be used to override default flags set by the +operator, or pass additional flags to mysqld_exporter. All entries must be +key-value string pairs of the form “flag”: “value”. The flag name should +not have any prefix (just “flag”, not “-flag”). To set a boolean flag, +set the string value to either “true” or “false”; the flag will be +automatically converted to the format expected by mysqld_exporter.

+ + + + resources
diff --git a/docs/api/index.html b/docs/api/index.html index 6059cb6e..cf34b8d3 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -1528,6 +1528,22 @@

MysqldExporterSpec +extraFlags
+ +map[string]string + + + +

ExtraFlags can optionally be used to override default flags set by the +operator, or pass additional flags to mysqld_exporter. All entries must be +key-value string pairs of the form “flag”: “value”. The flag name should +not have any prefix (just “flag”, not “-flag”). To set a boolean flag, +set the string value to either “true” or “false”; the flag will be +automatically converted to the format expected by mysqld_exporter.

+ + + + resources
 diff --git a/pkg/apis/planetscale/v2/vitessshard_types.go b/pkg/apis/planetscale/v2/vitessshard_types.go index 615e423f..67ee7e6f 100644 --- a/pkg/apis/planetscale/v2/vitessshard_types.go +++ b/pkg/apis/planetscale/v2/vitessshard_types.go @@ -317,6 +317,14 @@ type MysqldSpec struct { // MysqldExporterSpec configures the local MySQL exporter within a tablet. type MysqldExporterSpec struct { + // ExtraFlags can optionally be used to override default flags set by the + // operator, or pass additional flags to mysqld_exporter. All entries must be + // key-value string pairs of the form "flag": "value". The flag name should + // not have any prefix (just "flag", not "-flag"). To set a boolean flag, + // set the string value to either "true" or "false"; the flag will be + // automatically converted to the format expected by mysqld_exporter. + ExtraFlags map[string]string `json:"extraFlags,omitempty"` + // Resources specify the compute resources to allocate for just the MySQL Exporter. Resources corev1.ResourceRequirements `json:"resources"` } diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go index e82720c0..c81f9068 100644 --- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go +++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go @@ -433,6 +433,13 @@ func (in *LockserverStatus) DeepCopy() *LockserverStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MysqldExporterSpec) DeepCopyInto(out *MysqldExporterSpec) { *out = *in + if in.ExtraFlags != nil { + in, out := &in.ExtraFlags, &out.ExtraFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } in.Resources.DeepCopyInto(&out.Resources) } diff --git a/pkg/operator/vitess/flags.go b/pkg/operator/vitess/flags.go index 54a6770b..e5a2eebb 100644 --- a/pkg/operator/vitess/flags.go +++ b/pkg/operator/vitess/flags.go @@ -19,6 +19,7 @@ package vitess import ( "fmt" "sort" + "strings" ) // Flags represents values for flags to be passed to Vitess binaries. @@ -61,6 +62,44 @@ func (f Flags) FormatArgs() []string { return args } +// FormatArgsConvertBoolean returns the flags as a flattened list of +// command-line args with boolean values formatted as `--flag` or `--no-flag`. +// This format is used by some tools, like mysqld_exporter. +// Method is based on FormatArgs(). +func (f Flags) FormatArgsConvertBoolean() []string { + // Sort flag names so the ordering is deterministic, + // which is important when diffing object specs. + // This also makes it easier for humans to find things. + keys := make([]string, 0, len(f)) + for key := range f { + keys = append(keys, key) + } + sort.Strings(keys) + + // Make formatted args list. + args := make([]string, 0, len(f)) + for _, key := range keys { + // These args are passed to the command as a string array, + // so we don't need to worry about quotes or escaping. + // + // We use two dashes (--) even though the standard flag parser + // accepts either one or two dashes, because some wrappers like + // pflags require two dashes. + // + // All boolean values are formatted as `--flag` or `--no-flag`. + value := f[key].(string) + switch value := strings.ToLower(value); value { + case "true": + args = append(args, fmt.Sprintf("--%v", key)) + case "false": + args = append(args, fmt.Sprintf("--no-%v", key)) + default: + args = append(args, fmt.Sprintf("--%v=%v", key, value)) + } + } + return args +} + // Merge sets the given flags, overwriting duplicates. func (f Flags) Merge(flags Flags) Flags { for key, value := range flags { diff --git a/pkg/operator/vitess/flags_test.go b/pkg/operator/vitess/flags_test.go new file mode 100644 index 00000000..615f57dd --- /dev/null +++ b/pkg/operator/vitess/flags_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2019 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vitess + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestFormatArgs tests the FormatArgs method of the Flags type. +func TestFormatArgs(t *testing.T) { + tests := []struct { + name string + flags Flags + want []string + }{ + { + name: "empty flags", + flags: Flags{}, + want: []string{}, + }, + { + name: "single flag", + flags: Flags{ + "flag1": "value1", + }, + want: []string{"--flag1=value1"}, + }, + { + name: "multiple flags", + flags: Flags{ + "flag2": "value2", + "flag3": "value3", + }, + want: []string{"--flag2=value2", "--flag3=value3"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.flags.FormatArgs() + assert.Equal(t, tt.want, got) + }) + } +} + +// TestFormatArgsConvertBoolean tests the FormatArgsConvertBoolean method of the Flags type. +func TestFormatArgsConvertBoolean(t *testing.T) { + tests := []struct { + name string + flags Flags + want []string + }{ + { + name: "empty flags", + flags: Flags{}, + want: []string{}, + }, + { + name: "boolean flag true", + flags: Flags{ + "flag1": "true", + }, + want: []string{"--flag1"}, + }, + { + name: "boolean flag false", + flags: Flags{ + "flag2": "false", + }, + want: []string{"--no-flag2"}, + }, + { + name: "non-boolean flag", + flags: Flags{ + "flag3": "value3", + }, + want: []string{"--flag3=value3"}, + }, + { + name: "multiple flags", + flags: Flags{ + "flag4": "true", + "flag5": "false", + "flag6": "value6", + }, + want: []string{"--flag4", "--no-flag5", "--flag6=value6"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.flags.FormatArgsConvertBoolean() + assert.Equal(t, tt.want, got) + }) + } +} + +// TestMerge tests the Merge method of the Flags type. +func TestMerge(t *testing.T) { + tests := []struct { + name string + flags Flags + merge Flags + result Flags + }{ + { + name: "merge empty flags", + flags: Flags{ + "flag1": "value1", + }, + merge: Flags{}, + result: Flags{ + "flag1": "value1", + }, + }, + { + name: "merge non-empty flags", + flags: Flags{ + "flag1": "value1", + }, + merge: Flags{ + "flag2": "value2", + }, + result: Flags{ + "flag1": "value1", + "flag2": "value2", + }, + }, + { + name: "merge duplicate flags", + flags: Flags{ + "flag1": "value1", + }, + merge: Flags{ + "flag1": "value2", + }, + result: Flags{ + "flag1": "value2", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.flags.Merge(tt.merge) + assert.Equal(t, tt.result, got) + }) + } +} diff --git a/pkg/operator/vttablet/flags.go b/pkg/operator/vttablet/flags.go index 1ec42d0c..3201bb60 100644 --- a/pkg/operator/vttablet/flags.go +++ b/pkg/operator/vttablet/flags.go @@ -95,6 +95,18 @@ func init() { } }) + // Base mysqld_exporter flags. + mysqldExporterFlags.Add(func(s lazy.Spec) vitess.Flags { + spec := s.(*Spec) + return vitess.Flags{ + "config.my-cnf": spec.myCnfFilePath(), + // The default for `collect.info_schema.tables.databases` is `*`, + // which causes new time series to be created for each user table. + // This in turn causes scaling issues in Prometheus memory usage. + "collect.info_schema.tables.databases": "sys,_vt", + } + }) + // Base vtbackup flags. vtbackupFlags.Add(func(s lazy.Spec) vitess.Flags { backupSpec := s.(*BackupSpec) diff --git a/pkg/operator/vttablet/lazy_values.go b/pkg/operator/vttablet/lazy_values.go index b0aa84e8..ca63ba8e 100644 --- a/pkg/operator/vttablet/lazy_values.go +++ b/pkg/operator/vttablet/lazy_values.go @@ -52,6 +52,8 @@ var ( vttabletFlags lazy.VitessFlags // mysqlctldFlags are the flags for mysqlctld. mysqlctldFlags lazy.VitessFlags + // mysqldExporterFlags are the flags for mysqld_exporter. + mysqldExporterFlags lazy.VitessFlags // vtbackupFlags are the flags for vtbackup. vtbackupFlags lazy.VitessFlags ) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 55a0f373..dd99512a 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -200,21 +200,26 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { update.ResourceRequirements(&mysqldContainer.Resources, &spec.Mysqld.Resources) + // Compute all operator-generated mysqld_exporter flags first. + // Then apply user-provided overrides last so they take precedence. + mysqldExporterAllFlags := mysqldExporterFlags.Get(spec) + if spec.MysqldExporter != nil { + for key, value := range spec.MysqldExporter.ExtraFlags { + // We told users in the CRD API field doc not to put any leading '-', + // but people may not read that so we are liberal in what we accept. + key = strings.TrimLeft(key, "-") + mysqldExporterAllFlags[key] = value + } + } + // TODO: Can/should we still run mysqld_exporter pointing at external mysql? mysqldExporterContainer = &corev1.Container{ Name: mysqldExporterContainerName, Image: spec.Images.MysqldExporter, ImagePullPolicy: spec.ImagePullPolicies.MysqldExporter, Command: []string{mysqldExporterCommand}, - Args: []string{ - "--config.my-cnf=" + spec.myCnfFilePath(), - // The default for `collect.info_schema.tables.databases` is - // `*`, which causes new time series to be created for each user - // table. This in turn causes scaling issues in Prometheus - // memory usage. - "--collect.info_schema.tables.databases=sys,_vt", - }, - Env: mysqldExporterEnv, + Args: mysqldExporterAllFlags.FormatArgsConvertBoolean(), + Env: mysqldExporterEnv, Ports: []corev1.ContainerPort{ { Name: mysqldExporterPortName, From 32feb5e02b1b8470873cba5a7580cd76e6dbdc69 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Tue, 5 Nov 2024 13:50:42 +0100 Subject: [PATCH 2/5] Update year in license header Signed-off-by: Sergey Stankevich --- pkg/operator/vitess/flags_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/vitess/flags_test.go b/pkg/operator/vitess/flags_test.go index 615f57dd..91c9b33e 100644 --- a/pkg/operator/vitess/flags_test.go +++ b/pkg/operator/vitess/flags_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 PlanetScale Inc. +Copyright 2024 PlanetScale Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 128371919e122101356a68e2792e8d52ad526d86 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Tue, 5 Nov 2024 14:04:10 +0100 Subject: [PATCH 3/5] Regenerate operator.yaml Signed-off-by: Sergey Stankevich --- test/endtoend/operator/operator-latest.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml index d8e074ca..b765e8a7 100644 --- a/test/endtoend/operator/operator-latest.yaml +++ b/test/endtoend/operator/operator-latest.yaml @@ -3894,6 +3894,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -4335,6 +4339,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -5713,6 +5721,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -6154,6 +6166,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: @@ -7148,6 +7164,10 @@ spec: type: object mysqldExporter: properties: + extraFlags: + additionalProperties: + type: string + type: object resources: properties: claims: From 5796a5e60b071a3cbd155f72de239870374e9298 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Fri, 8 Nov 2024 14:09:15 +0100 Subject: [PATCH 4/5] Make mysqldExporter.resources optional It's possible that mysqldExporter.extraFlags are provided while mysqldExporter.resources are not. Signed-off-by: Sergey Stankevich --- deploy/crds/planetscale.com_vitessclusters.yaml | 4 ---- deploy/crds/planetscale.com_vitesskeyspaces.yaml | 4 ---- deploy/crds/planetscale.com_vitessshards.yaml | 2 -- pkg/apis/planetscale/v2/vitessshard_types.go | 2 +- pkg/operator/vttablet/pod.go | 7 +++++++ test/endtoend/operator/operator-latest.yaml | 10 ---------- 6 files changed, 8 insertions(+), 21 deletions(-) diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml index 11ddaf8a..931d11c8 100644 --- a/deploy/crds/planetscale.com_vitessclusters.yaml +++ b/deploy/crds/planetscale.com_vitessclusters.yaml @@ -2049,8 +2049,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -2494,8 +2492,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" diff --git a/deploy/crds/planetscale.com_vitesskeyspaces.yaml b/deploy/crds/planetscale.com_vitesskeyspaces.yaml index 4bd43977..1819b5fe 100644 --- a/deploy/crds/planetscale.com_vitesskeyspaces.yaml +++ b/deploy/crds/planetscale.com_vitesskeyspaces.yaml @@ -590,8 +590,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -1035,8 +1033,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" diff --git a/deploy/crds/planetscale.com_vitessshards.yaml b/deploy/crds/planetscale.com_vitessshards.yaml index b5f708b6..f75a594c 100644 --- a/deploy/crds/planetscale.com_vitessshards.yaml +++ b/deploy/crds/planetscale.com_vitessshards.yaml @@ -573,8 +573,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" diff --git a/pkg/apis/planetscale/v2/vitessshard_types.go b/pkg/apis/planetscale/v2/vitessshard_types.go index 67ee7e6f..d34755d1 100644 --- a/pkg/apis/planetscale/v2/vitessshard_types.go +++ b/pkg/apis/planetscale/v2/vitessshard_types.go @@ -326,7 +326,7 @@ type MysqldExporterSpec struct { ExtraFlags map[string]string `json:"extraFlags,omitempty"` // Resources specify the compute resources to allocate for just the MySQL Exporter. - Resources corev1.ResourceRequirements `json:"resources"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // VitessTabletPoolType represents the tablet types for which it makes sense diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index dd99512a..53cbead8 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -248,6 +248,13 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { corev1.ResourceMemory: *resource.NewQuantity(mysqldExporterMemoryLimitBytes, resource.BinarySI), }, } + // TODO(enisoc): Add readiness and liveness probes that make sense for mysqld-exporter. + // This depends on the exact semantics of each of mysqld-exporter's HTTP handlers, + // so we need to do more investigation. For now it's better to leave them empty. + } + + if spec.MysqldExporter != nil && (len(spec.MysqldExporter.Resources.Limits) > 0 || len(spec.MysqldExporter.Resources.Requests) > 0) { + update.ResourceRequirements(&mysqldExporterContainer.Resources, &spec.MysqldExporter.Resources) } } diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml index b765e8a7..62b55ef2 100644 --- a/test/endtoend/operator/operator-latest.yaml +++ b/test/endtoend/operator/operator-latest.yaml @@ -3931,8 +3931,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -4376,8 +4374,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -5758,8 +5754,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -6203,8 +6197,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" @@ -7201,8 +7193,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - required: - - resources type: object name: default: "" From 9b78217c6c8d068fbf68b31ac15bf5a1e43a18d7 Mon Sep 17 00:00:00 2001 From: Sergey Stankevich Date: Fri, 8 Nov 2024 17:01:34 +0100 Subject: [PATCH 5/5] Add mysqld_exporter flags to the upgrade test Signed-off-by: Sergey Stankevich --- test/endtoend/operator/cluster_upgrade.yaml | 6 ++++++ test/endtoend/upgrade_test.sh | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/test/endtoend/operator/cluster_upgrade.yaml b/test/endtoend/operator/cluster_upgrade.yaml index 5c9b66ee..eebcd73e 100644 --- a/test/endtoend/operator/cluster_upgrade.yaml +++ b/test/endtoend/operator/cluster_upgrade.yaml @@ -99,6 +99,12 @@ spec: cpu: 100m memory: 512Mi configOverrides: "innodb_fast_shutdown=0" + # Configure extra flags for mysqld_exporter and check them in the upgrade test + mysqldExporter: + extraFlags: + collect.info_schema.tables.databases: "*" # Override the default + collect.info_schema.innodb_cmpmem: "false" + collect.info_schema.tables: "true" dataVolumeClaimTemplate: accessModes: ["ReadWriteOnce"] resources: diff --git a/test/endtoend/upgrade_test.sh b/test/endtoend/upgrade_test.sh index f1151945..0a89fefc 100755 --- a/test/endtoend/upgrade_test.sh +++ b/test/endtoend/upgrade_test.sh @@ -252,6 +252,15 @@ function upgradeToLatest() { verifyDataCommerce } +function verifyResourceSpec() { + echo "Verifying resource spec" + + echo "mysqld_exporter flags:" + checkPodSpecBySelectorWithTimeout example "planetscale.com/component=vttablet" 3 "--no-collect.info_schema.innodb_cmpmem$" + checkPodSpecBySelectorWithTimeout example "planetscale.com/component=vttablet" 3 "--collect.info_schema.tables$" + checkPodSpecBySelectorWithTimeout example "planetscale.com/component=vttablet" 3 "--collect.info_schema.tables.databases=\*$" +} + # Test setup setupKindCluster cd test/endtoend/operator || exit 1 @@ -263,6 +272,7 @@ checkSemiSyncSetup verifyDurabilityPolicy "commerce" "semi_sync" upgradeToLatest verifyVtGateVersion "23.0.0" +verifyResourceSpec checkSemiSyncSetup # After upgrading, we verify that the durability policy is still semi_sync verifyDurabilityPolicy "commerce" "semi_sync"