Skip to content

Commit 4e4e92a

Browse files
SangJunBakSangJunBak
committed
Set replication factor to 0 for certain system clusters by default in self managed
1 parent da774ee commit 4e4e92a

File tree

8 files changed

+54
-13
lines changed

8 files changed

+54
-13
lines changed

‎misc/helm-charts/operator/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ The following table lists the configurable parameters of the Materialize operato
125125
| `operator.cloudProvider.providers.gcp` | GCP Configuration (placeholder for future use) | ``{"enabled":false}`` |
126126
| `operator.cloudProvider.region` | Common cloud provider settings | ``"kind"`` |
127127
| `operator.cloudProvider.type` | Specifies cloud provider. Valid values are 'aws', 'gcp', 'azure' , 'generic', or 'local' | ``"local"`` |
128+
| `operator.clusters.defaultReplicationFactor.probe` | | ``0`` |
129+
| `operator.clusters.defaultReplicationFactor.system` | | ``0`` |
128130
| `operator.clusters.defaultSizes.analytics` | | ``"25cc"`` |
129131
| `operator.clusters.defaultSizes.catalogServer` | | ``"50cc"`` |
130132
| `operator.clusters.defaultSizes.default` | | ``"25cc"`` |

‎misc/helm-charts/operator/templates/deployment.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ spec:
103103
{{ if .Values.operator.clusters.defaultSizes.analytics }}
104104
- "--bootstrap-builtin-analytics-cluster-replica-size={{ .Values.operator.clusters.defaultSizes.analytics }}"
105105
{{- end }}
106+
{{ if .Values.operator.clusters.defaultReplicationFactor.system }}
107+
- "--bootstrap-builtin-system-cluster-replication-factor={{ .Values.operator.clusters.defaultReplicationFactor.system }}"
108+
{{- end }}
109+
{{ if .Values.operator.clusters.defaultReplicationFactor.probe }}
110+
- "--bootstrap-builtin-probe-cluster-replication-factor={{ .Values.operator.clusters.defaultReplicationFactor.probe }}"
111+
{{- end }}
106112
{{- end }}
107113
- "--image-pull-policy={{ kebabcase .Values.operator.image.pullPolicy }}"
108114
{{- range $key, $value := .Values.environmentd.nodeSelector }}

‎misc/helm-charts/operator/values.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,11 @@ operator:
175175
support: 25cc
176176
catalogServer: 50cc
177177
analytics: 25cc
178+
defaultReplicationFactor:
179+
system: 0
180+
probe: 0
181+
support: 0
182+
analytics: 0
178183

179184
# Node selector to use for the operator pod
180185
nodeSelector: {}

‎src/environmentd/src/environmentd/main.rs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -549,39 +549,44 @@ pub struct Args {
549549
default_value = "1"
550550
)]
551551
bootstrap_builtin_analytics_cluster_replica_size: String,
552-
/// The size of the builtin system cluster replicas if bootstrapping.
552+
/// The replication factor of the builtin system cluster replicas if bootstrapping.
553553
#[clap(
554554
long,
555555
env = "BOOTSTRAP_BUILTIN_SYSTEM_CLUSTER_REPLICATION_FACTOR",
556-
default_value = SYSTEM_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string()
556+
default_value = SYSTEM_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string(),
557+
value_parser = clap::value_parser!(u32).range(0..=1)
557558
)]
558559
bootstrap_builtin_system_cluster_replication_factor: u32,
559560
/// The replication factor of the builtin catalog server cluster replicas if bootstrapping.
560561
#[clap(
561562
long,
562563
env = "BOOTSTRAP_BUILTIN_CATALOG_SERVER_CLUSTER_REPLICATION_FACTOR",
563-
default_value = CATALOG_SERVER_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string()
564+
default_value = CATALOG_SERVER_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string(),
565+
value_parser = clap::value_parser!(u32).range(0..=1)
564566
)]
565567
bootstrap_builtin_catalog_server_cluster_replication_factor: u32,
566568
/// The replication factor of the builtin probe cluster replicas if bootstrapping.
567569
#[clap(
568570
long,
569571
env = "BOOTSTRAP_BUILTIN_PROBE_CLUSTER_REPLICATION_FACTOR",
570-
default_value = PROBE_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string()
572+
default_value = PROBE_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string(),
573+
value_parser = clap::value_parser!(u32).range(0..=1)
571574
)]
572575
bootstrap_builtin_probe_cluster_replication_factor: u32,
573576
/// The replication factor of the builtin support cluster replicas if bootstrapping.
574577
#[clap(
575578
long,
576579
env = "BOOTSTRAP_BUILTIN_SUPPORT_CLUSTER_REPLICATION_FACTOR",
577-
default_value = SUPPORT_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string()
580+
default_value = SUPPORT_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string(),
581+
value_parser = clap::value_parser!(u32).range(0..=1)
578582
)]
579583
bootstrap_builtin_support_cluster_replication_factor: u32,
580584
/// The replication factor of the builtin analytics cluster replicas if bootstrapping.
581585
#[clap(
582586
long,
583587
env = "BOOTSTRAP_BUILTIN_ANALYTICS_CLUSTER_REPLICATION_FACTOR",
584-
default_value = ANALYTICS_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string()
588+
default_value = ANALYTICS_CLUSTER_DEFAULT_REPLICATION_FACTOR.to_string(),
589+
value_parser = clap::value_parser!(u32).range(0..=1)
585590
)]
586591
bootstrap_builtin_analytics_cluster_replication_factor: u32,
587592
/// An list of NAME=VALUE pairs used to override static defaults

‎src/environmentd/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ pub struct Config {
152152
pub environment_id: EnvironmentId,
153153
/// What role, if any, should be initially created with elevated privileges.
154154
pub bootstrap_role: Option<String>,
155-
/// The config of the default cluster replica if bootstrapping.
155+
/// The size of the default cluster replica if bootstrapping.
156156
pub bootstrap_default_cluster_replica_size: String,
157157
/// The config of the builtin system cluster replicas if bootstrapping.
158158
pub bootstrap_builtin_system_cluster_config: BootstrapBuiltinClusterConfig,

‎src/environmentd/tests/bootstrap_builtin_clusters.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,11 @@ fn test_zero_replication_factor_no_replicas() {
2222
let system_cluster = client
2323
.query_one(
2424
r#"
25-
SELECT c.id, c.name, c.replication_factor::integer, COUNT(cr.id)::integer as replica_count
25+
SELECT c.id, c.name, c.replication_factor::integer, COUNT(cr.id)::integer as replica_count
2626
FROM mz_clusters c
2727
LEFT JOIN mz_cluster_replicas cr ON c.id = cr.cluster_id
2828
WHERE c.name = 'mz_system'
29-
GROUP BY c.id, c.name, c.replication_factor
30-
"#,
29+
GROUP BY c.id, c.name, c.replication_factor"#,
3130
&[],
3231
)
3332
.unwrap();

‎src/orchestratord/src/controller/materialize.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,14 @@ pub struct MaterializeControllerArgs {
108108
bootstrap_builtin_catalog_server_cluster_replica_size: Option<String>,
109109
#[clap(long)]
110110
bootstrap_builtin_analytics_cluster_replica_size: Option<String>,
111+
#[clap(long)]
112+
bootstrap_builtin_system_cluster_replication_factor: Option<u32>,
113+
#[clap(long)]
114+
bootstrap_builtin_probe_cluster_replication_factor: Option<u32>,
115+
#[clap(long)]
116+
bootstrap_builtin_support_cluster_replication_factor: Option<u32>,
117+
#[clap(long)]
118+
bootstrap_builtin_analytics_cluster_replication_factor: Option<u32>,
111119

112120
#[clap(
113121
long,

‎src/orchestratord/src/controller/materialize/environmentd.rs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -833,13 +833,29 @@ fn create_environmentd_statefulset_object(
833833
config
834834
.bootstrap_builtin_catalog_server_cluster_replica_size
835835
.as_ref()
836-
.map(|size| {
837-
format!("--bootstrap-builtin-catalog-server-cluster-replica-size={size}")
838-
}),
836+
.map(|size| format!("--bootstrap-builtin-catalog-server-cluster-replica-size={size}")),
839837
config
840838
.bootstrap_builtin_analytics_cluster_replica_size
841839
.as_ref()
842840
.map(|size| format!("--bootstrap-builtin-analytics-cluster-replica-size={size}")),
841+
config
842+
.bootstrap_builtin_system_cluster_replication_factor
843+
.as_ref()
844+
.map(|replication_factor| {
845+
format!("--bootstrap-builtin-system-cluster-replication-factor={replication_factor}")
846+
}),
847+
config
848+
.bootstrap_builtin_probe_cluster_replication_factor
849+
.as_ref()
850+
.map(|replication_factor| format!("--bootstrap-builtin-probe-cluster-replication-factor={replication_factor}")),
851+
config
852+
.bootstrap_builtin_support_cluster_replication_factor
853+
.as_ref()
854+
.map(|replication_factor| format!("--bootstrap-builtin-support-cluster-replication-factor={replication_factor}")),
855+
config
856+
.bootstrap_builtin_analytics_cluster_replication_factor
857+
.as_ref()
858+
.map(|replication_factor| format!("--bootstrap-builtin-analytics-cluster-replication-factor={replication_factor}")),
843859
]
844860
.into_iter()
845861
.flatten(),

0 commit comments

Comments
 (0)