Skip to content

Commit 780f854

Browse files
tcarmettcarmet
committed
WKBCH-12: Add crr support
1 parent 99500bc commit 780f854

File tree

10 files changed

+160
-36
lines changed

10 files changed

+160
-36
lines changed

‎cmd/config.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,11 @@ type GlobalConfig struct {
6565
}
6666

6767
type FeatureConfig struct {
68-
Scuba ScubaFeatureConfig `yaml:"scuba"`
69-
BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
70-
Utapi UtapiFeatureConfig `yaml:"utapi"`
71-
Migration MigrationFeatureConfig `yaml:"migration"`
68+
Scuba ScubaFeatureConfig `yaml:"scuba"`
69+
BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
70+
CrossRegionReplication CrossRegionReplicationFeatureConfig `yaml:"cross_region_replication"`
71+
Utapi UtapiFeatureConfig `yaml:"utapi"`
72+
Migration MigrationFeatureConfig `yaml:"migration"`
7273
}
7374

7475
type ScubaFeatureConfig struct {
@@ -85,6 +86,10 @@ type BucketNotificationsFeatureConfig struct {
8586
} `yaml:"destinationAuth"`
8687
}
8788

89+
type CrossRegionReplicationFeatureConfig struct {
90+
Enabled bool `yaml:"enabled"`
91+
}
92+
8893
type UtapiFeatureConfig struct {
8994
Enabled bool `yaml:"enabled"`
9095
}
@@ -229,6 +234,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
229234
Migration: MigrationFeatureConfig{
230235
Enabled: false,
231236
},
237+
CrossRegionReplication: CrossRegionReplicationFeatureConfig{
238+
Enabled: false,
239+
},
232240
},
233241
Cloudserver: CloudserverConfig{},
234242
S3Metadata: MetadataConfig{

‎cmd/util.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string {
8484
profiles = append(profiles, "feature-migration")
8585
}
8686

87+
if cfg.Features.CrossRegionReplication.Enabled {
88+
profiles = append(profiles, "feature-crr")
89+
}
90+
8791
return profiles
8892
}
8993

‎templates/backbeat/config.json‎

Lines changed: 72 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,45 @@
11
{
22
"zookeeper": {
33
"connectionString": "127.0.0.1:2181/backbeat",
4-
"autoCreateNamespace": true
4+
"autoCreateNamespace": false
55
},
66
"kafka": {
77
"hosts": "127.0.0.1:9092",
8+
"compressionType": "none",
9+
"requiredAcks": 1,
810
"backlogMetrics": {
911
"zkPath": "/run/kafka-backlog-metrics",
1012
"intervalS": 60
11-
}
13+
},
14+
"maxRequestSize": 5000020
1215
},
1316
"s3": {
1417
"host": "127.0.0.1",
1518
"port": 8000
1619
},
20+
"vaultAdmin": {
21+
"host": "127.0.0.1",
22+
"port": 8500
23+
},
24+
"replicationGroupId": "RG001 ",
1725
"queuePopulator": {
1826
"cronRule": "*/5 * * * * *",
1927
"batchMaxRead": 10000,
28+
"batchTimeoutMs": 9000,
2029
"zookeeperPath": "/queue-populator",
2130
"logSource": "bucketd",
2231
"bucketd": {
2332
"host": "127.0.0.1",
2433
"port": 9000
2534
},
35+
"dmd": {
36+
"host": "127.0.0.1",
37+
"port": 9990
38+
},
39+
"kafka": {
40+
"topic": "backbeat-oplog",
41+
"consumerGroupId": "backbeat-qp-oplog-group"
42+
},
2643
"probeServer": {
2744
"bindAddress": "localhost",
2845
"port": 4042
@@ -37,27 +54,32 @@
3754
"port": 8000
3855
},
3956
"auth": {
40-
"type": "account",
41-
"account": "bart",
57+
"type": "role",
4258
"vault": {
4359
"host": "127.0.0.1",
4460
"port": 8500,
45-
"adminPort": 8600
61+
"adminPort": 8600,
62+
"adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
4663
}
4764
}
4865
},
4966
"destination": {
5067
"transport": "http",
5168
"bootstrapList": [
52-
{ "site": "sf", "servers": ["127.0.0.1:9443"], "echo": false },
53-
{ "site": "aws-location", "type": "aws_s3" }
69+
{ "site": "sf", "servers": ["127.0.0.1:8000"], "echo": false }
5470
],
5571
"auth": {
56-
"type": "account",
57-
"account": "lisa"
72+
"type": "role",
73+
"vault": {
74+
"host": "127.0.0.1",
75+
"port": 8500,
76+
"adminPort": 8600,
77+
"adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
78+
}
5879
}
5980
},
6081
"topic": "backbeat-replication",
82+
"dataMoverTopic": "backbeat-data-mover",
6183
"replicationStatusTopic": "backbeat-replication-status",
6284
"replicationFailedTopic": "backbeat-replication-failed",
6385
"monitorReplicationFailures": true,
@@ -70,26 +92,38 @@
7092
],
7193
"queueProcessor": {
7294
"groupId": "backbeat-replication-group",
73-
"retryTimeoutS": 300,
95+
"retry": {
96+
"scality": {
97+
"maxRetries": 5,
98+
"timeoutS": 300,
99+
"backoff": {
100+
"min": 1000,
101+
"max": 300000,
102+
"jitter": 0.1,
103+
"factor": 1.5
104+
}
105+
}
106+
},
74107
"concurrency": 10,
75108
"mpuPartsConcurrency": 10,
76-
"probeServer": [
77-
{
78-
"site": "aws-location",
79-
"bindAddress": "localhost",
80-
"port": "4043"
81-
}, {
82-
"site" : "us-east-1",
83-
"bindAddress": "localhost",
84-
"port": "4044"
85-
}
86-
]
109+
"probeServer": {
110+
"bindAddress": "localhost",
111+
"port": 4043
112+
}
87113
},
88114
"replicationStatusProcessor": {
89115
"groupId": "backbeat-replication-group",
90-
"retryTimeoutS": 300,
116+
"retry": {
117+
"maxRetries": 5,
118+
"timeoutS": 300,
119+
"backoff": {
120+
"min": 1000,
121+
"max": 300000,
122+
"jitter": 0.1,
123+
"factor": 1.5
124+
}
125+
},
91126
"concurrency": 10,
92-
"logConsumerMetricsIntervalS": 60,
93127
"probeServer": {
94128
"bindAddress": "localhost",
95129
"port": 4045
@@ -137,7 +171,11 @@
137171
"conductor": {
138172
"backlogControl": { "enabled": true },
139173
"cronRule": "*/5 * * * * *",
140-
"concurrency": 10
174+
"concurrency": 10,
175+
"probeServer": {
176+
"bindAddress": "0.0.0.0",
177+
"port": 8552
178+
}
141179
},
142180
"bucketProcessor": {
143181
"groupId": "backbeat-lifecycle-bucket-processor-group",
@@ -151,7 +189,11 @@
151189
"factor": 1.5
152190
}
153191
},
154-
"concurrency": 10
192+
"concurrency": 10,
193+
"probeServer": {
194+
"bindAddress": "0.0.0.0",
195+
"port": 8553
196+
}
155197
},
156198
"objectProcessor": {
157199
"groupId": "backbeat-lifecycle-object-processor-group",
@@ -165,7 +207,11 @@
165207
"factor": 1.5
166208
}
167209
},
168-
"concurrency": 10
210+
"concurrency": 10,
211+
"probeServer": {
212+
"bindAddress": "0.0.0.0",
213+
"port": 8554
214+
}
169215
}
170216
}
171217
},

‎templates/backbeat/env‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
EXPIRE_ONE_DAY_EARLIER=true
1+
export EXPIRE_ONE_DAY_EARLIER=true
2+
export CRASH_ON_BATCH_TIMEOUT=true
3+
export REMOTE_MANAGEMENT_DISABLE=true

‎templates/backbeat/supervisord.conf‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,47 @@ stderr_logfile_backups=7
4949
autorestart = true
5050
autostart = true
5151
{{ end }}
52+
53+
## Cross Region Replication
54+
{{ if .Features.CrossRegionReplication.Enabled }}
55+
[program:crr-queue-populator]
56+
command = bash -c "source /conf/env && exec npm run queue_populator"
57+
numprocs = 1
58+
process_name = %(program_name)s_%(process_num)s
59+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
60+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
61+
stdout_logfile_maxbytes=100MB
62+
stdout_logfile_backups=7
63+
stderr_logfile_maxbytes=100MB
64+
stderr_logfile_backups=7
65+
autorestart = true
66+
autostart = true
67+
68+
[program:crr-queue-processor]
69+
command = bash -c "source /conf/env && exec npm run queue_processor"
70+
environment = BOOTSTRAP_SITE_NAME=sf
71+
numprocs = 1
72+
process_name = %(program_name)s_%(process_num)s
73+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
74+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
75+
stdout_logfile_maxbytes=100MB
76+
stdout_logfile_backups=7
77+
stderr_logfile_maxbytes=100MB
78+
stderr_logfile_backups=7
79+
autorestart = true
80+
autostart = true
81+
82+
[program:crr-status-processor]
83+
command = bash -c "source /conf/env && exec npm run replication_status_processor"
84+
numprocs = 1
85+
process_name = %(program_name)s_%(process_num)s
86+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
87+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
88+
stdout_logfile_maxbytes=100MB
89+
stdout_logfile_backups=7
90+
stderr_logfile_maxbytes=100MB
91+
stderr_logfile_backups=7
92+
autorestart = true
93+
autostart = true
94+
95+
{{ end }}

‎templates/cloudserver/config-v7.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"port": 8000,
33
"listenOn": [],
4+
"metricsPort": 8002,
45
"replicationGroupId": "RG001",
56
"restEndpoints": {
67
"localhost": "us-east-1",
@@ -28,7 +29,7 @@
2829
],
2930
"replicationEndpoints": [
3031
{
31-
"site": "zenko",
32+
"site": "sf",
3233
"servers": ["127.0.0.1:8000"],
3334
"default": true
3435
},

‎templates/cloudserver/config-v9.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
],
3434
"replicationEndpoints": [
3535
{
36-
"site": "zenko",
36+
"site": "sf",
3737
"servers": ["127.0.0.1:8000"],
3838
"default": true
3939
},

‎templates/global/docker-compose.yaml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,16 @@ services:
214214
network_mode: host
215215
environment:
216216
KAFKA_PORT: '9092'
217-
TOPICS_TO_CREATE: backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks backbeat-bucket-notification
218-
CREATE_NOTIFICATION_PATHS: 'true'
217+
TOPICS_TO_CREATE: >
218+
backbeat-lifecycle-bucket-tasks
219+
backbeat-lifecycle-object-tasks
220+
backbeat-bucket-notification
221+
backbeat-replication
222+
backbeat-data-mover
223+
backbeat-replication-status
224+
backbeat-replication-failed
225+
backbeat-metrics-group-crr
226+
CREATE_ZOOKEEPER_PATHS: 'true'
219227
ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat
220228
depends_on:
221229
setup-vault:

‎templates/global/values.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ features:
1313
username: admin
1414
password: admin123
1515

16+
cross_region_replication:
17+
enabled: false
18+
1619
utapi:
1720
enabled: false
1821

‎templates/kafka/setup.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ if [[ -n "$TOPICS_TO_CREATE" ]]; then
3939
done
4040
fi
4141

42-
if [[ "$CREATE_NOTIFICATION_PATHS" == "true" ]]; then
42+
if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then
4343
if [[ -z "$ZOOKEEPER_ENDPOINT" ]]; then
4444
echo "[setup] Zookeeper endpoint not set"
4545
exit 1
@@ -56,6 +56,14 @@ create /bucket-notification/raft-id-dispatcher/provisions
5656
create /bucket-notification/raft-id-dispatcher/provisions/1
5757
create /bucket-notification/raft-id-dispatcher/provisions/2
5858
create /bucket-notification/raft-id-dispatcher/provisions/3
59+
create /queue-populator
60+
create /queue-populator/raft-id-dispatcher
61+
create /queue-populator/raft-id-dispatcher/owners
62+
create /queue-populator/raft-id-dispatcher/leaders
63+
create /queue-populator/raft-id-dispatcher/provisions
64+
create /queue-populator/raft-id-dispatcher/provisions/0
65+
create /queue-populator/raft-id-dispatcher/provisions/1
66+
create /queue-populator/raft-id-dispatcher/provisions/2
5967
quit
6068
EOF
6169
echo "[setup] Zookeeper paths created."

0 commit comments

Comments
 (0)