Skip to content

Commit 6b0cbd8

Browse files
tcarmettcarmet
committed
WKBCH-12: Add crr support
1 parent 99500bc commit 6b0cbd8

File tree

12 files changed

+195
-14
lines changed

12 files changed

+195
-14
lines changed

‎cmd/config.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,11 @@ type GlobalConfig struct {
6565
}
6666

6767
type FeatureConfig struct {
68-
Scuba ScubaFeatureConfig `yaml:"scuba"`
69-
BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
70-
Utapi UtapiFeatureConfig `yaml:"utapi"`
71-
Migration MigrationFeatureConfig `yaml:"migration"`
68+
Scuba ScubaFeatureConfig `yaml:"scuba"`
69+
BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
70+
CrossRegionReplication CrossRegionReplicationFeatureConfig `yaml:"cross_region_replication"`
71+
Utapi UtapiFeatureConfig `yaml:"utapi"`
72+
Migration MigrationFeatureConfig `yaml:"migration"`
7273
}
7374

7475
type ScubaFeatureConfig struct {
@@ -85,6 +86,10 @@ type BucketNotificationsFeatureConfig struct {
8586
} `yaml:"destinationAuth"`
8687
}
8788

89+
type CrossRegionReplicationFeatureConfig struct {
90+
Enabled bool `yaml:"enabled"`
91+
}
92+
8893
type UtapiFeatureConfig struct {
8994
Enabled bool `yaml:"enabled"`
9095
}
@@ -229,6 +234,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
229234
Migration: MigrationFeatureConfig{
230235
Enabled: false,
231236
},
237+
CrossRegionReplication: CrossRegionReplicationFeatureConfig{
238+
Enabled: false,
239+
},
232240
},
233241
Cloudserver: CloudserverConfig{},
234242
S3Metadata: MetadataConfig{

‎cmd/configure.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ type ConfigureCmd struct {
1515

1616
type configGenFunc func(cfg EnvironmentConfig, path string) error
1717

18+
type CloudserverTemplateData struct {
19+
EnvironmentConfig
20+
Port int
21+
MetricsPort int
22+
}
23+
1824
func (c *ConfigureCmd) Run() error {
1925
rc := RuntimeConfigFromFlags(c.EnvDir, c.Name)
2026
envPath := filepath.Join(rc.EnvDir, rc.EnvName)
@@ -107,16 +113,42 @@ func generateCloudserverConfig(cfg EnvironmentConfig, path string) error {
107113
}
108114
}
109115

116+
// Render the main config.json with port 8000
117+
templateData := CloudserverTemplateData{
118+
EnvironmentConfig: cfg,
119+
Port: 8000,
120+
MetricsPort: 8002,
121+
}
122+
110123
err := renderTemplateToFile(
111124
getTemplates(),
112125
configTemplate,
113-
cfg,
126+
templateData,
114127
filepath.Join(path, "cloudserver", "config.json"),
115128
)
116129
if err != nil {
117130
return err
118131
}
119132

133+
// Render config-destination.json with port 8001 if CRR is enabled
134+
if cfg.Features.CrossRegionReplication.Enabled {
135+
destTemplateData := CloudserverTemplateData{
136+
EnvironmentConfig: cfg,
137+
Port: 8001,
138+
MetricsPort: 8003,
139+
}
140+
141+
err = renderTemplateToFile(
142+
getTemplates(),
143+
configTemplate,
144+
destTemplateData,
145+
filepath.Join(path, "cloudserver", "config-destination.json"),
146+
)
147+
if err != nil {
148+
return err
149+
}
150+
}
151+
120152
return renderTemplateToFile(
121153
getTemplates(),
122154
"templates/cloudserver/locationConfig.json",
@@ -132,6 +164,8 @@ func generateBackbeatConfig(cfg EnvironmentConfig, path string) error {
132164
"config.json",
133165
"config.notification.json",
134166
"notificationCredentials.json",
167+
"create-service-user.sh",
168+
"Dockerfile.setup",
135169
}
136170

137171
return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates)

‎cmd/util.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string {
8484
profiles = append(profiles, "feature-migration")
8585
}
8686

87+
if cfg.Features.CrossRegionReplication.Enabled {
88+
profiles = append(profiles, "feature-crr")
89+
}
90+
8791
return profiles
8892
}
8993

‎templates/backbeat/Dockerfile.setup‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
ARG BASE_IMAGE
2+
3+
FROM $BASE_IMAGE
4+
5+
USER root
6+
7+
RUN apt-get update && apt-get install -y jq curl
8+
9+
COPY --chmod=755 create-service-user.sh /opt/
10+
11+
CMD ["/opt/create-service-user.sh"]

‎templates/backbeat/config.json‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,17 @@
4949
"destination": {
5050
"transport": "http",
5151
"bootstrapList": [
52-
{ "site": "sf", "servers": ["127.0.0.1:9443"], "echo": false },
52+
{ "site": "sf", "servers": ["127.0.0.1:8001"], "echo": false },
5353
{ "site": "aws-location", "type": "aws_s3" }
5454
],
5555
"auth": {
56-
"type": "account",
57-
"account": "lisa"
56+
"type": "service",
57+
"account": "service-replication",
58+
"vault": {
59+
"host": "127.0.0.1",
60+
"port": 8500,
61+
"adminPort": 8600
62+
}
5863
}
5964
},
6065
"topic": "backbeat-replication",

‎templates/backbeat/create-service-user.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env sh
2+
set -x
3+
4+
MANAGEMENT_ACCESS_KEY=$(jq -r '.accessKey' /secrets/management-creds.json)
5+
MANAGEMENT_SECRET_KEY=$(jq -r '.secretKey' /secrets/management-creds.json)
6+
7+
# === Create replication service user ===
8+
echo "[setup] Creating replication service user..."
9+
SERVICE_CREDS_JSON=$(AWS_ACCESS_KEY_ID="$MANAGEMENT_ACCESS_KEY" \
10+
AWS_SECRET_ACCESS_KEY="$MANAGEMENT_SECRET_KEY" \
11+
AWS_REGION="$REGION" \
12+
./bin/ensureServiceUser apply service-replication --iam-endpoint http://127.0.0.1:8600)
13+
14+
SERVICE_ACCESS_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.AccessKeyId')
15+
SERVICE_SECRET_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.SecretAccessKey')
16+
17+
echo "[setup] replication service user credentials:"
18+
echo "SERVICE_ACCESS_KEY=$SERVICE_ACCESS_KEY"
19+
echo "SERVICE_SECRET_KEY=$SERVICE_SECRET_KEY"
20+
echo
21+
22+
# === Update replication-service-creds.json ===
23+
echo "[setup] Updating replication-service-creds.json with service user credentials..."
24+
jq --null-input --arg ak "$SERVICE_ACCESS_KEY" --arg sk "$SERVICE_SECRET_KEY" \
25+
'{accessKey: $ak, secretKey: $sk}' > /secrets/replication-service-creds.json

‎templates/backbeat/supervisord.conf‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,49 @@ stderr_logfile_backups=7
4949
autorestart = true
5050
autostart = true
5151
{{ end }}
52+
53+
## Cross Region Replication
54+
{{ if .Features.CrossRegionReplication.Enabled }}
55+
[program:crr-queue-populator]
56+
command = bash -c "source /conf/env && exec npm run queue_populator"
57+
environment = CRASH_ON_BATCH_TIMEOUT=true REMOTE_MANAGEMENT_DISABLE=true
58+
numprocs = 1
59+
process_name = %(program_name)s_%(process_num)s
60+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
61+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
62+
stdout_logfile_maxbytes=100MB
63+
stdout_logfile_backups=7
64+
stderr_logfile_maxbytes=100MB
65+
stderr_logfile_backups=7
66+
autorestart = true
67+
autostart = true
68+
69+
[program:crr-queue-processor]
70+
command = bash -c "source /conf/env && exec npm run queue_processor"
71+
environment = CRASH_ON_BATCH_TIMEOUT=true REMOTE_MANAGEMENT_DISABLE=true BOOTSTRAP_SITE_NAME=sf
72+
numprocs = 1
73+
process_name = %(program_name)s_%(process_num)s
74+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
75+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
76+
stdout_logfile_maxbytes=100MB
77+
stdout_logfile_backups=7
78+
stderr_logfile_maxbytes=100MB
79+
stderr_logfile_backups=7
80+
autorestart = true
81+
autostart = true
82+
83+
[program:crr-status-processor]
84+
command = bash -c "source /conf/env && exec npm run replication_status_processor"
85+
environment = CRASH_ON_BATCH_TIMEOUT=true REMOTE_MANAGEMENT_DISABLE=true
86+
numprocs = 1
87+
process_name = %(program_name)s_%(process_num)s
88+
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
89+
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
90+
stdout_logfile_maxbytes=100MB
91+
stdout_logfile_backups=7
92+
stderr_logfile_maxbytes=100MB
93+
stderr_logfile_backups=7
94+
autorestart = true
95+
autostart = true
96+
97+
{{ end }}

‎templates/cloudserver/config-v7.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2-
"port": 8000,
2+
"port": {{ .Port }},
33
"listenOn": [],
4+
"metricsPort": {{ .MetricsPort }},
45
"replicationGroupId": "RG001",
56
"restEndpoints": {
67
"localhost": "us-east-1",

‎templates/cloudserver/config-v9.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
2-
"port": 8000,
2+
"port": {{ .Port }},
33
"listenOn": [],
4-
"metricsPort": 8002,
4+
"metricsPort": {{ .MetricsPort }},
55
"metricsListenOn": [],
66
"replicationGroupId": "RG001",
77
"restEndpoints": {

‎templates/global/docker-compose.yaml‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,26 @@ services:
6969
profiles:
7070
- base
7171

72+
cloudserver-destination:
73+
image: ${CLOUDSERVER_IMAGE}
74+
container_name: workbench-s3-destination
75+
network_mode: host
76+
command: node index.js
77+
environment:
78+
S3DATA: file
79+
S3METADATA: scality
80+
S3VAULT: scality
81+
BUCKETD_BOOTSTRAP: localhost:9000
82+
S3_CONFIG_FILE: /conf/config.json
83+
MPU_TESTING: 'yes'
84+
ENABLE_NULL_VERSION_COMPAT_MODE: ${CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE}
85+
REMOTE_MANAGEMENT_DISABLE: true
86+
volumes:
87+
- ./config/cloudserver/config-destination.json:/conf/config.json:ro
88+
- ./config/cloudserver/locationConfig.json:/conf/locationConfig.json:ro
89+
profiles:
90+
- feature-crr
91+
7292
vault:
7393
image: ${VAULT_IMAGE}
7494
container_name: workbench-vault
@@ -160,6 +180,22 @@ services:
160180
- feature-crr
161181
- feature-notifications
162182

183+
setup-backbeat:
184+
build:
185+
context: ./config/backbeat
186+
dockerfile: Dockerfile.setup
187+
args:
188+
BASE_IMAGE: ${BACKBEAT_IMAGE}
189+
container_name: workbench-setup-backbeat
190+
network_mode: host
191+
depends_on:
192+
setup-vault:
193+
condition: service_completed_successfully
194+
volumes:
195+
- ./config/vault/:/secrets
196+
profiles:
197+
- feature-crr
198+
163199
redis:
164200
image: ${REDIS_IMAGE}
165201
container_name: workbench-redis
@@ -214,8 +250,8 @@ services:
214250
network_mode: host
215251
environment:
216252
KAFKA_PORT: '9092'
217-
TOPICS_TO_CREATE: backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks backbeat-bucket-notification
218-
CREATE_NOTIFICATION_PATHS: 'true'
253+
TOPICS_TO_CREATE: backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks backbeat-bucket-notification backbeat-replications backbeat-replication-status
254+
CREATE_ZOOKEEPER_PATHS: 'true'
219255
ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat
220256
depends_on:
221257
setup-vault:

0 commit comments

Comments
 (0)