Skip to content

Commit d41d5cd

Browse files
ulya-sidorinaulya-sidorina
authored
refactor(configuration): modify config format (#165)
1 parent 45c1acf commit d41d5cd

File tree

8 files changed

+181
-81
lines changed

8 files changed

+181
-81
lines changed

‎cmd/ydbcp/config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ db_connection:
44
connection_string: "grpcs://localhost:2135/domain/database"
55
insecure: true
66
discovery: false
7+
enable_sdk_metrics: false
78

89
client_connection:
910
insecure: true

‎cmd/ydbcp/main.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ func main() {
6060
var wg sync.WaitGroup
6161

6262
var logger *xlog.LogConfig
63-
if configInstance.DuplicateLogToFile != "" {
64-
logger, err = xlog.SetupLoggingWithFile(configInstance.GRPCServer.LogLevel, configInstance.DuplicateLogToFile)
63+
if configInstance.GetDuplicateLogToFile() != "" {
64+
logger, err = xlog.SetupLoggingWithFile(configInstance.GetLogLevel(), configInstance.GetDuplicateLogToFile())
6565
} else {
66-
logger, err = xlog.SetupLogging(configInstance.GRPCServer.LogLevel)
66+
logger, err = xlog.SetupLogging(configInstance.GetLogLevel())
6767
}
6868
if err != nil {
6969
log.Error(err)
@@ -116,7 +116,7 @@ func main() {
116116
xlog.Info(ctx, "Initialized AuthProvider")
117117
metrics.InitializeMetricsRegistry(ctx, &wg, &configInstance.MetricsServer, clockwork.NewRealClock())
118118
xlog.Info(ctx, "Initialized metrics registry")
119-
audit.EventsDestination = configInstance.AuditEventsDestination
119+
audit.EventsDestination = configInstance.GetAuditEventsDestination()
120120
server, err := server.NewServer(&configInstance.GRPCServer, authProvider)
121121
if err != nil {
122122
xlog.Error(ctx, "failed to initialize GRPC server", zap.Error(err))
@@ -201,10 +201,10 @@ func main() {
201201
os.Exit(1)
202202
}
203203

204-
processor.NewOperationProcessor(ctx, &wg, configInstance.ProcessorIntervalSeconds, dbConnector, handlersRegistry)
204+
processor.NewOperationProcessor(ctx, &wg, configInstance.GetProcessorIntervalSeconds(), dbConnector, handlersRegistry)
205205
xlog.Info(ctx, "Initialized OperationProcessor")
206206

207-
if configInstance.DisableTTLDeletion {
207+
if configInstance.GetDisableTTLDeletion() {
208208
xlog.Info(ctx, "TtlWatcher is disabled, old backups won't be deleted")
209209
} else {
210210
ttl_watcher.NewTtlWatcher(ctx, &wg, dbConnector, queries.NewWriteTableQuery)

‎internal/config/config.go‎

Lines changed: 104 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,26 +23,26 @@ type S3Config struct {
2323
PathPrefix string `yaml:"path_prefix"`
2424
AccessKeyIDPath string `yaml:"access_key_id_path"`
2525
SecretAccessKeyPath string `yaml:"secret_access_key_path"`
26-
S3ForcePathStyle bool `yaml:"s3_force_path_style"`
26+
S3ForcePathStyle bool `yaml:"s3_force_path_style" default:"false"`
2727
IsMock bool
2828
}
2929

3030
type YDBConnectionConfig struct {
3131
ConnectionString string `yaml:"connection_string"`
32-
Insecure bool `yaml:"insecure"`
32+
Insecure bool `yaml:"insecure" default:"false"`
3333
Discovery bool `yaml:"discovery" default:"true"`
3434
DialTimeoutSeconds uint32 `yaml:"dial_timeout_seconds" default:"5"`
3535
OAuth2KeyFile string `yaml:"oauth2_key_file"`
36-
EnableSDKMetrics bool `yaml:"enable_sdk_metrics"`
36+
EnableSDKMetrics bool `yaml:"enable_sdk_metrics" default:"true"`
3737
}
3838

3939
type ClientConnectionConfig struct {
40-
Insecure bool `yaml:"insecure"`
40+
Insecure bool `yaml:"insecure" default:"false"`
4141
Discovery bool `yaml:"discovery" default:"true"`
4242
DialTimeoutSeconds uint32 `yaml:"dial_timeout_seconds" default:"5"`
4343
OAuth2KeyFile string `yaml:"oauth2_key_file"`
4444
AllowedEndpointDomains []string `yaml:"allowed_endpoint_domains"`
45-
AllowInsecureEndpoint bool `yaml:"allow_insecure_endpoint"`
45+
AllowInsecureEndpoint bool `yaml:"allow_insecure_endpoint" default:"false"`
4646
}
4747

4848
type AuthConfig struct {
@@ -55,7 +55,7 @@ type GRPCServerConfig struct {
5555
BindPort uint16 `yaml:"bind_port" default:"2135"`
5656
TLSCertificatePath string `yaml:"tls_certificate_path"`
5757
TLSKeyPath string `yaml:"tls_key_path"`
58-
LogLevel string `yaml:"log_level"`
58+
LogLevel string `yaml:"log_level" default:"DEBUG"`
5959
}
6060

6161
type MetricsServerConfig struct {
@@ -65,23 +65,109 @@ type MetricsServerConfig struct {
6565
TLSKeyPath string `yaml:"tls_key_path"`
6666
}
6767

68+
type FeatureFlagsConfig struct {
69+
DisableTTLDeletion bool `yaml:"disable_ttl_deletion" default:"false"`
70+
}
71+
72+
type LogConfig struct {
73+
DuplicateToFile string `yaml:"duplicate_to_file"`
74+
Level string `yaml:"level" default:"DEBUG"`
75+
}
76+
77+
type OperationProcessorConfig struct {
78+
OperationTtlSeconds int64 `yaml:"operation_ttl_seconds" default:"86400"`
79+
ProcessorIntervalSeconds int64 `yaml:"processor_interval_seconds" default:"10"`
80+
}
81+
82+
type AuditConfig struct {
83+
EventsDestination string `yaml:"events_destination"`
84+
}
85+
86+
type QuotaConfig struct {
87+
SchedulesPerDB int `yaml:"schedules_per_db" default:"10"`
88+
}
89+
6890
type Validatable interface {
6991
Validate() error
7092
}
7193

7294
type Config struct {
73-
DBConnection YDBConnectionConfig `yaml:"db_connection"`
74-
ClientConnection ClientConnectionConfig `yaml:"client_connection"`
75-
S3 S3Config `yaml:"s3"`
76-
OperationTtlSeconds int64 `yaml:"operation_ttl_seconds"`
77-
Auth AuthConfig `yaml:"auth"`
78-
GRPCServer GRPCServerConfig `yaml:"grpc_server"`
79-
MetricsServer MetricsServerConfig `yaml:"metrics_server"`
80-
SchedulesLimitPerDB int `yaml:"schedules_limit_per_db" default:"10"`
81-
ProcessorIntervalSeconds int64 `yaml:"processor_interval_seconds" default:"10"`
82-
DisableTTLDeletion bool `yaml:"disable_ttl_deletion" default:"false"`
83-
AuditEventsDestination string `yaml:"audit_events_destination"`
84-
DuplicateLogToFile string `yaml:"duplicate_log_to_file"`
95+
DBConnection YDBConnectionConfig `yaml:"db_connection"`
96+
ClientConnection ClientConnectionConfig `yaml:"client_connection"`
97+
S3 S3Config `yaml:"s3"`
98+
Auth AuthConfig `yaml:"auth"`
99+
GRPCServer GRPCServerConfig `yaml:"grpc_server"`
100+
MetricsServer MetricsServerConfig `yaml:"metrics_server"`
101+
OperationProcessor OperationProcessorConfig `yaml:"operation_processor"`
102+
Audit AuditConfig `yaml:"audit"`
103+
Log LogConfig `yaml:"log"`
104+
Quota QuotaConfig `yaml:"quota"`
105+
FeatureFlags FeatureFlagsConfig `yaml:"feature_flags"`
106+
107+
// TODO: remove these fields and their getters after migration to the new config format
108+
OperationTtlSeconds int64 `yaml:"operation_ttl_seconds" default:"86400"`
109+
SchedulesLimitPerDB int `yaml:"schedules_limit_per_db" default:"10"`
110+
ProcessorIntervalSeconds int64 `yaml:"processor_interval_seconds" default:"10"`
111+
DisableTTLDeletion bool `yaml:"disable_ttl_deletion" default:"false"`
112+
AuditEventsDestination string `yaml:"audit_events_destination"`
113+
DuplicateLogToFile string `yaml:"duplicate_log_to_file"`
114+
}
115+
116+
func (c Config) GetOperationTtlSeconds() int64 {
117+
if c.OperationTtlSeconds == 0 {
118+
return c.OperationProcessor.OperationTtlSeconds
119+
}
120+
121+
return c.OperationTtlSeconds
122+
}
123+
124+
func (c *Config) SetOperationTtlSeconds(val int64) {
125+
c.OperationTtlSeconds = val
126+
c.OperationProcessor.OperationTtlSeconds = val
127+
}
128+
129+
func (c Config) GetSchedulesLimitPerDB() int {
130+
if c.SchedulesLimitPerDB == 0 {
131+
return c.Quota.SchedulesPerDB
132+
}
133+
134+
return c.SchedulesLimitPerDB
135+
}
136+
137+
func (c Config) GetProcessorIntervalSeconds() int64 {
138+
if c.ProcessorIntervalSeconds == 0 {
139+
return c.OperationProcessor.ProcessorIntervalSeconds
140+
}
141+
142+
return c.ProcessorIntervalSeconds
143+
}
144+
145+
func (c Config) GetDisableTTLDeletion() bool {
146+
return c.DisableTTLDeletion || c.FeatureFlags.DisableTTLDeletion
147+
}
148+
149+
func (c Config) GetAuditEventsDestination() string {
150+
if len(c.AuditEventsDestination) == 0 {
151+
return c.Audit.EventsDestination
152+
}
153+
154+
return c.AuditEventsDestination
155+
}
156+
157+
func (c Config) GetDuplicateLogToFile() string {
158+
if len(c.DuplicateLogToFile) == 0 {
159+
return c.Log.DuplicateToFile
160+
}
161+
162+
return c.DuplicateLogToFile
163+
}
164+
165+
func (c Config) GetLogLevel() string {
166+
if len(c.GRPCServer.LogLevel) == 0 {
167+
return c.Log.Level
168+
}
169+
170+
return c.GRPCServer.LogLevel
85171
}
86172

87173
type ClusterConnectionConfig struct {

‎internal/handlers/delete_backup_test.go‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,10 @@ func TestDBOperationHandlerDeadlineExceededForRunningOperation(t *testing.T) {
5151

5252
s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap)
5353

54+
config := config.Config{}
55+
config.SetOperationTtlSeconds(0)
5456
handler := NewDBOperationHandler(
55-
dbConnector, s3Connector, config.Config{
56-
OperationTtlSeconds: 0,
57-
}, queries.NewWriteTableQueryMock,
57+
dbConnector, s3Connector, config, queries.NewWriteTableQueryMock,
5858
)
5959

6060
err := handler(ctx, &dbOp)
@@ -123,10 +123,10 @@ func TestDBOperationHandlerPendingOperationCompletedSuccessfully(t *testing.T) {
123123
s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap)
124124

125125
metrics.InitializeMockMetricsRegistry()
126+
config := config.Config{}
127+
config.SetOperationTtlSeconds(1000)
126128
handler := NewDBOperationHandler(
127-
dbConnector, s3Connector, config.Config{
128-
OperationTtlSeconds: 1000,
129-
}, queries.NewWriteTableQueryMock,
129+
dbConnector, s3Connector, config, queries.NewWriteTableQueryMock,
130130
)
131131

132132
err := handler(ctx, &dbOp)
@@ -203,10 +203,10 @@ func TestDBOperationHandlerRunningOperationCompletedSuccessfully(t *testing.T) {
203203
s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap)
204204

205205
metrics.InitializeMockMetricsRegistry()
206+
config := config.Config{}
207+
config.SetOperationTtlSeconds(1000)
206208
handler := NewDBOperationHandler(
207-
dbConnector, s3Connector, config.Config{
208-
OperationTtlSeconds: 1000,
209-
}, queries.NewWriteTableQueryMock,
209+
dbConnector, s3Connector, config, queries.NewWriteTableQueryMock,
210210
)
211211

212212
err := handler(ctx, &dbOp)
@@ -280,11 +280,10 @@ func TestDBOperationHandlerUnexpectedBackupStatus(t *testing.T) {
280280
)
281281

282282
s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap)
283-
283+
config := config.Config{}
284+
config.SetOperationTtlSeconds(1000)
284285
handler := NewDBOperationHandler(
285-
dbConnector, s3Connector, config.Config{
286-
OperationTtlSeconds: 1000,
287-
}, queries.NewWriteTableQueryMock,
286+
dbConnector, s3Connector, config, queries.NewWriteTableQueryMock,
288287
)
289288

290289
err := handler(ctx, &dbOp)
@@ -351,10 +350,10 @@ func TestDBOperationHandlerDeleteMoreThanAllowedLimit(t *testing.T) {
351350
s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap)
352351

353352
metrics.InitializeMockMetricsRegistry()
353+
config := config.Config{}
354+
config.SetOperationTtlSeconds(1000)
354355
handler := NewDBOperationHandler(
355-
dbConnector, s3Connector, config.Config{
356-
OperationTtlSeconds: 1000,
357-
}, queries.NewWriteTableQueryMock,
356+
dbConnector, s3Connector, config, queries.NewWriteTableQueryMock,
358357
)
359358

360359
err := handler(ctx, &dbOp)

‎internal/handlers/restore_backup_test.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,12 @@ func TestRBOperationHandlerRunningOperationInProgress(t *testing.T) {
132132
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
133133

134134
// try to handle pending rb operation with ttl
135+
config := config.Config{}
136+
config.SetOperationTtlSeconds(1000)
135137
handler := NewRBOperationHandler(
136138
dbConnector,
137139
clientConnector,
138-
config.Config{OperationTtlSeconds: 1000},
140+
config,
139141
)
140142
err := handler(ctx, &rbOp)
141143
assert.Empty(t, err)
@@ -184,10 +186,12 @@ func TestRBOperationHandlerRunningOperationCompletedSuccessfully(t *testing.T) {
184186
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
185187
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
186188

189+
config := config.Config{}
190+
config.SetOperationTtlSeconds(1000)
187191
handler := NewRBOperationHandler(
188192
dbConnector,
189193
clientConnector,
190-
config.Config{OperationTtlSeconds: 1000},
194+
config,
191195
)
192196
err := handler(ctx, &rbOp)
193197
assert.Empty(t, err)
@@ -235,10 +239,12 @@ func TestRBOperationHandlerRunningOperationCancelled(t *testing.T) {
235239
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
236240
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
237241

242+
config := config.Config{}
243+
config.SetOperationTtlSeconds(10)
238244
handler := NewRBOperationHandler(
239245
dbConnector,
240246
clientConnector,
241-
config.Config{OperationTtlSeconds: 10},
247+
config,
242248
)
243249
err := handler(ctx, &rbOp)
244250
assert.Empty(t, err)
@@ -337,10 +343,12 @@ func TestRBOperationHandlerCancellingOperationInProgress(t *testing.T) {
337343
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
338344
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
339345

346+
config := config.Config{}
347+
config.SetOperationTtlSeconds(1000)
340348
handler := NewRBOperationHandler(
341349
dbConnector,
342350
clientConnector,
343-
config.Config{OperationTtlSeconds: 1000},
351+
config,
344352
)
345353
err := handler(ctx, &rbOp)
346354
assert.Empty(t, err)
@@ -389,10 +397,12 @@ func TestRBOperationHandlerCancellingOperationCompletedSuccessfully(t *testing.T
389397
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
390398
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
391399

400+
config := config.Config{}
401+
config.SetOperationTtlSeconds(10)
392402
handler := NewRBOperationHandler(
393403
dbConnector,
394404
clientConnector,
395-
config.Config{OperationTtlSeconds: 10},
405+
config,
396406
)
397407
err := handler(ctx, &rbOp)
398408
assert.Empty(t, err)
@@ -441,10 +451,12 @@ func TestRBOperationHandlerCancellingOperationCancelled(t *testing.T) {
441451
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
442452
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
443453

454+
config := config.Config{}
455+
config.SetOperationTtlSeconds(10)
444456
handler := NewRBOperationHandler(
445457
dbConnector,
446458
clientConnector,
447-
config.Config{OperationTtlSeconds: 10},
459+
config,
448460
)
449461
err := handler(ctx, &rbOp)
450462
assert.Empty(t, err)
@@ -493,10 +505,12 @@ func TestRBOperationHandlerRetriableErrorForRunningOperation(t *testing.T) {
493505
clientConnector := client.NewMockClientConnector(client.WithOperations(ydbOpMap))
494506
dbConnector := db.NewMockDBConnector(db.WithOperations(opMap))
495507

508+
config := config.Config{}
509+
config.SetOperationTtlSeconds(10)
496510
handler := NewRBOperationHandler(
497511
dbConnector,
498512
clientConnector,
499-
config.Config{OperationTtlSeconds: 10},
513+
config,
500514
)
501515
err := handler(ctx, &rbOp)
502516
assert.Empty(t, err)

0 commit comments

Comments
 (0)