Skip to content

Commit 7133a98

Browse files
chupechupe
committed
PCSM-219: Move to mapstruct for config
PCSM-219: Update VSCode settings and README with additional config options PCSM-219: Fix
1 parent 7547453 commit 7133a98

File tree

13 files changed

+335
-259
lines changed

13 files changed

+335
-259
lines changed

‎.vscode/settings.json‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,31 @@
3939
"tests"
4040
],
4141
"python.testing.pytestEnabled": true,
42-
"python.testing.unittestEnabled": false
42+
"python.testing.unittestEnabled": false,
43+
"cSpell.words": [
44+
"bson",
45+
"clustersync",
46+
"cmdutil",
47+
"codegen",
48+
"colls",
49+
"connstring",
50+
"contextcheck",
51+
"Debugf",
52+
"errgroup",
53+
"errorlint",
54+
"Infof",
55+
"keygen",
56+
"mapstructure",
57+
"nolint",
58+
"opencode",
59+
"pcsm",
60+
"pipefail",
61+
"readconcern",
62+
"readpref",
63+
"Warnf",
64+
"wrapcheck",
65+
"Wrapf",
66+
"writeconcern",
67+
"zerolog"
68+
]
4369
}

‎README.md‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -145,13 +145,15 @@ curl http://localhost:2242/status
145145

146146
When starting the PCSM server, you can use the following options:
147147

148-
- `--port`: The port on which the server will listen (default: 2242)
149-
- `--source`: The MongoDB connection string for the source cluster
150-
- `--target`: The MongoDB connection string for the target cluster
151-
- `--log-level`: The log level (default: "info")
152-
- `--log-json`: Output log in JSON format with disabled color
153-
- `--no-color`: Disable log ASCI color
154-
- `--mongodb-cli-operation-timeout`: Timeout for MongoDB operations (e.g., `30s`, `5m`)
148+
| Option | CLI Flag | Default | Description |
149+
|----------------------------------|------------------------------------|---------|--------------------------------------|
150+
| Port | `--port` | 2242 | Port on which the server listens |
151+
| Source URI | `--source` | - | MongoDB connection string for source |
152+
| Target URI | `--target` | - | MongoDB connection string for target |
153+
| Log Level | `--log-level` | info | Log level (trace/debug/info/warn/error/fatal/panic) |
154+
| Log JSON | `--log-json` | false | Output log in JSON format |
155+
| No Color | `--no-color` | false | Disable log ASCII color |
156+
| MongoDB Operation Timeout | `--mongodb-cli-operation-timeout` | 5m | Timeout for MongoDB operations |
155157

156158
Example:
157159

@@ -169,16 +171,16 @@ bin/pcsm \
169171

170172
The following environment variables are supported:
171173

172-
| Option | Env Var | Default |
173-
|---------------------------|--------------------------------------|---------|
174-
| Source URI | `PCSM_SOURCE_URI` | - |
175-
| Target URI | `PCSM_TARGET_URI` | - |
176-
| Port | `PCSM_PORT` | 2242 |
177-
| Log Level | `PCSM_LOG_LEVEL` | info |
178-
| Log JSON | `PCSM_LOG_JSON` | false |
179-
| No Color | `PCSM_NO_COLOR` | false |
180-
| MongoDB Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m |
181-
| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false |
174+
| Option | Env Var | Default | Description |
175+
|----------------------------|--------------------------------------|---------|--------------------------------------|
176+
| Source URI | `PCSM_SOURCE_URI` | - | MongoDB connection string for source |
177+
| Target URI | `PCSM_TARGET_URI` | - | MongoDB connection string for target |
178+
| Port | `PCSM_PORT` | 2242 | Port on which the server listens |
179+
| Log Level | `PCSM_LOG_LEVEL` | info | Log level |
180+
| Log JSON | `PCSM_LOG_JSON` | false | Output log in JSON format |
181+
| No Color | `PCSM_NO_COLOR` | false | Disable log ASCII color |
182+
| MongoDB Operation Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m | Timeout for MongoDB operations |
183+
| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false | Use collection-level bulk write (internal) |
182184

183185
> **Note**: Clone tuning options (see below) are intentionally NOT supported via environment variables. They are configurable via CLI flags and HTTP request parameters only.
184186

@@ -187,15 +189,16 @@ The following environment variables are supported:
187189
Advanced tuning options for the clone process. These are available via CLI flags
188190
and HTTP request parameters, but NOT via environment variables.
189191

190-
| CLI Flag | HTTP Parameter | Default | Description |
191-
|------------------------------------|-------------------------------|---------|------------------------------------------|
192-
| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | Collections to clone in parallel (0-100) |
193-
| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto | Read workers during clone (0-1000) |
194-
| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto | Insert workers during clone (0-1000) |
195-
| `--clone-segment-size` | `cloneSegmentSize` | auto | Segment size (min ~475MB, max 64GiB) |
196-
| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | Read batch size (16MiB - 2GiB) |
192+
| CLI Flag | HTTP Parameter | Default | Range | Description |
193+
|------------------------------------|-------------------------------|----------|-------------|------------------------------------|
194+
| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | 0-100 | Collections to clone in parallel |
195+
| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto (0) | 0-1000 | Read workers during clone |
196+
| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto (0) | 0-1000 | Insert workers during clone |
197+
| `--clone-segment-size` | `cloneSegmentSize` | auto | ~475MB-64GB | Segment size for parallel cloning |
198+
| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | 16MiB-2GiB | Read cursor batch size |
197199

198200
> **Note**: These CLI flags are hidden from `--help` output. They are intended for advanced tuning only.
201+
> Setting a value to 0 or empty string uses the automatic/default behavior.
199202

200203
Example CLI usage:
201204

‎config/config.go‎

Lines changed: 43 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,57 +2,48 @@
22
package config
33

44
import (
5+
"os"
6+
"slices"
57
"strings"
68

79
"github.com/spf13/cobra"
810
"github.com/spf13/viper"
11+
12+
"github.com/percona/percona-clustersync-mongodb/errors"
13+
"github.com/percona/percona-clustersync-mongodb/validate"
914
)
1015

11-
// Init initializes Viper configuration binding with Cobra command flags.
12-
// This should be called in PersistentPreRun to ensure all flags are bound before use.
13-
func Init(cmd *cobra.Command) {
14-
// Set environment variable prefix
16+
// Load initializes Viper and returns a validated Config.
17+
func Load(cmd *cobra.Command) (*Config, error) {
1518
viper.SetEnvPrefix("PCSM")
16-
17-
// Replace hyphens with underscores for env var lookup
18-
// e.g., "log-level" -> "PCSM_LOG_LEVEL"
1919
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
20-
21-
// Automatically read matching env vars
2220
viper.AutomaticEnv()
2321

24-
// Bind CLI flags to Viper
2522
if cmd.PersistentFlags() != nil {
2623
_ = viper.BindPFlags(cmd.PersistentFlags())
2724
}
2825
if cmd.Flags() != nil {
2926
_ = viper.BindPFlags(cmd.Flags())
3027
}
3128

32-
// Bind specific env var names
3329
bindEnvVars()
30+
31+
var cfg Config
32+
33+
err := viper.Unmarshal(&cfg)
34+
if err != nil {
35+
return nil, errors.Wrap(err, "unmarshal config")
36+
}
37+
38+
err = validate.Struct(&cfg)
39+
if err != nil {
40+
return nil, errors.Wrap(err, "validate config")
41+
}
42+
43+
return &cfg, nil
3444
}
3545

3646
// bindEnvVars binds environment variable names to Viper keys.
37-
//
38-
// Configuration Reference:
39-
//
40-
// | Viper Key | CLI Flag | Env Var | Default |
41-
// |--------------------------------|-----------------------------------|--------------------------------------|---------|
42-
// | source | --source | PCSM_SOURCE_URI | - |
43-
// | target | --target | PCSM_TARGET_URI | - |
44-
// | port | --port | PCSM_PORT | 2242 |
45-
// | log-level | --log-level | PCSM_LOG_LEVEL | info |
46-
// | log-json | --log-json | PCSM_LOG_JSON | false |
47-
// | no-color | --no-color | PCSM_NO_COLOR | false |
48-
// | mongodb-cli-operation-timeout | --mongodb-cli-operation-timeout | PCSM_MONGODB_CLI_OPERATION_TIMEOUT | 5m |
49-
// | use-collection-bulk-write | --use-collection-bulk-write | PCSM_USE_COLLECTION_BULK_WRITE | false |
50-
// | clone-num-parallel-collections | --clone-num-parallel-collections | - | 0 |
51-
// | clone-num-read-workers | --clone-num-read-workers | - | 0 |
52-
// | clone-num-insert-workers | --clone-num-insert-workers | - | 0 |
53-
// | clone-segment-size | --clone-segment-size | - | 0 |
54-
// | clone-read-batch-size | --clone-read-batch-size | - | 0 |
55-
//
5647
// Note: Clone tuning options are CLI/HTTP only (no env var support).
5748
func bindEnvVars() {
5849
// Server connection URIs
@@ -65,3 +56,25 @@ func bindEnvVars() {
6556
// Bulk write option (hidden, internal use)
6657
_ = viper.BindEnv("use-collection-bulk-write", "PCSM_USE_COLLECTION_BULK_WRITE")
6758
}
59+
60+
// UseTargetClientCompressors returns a list of enabled compressors (from "zstd", "zlib", "snappy")
61+
// for the target MongoDB client connection, as specified by the comma-separated environment
62+
// variable PCSM_DEV_TARGET_CLIENT_COMPRESSORS. If unset or empty, returns nil.
63+
func UseTargetClientCompressors() []string {
64+
s := strings.TrimSpace(os.Getenv("PCSM_DEV_TARGET_CLIENT_COMPRESSORS"))
65+
if s == "" {
66+
return nil
67+
}
68+
69+
allowCompressors := []string{"zstd", "zlib", "snappy"}
70+
71+
rv := make([]string, 0, min(len(s), len(allowCompressors)))
72+
for a := range strings.SplitSeq(s, ",") {
73+
a = strings.TrimSpace(a)
74+
if slices.Contains(allowCompressors, a) && !slices.Contains(rv, a) {
75+
rv = append(rv, a)
76+
}
77+
}
78+
79+
return rv
80+
}

‎config/schema.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package config
2+
3+
import (
4+
"math"
5+
"time"
6+
7+
"github.com/dustin/go-humanize"
8+
)
9+
10+
// Config holds all PCSM configuration.
11+
type Config struct {
12+
// Connection
13+
Port int `mapstructure:"port" validate:"omitempty,gte=1024,lte=65535"`
14+
Source string `mapstructure:"source"`
15+
Target string `mapstructure:"target"`
16+
17+
// Logging (squash keeps flat keys)
18+
Log LogConfig `mapstructure:",squash"`
19+
20+
// MongoDB client options
21+
MongoDB MongoDBConfig `mapstructure:",squash"`
22+
23+
// Clone tuning (CLI/HTTP only)
24+
Clone CloneConfig `mapstructure:",squash"`
25+
26+
// Internal options
27+
UseCollectionBulkWrite bool `mapstructure:"use-collection-bulk-write"`
28+
29+
// Hidden startup flags
30+
Start bool `mapstructure:"start"`
31+
ResetState bool `mapstructure:"reset-state"`
32+
PauseOnInitialSync bool `mapstructure:"pause-on-initial-sync"`
33+
}
34+
35+
// LogConfig holds logging configuration.
36+
type LogConfig struct {
37+
Level string `mapstructure:"log-level" validate:"omitempty,oneof=trace debug info warn error fatal panic"`
38+
JSON bool `mapstructure:"log-json"`
39+
NoColor bool `mapstructure:"no-color"`
40+
}
41+
42+
// MongoDBConfig holds MongoDB client configuration.
43+
type MongoDBConfig struct {
44+
OperationTimeout string `mapstructure:"mongodb-cli-operation-timeout" validate:"omitempty,duration"`
45+
}
46+
47+
// OperationTimeoutDuration returns the parsed timeout or default.
48+
func (m *MongoDBConfig) OperationTimeoutDuration() time.Duration {
49+
if m.OperationTimeout != "" {
50+
d, err := time.ParseDuration(m.OperationTimeout)
51+
if err == nil && d > 0 {
52+
return d
53+
}
54+
}
55+
56+
return DefaultMongoDBCliOperationTimeout
57+
}
58+
59+
// CloneConfig holds clone tuning configuration.
60+
type CloneConfig struct {
61+
NumParallelCollections int `mapstructure:"clone-num-parallel-collections" validate:"omitempty,gte=0,lte=100"`
62+
NumReadWorkers int `mapstructure:"clone-num-read-workers" validate:"omitempty,gte=0,lte=1000"`
63+
NumInsertWorkers int `mapstructure:"clone-num-insert-workers" validate:"omitempty,gte=0,lte=1000"`
64+
SegmentSize string `mapstructure:"clone-segment-size" validate:"omitempty,bytesize"`
65+
ReadBatchSize string `mapstructure:"clone-read-batch-size" validate:"omitempty,bytesize"`
66+
}
67+
68+
// SegmentSizeBytes parses and returns the segment size in bytes.
69+
func (c *CloneConfig) SegmentSizeBytes() int64 {
70+
if c.SegmentSize == "" {
71+
return AutoCloneSegmentSize
72+
}
73+
74+
bytes, err := humanize.ParseBytes(c.SegmentSize)
75+
if err == nil && bytes > 0 {
76+
return int64(min(bytes, math.MaxInt64)) //nolint:gosec
77+
}
78+
79+
return AutoCloneSegmentSize
80+
}
81+
82+
// ReadBatchSizeBytes parses and returns the read batch size in bytes.
83+
func (c *CloneConfig) ReadBatchSizeBytes() int32 {
84+
if c.ReadBatchSize == "" {
85+
return 0
86+
}
87+
88+
bytes, err := humanize.ParseBytes(c.ReadBatchSize)
89+
if err == nil {
90+
return int32(min(bytes, math.MaxInt32)) //nolint:gosec
91+
}
92+
93+
return 0
94+
}

0 commit comments

Comments
 (0)