Merge pull request #300 from databacker/restructure-config-file

restructure config file

Author: deitch

TODO.md

@@ -15,7 +15,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"gopkg.in/yaml.v3"
 )
 
 type execs interface {
@@ -32,7 +31,7 @@ var subCommands = []subCommand{dumpCmd, restoreCmd, pruneCmd}
 type cmdConfiguration struct {
 	dbconn        database.Connection
 	creds         credentials.Creds
-	configuration *config.Config
+	configuration *config.ConfigSpec
 }
 
 const (
@@ -71,41 +70,43 @@ func rootCmd(execs execs) (*cobra.Command, error) {
 			// read the config file, if needed; the structure of the config differs quite some
 			// from the necessarily flat env vars/CLI flags, so we can't just use viper's
 			// automatic config file support.
-			if configFile := v.GetString("config-file"); configFile != "" {
+			var actualConfig *config.ConfigSpec
+
+			if configFilePath := v.GetString("config-file"); configFilePath != "" {
 				var (
-					f      *os.File
-					err    error
-					config config.Config
+					f   *os.File
+					err error
 				)
-				if f, err = os.Open(configFile); err != nil {
+				if f, err = os.Open(configFilePath); err != nil {
 					return fmt.Errorf("fatal error config file: %w", err)
 				}
 				defer f.Close()
-				decoder := yaml.NewDecoder(f)
-				if err := decoder.Decode(&config); err != nil {
-					return fmt.Errorf("fatal error config file: %w", err)
+				actualConfig, err = config.ProcessConfig(f)
+				if err != nil {
+					return fmt.Errorf("unable to read provided config: %w", err)
 				}
-				cmdConfig.configuration = &config
 			}
 
 			// the structure of our config file is more complex and with relationships than our config/env var
 			// so we cannot use a single viper structure, as described above.
 
 			// set up database connection
-			if cmdConfig.configuration != nil {
-				if cmdConfig.configuration.Database.Server != "" {
-					cmdConfig.dbconn.Host = cmdConfig.configuration.Database.Server
+			if actualConfig != nil {
+				if actualConfig.Database.Server != "" {
+					cmdConfig.dbconn.Host = actualConfig.Database.Server
 				}
-				if cmdConfig.configuration.Database.Port != 0 {
-					cmdConfig.dbconn.Port = cmdConfig.configuration.Database.Port
+				if actualConfig.Database.Port != 0 {
+					cmdConfig.dbconn.Port = actualConfig.Database.Port
 				}
-				if cmdConfig.configuration.Database.Credentials.Username != "" {
-					cmdConfig.dbconn.User = cmdConfig.configuration.Database.Credentials.Username
+				if actualConfig.Database.Credentials.Username != "" {
+					cmdConfig.dbconn.User = actualConfig.Database.Credentials.Username
 				}
-				if cmdConfig.configuration.Database.Credentials.Password != "" {
-					cmdConfig.dbconn.Pass = cmdConfig.configuration.Database.Credentials.Password
+				if actualConfig.Database.Credentials.Password != "" {
+					cmdConfig.dbconn.Pass = actualConfig.Database.Credentials.Password
 				}
+				cmdConfig.configuration = actualConfig
 			}
+
 			// override config with env var or CLI flag, if set
 			dbHost := v.GetString("server")
 			if dbHost != "" && v.IsSet("server") {

cmd/root.go

@@ -1,21 +1,25 @@
-database:
-  server: abcd
-  port: 3306
-  credentials:
-    username: user2
-    password: xxxx2
+version: config.databack.io/v1
+kind: local
 
-targets:
-  local:
-    type: file
-    url: file:///foo/bar
-  other:
-    type: file
-    url: /foo/bar
+spec: 
+  database:
+    server: abcd
+    port: 3306
+    credentials:
+      username: user2
+      password: xxxx2
 
-dump:
   targets:
-  - local
+    local:
+      type: file
+      url: file:///foo/bar
+    other:
+      type: file
+      url: /foo/bar
 
-prune:
-  retention: "1h"
+  dump:
+    targets:
+    - local
+
+  prune:
+    retention: "1h"

cmd/testdata/config.yml

@@ -66,15 +66,15 @@ The following are the environment variables, CLI flags and configuration file op
 | port to use to connect to database. Optional. | BR | `port` | `DB_PORT` | `database.port` | 3306 |
 | username for the database | BR | `user` | `DB_USER` | `database.credentials.username` |  |
 | password for the database | BR | `pass` | `DB_PASS` | `database.credentials.password` |  |
-| names of databases to dump, comma-separated | B | `include` | `DB_NAMES` | `database.include` | all databases in the server |
-| names of databases to exclude from the dump | B | `exclude` | `DB_NAMES_EXCLUDE` | `database.exclude` |  |
-| do not include `USE <database>;` statement in the dump | B | `no-database-name` | `NO_DATABASE_NAME` | `database.no-database-name` | `false` |
+| names of databases to dump, comma-separated | B | `include` | `DB_NAMES` | `dump.include` | all databases in the server |
+| names of databases to exclude from the dump | B | `exclude` | `DB_NAMES_EXCLUDE` | `dump.exclude` |  |
+| do not include `USE <database>;` statement in the dump | B | `no-database-name` | `NO_DATABASE_NAME` | `dump.no-database-name` | `false` |
 | restore to a specific database | R | `restore --database` | `RESTORE_DATABASE` | `restore.database` |  |
 | how often to do a dump or prune, in minutes | BP | `dump --frequency` | `DB_DUMP_FREQ` | `dump.schedule.frequency` | `1440` (in minutes), i.e. once per day |
 | what time to do the first dump or prune | BP | `dump --begin` | `DB_DUMP_BEGIN` | `dump.schedule.begin` | `0`, i.e. immediately |
 | cron schedule for dumps or prunes | BP | `dump --cron` | `DB_DUMP_CRON` | `dump.schedule.cron` |  |
 | run the backup or prune a single time and exit | BP | `dump --once` | `RUN_ONCE` | `dump.schedule.once` | `false` |
-| enable debug logging | BRP | `debug` | `DEBUG` | `logging: debug` | `false` |
+| enable debug logging | BRP | `debug` | `DEBUG` | `logging` | `false` |
 | where to put the dump file; see [backup](./backup.md) | BP | `dump --target` | `DB_DUMP_TARGET` | `dump.targets` |  |
 | where the restore file exists; see [restore](./restore.md) | R | `restore --target` | `DB_RESTORE_TARGET` | `restore.target` |  |
 | replace any `:` in the dump filename with `-` | BP | `dump --safechars` | `DB_DUMP_SAFECHARS` | `database.safechars` | `false` |
@@ -89,6 +89,103 @@ The following are the environment variables, CLI flags and configuration file op
 | filename to save the target backup file | B | `dump --filename-pattern` | `DB_DUMP_FILENAME_PATTERN` | `dump.filename-pattern` |  |
 | directory with scripts to execute before backup | B | `dump --pre-backup-scripts` | `DB_DUMP_PRE_BACKUP_SCRIPTS` | `dump.scripts.pre-backup` | in container, `/scripts.d/pre-backup/` |
 | directory with scripts to execute after backup | B | `dump --post-backup-scripts` | `DB_DUMP_POST_BACKUP_SCRIPTS` | `dump.scripts.post-backup` | in container, `/scripts.d/post-backup/` |
-| directory with scripts to execute before restore | R | `restore --pre-restore-scripts` | `DB_DUMP_PRE_RESTORE_SCRIPTS` | `dump.pre-restore-scripts` | in container, `/scripts.d/pre-restore/` |
-| directory with scripts to execute after restore | R | `restore --post-restore-scripts` | `DB_DUMP_POST_RESTORE_SCRIPTS` | `dump.post-restore-scripts` | in container, `/scripts.d/post-restore/` |
+| directory with scripts to execute before restore | R | `restore --pre-restore-scripts` | `DB_DUMP_PRE_RESTORE_SCRIPTS` | `restore.pre-restore-scripts` | in container, `/scripts.d/pre-restore/` |
+| directory with scripts to execute after restore | R | `restore --post-restore-scripts` | `DB_DUMP_POST_RESTORE_SCRIPTS` | `restore.post-restore-scripts` | in container, `/scripts.d/post-restore/` |
 | retention policy for backups | BP | `dump --retention` | `RETENTION` | `prune.retention` | Infinite |
+
+## Configuration File
+
+### Format
+
+The config file is a YAML file. You can write the yaml configuration file by hand. Alternatively, you can use an online service
+to generate one for you. Referenced services will be listed here in the future.
+
+The keys are:
+
+* `version`: the version of configuration, must be `config.databack.io/v1`
+* `kind`: the kind of configuration, must be one of:
+  * `local`: local configuration
+  * `remote`: remote configuration
+* `metadata`: metadata about the configuration. Not required. Used primarily for validating or optional information.
+  * `name` (optional): the name of the configuration
+  * `description` (optional): a description of the configuration
+  * `digest` (optional): the digest of the configuration, excluding the `digest` key itself. Everything else, including optional metadata, is included.
+  * `created` (optional): the date the configuration was created in [ISO8601 date format](https://en.wikipedia.org/wiki/ISO_8601), e.g. `2021-01-01T00:00:00Z`. The timezone always should be `Z` for UTC.
+* `spec`: the specification. Varies by the `kind` of configuration.
+
+The contents of `spec` depend on the kind of configuration.
+
+#### Local Configuration
+
+For local configuration, the `spec` is composed of the following. See the [Configuration Options](#configuration-options)
+for details of each.
+
+* `dump`: the dump configuration
+  * `include`: list of tables to include
+  * `exclude`: list of tables to exclude
+  * `safechars`: safe characters in filename
+  * `no-database-name`: remove `USE <database>` from dumpfile
+  * `schedule`: the schedule configuration
+    * `frequency`: the frequency of the schedule
+    * `begin`: the time to begin the schedule
+    * `cron`: the cron schedule
+    * `once`: run once and exit
+  * `compression`: the compression to use
+  * `compact`: compact the dump
+  * `max-allowed-packet`: max packet size
+  * `filename-pattern`: the filename pattern
+  * `scripts`:
+    * `pre-backup`: path to directory with pre-backup scripts
+    * `post-backup`: path to directory with post-backup scripts
+  * `targets`: list of names of known targets, defined in the `targets` section, where to save the backup
+* `restore`: the restore configuration
+  * `scripts`:
+    * `pre-restore`: path to directory with pre-restore scripts
+    * `post-restore`: path to directory with post-restore scripts
+* `database`: the database configuration
+  * `server`: host:port
+  * `port`: port (deprecated)
+  * `credentials`: access credentials for the database
+    * `username`: user
+    * `password`: password
+* `prune`: the prune configuration
+  * `retention`: retention policy
+* `targets`: target configurations, each of which can be reference by other sections. Key is the name of the target that is referenced elsewhere. Each one has the following structure:
+  * `type`: the type of target, one of: file, s3, smb
+  * `url`: the URL of the target
+  * `details`: access details for the target, depends on target type:
+    * Type s3:
+      * `region`: the region
+      * `endpoint`: the endpoint
+      * `access-key-id`: the access key ID (s3)
+      * `secret-access-key`: the secret access key (s3)
+    * Type smb:
+      * `domain`: the domain (smb)
+      * `username`: the username (smb)
+      * `password`: the password (smb)
+* `logging`: the log level, one of: error,warning,info,debug,trace; default is info
+* `telemetry`: configuration for sending telemetry data (optional)
+  * `url`: URL to telemetry service
+  * `certificate`: the certificate for the telemetry server or a CA that signed the server's TLS certificate. Not required if telemetry server does not use TLS, or if the system's certificate store already contains the server's cert or CA.
+  * `credentials`: unique token provided by the remote service as credentials, base64-encoded
+
+#### Remote Configuration
+
+For remote configuration, the `spec` is composed of the following:
+
+* `url`: the URL of the remote configuration; required
+* `certificate`: the certificate for the server or a CA that signed the server's TLS certificate. Not required if remote server does not use TLS, or if the system's certificate store already contains the server's cert or CA.
+* `credentials`: unique token provided by the remote service as credentials, base64-encoded
+
+The configuration file retrieved from a remote **always** has the same structure as any config file. It even can be
+saved locally and used as a local configuration. This means it also can
+reference another remote configuration, just like a local one. That can in turn reference another
+and so on, ad infinitum. In practice, remote service will avoid this.
+
+### Multiple Configurations
+
+As of version 1.0 of `mysql-backup`, there is support only for one config file. This means:
+
+* The `--config` flag can be used only once.
+* The config file does not support [multiple yaml documents in a single file](https://yaml.org/spec/1.2.2/). If you ask it to read a yaml file with multiple documents sepaarted by `---`, it will read only the first one.
+* You can have chaining, as described in the [remote configuration](#remote-configuration) section, where one file of kind `remote` references another, which itself is `remote`, etc. But only the final one will be used. It is not merging.

docs/configuration.md

@@ -0,0 +1,43 @@
+# Security
+
+## Database and Targets
+
+`mysql-backup` uses standard libraries for accessing remote services, including the database to backup
+or restore, and targets for saving backups, restoring backups, or pruning.
+
+## Logs
+
+Logs never should include credentials or other secrets, including at the most detailed level like `trace`. If, despite our efforts,
+you see confidential information in logs, please report an issue immediately. 
+
+## Telemetry
+
+Remote telemetry services store your logs, as well as details about when backups occurred, if there were any errors,
+and how long they took. This means that the telemetry services knows:
+
+* The names of the databases you back up
+* The names of the targets you use
+* The times of backups
+* The duration of backups
+* The success or failure of backups
+* Backup logs. As described above in [Logs](#logs), logs never should include credentials or other secrets.
+
+Telemetry services do not store your credentials or other secrets, nor do they store the contents of your backups.
+They _do_ know the names of your database tables, as those appear in the logs.
+
+## Remote Configuration
+
+Remote configuration services store your configuration, including the names of your databases and targets, as well as
+credentials. However, they only have that data encrypted in a way that only you can decrypt. When you load configuration
+into the remote service, it is encrypted locally to you, and then stored as an encrypted blob. The remote service never
+sees your unencrypted data.
+
+The data is decrypted by `mysql-backup` locally on your machine, when you retrieve the configuration.
+
+Your access token to the remote service, stored in your local configuration file, is a
+[Curve25519 private key](https://en.wikipedia.org/wiki/Curve25519), which authenticates
+you to the remote service. The remote service never sees this key, only the public key, which is used to verify your identity.
+
+This key is then used to decrypt the configuration blob, which is used to configure `mysql-backup`.
+
+In configuration files, the key is stored base64-encoded.

docs/security.md

@@ -22,15 +22,16 @@ require (
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.29
 	github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67
+	github.com/dsnet/compress v0.0.1
 	github.com/go-test/deep v1.1.0
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/aws/aws-sdk-go v1.44.256 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.29 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 // indirect
@@ -48,7 +49,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect