Support _FILE based configuration values (#264)

* Replace envconfig with env

* Adjust config options and processing

* Added _FILE variant for all password vars.

* Try pathenvconfig

* Revert everything so far

* Use our fork of envconfig with custom lookup

* Use our fork of envconfig with custom lookup

* Test compose timeout option

* Remove secret resolving and specific _FILE config

* Fix timing issue in swarm tests

* Revert "Test compose timeout option"

This reverts commit ab50b21, reversing
changes made to 0282514.

Revert "Test compose timeout option"

This reverts commit 0282514.

* Use offen/envconfig v1.5.0

* Add info about _FILE in README

* Value > File. Panic on file error. Panic on duplicate presence.

* Test panic on duplicate vars and panic on file error.

Author: MaxJa4

README.md

@@ -139,6 +139,9 @@ Documentation references Docker Hub, but all examples will work using ghcr.io ju
 ## Configuration reference
 
 Backup targets, schedule and retention are configured in environment variables.
+
+Note: You can use any environment variable from below also with a `_FILE` suffix to be able to load the value from a file. This is usually useful when using [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/) or similar.
+
 You can populate below template according to your requirements and use it as your `env_file`:
 
 ```ini
@@ -233,14 +236,6 @@ You can populate below template according to your requirements and use it as you
 # AWS_ACCESS_KEY_ID="<xxx>"
 # AWS_SECRET_ACCESS_KEY="<xxx>"
 
-# It is possible to provide the keys in files, allowing to hide the sensitive data.
-# These values have a higher priority than the ones above, meaning if both are set
-# the values from the files will be used.
-# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/).
-
-# AWS_ACCESS_KEY_ID_FILE="/path/to/file"
-# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file"
-
 # Instead of providing static credentials, you can also use IAM instance profiles
 # or similar to provide authentication. Some possible configuration options on AWS:
 # - EC2: http://169.254.169.254

cmd/backup/config.go

@@ -24,9 +24,7 @@ type Config struct {
 	AwsEndpointCACert             CertDecoder     `envconfig:"AWS_ENDPOINT_CA_CERT"`
 	AwsStorageClass               string          `split_words:"true"`
 	AwsAccessKeyID                string          `envconfig:"AWS_ACCESS_KEY_ID"`
-	AwsAccessKeyIDFile            string          `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
 	AwsSecretAccessKey            string          `split_words:"true"`
-	AwsSecretAccessKeyFile        string          `split_words:"true"`
 	AwsIamRoleEndpoint            string          `split_words:"true"`
 	AwsPartSize                   int64           `split_words:"true"`
 	BackupCompression             CompressionType `split_words:"true" default:"gz"`
@@ -80,17 +78,6 @@ type Config struct {
 	DropboxConcurrencyLevel       NaturalNumber   `split_words:"true" default:"6"`
 }
 
-func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
-	if secretPath == "" {
-		return envVar, nil
-	}
-	data, err := os.ReadFile(secretPath)
-	if err != nil {
-		return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
-	}
-	return string(data), nil
-}
-
 type CompressionType string
 
 func (c *CompressionType) Decode(v string) error {

cmd/backup/script.go

@@ -35,8 +35,8 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
-	"github.com/kelseyhightower/envconfig"
 	"github.com/leekchan/timeutil"
+	"github.com/offen/envconfig"
 	"github.com/otiai10/copy"
 	"golang.org/x/sync/errgroup"
 )
@@ -89,6 +89,28 @@ func newScript() (*script, error) {
 		return nil
 	})
 
+	envconfig.Lookup = func(key string) (string, bool) {
+		value, okValue := os.LookupEnv(key)
+		location, okFile := os.LookupEnv(key + "_FILE")
+
+		switch {
+		case okValue && !okFile: // only value
+			return value, true
+		case !okValue && okFile: // only file
+			contents, err := os.ReadFile(location)
+			if err != nil {
+				s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
+				return "", false
+			}
+			return string(contents), true
+		case okValue && okFile: // both
+			s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
+			return "", false
+		default: // neither, ignore
+			return "", false
+		}
+	}
+
 	if err := envconfig.Process("", s.c); err != nil {
 		return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
 	}
@@ -137,18 +159,10 @@ func newScript() (*script, error) {
 	}
 
 	if s.c.AwsS3BucketName != "" {
-		accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
-		}
-		secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
-		if err != nil {
-			return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
-		}
 		s3Config := s3.Config{
 			Endpoint:         s.c.AwsEndpoint,
-			AccessKeyID:      accessKeyID,
-			SecretAccessKey:  secretAccessKey,
+			AccessKeyID:      s.c.AwsAccessKeyID,
+			SecretAccessKey:  s.c.AwsSecretAccessKey,
 			IamRoleEndpoint:  s.c.AwsIamRoleEndpoint,
 			EndpointProto:    s.c.AwsEndpointProto,
 			EndpointInsecure: s.c.AwsEndpointInsecure,

go.mod

@@ -9,10 +9,10 @@ require (
 	github.com/cosiner/argv v0.1.0
 	github.com/docker/docker v24.0.5+incompatible
 	github.com/gofrs/flock v0.8.1
-	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/klauspost/compress v1.16.7
 	github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
 	github.com/minio/minio-go/v7 v7.0.62
+	github.com/offen/envconfig v1.5.0
 	github.com/otiai10/copy v1.11.0
 	github.com/pkg/sftp v1.13.6
 	github.com/studio-b12/gowebdav v0.9.0

go.sum

@@ -451,8 +451,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
-github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
@@ -527,6 +525,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/offen/envconfig v1.5.0 h1:LHL4wYIDVeoGxSDI40MShmWfss3gYUlCdstfSiSq4Fk=
+github.com/offen/envconfig v1.5.0/go.mod h1:L7ny7R+4JWH3VVnZ+ARHvZysWUiZ2eQcm3L0imU9ACY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=

test/secrets/run.sh

@@ -31,6 +31,16 @@ pass "Found relevant files in untared backup."
 sleep 5
 expect_running_containers "5"
 
+docker exec -e AWS_ACCESS_KEY_ID=test $(docker ps -q -f name=backup) backup \
+  && fail "Backup should have failed due to duplicate env variables."
+
+pass "Backup failed due to duplicate env variables."
+
+docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=backup) backup \
+  && fail "Backup should have failed due to non existing file env variable."
+
+pass "Backup failed due to non existing file env variable."
+
 docker stack rm test_stack
 
 docker secret rm minio_root_password

test/swarm/run.sh

@@ -29,6 +29,7 @@ sleep 5
 expect_running_containers "5"
 
 docker stack rm test_stack
+sleep 1
 docker swarm leave --force
 
 sleep 10