Merge pull request #76 from liamstevens/feat/add-multiple-secret-file-support

Feat/add multiple secret file support

Author: lizrabuya

README.md

@@ -7,6 +7,12 @@ Different types of secrets are supported and exposed to your builds in appropria
 - `ssh-agent` for SSH Private Keys
 - Environment Variables for strings
 - `git-credential` via git's credential.helper
+- Other secrets, which must be suffixed with one of the following:
+  - `_SECRET`
+  - `_SECRET_KEY`
+  - `_PASSWORD`
+  - `_TOKEN`
+  - `_ACCESS_KEY`
 
 ## Installation
 
@@ -55,9 +61,10 @@ When run via the agent environment and pre-exit hook, your builds will check in
 - `s3://{bucket_name}/private_ssh_key`
 - `s3://{bucket_name}/environment` or `s3://{bucket_name}/env`
 - `s3://{bucket_name}/git-credentials`
+- `s3://{bucket_name}/secret-files/`
 
 The private key is exposed to both the checkout and the command as an ssh-agent instance.
-The secrets in the env file are exposed as environment variables.
+The secrets in the env file are exposed as environment variables, as are individual secret files.
 The locations of git-credentials are passed via `GIT_CONFIG_PARAMETERS` environment to git.
 
 ## Uploading Secrets
@@ -100,6 +107,16 @@ Key values pairs can also be uploaded.
 aws s3 cp --acl private --sse aws:kms <(echo "MY_SECRET=blah") "s3://${secrets_bucket}/environment"
 ```
 
+### Individual Secrets
+
+Individual secrets with a suffix of `_SECRET`, `_SECRET_KEY`, `_PASSWORD`, `_TOKEN`, or `_ACCESS_KEY` can be uploaded to the same location as the rest of your configuration, under an additional prefix of `/secret-files/`.
+
+The file contents should be the secret value, and the object key becomes the environment variable name. For example:
+
+```bash
+aws s3 cp --acl private --sse aws:kms <(echo "<SECRET_VALUE>") "s3://${secrets_bucket}/secret-files/SPECIAL_SECRET"
+```
+
 ## Options
 
 ### `bucket`

s3secrets-helper/s3/s3.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log"
 	"io/ioutil"
+	"log"
 	"os"
+	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -20,7 +21,7 @@ import (
 )
 
 type Client struct {
-	s3 *s3.Client
+	s3     *s3.Client
 	bucket string
 	region string
 }
@@ -82,17 +83,17 @@ func New(log *log.Logger, bucket string, regionHint string) (*Client, error) {
 	}
 
 	return &Client{
-		s3: s3.NewFromConfig(awsConfig),
+		s3:     s3.NewFromConfig(awsConfig),
 		bucket: bucket,
 		region: awsConfig.Region,
 	}, nil
 }
 
-func (c *Client) Bucket() (string) {
+func (c *Client) Bucket() string {
 	return c.bucket
 }
 
-func (c *Client) Region() (string) {
+func (c *Client) Region() string {
 	return c.region
 }
 
@@ -128,6 +129,32 @@ func (c *Client) Get(key string) ([]byte, error) {
 	return ioutil.ReadAll(out.Body)
 }
 
+// ListSuffix returns a list of keys in the bucket that have the given prefix and suffix.
+// This has a maximum of 1000 keys, for now. This can be expanded by using the continuation token.
+func (c *Client) ListSuffix(prefix string, suffixes []string) ([]string, error) {
+	var resp *s3.ListObjectsV2Output
+	var keys []string
+	resp, err := c.s3.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: &c.bucket,
+		Prefix: &prefix,
+	})
+
+	// Iterate over all objects at the prefix and find those who match our suffix
+	for _, object := range resp.Contents {
+		for _, suffix := range suffixes {
+			if strings.HasSuffix(*object.Key, suffix) {
+				keys = append(keys, *object.Key)
+				break //break out of the suffix loop
+			}
+		}
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("Could not ListObjectsV2 (%s) in bucket (%s). Ensure your IAM Identity has s3:ListBucket permission for this bucket. (%v)", prefix, c.bucket, err)
+	}
+	return keys, nil
+}
+
 // BucketExists returns whether the bucket exists.
 // 200 OK returns true without error.
 // 404 Not Found and 403 Forbidden return false without error.

s3secrets-helper/secrets/secrets.go

@@ -12,9 +12,10 @@ import (
 
 // Client represents interaction with AWS S3
 type Client interface {
-	Bucket() (string)
-	Region() (string)
+	Bucket() string
+	Region() string
 	Get(key string) ([]byte, error)
+	ListSuffix(prefix string, suffix []string) ([]string, error)
 	BucketExists() (bool, error)
 }
 
@@ -52,6 +53,10 @@ type Config struct {
 
 	// GitCredentialHelper is the path to git-credential-s3-secrets
 	GitCredentialHelper string
+
+	// Secret suffixes to look for in S3.
+	// Defaults to "_SECRET", "_SECRET_KEY", "_PASSWORD", "_TOKEN", and "_ACCESS_KEY"
+	SecretSuffixes []string
 }
 
 // Run is the programmatic (as opposed to CLI) entrypoint to all
@@ -81,6 +86,9 @@ func Run(conf Config) error {
 	resultsGit := make(chan getResult)
 	getGitCredentials(conf, resultsGit)
 
+	resultsSecrets := make(chan getResult)
+	getSecrets(conf, resultsSecrets)
+
 	if err := handleSSHKeys(conf, resultsSSH); err != nil {
 		return err
 	}
@@ -90,6 +98,9 @@ func Run(conf Config) error {
 	if err := handleGitCredentials(conf, resultsGit); err != nil {
 		return err
 	}
+	if err := handleSecrets(conf, resultsSecrets); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -121,6 +132,22 @@ func getEnvs(conf Config, results chan<- getResult) {
 	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
 }
 
+func getSecrets(conf Config, results chan<- getResult) {
+	suffixes := append(conf.SecretSuffixes, []string{
+		"_SECRET",
+		"_SECRET_KEY",
+		"_PASSWORD",
+		"_TOKEN",
+		"_ACCESS_KEY",
+	}...)
+	secretPrefix := conf.Prefix + "/secret-files"
+	keys, err := conf.Client.ListSuffix(secretPrefix, suffixes)
+	if err != nil {
+		fmt.Errorf("listing matching secrets: %w", err)
+	}
+	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
+}
+
 func getGitCredentials(conf Config, results chan<- getResult) {
 	keys := []string{
 		"git-credentials",
@@ -227,7 +254,7 @@ func handleGitCredentials(conf Config, results <-chan getResult) error {
 		// Replace backslash '\' with double backslash '\\'
 		helper = strings.ReplaceAll(helper, "\\", "\\\\")
 
-		singleQuotedHelpers = append(singleQuotedHelpers, "'" + helper + "'")
+		singleQuotedHelpers = append(singleQuotedHelpers, "'"+helper+"'")
 	}
 	env := "GIT_CONFIG_PARAMETERS=\"" + strings.Join(singleQuotedHelpers, " ") + "\"\n"
 
@@ -237,6 +264,40 @@ func handleGitCredentials(conf Config, results <-chan getResult) error {
 	return nil
 }
 
+// handleSecrets loads secrets into the environment.
+// The key is the last part of the S3 key.
+func handleSecrets(conf Config, results <-chan getResult) error {
+	log := conf.Logger
+	// Build an environment variable for interpretation by a shell
+	var singleQuotedSecrets []string
+	var envString string
+	for r := range results {
+		if r.err != nil {
+			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
+				log.Printf("+++ :warning: Failed to download secret %s/%s: %v", r.bucket, r.key, r.err)
+			}
+			continue
+		}
+		log.Printf("Adding secret %s/%s to environment", r.bucket, r.key)
+		envKey := strings.Split(r.key, "/")[len(strings.Split(r.key, "/"))-1]
+
+		// Replace backslash '\' with double backslash '\\'
+		value := strings.ReplaceAll(string(r.data), "\\", "\\\\")
+
+		singleQuotedSecrets = append(singleQuotedSecrets, envKey+"='"+value+"'")
+
+	}
+	if len(singleQuotedSecrets) == 0 {
+		log.Printf("No secrets found in %q", conf.Prefix)
+		return nil
+	}
+	envString = strings.Join(singleQuotedSecrets, "\n") + "\n"
+	if _, err := io.WriteString(conf.EnvSink, envString); err != nil {
+		return fmt.Errorf("writing SECRETS to env: %w", err)
+	}
+	return nil
+}
+
 type getResult struct {
 	bucket string
 	key    string

s3secrets-helper/secrets/secrets_test.go

@@ -16,18 +16,19 @@ import (
 )
 
 type FakeClient struct {
-	t    *testing.T
-	data map[string]FakeObject
+	t      *testing.T
+	data   map[string]FakeObject
+	bucket string
 }
 
 type FakeObject struct {
 	data []byte
 	err  error
 }
 
-func (c *FakeClient) Get(bucket, key string) ([]byte, error) {
+func (c *FakeClient) Get(key string) ([]byte, error) {
 	time.Sleep(time.Duration(rand.Int()%100) * time.Millisecond)
-	path := bucket + "/" + key
+	path := c.bucket + "/" + key
 	if result, ok := c.data[path]; ok {
 		c.t.Logf("FakeClient Get %s: %d bytes, error: %v", path, len(result.data), result.err)
 		return result.data, result.err
@@ -36,10 +37,23 @@ func (c *FakeClient) Get(bucket, key string) ([]byte, error) {
 	return nil, sentinel.ErrNotFound
 }
 
-func (c *FakeClient) BucketExists(bucket string) (bool, error) {
+func (c *FakeClient) BucketExists() (bool, error) {
 	return true, nil
 }
 
+func (c *FakeClient) Bucket() string {
+	return c.bucket
+}
+
+func (c *FakeClient) ListSuffix(prefix string, suffix []string) ([]string, error) {
+	fakeSecrets := []string{"pipeline/secret-files/BUILDKITE_ACCESS_KEY", "pipeline/secret-files/DATABASE_SECRET", "pipeline/secret-files/EXTERNAL_API_SECRET_KEY", "pipeline/secret-files/PRIVILEGED_PASSWORD", "pipeline/secret-files/SERVICE_TOKEN"}
+	return fakeSecrets, nil
+}
+
+func (c *FakeClient) Region() string {
+	return "us-west-2"
+}
+
 type FakeAgent struct {
 	t    *testing.T
 	keys []string
@@ -91,6 +105,12 @@ func TestRun(t *testing.T) {
 
 		"bkt/git-credentials":          {[]byte("general git key"), nil},
 		"bkt/pipeline/git-credentials": {[]byte("pipeline git key"), nil},
+
+		"bkt/pipeline/secret-files/BUILDKITE_ACCESS_KEY":    {[]byte("buildkite access key"), nil},
+		"bkt/pipeline/secret-files/DATABASE_SECRET":         {[]byte("database secret"), nil},
+		"bkt/pipeline/secret-files/EXTERNAL_API_SECRET_KEY": {[]byte("external api secret"), nil},
+		"bkt/pipeline/secret-files/PRIVILEGED_PASSWORD":     {[]byte("privileged password"), nil},
+		"bkt/pipeline/secret-files/SERVICE_TOKEN":           {[]byte("service token"), nil},
 	}
 	logbuf := &bytes.Buffer{}
 	fakeAgent := &FakeAgent{t: t}
@@ -100,7 +120,7 @@ func TestRun(t *testing.T) {
 		Repo:                "[email protected]:buildkite/bash-example.git",
 		Bucket:              "bkt",
 		Prefix:              "pipeline",
-		Client:              &FakeClient{t: t, data: fakeData},
+		Client:              &FakeClient{t: t, data: fakeData, bucket: "bkt"},
 		Logger:              log.New(logbuf, "", log.LstdFlags),
 		SSHAgent:            fakeAgent,
 		EnvSink:             envSink,
@@ -115,8 +135,8 @@ func TestRun(t *testing.T) {
 
 	// verify env
 	gitCredentialHelpers := strings.Join([]string{
-		`'credential.helper=/path/to/git-credential-s3-secrets bkt git-credentials'`,
-		`'credential.helper=/path/to/git-credential-s3-secrets bkt pipeline/git-credentials'`,
+		`'credential.helper=/path/to/git-credential-s3-secrets bkt us-west-2 git-credentials'`,
+		`'credential.helper=/path/to/git-credential-s3-secrets bkt us-west-2 pipeline/git-credentials'`,
 	}, " ")
 	expected := strings.Join([]string{
 		// because an SSH key was found, ssh-agent was started:
@@ -130,7 +150,13 @@ func TestRun(t *testing.T) {
 		// because git-credentials were found:
 		// (wrap in double quotes so that bash eval doesn't consume the inner single quote.
 		`GIT_CONFIG_PARAMETERS="` + gitCredentialHelpers + `"`,
+		"BUILDKITE_ACCESS_KEY='buildkite access key'",
+		"DATABASE_SECRET='database secret'",
+		"EXTERNAL_API_SECRET_KEY='external api secret'",
+		"PRIVILEGED_PASSWORD='privileged password'",
+		"SERVICE_TOKEN='service token'",
 	}, "\n") + "\n"
+
 	if actual := envSink.String(); expected != actual {
 		t.Errorf("unexpected env written:\n-%q\n+%q", expected, actual)
 	}