Merge pull request #37 from buildkite/go

10–100x speed-up: s3secrets-helper written in Go

Author: pda

.buildkite/test_credentials.sh

@@ -7,6 +7,21 @@ pre_exit() {
   source hooks/pre-exit
 }
 
+# hooks/environment assumes `s3secrets-helper` command is available, normally
+# by way of that binary existing in $PATH. But a shell function works too.
+# Rather than build/install s3secrets-helper onto the build agent, build and
+# run it inside docker.
+# Feel free to replace this terrible hack with something better.
+s3secrets-helper() {
+  docker run \
+    --rm \
+    --volume $(pwd)/s3secrets-helper:/s3secrets-helper \
+    --workdir /s3secrets-helper \
+    --env-file <(env | egrep 'AWS|BUILDKITE') \
+    golang:1.15 \
+    bash -c 'go build && ./s3secrets-helper'
+}
+
 trap pre_exit EXIT
 source hooks/environment
 

README.md

@@ -12,9 +12,12 @@ Different types of secrets are supported and exposed to your builds in appropria
 
 The hooks needs to be installed directly in the agent so that secrets can be downloaded before jobs attempt checking out your repository. We are going to assume that buildkite has been installed at `/buildkite`, but this will vary depending on your operating system. Change the instructions accordingly.
 
+The core of the hook is an `s3secrets-helper` binary, the result of `go build` in the `s3secrets-helper/` subdirectory of this repository. It must be placed in `$PATH` to be found by the `hooks/environment` wrapper script.
+
 ```bash
 # clone to a path your buildkite-agent can access
 git clone https://github.com/buildkite-plugins/s3-secrets-buildkite-plugin.git /buildkite/s3_secrets
+(cd /buildkite/s3_secrets/s3secrets-helper && go build -o /usr/local/bin/s3secrets-helper)
 ```
 
 Modify your agent's global hooks (see [https://buildkite.com/docs/agent/v3/hooks#global-hooks](https://buildkite.com/docs/agent/v3/hooks#global-hooks)):
@@ -50,7 +53,9 @@ When run via the agent environment and pre-exit hook, your builds will check in
 - `s3://{bucket_name}/environment` or `s3://{bucket_name}/env`
 - `s3://{bucket_name}/git-credentials`
 
-The private key is exposed to both the checkout and the command as an ssh-agent instance. The secrets in the env file are exposed as environment variables.
+The private key is exposed to both the checkout and the command as an ssh-agent instance.
+The secrets in the env file are exposed as environment variables.
+The locations of git-credentials are passed via `GIT_CONFIG_PARAMETERS` environment to git.
 
 ## Uploading Secrets
 

hooks/environment

@@ -1,107 +1,20 @@
 #!/bin/bash
-
-set -eu -o pipefail
+set -e -o pipefail -u
 
 basedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+credhelper="$basedir/git-credential-s3-secrets"
 
-# shellcheck disable=SC1090
-. "$basedir/lib/shared.bash"
-
-# For resiliency, increase the number of attempts to retrieve credentials for IAM roles
-# The default value is 1
-export AWS_METADATA_SERVICE_NUM_ATTEMPTS=3
-
-TMPDIR=${TMPDIR:-/tmp}
-AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1}
-
-s3_bucket="${BUILDKITE_PLUGIN_S3_SECRETS_BUCKET:-}"
-s3_bucket_prefix="${BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX:-$BUILDKITE_PIPELINE_SLUG}"
-
-if [[ -z "$s3_bucket" ]] ; then
-  exit 0
-fi
-
-echo "~~~ Downloading secrets from :s3: $s3_bucket" >&2;
-
-if ! s3_bucket_exists "$s3_bucket" ; then
-  echo "+++ :warning: Bucket $s3_bucket doesn't exist" >&2;
-  exit 1
-fi
-
-ssh_key_paths=(
-  "$s3_bucket_prefix/private_ssh_key"
-  "$s3_bucket_prefix/id_rsa_github"
-  "private_ssh_key"
-  "id_rsa_github"
-)
-
-for key in ${ssh_key_paths[*]} ; do
-  echo "Checking ${key}" >&2
-  if s3_exists "$s3_bucket" "$key" ; then
-    echo "Found ${key}, downloading" >&2;
-    if ! ssh_key=$(s3_download "${s3_bucket}" "$key") ; then
-      echo "+++ :warning: Failed to download ssh-key $key" >&2;
-      exit 1
-    fi
-    echo "Downloaded ${#ssh_key} bytes of ssh key"
-    add_ssh_private_key_to_agent "$ssh_key"
-    key_found=1
-  elif [[ $? -eq 2 ]] ; then
-    echo "+++ :warning: Failed to check if $key exists" >&2;
-    exit 1
-  fi
-done
-
-if [[ -z "${key_found:-}" ]] && [[ "${BUILDKITE_REPO:-}" =~ ^git@ ]] ; then
-  echo >&2 "+++ :warning: Failed to find an SSH key in secret bucket"
-  echo >&2 "The repository '$BUILDKITE_REPO' appears to use SSH for transport, but the elastic-ci-stack-s3-secrets-hooks plugin did not find any SSH keys in the $s3_bucket S3 bucket."
-  echo >&2 "See https://github.com/buildkite/elastic-ci-stack-for-aws#build-secrets for more information."
-fi
-
-env_paths=(
-  "env"
-  "environment"
-  "${s3_bucket_prefix}/env"
-  "${s3_bucket_prefix}/environment"
-)
+# s3secrets-helper must be in PATH
+envscript="$(
+  BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER="$credhelper" \
+    s3secrets-helper
+)"
 
 env_before="$(env | sort)"
-
-for key in ${env_paths[*]} ; do
-  echo "Checking ${key}" >&2
-  if s3_exists "$s3_bucket" "$key" ; then
-    echo "Downloading env file from ${key}" >&2;
-    if ! envscript=$(s3_download "${s3_bucket}" "$key") ; then
-      echo "+++ :warning: Failed to download env from $key" >&2;
-      exit 1
-    fi
-    echo "Evaluating ${#envscript} bytes of env"
-    set -o allexport
-    eval "$envscript"
-    set +o allexport
-  elif [[ $? -eq 2 ]] ; then
-    echo "Failed to check if $key exists" >&2;
-  fi
-done
-
-git_credentials_paths=(
-  "git-credentials"
-  "${s3_bucket_prefix}/git-credentials"
-)
-
-git_credentials=()
-
-for key in ${git_credentials_paths[*]} ; do
-  if s3_exists "$s3_bucket" "$key" ; then
-    echo "Adding git-credentials in $key as a credential helper" >&2;
-    git_credentials+=("'credential.helper=$basedir/git-credential-s3-secrets ${s3_bucket} ${key}'")
-  fi
-done
-
-if [[ "${#git_credentials[@]}" -gt 0 ]] ; then
-  export GIT_CONFIG_PARAMETERS
-  GIT_CONFIG_PARAMETERS=$( IFS=' '; echo -n "${git_credentials[*]}" )
-fi
+echo "Evaluating ${#envscript} bytes of env"
+set -o allexport
+eval "$envscript"
+set +o allexport
 
 if [[ "${BUILDKITE_PLUGIN_S3_SECRETS_DUMP_ENV:-}" =~ ^(true|1)$ ]] ; then
   echo "~~~ Environment variables that were set" >&2;

lib/shared.bash

@@ -0,0 +1,2 @@
+# default `go build` binary
+/s3secrets-helper

s3secrets-helper/.gitignore

@@ -0,0 +1,5 @@
+module github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2
+
+go 1.15
+
+require github.com/aws/aws-sdk-go v1.35.14

s3secrets-helper/go.mod

@@ -0,0 +1,16 @@
+github.com/aws/aws-sdk-go v1.35.14 h1:nucVVXXjAr9UkmYCBWxQWRuYa5KOlaXjuJGg2ulW0K0=
+github.com/aws/aws-sdk-go v1.35.14/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/buildkite/elastic-ci-stack-s3-secrets-hooks v1.3.0 h1:CRdPqLjYECJY0cgs24E4ZwUJ2qlmcavN7W1jo+5ujnQ=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

s3secrets-helper/go.sum

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/s3"
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/secrets"
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sshagent"
+)
+
+const (
+	envBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
+	envPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
+	envPipeline   = "BUILDKITE_PIPELINE_SLUG"
+	envRepo       = "BUILDKITE_REPO"
+	envCredHelper = "BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER"
+
+	envDefaultRegion = "AWS_DEFAULT_REGION"
+	defaultRegion    = "us-east-1"
+)
+
+func main() {
+	log := log.New(os.Stderr, "", log.Lmsgprefix)
+	if err := mainWithError(log); err != nil {
+		log.Fatalf("fatal error: %v", err)
+	}
+}
+
+func mainWithError(log *log.Logger) error {
+	bucket := os.Getenv(envBucket)
+	if bucket == "" {
+		return nil
+	}
+
+	prefix := os.Getenv(envPrefix)
+	if prefix == "" {
+		prefix = os.Getenv(envPipeline)
+	}
+	if prefix == "" {
+		return fmt.Errorf("%s or %s required", envPrefix, envPipeline)
+	}
+
+	region := os.Getenv(envDefaultRegion)
+	if region == "" {
+		region = defaultRegion
+	}
+	client, err := s3.New(region)
+	if err != nil {
+		return err
+	}
+
+	agent := &sshagent.Agent{}
+
+	credHelper := os.Getenv(envCredHelper)
+	if credHelper == "" {
+		return fmt.Errorf("%s required", envCredHelper)
+	}
+
+	return secrets.Run(secrets.Config{
+		Repo:                os.Getenv(envRepo),
+		Bucket:              bucket,
+		Prefix:              prefix,
+		Client:              client,
+		Logger:              log,
+		SSHAgent:            agent,
+		EnvSink:             os.Stdout,
+		GitCredentialHelper: credHelper,
+	})
+}

s3secrets-helper/main.go

@@ -0,0 +1,77 @@
+package s3
+
+import (
+	"io/ioutil"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
+)
+
+type Client struct {
+	s3 *s3.S3
+}
+
+func New(region string) (*Client, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region: &region,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		s3: s3.New(sess),
+	}, nil
+}
+
+// Get downloads an object from S3.
+// Intended for small files; object is fully read into memory.
+// sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases.
+// Other errors are returned verbatim.
+func (c *Client) Get(bucket, key string) ([]byte, error) {
+	out, err := c.s3.GetObject(&s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &key,
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case "NoSuchKey":
+				return nil, sentinel.ErrNotFound
+			case "Forbidden":
+				return nil, sentinel.ErrForbidden
+			default:
+				return nil, aerr
+			}
+		} else {
+			return nil, err
+		}
+	}
+	defer out.Body.Close()
+	// we probably should return io.Reader or io.ReadCloser rather than []byte,
+	// maybe somebody should refactor that (and all the tests etc) one day.
+	return ioutil.ReadAll(out.Body)
+}
+
+// BucketExists returns whether the bucket exists.
+// 200 OK returns true without error.
+// 404 Not Found and 403 Forbidden return false without error.
+// Other errors result in false with an error.
+func (c *Client) BucketExists(bucket string) (bool, error) {
+	if _, err := c.s3.HeadBucket(&s3.HeadBucketInput{Bucket: &bucket}); err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			// https://github.com/aws/aws-sdk-go/issues/2593#issuecomment-491436818
+			case "NoSuchBucket", "NotFound":
+				return false, nil
+			default: // e.g. NoCredentialProviders, Forbidden
+				return false, aerr
+			}
+		} else {
+			return false, err
+		}
+	}
+	return true, nil
+}