Merge branch 'master' into MaxInstanceLifetime

Author: nitrocode

.buildkite/pipeline.yml

@@ -16,63 +16,6 @@ steps:
         run: unit-tests
         config: docker-compose.unit-tests.yml
 
-  - id: "s3secrets-helper-linux-amd64"
-    name: ":golang: :linux: s3secrets-helper-linux-amd64"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
-    plugins:
-      docker#v3.7.0:
-        image: "golang:1.15"
-        mount-checkout: false
-        volumes:
-          - "./build:/build:rw"
-          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
-        workdir: /s3secrets-helper
-        environment:
-          - "GOOS=linux"
-          - "GOARCH=amd64"
-        command: ["go", "build", "-o", "/build/s3secrets-helper-linux-amd64"]
-    artifact_paths:
-      - build/s3secrets-helper-linux-amd64
-
-  - id: "s3secrets-helper-linux-arm64"
-    name: ":golang: :linux: s3secrets-helper-linux-arm64"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
-    plugins:
-      docker#v3.7.0:
-        image: "golang:1.15"
-        mount-checkout: false
-        volumes:
-          - "./build:/build:rw"
-          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
-        workdir: /s3secrets-helper
-        environment:
-          - "GOOS=linux"
-          - "GOARCH=arm64"
-        command: ["go", "build", "-o", "/build/s3secrets-helper-linux-arm64"]
-    artifact_paths:
-      - build/s3secrets-helper-linux-arm64
-
-  - id: "s3secrets-helper-windows-amd64"
-    name: ":golang: :windows: s3secrets-helper-windows-amd64.exe"
-    agents:
-      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
-    plugins:
-      docker#v3.7.0:
-        image: "golang:1.15"
-        mount-checkout: false
-        volumes:
-          - "./build:/build:rw"
-          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
-        workdir: /s3secrets-helper
-        environment:
-          - "GOOS=windows"
-          - "GOARCH=amd64"
-        command: ["go", "build", "-o", "/build/s3secrets-helper-windows-amd64.exe"]
-    artifact_paths:
-      - build/s3secrets-helper-windows-amd64.exe
-
   - id: "packer-windows"
     name: ":packer: :windows:"
     command: .buildkite/steps/packer.sh windows
@@ -83,7 +26,6 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
-      - "s3secrets-helper-windows-amd64"
 
   - id: "launch-windows"
     name: ":cloudformation: :windows: Launch"
@@ -112,7 +54,6 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
-      - "s3secrets-helper-linux-amd64"
 
   - id: "launch-linux-amd64"
     name: ":cloudformation: :linux: AMD64 Launch"
@@ -141,7 +82,6 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
-      - "s3secrets-helper-linux-arm64"
 
   - id: "launch-linux-arm64"
     name: ":cloudformation: :linux: ARM64 Launch"
@@ -178,6 +118,7 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     concurrency_group: "aws-stack-publish"
     concurrency: 1
+    concurrency_method: eager
     artifact_paths: "build/*.yml"
     depends_on: "copy-ami"
 

.buildkite/steps/cleanup.sh

@@ -48,6 +48,15 @@ aws s3api list-buckets \
   | grep -E 'buildkite-aws-stack-test-.*-managedsecretsbucket' \
   | xargs -n1 -t -I% aws s3 rb s3://% --force
 
+# Do this before deleting the stacks so we don't race with stack-managed log
+# groups
+echo "--- Deleting old lambda logs after ${cutoff_date_milli}"
+aws logs describe-log-groups \
+  --log-group-name-prefix "/aws/lambda/buildkite-aws-stack-test-" \
+  --query "$(printf 'logGroups[?creationTime<`%s`].[logGroupName]' "$cutoff_date_milli" )" \
+  --output text \
+  | xargs -n1 -t -I% aws logs delete-log-group --log-group-name "%"
+
 echo "--- Deleting old cloudformation stacks"
 aws cloudformation describe-stacks \
   --output text \
@@ -62,10 +71,3 @@ aws ec2 describe-instances \
   --query "$(printf 'Reservations[].Instances[?LaunchTime<`%s`].[InstanceId]' "$cutoff_date")" \
   --output text \
   | xargs -n1 -t -I% aws ec2 terminate-instances --instance-ids "%"
-
-echo "--- Deleting old lambda logs after ${cutoff_date_milli}"
-aws logs describe-log-groups \
-  --log-group-name-prefix "/aws/lambda/buildkite-aws-stack-test-" \
-  --query "$(printf 'logGroups[?creationTime<`%s`].[logGroupName]' "$cutoff_date_milli" )" \
-  --output text \
-  | xargs -n1 -t -I% aws logs delete-log-group --log-group-name "%"

.buildkite/steps/packer.sh

@@ -9,17 +9,13 @@ fi
 os="${1:-linux}"
 arch="${2:-amd64}"
 agent_binary="buildkite-agent-${os}-${arch}"
-s3secrets_binary="s3secrets-helper-${os}-${arch}"
 
 if [[ "$os" == "windows" ]] ; then
   agent_binary+=".exe"
-  s3secrets_binary+=".exe"
 fi
 
 mkdir -p "build/"
 
-buildkite-agent artifact download "build/$s3secrets_binary" .
-
 # Build a hash of packer files and the agent versions
 packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
 stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")

.gitmodules

@@ -1,11 +1,14 @@
 [submodule "plugins/secrets"]
 	path = plugins/secrets
 	url = https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks.git
+	branch = v2.1.4
 
 [submodule "plugins/ecr"]
 	path = plugins/ecr
 	url = https://github.com/buildkite-plugins/ecr-buildkite-plugin.git
+	branch = v2.4.0
 
 [submodule "plugins/docker-login"]
 	path = plugins/docker-login
 	url = https://github.com/buildkite-plugins/docker-login-buildkite-plugin.git
+	branch = v2.0.2

CHANGELOG.md

@@ -4,6 +4,87 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v5.6.1](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.6.0...v5.6.1) (2021-09-02)
+
+## Fixed
+
+* Missed parameter `BuildkiteAgentTokenParameterStoreKMSKey` in `Autoscaling` nested cloudformation template [#901](https://github.com/buildkite/elastic-ci-stack-for-aws/issues/901)
+
+## [v5.6.0](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.5.1...v5.6.0) (2021-08-31)
+
+### Added
+
+* Cross-region secrets bucket support to git-credentials-s3-secrets [elastic-ci-stack-s3-secrets-hooks#48](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/pull/48)
+* AssumeRole support in the ECR Login plug-in [ecr-buildkite-plugin#69](https://github.com/buildkite-plugins/ecr-buildkite-plugin/pull/69)
+
+### Changed
+
+* Instance IAM Profile role permissions to be more tightly scoped [#800](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/800) ([@nitrocode](https://github.com/nitrocode))
+* Import buildkite-lambda-scaler from the Severless Application Repository [#685](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/685)
+* The built-in environment hook no longer overwrites `AWS_REGION` and `AWS_DEFAULT_REGION` if already present [#892](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/892) ([@toothbrush](https://github.com/toothbrush))
+* Included buildkite-agent from 3.32.1 to 3.32.3
+
+### Fixed
+
+* Hourly disk check script on Linux [#898](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/898)
+* git-credentials-s3-secrets on Windows [elastic-ci-stack-s3-secrets-hooks#47](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/pull/47)
+* PowerShell hook support on Windows [agent#1497](https://github.com/buildkite/agent/pull/1497)
+
+## [v5.5.1](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.5.0...v5.5.1) (2021-08-06)
+
+### Changed
+
+* Included buildkite-agent from 3.32.0 to 3.32.1
+
+### Fixed
+
+* A source of unexpected instance termination causing build failures [#888](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/888)
+
+## [v5.5.0](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.4.0...v5.5.0) (2021-07-30)
+
+### Added
+
+* Template validation rules for the Buildkite Agent token [#873](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/873)
+* Secret redaction in build logs [agent#1452](https://github.com/buildkite/agent/pull/1452)
+* Support for the `pre-bootstrap` Buildkite Agent Lifecycle Hook [agent#1456](https://github.com/buildkite/agent/pull/1456)
+
+### Changed
+
+* Included buildkite-agent from 3.30.0 to 3.32.0 [#876](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/876) ([keithduncan](https://github.com/keithduncan))
+
+### Fixed
+
+* Remove logging of the Buildkite Agent token to CloudWatch Logs [#879](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/879)
+* Cross-region S3 bucket access for secrets [#875](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/875)
+* An error when handling zero length `environment` files [elastic-ci-stack-s3-secrets-hooks#42](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/pull/42)
+* A hang when loading ssh keys without a trailing newline [elastic-ci-stack-s3-secrets-hooks#44](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/pull/44)
+
+## [v5.4.0](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.3.2...v5.4.0) (2021-06-30)
+
+### Added
+
+* Docker Buildx [#871](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/871)
+* Docs on which user SSH access applies to [#863](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/863) ([@Temikus](https://github.com/Temikus))
+
+### Changed
+
+* Update Buildkite Agent to version 3.30.0 [#868](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/868) ([@esalter](https://github.com/esalter))
+* The HttpPutResponseHopLimit from 1 to 2 [#858](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/858)
+
+### Fixed
+
+* The default cost allocation tag value [#859](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/859)
+
+## [v5.3.2](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.3.1...v5.3.2) (2021-06-11)
+
+### Fixed
+* Fix s3secrets-helper for Windows [#846](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/846) ([DuBistKomisch](https://github.com/DuBistKomisch))
+* Pin Docker systemd configuration to the same Docker version [#849](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/849) ([cmanou](https://github.com/cmanou))
+* Excessive instance scaling while waiting for instances to boot
+
+### Changed
+* Create S3 secrets bucket only when needed [#844](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/844) ([vgrigoruk](https://github.com/vgrigoruk))
+
 ## [v5.3.1](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.3.0...v5.3.1) (2021-05-05)
 
 ### Fixed
@@ -109,7 +190,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Dependencies updated
 * Bump Buildkite Agent to v3.25.0 [#749](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/749) ([JuanitoFatas](https://github.com/JuanitoFatas))
-* Bump Buildkite Agent Scaler to v1.0.2 [#724](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/724) ([JuanitoFatas](https://github.com/JuanitoFatas)) [4fafd8e](https://github.com/buildkite/elastic-ci-stack-for-aws/commit/4fafd8e85a888f0d7b23bb3a1420332fe4e9063c) ([JuanitoFatas](https://github.com/JuanitoFatas)) 
+* Bump Buildkite Agent Scaler to v1.0.2 [#724](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/724) ([JuanitoFatas](https://github.com/JuanitoFatas)) [4fafd8e](https://github.com/buildkite/elastic-ci-stack-for-aws/commit/4fafd8e85a888f0d7b23bb3a1420332fe4e9063c) ([JuanitoFatas](https://github.com/JuanitoFatas))
 * Bump docker to v19.03.13 (linux) and v19.03.12 (windows) and docker-compose to v1.27.4 (linux, windows uses [latest choco version](https://chocolatey.org/packages/docker-comp…)) [#719](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/719) ([yob](https://github.com/yob)) [#723](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/723) ([JuanitoFatas](https://github.com/JuanitoFatas))
 * Bump bundled plugins to the latest versions [secrets](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/740) [ecr](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/741) [docker login](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/744)
 

Makefile

@@ -75,7 +75,7 @@ build/linux-amd64-ami.txt: packer-linux-amd64.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build linux packer image
-packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/s3secrets-helper-linux-amd64
+packer-linux-amd64.output: $(PACKER_LINUX_FILES)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -96,7 +96,7 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build linuxarm64 packer image
-packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/s3secrets-helper-linux-arm64
+packer-linux-arm64.output: $(PACKER_LINUX_FILES)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -117,7 +117,7 @@ build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build windows packer image
-packer-windows-amd64.output: $(PACKER_WINDOWS_FILES) build/s3secrets-helper-windows-amd64.exe
+packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -146,15 +146,15 @@ create-stack: build/aws-stack.yml env-STACK_NAME
 		--stack-name $(STACK_NAME) \
 		--disable-rollback \
 		--template-body "file://$(PWD)/build/aws-stack.yml" \
-		--capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \
+		--capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \
 		--parameters "$$(cat config.json)"
 
 update-stack: build/aws-stack.yml env-STACK_NAME
 	aws cloudformation update-stack \
 		--output text \
 		--stack-name $(STACK_NAME) \
 		--template-body "file://$(PWD)/build/aws-stack.yml" \
-		--capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \
+		--capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \
 		--parameters "$$(cat config.json)"
 
 
@@ -179,12 +179,3 @@ validate: build/aws-stack.yml
 generate-toc:
 	docker run -it --rm -v "$(PWD):/app" node:slim bash \
 		-c "npm install -g markdown-toc && cd /app && markdown-toc -i README.md"
-
-build/s3secrets-helper-linux-amd64:
-	cd plugins/secrets/s3secrets-helper && GOOS=linux GOARCH=amd64 go build -o ../../../$@
-
-build/s3secrets-helper-linux-arm64:
-	cd plugins/secrets/s3secrets-helper && GOOS=linux GOARCH=arm64 go build -o ../../../$@
-
-build/s3secrets-helper-windows-amd64.exe:
-	cd plugins/secrets/s3secrets-helper && GOOD=windows GOARCH=amd64 go build -o ../../../$@