Merge remote-tracking branch 'origin/master' into add-instance-storage

Author: keithduncan

.buildkite/pipeline.yml

@@ -118,6 +118,7 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     concurrency_group: "aws-stack-publish"
     concurrency: 1
+    concurrency_method: eager
     artifact_paths: "build/*.yml"
     depends_on: "copy-ami"
 

README.md

@@ -6,7 +6,10 @@
 
 [Buildkite](https://buildkite.com/) is a platform for running fast, secure, and scalable continuous integration pipelines on your own infrastructure.
 
-The Buildkite Elastic CI Stack gives you a private, autoscaling [Buildkite Agent](https://buildkite.com/docs/agent) cluster. Use it to parallelize legacy tests across hundreds of nodes, run tests and deployments for all your Linux-based services and apps, or run AWS ops tasks.
+The Buildkite Elastic CI Stack for AWS gives you a private, autoscaling
+[Buildkite Agent](https://buildkite.com/docs/agent) cluster. Use it to parallelize
+large test suites across thousands of nodes, run tests and deployments for Linux or Windows
+based services and apps, or run AWS ops tasks.
 
 ## Getting started
 
@@ -31,6 +34,30 @@ aws cloudformation create-stack \
   --parameters "$(cat config.json)"
 ```
 
+## Supported Features
+
+Most features are supported across both Linux and Windows. See below for details
+of per-operating system support:
+
+Feature | Linux | Windows
+--- | --- | ---
+Docker | ✅ | ✅ 
+Docker Compose | ✅ | ✅ 
+AWS CLI | ✅ | ✅ 
+S3 Secrets Bucket | ✅ | ✅ 
+ECR Login | ✅ | ✅ 
+Docker Login | ✅ | ✅ 
+CloudWatch Logs Agent | ✅ | ✅ 
+Per-Instance Bootstrap Script | ✅ | ✅ 
+🧑‍🔬 git-mirrors experiment | ✅ | ✅ 
+SSM Access | ✅ | ✅ 
+SSH Access | ✅ | 
+Periodic authorized_keys Refresh | ✅ | 
+Periodic Instance Health Check | ✅ | 
+git lfs | ✅ | 
+Additional sudo Permissions | ✅ | 
+RDP Access | | ✅ 
+
 ## Security
 
 This repository hasn't been reviewed by security researchers so exercise caution and careful thought with what credentials you make available to your builds.
@@ -71,9 +98,9 @@ make packer
 This will boot and image three AWS EC2 instances in your account’s `us-east-1`
 default VPC:
 
-- Linux amd64
-- Linux arm64
-- Windows amd64
+- Linux (64-bit x86)
+- Linux (64-bit Arm)
+- Windows (64-bit x86)
 
 ## Support Policy
 
@@ -92,9 +119,9 @@ We aim to support new regions within one month of general availability.
 
 We build and deploy the following AMIs to all our supported regions:
 
-- Amazon Linux 2 (x86_64)
-- Amazon Linux 2 (arm64)
-- Windows Server 2019 (x86_64)
+- Amazon Linux 2 (64-bit x86)
+- Amazon Linux 2 (64-bit Arm)
+- Windows Server 2019 (64-bit x86)
 
 ### Buildkite Agent
 

goss.yaml

@@ -44,10 +44,6 @@ service:
     enabled: true
     running: true
 
-  lifecycled:
-    enabled: true
-    running: true
-
   sshd:
     enabled: true
     running: true
@@ -93,9 +89,6 @@ process:
   buildkite-agent:
     running: true
 
-  lifecycled:
-    running: true
-
   sshd:
     running: true
 

packer/linux/buildkite-ami.json

@@ -45,10 +45,6 @@
       "type": "shell",
       "script": "scripts/install-cloudwatch-agent.sh"
     },
-    {
-      "type": "shell",
-      "script": "scripts/install-lifecycled.sh"
-    },
     {
       "type": "shell",
       "script": "scripts/install-docker.sh"

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -193,15 +193,6 @@ if [[ -n "${BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT}" ]] ; then
 	rm /tmp/elastic_bootstrap
 fi
 
-cat << EOF > /etc/lifecycled
-AWS_REGION=${AWS_REGION}
-LIFECYCLED_HANDLER=/usr/local/bin/stop-agent-gracefully
-LIFECYCLED_CLOUDWATCH_GROUP=/buildkite/lifecycled
-EOF
-
-systemctl enable lifecycled.service
-systemctl start lifecycled
-
 # wait for docker to start
 next_wait_time=0
 until docker ps || [ $next_wait_time -eq 5 ]; do

packer/linux/scripts/install-lifecycled.sh

@@ -44,10 +44,6 @@
       "type": "powershell",
       "script": "scripts/install-cloudwatch-agent.ps1"
     },
-    {
-      "type": "powershell",
-      "script": "scripts/install-lifecycled.ps1"
-    },
     {
       "type": "powershell",
       "script": "scripts/install-docker.ps1"

packer/windows/buildkite-ami.json

@@ -139,10 +139,6 @@ disconnect-after-job=${Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB}
 "@
 $OFS=" "
 
-nssm set lifecycled AppEnvironmentExtra :AWS_REGION=$Env:AWS_REGION
-nssm set lifecycled AppEnvironmentExtra +LIFECYCLED_HANDLER="C:\buildkite-agent\bin\stop-agent-gracefully.ps1"
-Restart-Service lifecycled
-
 # wait for docker to start
 $next_wait_time=0
 do {

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -22,11 +22,6 @@
             "file_path": "C:\\buildkite-agent\\buildkite-agent.log",
             "log_group_name": "/buildkite/buildkite-agent",
             "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
-          },
-          {
-            "file_path": "C:\\lifecycled\\lifecycled.log",
-            "log_group_name": "/buildkite/lifecycled",
-            "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
           }
         ]
       },