Merge pull request #1554 from buildkite/SUP-4519/Adding-artifcats-bucket-env-var

Add support for ArtifactsBucket region, with bucket ACL and Region configs

Author: omehegan

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -145,6 +145,9 @@ set_always         "BUILDKITE_DOCKER_EXPERIMENTAL" "$DOCKER_EXPERIMENTAL"
 set_always         "DOCKER_USERNS_REMAP" "$DOCKER_USERNS_REMAP"
 set_always         "DOCKER_VERSION" "$DOCKER_VERSION"
 set_always         "PLUGINS_ENABLED" "${PLUGINS_ENABLED[*]-}"
+set_always         "BUILDKITE_ARTIFACTS_BUCKET" "$BUILDKITE_ARTIFACTS_BUCKET"
+set_always         "BUILDKITE_S3_DEFAULT_REGION" "$BUILDKITE_S3_DEFAULT_REGION"
+set_always         "BUILDKITE_S3_ACL" "$BUILDKITE_S3_ACL"
 set_unless_present "AWS_DEFAULT_REGION" "$AWS_REGION"
 set_unless_present "AWS_REGION" "$AWS_REGION"
 EOF

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -84,6 +84,9 @@ set_always         "BUILDKITE_STACK_VERSION" "$Env:BUILDKITE_STACK_VERSION"
 set_always         "BUILDKITE_DOCKER_EXPERIMENTAL" "$DOCKER_EXPERIMENTAL"
 set_always         "DOCKER_VERSION" "$DOCKER_VERSION"
 set_always         "PLUGINS_ENABLED" "$PLUGINS_ENABLED"
+set_always         "BUILDKITE_ARTIFACTS_BUCKET" "$Env:BUILDKITE_ARTIFACTS_BUCKET"
+set_always         "BUILDKITE_S3_DEFAULT_REGION" "$Env:BUILDKITE_S3_DEFAULT_REGION"
+set_always         "BUILDKITE_S3_ACL" "$Env:BUILDKITE_S3_ACL"
 set_unless_present "AWS_DEFAULT_REGION" "$Env:AWS_REGION"
 set_unless_present "AWS_REGION" "$Env:AWS_REGION"
 set_unless_present "BUILDKITE_AGENT_ENDPOINT" "https://agent.buildkite.com/v3"

templates/aws-stack.yml

@@ -86,6 +86,8 @@ Metadata:
         - SecretsBucketRegion
         - SecretsBucketEncryption
         - ArtifactsBucket
+        - ArtifactsBucketRegion
+        - ArtifactsS3ACL
         - AuthorizedUsersUrl
         - BootstrapScriptUrl
         - AgentEnvFileUrl
@@ -409,6 +411,24 @@ Parameters:
     Type: String
     Default: ""
 
+  ArtifactsBucketRegion:
+    Description: Optional - Region for the ArtifactsBucket. If blank the bucket's region is dynamically discovered.
+    Type: String
+    Default: ""
+
+  ArtifactsS3ACL:
+    Description: Optional - ACL to use for S3 artifact uploads
+    Type: String
+    AllowedValues:
+      - "private"
+      - "public-read"
+      - "public-read-write"
+      - "authenticated-read"
+      - "aws-exec-read"
+      - "bucket-owner-read"
+      - "bucket-owner-full-control"
+    Default: "private"
+
   BootstrapScriptUrl:
     Description: Optional - HTTPS or S3 URL for a script to run on each instance during boot
     Type: String
@@ -866,6 +886,8 @@ Conditions:
 
     UseArtifactsBucket:
       !Not [ !Equals [ !Ref ArtifactsBucket, "" ] ]
+    IsArtifactsBucketRegionEmpty:
+      !Equals [ !Ref ArtifactsBucketRegion, "" ]
 
     HasImageId:
       !Not [ !Equals [ !Ref ImageId, "" ] ]
@@ -895,13 +917,13 @@ Conditions:
 
     UseInstanceType7:
       !Not [ !Equals [ !Select [ "6", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType8:
       !Not [ !Equals [ !Select [ "7", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType9:
       !Not [ !Equals [ !Select [ "8", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType10:
       !Not [ !Equals [ !Select [ "9", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
 
@@ -910,13 +932,13 @@ Conditions:
 
     UseInstanceType12:
       !Not [ !Equals [ !Select [ "11", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType13:
       !Not [ !Equals [ !Select [ "12", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
 
     UseInstanceType14:
       !Not [ !Equals [ !Select [ "13", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType15:
       !Not [ !Equals [ !Select [ "14", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
 
@@ -925,13 +947,13 @@ Conditions:
 
     UseInstanceType17:
       !Not [ !Equals [ !Select [ "16", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType18:
       !Not [ !Equals [ !Select [ "17", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType19:
       !Not [ !Equals [ !Select [ "18", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType20:
       !Not [ !Equals [ !Select [ "19", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
 
@@ -943,7 +965,7 @@ Conditions:
 
     UseInstanceType23:
       !Not [ !Equals [ !Select [ "22", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
-    
+
     UseInstanceType24:
       !Not [ !Equals [ !Select [ "23", !Split [ ",", !Join [ ",", [ !Ref InstanceTypes, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] ] ] ], ""] ]
 
@@ -1572,6 +1594,9 @@ Resources:
                   $Env:BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}"
                   $Env:BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}"
                   $Env:BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}"
+                  $Env:BUILDKITE_ARTIFACTS_BUCKET="${ArtifactsBucket}"
+                  $Env:BUILDKITE_S3_DEFAULT_REGION="${LocalArtifactsBucketRegion}"
+                  $Env:BUILDKITE_S3_ACL="${ArtifactsS3ACL}"
                   $Env:BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}"
                   $Env:BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}"
                   $Env:BUILDKITE_AGENT_ENDPOINT="${AgentEndpoint}"
@@ -1607,6 +1632,13 @@ Resources:
                     - CreateSecretsBucket
                     - !Ref "AWS::Region"
                     - !Ref SecretsBucketRegion
+                  LocalArtifactsBucketRegion: !If
+                    - UseArtifactsBucket
+                    - !If
+                      - IsArtifactsBucketRegionEmpty
+                      - !Ref "AWS::Region"
+                      - !Ref ArtifactsBucketRegion
+                    - !Ref "AWS::Region"
                   AgentTokenPath: !If
                     - UseCustomerManagedParameterPath
                     - !Ref BuildkiteAgentTokenParameterStorePath
@@ -1649,6 +1681,9 @@ Resources:
                   BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" \
                   BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}" \
                   BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}" \
+                  BUILDKITE_ARTIFACTS_BUCKET="${ArtifactsBucket}" \
+                  BUILDKITE_S3_DEFAULT_REGION="${LocalArtifactsBucketRegion}" \
+                  BUILDKITE_S3_ACL="${ArtifactsS3ACL}" \
                   BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}" \
                   BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}" \
                   BUILDKITE_AGENT_ENDPOINT="${AgentEndpoint}" \
@@ -1696,6 +1731,13 @@ Resources:
                     - CreateSecretsBucket
                     - !Ref "AWS::Region"
                     - !Ref SecretsBucketRegion
+                  LocalArtifactsBucketRegion: !If
+                    - UseArtifactsBucket
+                    - !If
+                      - IsArtifactsBucketRegionEmpty
+                      - !Ref "AWS::Region"
+                      - !Ref ArtifactsBucketRegion
+                    - !Ref "AWS::Region"
                   AgentTokenPath: !If
                     - UseCustomerManagedParameterPath
                     - !Ref BuildkiteAgentTokenParameterStorePath