fix: make govcloud builds possible (#2621)

Make various fixes so that govcloud builds work:

* Make latest-binaries.sh handle fetching cross-partition
* Change s3 copy logic to not check the AWS_ACCESS_KEY_ID variable,
  which is a fundamentally flawed check, and instead just attempt a copy
  using the environment's credentials and fall back to --no-sign-request
  if the initial copy fails.
* update the cache-pause-container script to use the eks-distro public mirror

The net effect of this change is that by default, none of the
provisioners rely on the caller's AWS credentials. I'm leaving in the
logic that passes through the AWS credentials via environment variables
since removing is a breaking change, but none of them are needed.

In a future major release, it would be nice to remove those entirely and
require people to use instance roles if they need permissions from the
provisioners (e.g. for private artifact buckets).

Author: fletcherw

Makefile

@@ -14,6 +14,7 @@ K8S_VERSION_MINOR := $(word 1,${K8S_VERSION_PARTS}).$(word 2,${K8S_VERSION_PARTS
 
 AMI_VARIANT ?= amazon-eks
 AMI_VERSION ?= v$(shell date '+%Y%m%d')
+aws_region ?= us-west-2
 os_distro ?= al2
 arch ?= x86_64
 
@@ -41,10 +42,9 @@ endif
 
 ami_name ?= $(AMI_VARIANT)-node-$(K8S_VERSION_MINOR)-$(AMI_VERSION)
 
-# ami owner overrides for cn/gov-cloud
 ifeq ($(aws_region), cn-northwest-1)
 	source_ami_owners ?= 141808717104
-else ifeq ($(aws_region), us-gov-west-1)
+else ifneq ($(filter $(aws_region),us-gov-west-1 us-gov-east-1),)
 	source_ami_owners ?= 045324592363
 endif
 
@@ -53,7 +53,7 @@ k8s=1.28
 
 .PHONY: build
 build: ## Build EKS Optimized AMI, default using AL2, use os_distro=al2023 for AL2023 AMI
-	$(MAKE) k8s $(shell hack/latest-binaries.sh $(k8s))
+	$(MAKE) k8s $(shell hack/latest-binaries.sh $(k8s) $(aws_region))
 
 .PHONY: fmt
 fmt: ## Format the source files
@@ -112,27 +112,27 @@ k8s: validate ## Build default K8s version of EKS Optimized AMI
 
 .PHONY: 1.23
 1.23: ## Build EKS Optimized AMI - K8s 1.23 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.23)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.23 $(aws_region))
 
 .PHONY: 1.24
 1.24: ## Build EKS Optimized AMI - K8s 1.24 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.24)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.24 $(aws_region))
 
 .PHONY: 1.25
 1.25: ## Build EKS Optimized AMI - K8s 1.25 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.25)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.25 $(aws_region))
 
 .PHONY: 1.26
 1.26: ## Build EKS Optimized AMI - K8s 1.26 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.26)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.26 $(aws_region))
 
 .PHONY: 1.27
 1.27: ## Build EKS Optimized AMI - K8s 1.27 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.27)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.27 $(aws_region))
 
 .PHONY: 1.28
 1.28: ## Build EKS Optimized AMI - K8s 1.28 - DEPRECATED: use the `k8s` variable instead
-	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.28)
+	$(MAKE) k8s $(shell hack/latest-binaries.sh 1.28 $(aws_region))
 
 .PHONY: lint-docs
 lint-docs: ## Lint the docs

doc/usage/overview.md

@@ -24,11 +24,11 @@ When building the AMI, binaries such as `kubelet`, `aws-iam-authenticator`, and
 It is recommended that the latest available binaries are used, as they may contain important fixes for bugs or security issues.
 The latest binaries can be discovered with the following script:
 ```bash
-hack/latest-binaries.sh $KUBERNETES_MINOR_VERSION
+hack/latest-binaries.sh $KUBERNETES_MINOR_VERSION $AWS_REGION
 ```
 This script will return the values for the binary-related AMI template variables, for example:
 ```bash
-> hack/latest-binaries.sh 1.28
+> hack/latest-binaries.sh 1.28 us-west-2
 
 kubernetes_version=1.28.1 kubernetes_build_date=2023-10-01
 ```

hack/latest-binaries.sh

@@ -4,16 +4,24 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-if [ "$#" -ne 1 ]; then
-  echo "usage: $0 KUBERNETES_MINOR_VERSION"
+if [ "$#" -ne 2 ]; then
+  echo "usage: $0 KUBERNETES_MINOR_VERSION REGION"
   exit 1
 fi
 
 MINOR_VERSION="${1}"
+REGION="${2}"
+
+S3_EXTRA_ARGS=""
+if [[ "${REGION}" == us-gov* || "${REGION}" == eusc* ]]; then
+  # Pull cross-partition from aws since there's no bucket in this partition.
+  S3_EXTRA_ARGS="--no-sign-request --region us-west-2"
+fi
 
 # retrieve the available "VERSION/BUILD_DATE" prefixes (e.g. "1.28.1/2023-09-14")
 # from the binary object keys, sorted in descending semver order, and pick the first one
-LATEST_BINARIES=$(aws s3api list-objects-v2 --bucket amazon-eks --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text | grep linux | cut -d'/' -f-2 | sort -Vru | head -n1)
+LATEST_BINARIES=$(aws s3api list-objects-v2 ${S3_EXTRA_ARGS} --bucket amazon-eks --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text \
+  | grep linux | cut -d'/' -f-2 | sort -Vru | head -n1)
 
 if [ "${LATEST_BINARIES}" == "None" ]; then
   echo >&2 "No binaries available for minor version: ${MINOR_VERSION}"

templates/al2023/provisioners/install-worker.sh

@@ -163,22 +163,23 @@ sudo mkdir -p /opt/cni/bin
 
 echo "Downloading binaries from: s3://$BINARY_BUCKET_NAME"
 AWS_DOMAIN=$(imds "/latest/meta-data/services/domain")
-S3_URL_BASE="https://$BINARY_BUCKET_NAME.s3.$BINARY_BUCKET_REGION.$AWS_DOMAIN/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 S3_PATH="s3://$BINARY_BUCKET_NAME/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 
 BINARIES=(
   kubelet
 )
 for binary in "${BINARIES[@]}"; do
-  if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-    echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary.sha256 .
-  else
-    echo "AWS cli missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
-    sudo wget $S3_URL_BASE/$binary
-    sudo wget $S3_URL_BASE/$binary.sha256
-  fi
+  FILES=(
+    "$binary"
+    "$binary.sha256"
+  )
+  for file in "${FILES[@]}"; do
+    if ! aws s3 cp --region $BINARY_BUCKET_REGION "$S3_PATH/$file" .; then
+      echo "Fetching ${file} from s3 failed, trying again with unauthenticated request."
+      aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION "$S3_PATH/$file" .
+    fi
+  done
+
   sudo sha256sum -c $binary.sha256
   sudo chmod +x $binary
   sudo chown root:root $binary
@@ -198,12 +199,9 @@ sudo systemctl enable ebs-initialize-bin@kubelet
 
 ECR_CREDENTIAL_PROVIDER_BINARY="ecr-credential-provider"
 
-if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-  echo "AWS cli present - using it to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
-  aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
-else
-  echo "AWS cli missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
-  sudo wget "$S3_URL_BASE/$ECR_CREDENTIAL_PROVIDER_BINARY"
+if ! aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .; then
+  echo "Fetching ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3 failed, trying again with unauthenticated request."
+  aws s3 cp --no-sign-request --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
 fi
 
 sudo chmod +x $ECR_CREDENTIAL_PROVIDER_BINARY

templates/al2023/variables-default.json

@@ -24,7 +24,7 @@
     "nvidia_driver_major_version": "580",
     "nvidia_repository_url": null,
     "nvidia_grid_runfile_bucket_name": "ec2-linux-nvidia-drivers",
-    "pause_container_image": "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/pause:3.10",
+    "pause_container_image": "public.ecr.aws/eks-distro/kubernetes/pause:3.10",
     "remote_folder": "/tmp",
     "runc_version": "*",
     "security_group_id": "",

templates/shared/runtime/bin/cache-pause-container

@@ -23,6 +23,12 @@ done
 PULL_ARGS=""
 if [[ "${PAUSE_CONTAINER_IMAGE}" == *"dkr.ecr"* ]]; then
   PULL_ARGS="${PULL_ARGS} --user AWS:$(aws ecr get-login-password)"
+elif [[ "${PAUSE_CONTAINER_IMAGE}" == *"public.ecr.aws"* ]]; then
+  if ECR_PUBLIC_PASSWORD=$(aws ecr-public get-login-password); then
+    PULL_ARGS="${PULL_ARGS} --user AWS:${ECR_PUBLIC_PASSWORD}"
+  else
+    echo "Failed to authenticate to public ECR, retrying with unauthenticated pull" >&2
+  fi
 fi
 sudo ctr --namespace k8s.io image pull ${PULL_ARGS} ${PAUSE_CONTAINER_IMAGE}
 sudo ctr --namespace k8s.io image tag ${PAUSE_CONTAINER_IMAGE} ${TAG}