Merge pull request #1231 from buildkite/pdp-1788-fix-qemu-binfmt-image-is-pulled-during-elastic-ci-stack

Author: triarius

packer/linux/conf/bin/bk-configure-docker.sh

@@ -25,11 +25,31 @@ trap '[[ $? = 0 ]] && on_exit' EXIT
 # See https://alestic.com/2010/12/ec2-user-data-output/
 exec > >(tee -a /var/log/elastic-stack.log | logger -t user-data -s 2>/dev/console) 2>&1
 
+echo "Starting ${BASH_SOURCE[0]}..."
 
-echo Reading variables from AMI creation...
+echo Sourcing /usr/local/lib/bk-configure-docker.sh...
+echo This file is written by the scripts in packer/scripts.
+echo Note that the path is /usr/local/lib, not /usr/local/bin.
+echo Contents of /usr/local/lib/bk-configure-docker.sh:
+cat /usr/local/lib/bk-configure-docker.sh
 # shellcheck disable=SC1091
 source /usr/local/lib/bk-configure-docker.sh
 
+echo Installing qemu binfmt for multiarch...
+if ! docker run \
+  --privileged \
+  --userns=host \
+  --pull=never \
+  --rm \
+  "tonistiigi/binfmt@${QEMU_BINFMT_DIGEST}" \
+    --install all
+then
+  echo Failed to install binfmt.
+  echo Avaliable docker images:
+  docker image ls
+  exit 1
+fi
+
 if [[ "${DOCKER_USERNS_REMAP:-false}" == "true" ]]; then
   echo Configuring user namespace remapping...
 
@@ -72,17 +92,11 @@ cat <<<"$(jq \
   /etc/docker/daemon.json \
 )" >/etc/docker/daemon.json
 
-# See https://docs.docker.com/build/building/multi-platform/
-echo Installing qemu binfmt for multiarch...
-docker run \
-  --privileged \
-  --userns=host \
-  --rm \
-  "tonistiigi/binfmt:${QEMU_BINFMT_TAG}" \
-    --install all
-
 echo Cleaning up docker images...
 systemctl start docker-low-disk-gc.service
 
+echo Enabling docker-gc timers...
+systemctl enable docker-gc.timer docker-low-disk-gc.timer
+
 echo Restarting docker daemon...
 systemctl restart docker

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -41,6 +41,8 @@ trap '[[ $? = 0 ]] && on_exit' EXIT
 # See https://alestic.com/2010/12/ec2-user-data-output/
 exec > >(tee -a /var/log/elastic-stack.log | logger -t user-data -s 2>/dev/console) 2>&1
 
+echo "Starting ${BASH_SOURCE[0]}..."
+
 # This needs to happen first so that the error reporting works
 token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" --fail --silent --show-error --location http://169.254.169.254/latest/api/token)
 INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location http://169.254.169.254/latest/meta-data/instance-id)

packer/linux/conf/bin/bk-mount-instance-storage.sh

@@ -22,6 +22,8 @@ trap '[[ $? = 0 ]] && on_exit' EXIT
 # See https://alestic.com/2010/12/ec2-user-data-output/
 exec > >(tee -a /var/log/elastic-stack.log | logger -t user-data -s 2>/dev/console) 2>&1
 
+echo "Starting ${BASH_SOURCE[0]}..."
+
 # Mount instance storage if we can
 # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html
 

packer/linux/conf/docker/systemd/docker-gc.service

@@ -5,6 +5,3 @@ Wants=docker-gc.timer
 [Service]
 Type=oneshot
 ExecStart=/usr/local/bin/docker-gc
-
-[Install]
-WantedBy=multi-user.target

packer/linux/conf/docker/systemd/docker-low-disk-gc.service

@@ -5,6 +5,3 @@ Wants=docker-low-disk-gc.timer
 [Service]
 Type=oneshot
 ExecStart=/usr/local/bin/docker-low-disk-gc
-
-[Install]
-WantedBy=multi-user.target

packer/linux/scripts/install-docker.sh

@@ -20,8 +20,6 @@ echo "Adding docker systemd timers..."
 sudo cp /tmp/conf/docker/scripts/* /usr/local/bin
 sudo cp /tmp/conf/docker/systemd/docker-* /etc/systemd/system
 sudo chmod +x /usr/local/bin/docker-*
-sudo systemctl daemon-reload
-sudo systemctl enable docker-gc.timer docker-low-disk-gc.timer
 
 echo "Installing docker buildx..."
 DOCKER_CLI_DIR=/usr/libexec/docker/cli-plugins
@@ -47,11 +45,22 @@ sudo cp /tmp/conf/bin/docker-compose /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 docker-compose version
 
+# Writing QEMU container version info to /usr/local/lib/bk-configure-docker.sh.
+# We only pull this image when we build the AMI. It will be run in
+# /usr/local/bin/bk-configure-docker.sh, but it needs to know the image digest
+# to make sure it does not pull in another image instead.
+# NOTE: the executable file is in /usr/local/bin and it sources as file of the
+# same name in /usr/local/lib. These are not the same file.
 # See https://docs.docker.com/build/building/multi-platform/
+
+echo Contents of /usr/local/lib/bk-configure-docker.sh:
+cat <<'EOF' | sudo tee -a /usr/local/lib/bk-configure-docker.sh
 QEMU_BINFMT_VERSION=7.0.0-28
 QEMU_BINFMT_DIGEST=sha256:66e11bea77a5ea9d6f0fe79b57cd2b189b5d15b93a2bdb925be22949232e4e55
 QEMU_BINFMT_TAG="qemu-v${QEMU_BINFMT_VERSION}@${QEMU_BINFMT_DIGEST}"
+EOF
+# shellcheck disable=SC1091
+source /usr/local/lib/bk-configure-docker.sh
 sudo mkdir -p /usr/local/lib
-echo "QEMU_BINFMT_TAG=\"$QEMU_BINFMT_TAG\"" | sudo tee -a /usr/local/lib/bk-configure-docker.sh
 echo Pulling qemu binfmt for multiarch...
 sudo docker pull "tonistiigi/binfmt:${QEMU_BINFMT_TAG}"