feat: fixed various issues in the ARM64 build pipeline (#1198)

Author: michaelawyu

.github/workflows/build-publish-mcr.yml

@@ -14,7 +14,7 @@ permissions:
   contents: read
 
 env:
-  # `public` indicates images to MCR wil be publicly available, and will be removed in the final MCR images
+  # `public` indicates images to MCR will be publicly available, and will be removed in the final MCR images
   REGISTRY_REPO: public/aks/fleet
   ARC_REGISTRY_REPO: public/microsoft.fleetmember
 
@@ -106,7 +106,7 @@ jobs:
           MCS_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }}
           MEMBER_NET_CONTROLLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.fleet_networking_version }}
           REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.ARC_REGISTRY_REPO}}
-  
+
   publish-images-arm64:
     runs-on:
       labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu-arm64"]
@@ -116,36 +116,70 @@ jobs:
         with:
           ref: ${{ needs.prepare-variables.outputs.release_tag }}
       - name: 'Install the Azure CLI'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Azure CLI installed by default;
+        # install it manually here.
         run:
           curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      - name: 'Set up build essential meta package'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have the common build
+        # tools (e.g., make) installed by default; install the build-essential meta package to set them up.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+      - name: 'Set up Docker'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Docker installed by default,
+        # and cannot have Docker installed via the docker/setup-docker-action Github Action, hence the manual setup
+        # steps here.
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
       - name: 'Login the ACR'
         run: |
-          az login --identity 
+          az login --identity
           az acr login -n ${{ secrets.AZURE_REGISTRY }}
+      - name: 'Verify Docker CLI'
+        # Note (chenyu1): the Docker installation has to be invoked with root privileges by default; for
+        # simplicity reasons in this pipeline we will make no attempt to enable rootless Docker usage.
+        run: |
+          sudo docker version
+          sudo docker info
       - name: Build and publish hub-agent
+        # Note (chenyu1): must preserve the environment here.
         run: |
-          make docker-build-hub-agent
+          sudo -E make docker-build-hub-agent
         env:
           HUB_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }}-arm64
           REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}
           TARGET_ARCH: arm64
       - name: Build and publish member-agent
+        # Note (chenyu1): must preserve the environment here.
         run: |
-          make docker-build-member-agent
+          sudo -E make docker-build-member-agent
         env:
           MEMBER_AGENT_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }}-arm64
           REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}
-          TARGET_ARCH: linux/arm64
+          TARGET_ARCH: arm64
       - name: Build and publish refresh-token
+        # Note (chenyu1): must preserve the environment here.
         run: |
-          make docker-build-refresh-token
+          sudo -E make docker-build-refresh-token
         env:
           REFRESH_TOKEN_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }}-arm64
           REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}
           TARGET_ARCH: arm64
       - name: Build and publish crd-installer
+        # Note (chenyu1): must preserve the environment here.
         run: |
-          make docker-build-crd-installer
+          sudo -E make docker-build-crd-installer
         env:
           CRD_INSTALLER_IMAGE_VERSION: ${{ needs.prepare-variables.outputs.release_tag }}-arm64
           REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}

Makefile

@@ -300,10 +300,24 @@ push:
 	$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token docker-build-crd-installer
 
 # By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error
+#
+# Note (chenyu1): the step below sets up emulation for building/running non-native binaries on the host. The original
+# setup assumes that the Makefile is always run on an x86_64 platform, and adds support for non-x86_64 hosts. Here
+# we keep the original setup if the build target is x86_64 platforms (default) for compatibility reasons, but will switch to
+# a more general setup for non-x86_64 hosts.
+#
+# On some systems the emulation setup might not work at all (e.g., macOS on Apple Silicon -> Rosetta 2 will be used 
+# by Docker Desktop as the default emulation option for AMD64 on ARM64 container compatibility).
 .PHONY: docker-buildx-builder
 docker-buildx-builder:
 	@if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \
-		docker run --rm --privileged mcr.microsoft.com/mirror/docker/multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \
+		if [ "$(TARGET_ARCH)" = "amd64" ] ; then \
+			echo "The target is an x86_64 platform; setting up emulation for other known architectures"; \
+			docker run --rm --privileged mcr.microsoft.com/mirror/docker/multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \
+		else \
+			echo "Setting up emulation for known architectures"; \
+			docker run --rm --privileged tonistiigi/binfmt --install all; \
+		fi ;\
 		docker buildx create --driver-opt image=mcr.microsoft.com/oss/v2/moby/buildkit:$(BUILDKIT_VERSION) --name $(BUILDX_BUILDER_NAME) --use; \
 		docker buildx inspect $(BUILDX_BUILDER_NAME) --bootstrap; \
 	fi