build: kernel: force target arch on cross-built kernel docker image manifest

- our kernel builds are done in arch-independent Dockerfiles
- but those get the build-host's architecture, despite the contents being correct
- when locally developing on a kernel that is != host-arch
  - those get the host-arch in the image
  - but LinuxKit refuses to use it due to arch mismatch
- (when pushed to a registry, the arch info is discarded, and LK is ok with that)
- thus
  - introduce `ensure_docker_image_architecture(imagetag, arch)`
    - which just hacks at manifests via a docker save/docker load
  - call it from both default and armbian kernel builds

Signed-off-by: Ricardo Pardini <[email protected]>

Author: rpardini

bash/docker.sh

@@ -123,3 +123,70 @@ function produce_dockerfile_helper_apt_oras() {
 	DOWNLOAD_HELPER_SCRIPT
 	log debug "Created apt-oras helper script '${fn}'"
 }
+
+# A huge hack to force the architecture of a Docker image to a specific value.
+# This is required for the LinuxKit kernel images: LK expects them to have the correct arch, despite the
+#  actual contents always being the same. Docker's buildkit tags a locally built image with the host arch.
+# Thus change the host arch to the expected arch in the image's manifests via a dump/reimport.
+function ensure_docker_image_architecture() {
+	declare kernel_oci_image="$1"
+	declare expected_arch="$2"
+
+	# If the host arch is the same as the expected arch, no need to do anything
+	if [[ "$(uname -m)" == "${expected_arch}" ]]; then
+		log info "Host architecture is already ${expected_arch}, no need to rewrite Docker image ${kernel_oci_image}"
+		return 0
+	fi
+
+	log info "Rewriting Docker image ${kernel_oci_image} to architecture ${expected_arch}, wait..."
+
+	# Create a temporary directory, use mktemp
+	declare -g tmpdir
+	tmpdir="$(mktemp -d)"
+	log debug "Created temporary directory: ${tmpdir}"
+
+	# Export the image to a tarball
+	docker save -o "${tmpdir}/original.tar" "${kernel_oci_image}"
+
+	# Untag the hostarch image
+	docker rmi "${kernel_oci_image}"
+
+	# Create a working dir under the tmpdir
+	mkdir -p "${tmpdir}/working"
+
+	# Extract the tarball into the working dir
+	tar -xf "${tmpdir}/original.tar" -C "${tmpdir}/working"
+	log debug "Extracted tarball to ${tmpdir}/working"
+
+	# Remove the original tarball
+	rm -f "${tmpdir}/original.tar"
+
+	declare working_blobs_dir="${tmpdir}/working/blobs/sha256"
+
+	# Find all files under working_blobs_dir which are smaller than 2048 bytes
+	# Use mapfile to create an array of files
+	declare -a small_files
+	mapfile -t small_files < <(find "${working_blobs_dir}" -type f -size -2048c)
+	log debug "Found small blob files: ${small_files[*]}"
+
+	# Replace the architecture in each of the small files
+	for file in "${small_files[@]}"; do
+		log debug "Replacing architecture in ${file}"
+		sed -i "s|\"architecture\":\".....\"|\"architecture\":\"${expected_arch}\"|g" "${file}" # 🤮
+	done
+
+	# Create a new tarball with the modified files
+	tar -cf "${tmpdir}/modified.tar" -C "${tmpdir}/working" .
+	log debug "Created modified tarball: ${tmpdir}/modified.tar"
+
+	# Remove the working directory
+	rm -rf "${tmpdir}/working"
+
+	# Import the modified tarball back into the local cache
+	docker load -i "${tmpdir}/modified.tar"
+
+	# Remove the temporary directory, completely
+	rm -rf "${tmpdir}"
+
+	log info "Rewrote Docker image ${kernel_oci_image} to architecture ${expected_arch}."
+}

bash/kernel.sh

@@ -33,7 +33,7 @@ function kernel_build() {
 	log debug "Kernel build method: ${kernel_info[BUILD_FUNC]}"
 	"${kernel_info[BUILD_FUNC]}"
 
-	# Push it to the OCI registry
+	# Push it to the OCI registry; this discards the os/arch information that BuildKit generates
 	if [[ "${DO_PUSH:-"no"}" == "yes" ]]; then
 		log info "Kernel built; pushing to ${kernel_oci_image}"
 		docker push "${kernel_oci_image}"

bash/kernel/kernel_armbian.sh

@@ -25,7 +25,6 @@ function calculate_kernel_version_armbian() {
 	ARMBIAN_KERNEL_MAJOR_MINOR_POINT="$(echo -n "${ARMBIAN_KERNEL_VERSION}" | cut -d "-" -f 1)"
 	log info "ARMBIAN_KERNEL_MAJOR_MINOR_POINT: ${ARMBIAN_KERNEL_MAJOR_MINOR_POINT}"
 
-
 	# A helper script, as escaping bash into a RUN command in Dockerfile is a pain; included in input_hash later
 	declare dockerfile_helper_filename="undefined.sh"
 	produce_dockerfile_helper_apt_oras "kernel/" # will create the helper script in kernel/ directory; sets helper_name
@@ -99,8 +98,10 @@ function build_kernel_armbian() {
 	log info "Building armbian kernel from deb-tar at ${ARMBIAN_KERNEL_FULL_ORAS_REF_DEB_TAR}"
 	log info "Will build Dockerfile ${ARMBIAN_KERNEL_DOCKERFILE}"
 
-	# Build the Dockerfile; don't specify platform, our Dockerfile is multiarch, thus you can get build x86 kernels in arm64 hosts and vice-versa
+	# Don't specify platform, our Dockerfile is multiarch, thus you can build x86 kernels in arm64 hosts and vice-versa ...
 	docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" -t "${kernel_oci_image}" -f "${ARMBIAN_KERNEL_DOCKERFILE}" kernel
+	# .. but enforce the target arch for LK in the final image via dump/edit-manifests/reimport trick
+	ensure_docker_image_architecture "${kernel_oci_image}" "${kernel_info['DOCKER_ARCH']}"
 }
 
 function configure_kernel_armbian() {

bash/kernel/kernel_default.sh

@@ -131,9 +131,8 @@ function build_kernel_default() {
 	common_build_args_kernel_default
 	log info "Will build with: ${build_args[*]}"
 
-	(
-		cd kernel
-		docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" "${build_args[@]}" -t "${kernel_oci_image}" .
-	)
-
+	# Don't specify platform, our Dockerfile is multiarch, thus you can build x86 kernels in arm64 hosts and vice-versa ...
+	docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" "${build_args[@]}" -t "${kernel_oci_image}" -f kernel/Dockerfile kernel
+	# .. but enforce the target arch for LK in the final image via dump/edit-manifests/reimport trick
+	ensure_docker_image_architecture "${kernel_oci_image}" "${kernel_info['DOCKER_ARCH']}"
 }