Merge pull request moby#3666 from crazy-max/bake-binaries

hack: binaries and cross bake targets

Author: crazy-max

.github/workflows/buildkit.yml

@@ -26,6 +26,7 @@ env:
   SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest"
   IMAGE_NAME: "moby/buildkit"
   PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64"
+  DESTDIR: "./bin"
 
 jobs:
   test:
@@ -59,6 +60,9 @@ jobs:
       push: ${{ steps.prep.outputs.push }}
       platforms: ${{ steps.prep.outputs.platforms }}
     steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
       -
         name: Prepare
         id: prep
@@ -79,10 +83,10 @@ jobs:
           fi
           echo "tag=${TAG}" >>${GITHUB_OUTPUT}
           echo "push=${PUSH}" >>${GITHUB_OUTPUT}
-          platforms=$(jq -c -n --argjson str '"${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64"' '$str|split(",")')
+          platforms=$(docker buildx bake release --print | jq -cr '.target."release".platforms')
           echo "platforms=$platforms" >>${GITHUB_OUTPUT}
 
-  cross:
+  binaries:
     runs-on: ubuntu-20.04
     needs:
       - prepare
@@ -99,6 +103,8 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       -
         name: Expose GitHub Runtime
         uses: crazy-max/ghaction-github-runtime@v2
@@ -115,18 +121,18 @@ jobs:
       -
         name: Build
         run: |
-          ./hack/release-tar "${{ needs.prepare.outputs.tag }}" release-out
+          make release
         env:
           RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
           PLATFORMS: ${{ matrix.platform }}
-          CACHE_FROM: type=gha,scope=cross-${{ env.PLATFORM_PAIR }}
-          CACHE_TO: type=gha,scope=cross-${{ env.PLATFORM_PAIR }}
+          CACHE_FROM: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }}
+          CACHE_TO: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }}
       -
         name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           name: buildkit
-          path: ./release-out/*
+          path: ${{ env.DESTDIR }}/*
           if-no-files-found: error
 
   image:
@@ -179,19 +185,19 @@ jobs:
     needs:
       - prepare
       - test
-      - cross
+      - binaries
       - image
     steps:
       -
         name: Download artifacts
         uses: actions/download-artifact@v3
         with:
           name: buildkit
-          path: ./release-out/*
+          path: ${{ env.DESTDIR }}
       -
         name: List artifacts
         run: |
-          tree -nh ./release-out/
+          tree -nh ${{ env.DESTDIR }}
       -
         name: GitHub Release
         if: startsWith(github.ref, 'refs/tags/v')
@@ -200,5 +206,5 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           draft: true
-          files: ./release-out/*
+          files: ${{ env.DESTDIR }}/*
           name: ${{ needs.prepare.outputs.tag }}

.gitignore

@@ -3,6 +3,5 @@
 #  please consider a global .gitignore https://help.github.com/articles/ignoring-files
 bin
 coverage
-release-out
 .certs
 .tmp

Makefile

@@ -5,7 +5,11 @@ export BUILDX_CMD ?= docker buildx
 
 .PHONY: binaries
 binaries:
-	hack/binaries
+	$(BUILDX_CMD) bake binaries
+
+.PHONY: cross
+cross:
+	$(BUILDX_CMD) bake binaries-cross
 
 .PHONY: images
 images:
@@ -16,7 +20,11 @@ images:
 .PHONY: install
 install:
 	mkdir -p $(DESTDIR)$(bindir)
-	install bin/* $(DESTDIR)$(bindir)
+	install bin/build/* $(DESTDIR)$(bindir)
+
+.PHONY: release
+release:
+	./hack/release
 
 .PHONY: clean
 clean:

docker-bake.hcl

@@ -10,6 +10,19 @@ variable "NODE_VERSION" {
   default = null
 }
 
+variable "BUILDKITD_TAGS" {
+  default = null
+}
+
+# Defines the output folder
+variable "DESTDIR" {
+  default = ""
+}
+function "bindir" {
+  params = [defaultdir]
+  result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
+}
+
 target "_common" {
   args = {
     ALPINE_VERSION = ALPINE_VERSION
@@ -19,6 +32,42 @@ target "_common" {
   }
 }
 
+group "default" {
+  targets = ["binaries"]
+}
+
+target "binaries" {
+  inherits = ["_common"]
+  target = "binaries"
+  args = {
+    BUILDKITD_TAGS = BUILDKITD_TAGS
+  }
+  output = [bindir("build")]
+}
+
+target "binaries-cross" {
+  inherits = ["binaries"]
+  output = [bindir("cross")]
+  platforms = [
+    "darwin/amd64",
+    "darwin/arm64",
+    "linux/amd64",
+    "linux/arm/v7",
+    "linux/arm64",
+    "linux/s390x",
+    "linux/ppc64le",
+    "linux/riscv64",
+    "windows/amd64",
+    "windows/arm64"
+  ]
+}
+
+target "release" {
+  inherits = ["binaries-cross"]
+  target = "release"
+  output = [bindir("release")]
+}
+
 group "validate" {
   targets = ["lint", "validate-vendor", "validate-doctoc", "validate-generated-files", "validate-shfmt"]
 }

hack/binaries

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+: "${GITHUB_ACTIONS=}"
+: "${GITHUB_REPOSITORY=}"
+: "${GITHUB_RUN_ID=}"
+
+: "${BUILDX_CMD=docker buildx}"
+: "${DESTDIR=./bin/release}"
+: "${CACHE_FROM=}"
+: "${CACHE_TO=}"
+: "${RELEASE=false}"
+: "${PLATFORMS=}"
+
+if [ -n "$CACHE_FROM" ]; then
+  for cfrom in $CACHE_FROM; do
+    setFlags+=(--set "*.cache-from=$cfrom")
+  done
+fi
+if [ -n "$CACHE_TO" ]; then
+  for cto in $CACHE_TO; do
+    setFlags+=(--set "*.cache-to=$cto")
+  done
+fi
+if [ -n "$PLATFORMS" ]; then
+  setFlags+=(--set "*.platform=$PLATFORMS")
+fi
+if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then
+  prvattrs="mode=max"
+  if [ "$GITHUB_ACTIONS" = "true" ]; then
+    prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+  fi
+  setFlags+=(--set "*.attest=type=sbom")
+  setFlags+=(--set "*.attest=type=provenance,$prvattrs")
+fi
+if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
+  setFlags+=(--set "*.no-cache-filter=git,gobuild-base")
+fi
+
+output=$(mktemp -d -t buildkit-output.XXXXXXXXXX)
+
+(
+  set -x
+  ${BUILDX_CMD} bake "${setFlags[@]}" --set "*.args.BUILDKIT_MULTI_PLATFORM=true" --set "*.output=$output" release
+)
+
+for pdir in "${output}"/*/; do
+  (
+    cd "$pdir"
+    releasetar=$(find . -name '*.tar.gz')
+    filename=$(basename "${releasetar%.tar.gz}")
+    mv "provenance.json" "${filename}.provenance.json"
+    mv "sbom-binaries.spdx.json" "${filename}.sbom.json"
+    find . -name 'sbom*.json' -exec rm {} \;
+  )
+done
+
+mkdir -p "$DESTDIR"
+mv "$output"/**/* "$DESTDIR/"
+rm -rf "$output"