test: allow testing frontends from different version

This allows running Dockerfile tests so that the Dockerfile
version and the BuildKit version are from different commits so
that we can test that old Dockerfile releases remain compatible
with the latest BuildKit.

The tests are based on the commit of the Dockerfile frontend as
we can't expect that new test would work on old frontends. In future
we might consider doing it the other way as well but then we need
a way to mark tests that can be ignored if they are not expected to
pass because of a new feature dependency.

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

.github/workflows/.test.yml

@@ -104,7 +104,6 @@ jobs:
       TESTFLAGS: "-v --parallel=6 --timeout=30m"
       GOTESTSUM_FORMAT: "standard-verbose"
       TEST_IMAGE_BUILD: "0"
-      TEST_IMAGE_ID: "buildkit-tests"
     strategy:
       fail-fast: false
       matrix:
@@ -161,7 +160,6 @@ jobs:
           targets: integration-tests
           set: |
             *.cache-from=type=gha,scope=${{ inputs.cache_scope }}
-            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
         env:
           BUILDKITD_TAGS: ${{ matrix.tags }}
           TEST_COVERAGE: 1

docker-bake.hcl

@@ -51,6 +51,24 @@ variable "TEST_COVERAGE" {
   default = null
 }
 
+variable "TEST_IMAGE_NAME" {
+  default = "buildkit-tests"
+}
+
+variable "TEST_CONTEXT" {
+  default = "."
+  description = "Context for building the test image"
+}
+
+variable "TEST_BINARIES_CONTEXT" {
+  default = TEST_CONTEXT
+  description = "Context for building the buildkitd for test image"
+}
+
+variable "BUILDKIT_SYNTAX" {
+  default = null
+}
+
 function "bindir" {
   params = [defaultdir]
   result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
@@ -117,12 +135,26 @@ target "integration-tests-base" {
   output = ["type=cacheonly"]
 }
 
+target "integration-tests-binaries" {
+  inherits = ["_common"]
+  target = "binaries"
+  context = TEST_BINARIES_CONTEXT
+}
+
 target "integration-tests" {
   inherits = ["integration-tests-base"]
   target = "integration-tests"
+  context = TEST_CONTEXT
+  contexts = TEST_CONTEXT != TEST_BINARIES_CONTEXT ? {
+    "binaries" = "target:integration-tests-binaries"
+  } : null
   args = {
     GOBUILDFLAGS = TEST_COVERAGE == "1" ? "-cover" : null
+    BUILDKIT_SYNTAX = BUILDKIT_SYNTAX
   }
+  output = [
+    "type=docker,name=${TEST_IMAGE_NAME}",
+  ]
 }
 
 group "validate" {

frontend/dockerfile/cmd/dockerfile-frontend/hack/release

@@ -104,6 +104,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "${outputFlag},name=$REPO:$pushTag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext
@@ -125,6 +126,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "$outputFlag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext

hack/images

@@ -112,5 +112,5 @@ if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
   nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless"
 fi
 
-buildxCmd build --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
+buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
   $currentcontext

hack/test

@@ -15,14 +15,15 @@ set -eu -o pipefail
 
 : "${TEST_COVERAGE=}"
 : "${TEST_IMAGE_BUILD=1}"
-: "${TEST_IMAGE_ID=buildkit-tests}"
+: "${TEST_IMAGE_NAME=buildkit-tests}"
 : "${TEST_INTEGRATION=}"
 : "${TEST_GATEWAY=}"
 : "${TEST_DOCKERFILE=}"
 : "${TEST_DOCKERD=}"
 : "${TEST_DOCKERD_BINARY=$(which dockerd)}"
 : "${TEST_REPORT_SUFFIX=}"
 : "${TEST_KEEP_CACHE=}"
+: "${TEST_SUITE_CONTEXT=}"
 : "${TESTFLAGS=}"
 
 : "${DOCKERFILE_RELEASES=}"
@@ -87,20 +88,15 @@ if [ "$TEST_COVERAGE" == "1" ]; then
   export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt"
 fi
 
+if [ -n "$TEST_SUITE_CONTEXT" ]; then
+  export TEST_BINARIES_CONTEXT=$currentcontext
+  # FIXME: something breaks with the syntax when using the context
+  export BUILDKIT_SYNTAX="docker/dockerfile:1.10.0"
+  currentcontext=$TEST_SUITE_CONTEXT
+fi
+
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
-  buildxCmd build $cacheFromFlags \
-    --build-arg ALPINE_VERSION \
-    --build-arg GO_VERSION \
-    --build-arg BUILDKITD_TAGS \
-    --build-arg HTTP_PROXY \
-    --build-arg HTTPS_PROXY \
-    --build-arg NO_PROXY \
-    --build-arg GOBUILDFLAGS \
-    --build-arg VERIFYFLAGS \
-    --build-arg CGO_ENABLED \
-    --target "integration-tests" \
-    --output "type=docker,name=$TEST_IMAGE_ID" \
-    $currentcontext
+  TEST_CONTEXT=$currentcontext buildxCmd bake integration-tests
 fi
 
 cacheVolume="buildkit-test-cache"
@@ -131,7 +127,7 @@ if [ "$TEST_INTEGRATION" == 1 ]; then
   cid=$(dockerCmd create $baseCreateFlags \
     ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \
     -e SKIP_INTEGRATION_TESTS \
-    $TEST_IMAGE_ID \
+    $TEST_IMAGE_NAME \
     gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
@@ -141,8 +137,8 @@ fi
 
 if [ "$TEST_GATEWAY" == 1 ]; then
   # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI
-  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_ID, which is a direct child of buildkit-base.
-  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_ID go build -v ./frontend/gateway/client)
+  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_NAME, which is a direct child of buildkit-base.
+  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_NAME go build -v ./frontend/gateway/client)
   dockerCmd start -a $cid
 fi
 
@@ -165,7 +161,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
     tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
 
     buildxCmd build $cacheFromFlags \
-      --build-arg "BUILDTAGS=$buildtags" \
+      --build-arg "BUILDTAGS=$buildtags" --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
       --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
       --output "type=oci,dest=$tarout" \
       $currentcontext
@@ -176,7 +172,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
           -e BUILDKIT_WORKER_RANDOM \
           -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \
           -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \
-          $TEST_IMAGE_ID \
+          $TEST_IMAGE_NAME \
           gotestsumandcover $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         dockerCmd cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then

hack/util

@@ -36,12 +36,11 @@ buildAttestFlags() {
   fi
 }
 
-currentref=""
 currentcontext="."
 cacheFromFlags=""
 cacheToFlags=""
 if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; then
-  currentref="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
+  currentcontext="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
   if [ -n "$CACHE_FROM" ]; then
     for cfrom in $CACHE_FROM; do
       if [[ $cfrom == *"type=gha"* ]]; then
@@ -69,9 +68,6 @@ if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ];
     done
   fi
 fi
-if [ -n "$currentref" ]; then
-  currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref"
-fi
 if [ -n "$CONTEXT" ]; then
   currentcontext=$CONTEXT
 fi