Merge pull request #25 from gitpod-io/rl/add-integration-tests

Add integration tests

Author: rl-gitpod

.github/workflows/build.yaml

@@ -62,6 +62,13 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - build-go
+    services:
+      # Local registry
+      registr:
+        # Docker Hub image
+        image: registry:2
+        ports:
+          - 5000:5000
     steps:
       - name: Create checkout directory
         run: mkdir -p ~/go/src/github.com/gitpod-io
@@ -81,7 +88,25 @@ jobs:
       - name: Download and vendor all required packages
         run: |
           go mod download
+      - name: Build the test runner
+        run: pushd pkg/test/runner; ./build.sh; popd
+      - name: Install the latest buildkit release
+        run: |
+          BUILDKIT_URL="$(curl -sL https://api.github.com/repos/moby/buildkit/releases \
+            | jq -r 'map(select(.name|startswith("v")))|sort_by(.name)[-1].assets[]|select(.name|endswith(".linux-amd64.tar.gz")).browser_download_url')"
+          curl -L "${BUILDKIT_URL}" | sudo tar -xz -C /usr/local
+      - name: Start buildkit daemon
+        run: |
+          sudo --non-interactive --shell <<END_SUDO
+            install -d -m 0750 -o root -g docker /run/buildkit
+            buildkitd &
+            while ! test -S /run/buildkit/buildkitd.sock; do sleep 0.1; done
+            chgrp docker /run/buildkit/buildkitd.sock
+          END_SUDO
       - name: Run all unit tests
+        env:
+          BUILDKIT_ADDR: unix:///run/buildkit/buildkitd.sock
+          TARGET_REF: 127.0.0.1:5000
         run: go test -v -coverprofile=coverage.out $(go list ./...)
       - name: Generate code coverage artifacts
         uses: actions/upload-artifact@v2

.vscode/launch.json

@@ -27,6 +27,21 @@
             "args": [
                 "eyJEZXNjIjoiaXQgc2hvdWxkIGhhdmUgR28gaW4gdmVyc2lvbiAxLjEzIiwiU2tpcCI6ZmFsc2UsIlVzZXIiOiIiLCJDb21tYW5kIjpbImdvIiwidmVyc2lvbiJdLCJFbnRyeXBvaW50IjpudWxsLCJFbnYiOm51bGwsIkFzc2VydGlvbnMiOlsic3Rkb3V0LmluZGV4T2YoXCJnbzEuMTFcIikgIT0gLTEiXX0="
             ]
+        },
+        {
+            "name": "Launch build integration test",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${fileWorkspaceFolder}/pkg/dazzle/build_test.go",
+            "args": [
+                "-test.run",
+                "TestProjectChunk_test_integration"
+            ],
+            "env": {
+                "BUILDKIT_ADDR": "unix:///run/buildkit/buildkitd.sock",
+                "TARGET_REF": "127.0.0.1:5000",
+            }
         }
     ]
 }

README.md

@@ -204,3 +204,15 @@ They can be extracted using jq e.g.:
 $ go run pkg/test/runner/main.go eyJEZXNjIjoiaXQgc2hvdWxkIGhhdmUgR28gaW4gdmVyc2lvbiAxLjEzIiwiU2tpcCI6ZmFsc2UsIlVzZXIiOiIiLCJDb21tYW5kIjpbImdvIiwidmVyc2lvbiJdLCJFbnRyeXBvaW50IjpudWxsLCJFbnYiOm51bGwsIkFzc2VydGlvbnMiOlsic3Rkb3V0LmluZGV4T2YoXCJnbzEuMTFcIikgIT0gLTEiXX0= | jq -r '.Stdout | @base64d'
 go version go1.16.4 linux/amd64
 ```
+
+### Integration tests
+There is an integration test for the build command in pkg/dazzle/build_test.go - TestProjectChunk_test_integration and a shell script to run it. 
+The integration test does an end-to-end check along with editing a test and re-running to ensure only the test image is updated.
+
+It requires a running Buildkitd instance at unix:///run/buildkit/buildkitd.sock and a docker registry on 127.0.0.1:5000 (i.e. as this workspace is setup on startup).
+
+Override the env vars BUILDKIT_ADDR and TARGET_REF as required prior to running in a different environment.
+
+```bash
+$ ./integration_tests.sh
+```

integration_tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the integration tests
+# NOTE: the TARGET_REF is hostname:port 
+# shellcheck disable=SC2034
+BUILDKIT_ADDR=unix:///run/buildkit/buildkitd.sock TARGET_REF=127.0.0.1:5000 go test -count 1 -run ^Test.*_integration$ github.com/csweichel/dazzle/pkg/dazzle -v

pkg/dazzle/build.go

@@ -168,7 +168,7 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error {
 	session.baseBuildFinished(absbaseref, basemf, basecfg)
 
 	for _, chk := range p.Chunks {
-		_, err := chk.test(ctx, session)
+		_, _, err := chk.test(ctx, session)
 		if err != nil {
 			return fmt.Errorf("cannot build chunk %s: %w", chk.Name, err)
 		}
@@ -546,52 +546,52 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se
 	return resref, nil
 }
 
-func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, err error) {
+func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, didRun bool, err error) {
 	if sess == nil {
-		return false, errors.New("cannot test without a session")
+		return false, false, errors.New("cannot test without a session")
 	}
 	if sess.opts.NoTests || len(p.Tests) == 0 {
-		return true, nil
+		return true, false, nil
 	}
 
 	resultRef, err := p.ImageName(imageTypeTestResult, sess)
 	if err != nil {
-		return false, err
+		return false, false, err
 	}
 	r, err := pullTestResult(ctx, sess.opts.Registry, resultRef)
 	if err != nil && !errdefs.IsNotFound(err) {
-		return false, err
+		return false, false, err
 	}
 	if r != nil && r.Passed {
 		// tests have run before and have passed
-		return true, nil
+		return true, false, nil
 	}
 
 	// build temp image for testing
 	testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess)
 	if err != nil {
-		return false, err
+		return false, false, err
 	}
 
 	_, _, imgcfg, err := getImageMetadata(ctx, testRef, sess.opts.Registry)
 	if err != nil {
-		return false, err
+		return false, false, err
 	}
 
 	log.WithField("chunk", p.Name).Warn("running tests")
 	executor := buildkit.NewExecutor(sess.Client, testRef.String(), imgcfg)
 	_, ok = test.RunTests(ctx, executor, p.Tests)
 	if !ok {
-		return false, fmt.Errorf("%s: tests failed", p.Name)
+		return false, true, fmt.Errorf("%s: tests failed", p.Name)
 	}
 
 	// tests have passed - mark them as such
 	_, err = pushTestResult(ctx, sess.opts.Registry, resultRef, StoredTestResult{true})
 	if err != nil && !errdefs.IsAlreadyExists(err) {
-		return true, err
+		return true, true, err
 	}
 
-	return true, nil
+	return true, true, nil
 }
 
 func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef reference.NamedTagged, didBuild bool, err error) {