ci: ensure we always get the test results even if they fail

The previous strucuture meant that a failure skipped the result extraction.

I considered switching to using `-v` to mount the output dirs to the host, but
this would result in root owned files. Instead I split the `Makefile` target
into two parts which the CI can call independently, the original target remains
with the same overall behaviour for end user use.

Signed-off-by: Ian Campbell <[email protected]>

Author: Ian Campbell

Jenkinsfile

@@ -115,7 +115,8 @@ pipeline {
                                 sh 'docker load -i coverage-invocation-image.tar'
                             }
                             ansiColor('xterm') {
-                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage'
+                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage-run || true'
+                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage-results'
                             }
                             archiveArtifacts '_build/ci-cov/all.out'
                             archiveArtifacts '_build/ci-cov/coverage.html'
@@ -146,7 +147,8 @@ pipeline {
                                 sh 'docker load -i coverage-experimental-invocation-image.tar'
                             }
                             ansiColor('xterm') {
-                                sh 'make EXPERIMENTAL=on TEST_RESULTS_PREFIX="experimental-" -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage-experimental coverage'
+                                sh 'make EXPERIMENTAL=on TEST_RESULTS_PREFIX="experimental-" -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage-experimental coverage-run || true'
+                                sh 'make EXPERIMENTAL=on TEST_RESULTS_PREFIX="experimental-" -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage-experimental coverage-results'
                             }
                         }
                     }

Jenkinsfile.baguette

@@ -98,7 +98,8 @@ pipeline {
                                 sh 'docker load -i coverage-invocation-image.tar'
                             }
                             ansiColor('xterm') {
-                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage'
+                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage-run || true'
+                                sh 'make -f docker.Makefile BUILD_TAG=$BUILD_TAG-coverage coverage-results'
                             }
                             archiveArtifacts '_build/ci-cov/all.out'
                             archiveArtifacts '_build/ci-cov/coverage.html'

docker.Makefile

@@ -103,12 +103,15 @@ test-e2e: build_dev_image invocation-image ## run end-to-end tests
 	docker run -v /var/run:/var/run:ro --rm --network="host" $(DEV_IMAGE_NAME) make EXPERIMENTAL=$(EXPERIMENTAL) TEST_RESULTS_PREFIX=$(TEST_RESULTS_PREFIX) bin/$(BIN_NAME) test-e2e
 
 COV_LABEL := com.docker.app.cov-run=$(TAG)
-coverage: build_dev_image ## run tests with coverage
+coverage-run: build_dev_image ## run tests with coverage
 	@$(call mkdir,_build)
 	docker run -v /var/run:/var/run:ro --name $(COV_CTNR_NAME) --network="host" -t $(DEV_IMAGE_NAME) make COMMIT=${COMMIT} TAG=${TAG} EXPERIMENTAL=$(EXPERIMENTAL) TEST_RESULTS_PREFIX=$(TEST_RESULTS_PREFIX) coverage
+coverage-results:
 	docker cp $(COV_CTNR_NAME):$(PKG_PATH)/_build/cov/ ./_build/ci-cov
 	docker cp $(COV_CTNR_NAME):$(PKG_PATH)/_build/test-results/ ./_build/test-results
 	docker rm $(COV_CTNR_NAME)
+# coverage is split in two like this so that CI can extract the results even on failure (which will be detected via the junit) using the individual steps, but for end users running we want the overall failure.
+coverage: coverage-run coverage-results
 
 gradle-test:
 	tar cf - Dockerfile.gradle bin/docker-app-linux integrations/gradle | docker build -t $(GRADLE_IMAGE_NAME) -f Dockerfile.gradle -
@@ -165,4 +168,4 @@ push-invocation-image:
 help: ## this help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
 
-.PHONY: lint test-e2e test-unit test cli-cross cross e2e-cross coverage gradle-test shell build_dev_image tars vendor check-vendor schemas help invocation-image save-invocation-image save-invocation-image-tag push-invocation-image
+.PHONY: lint test-e2e test-unit test cli-cross cross e2e-cross coverage coverage-run coverage-results gradle-test shell build_dev_image tars vendor check-vendor schemas help invocation-image save-invocation-image save-invocation-image-tag push-invocation-image