Merge pull request #391 from sapcc/docker-test-target

Add docker test target which can be easily run in concourse

Author: majewsky

README.md

@@ -170,6 +170,8 @@ Since only entire packages (not single source files) can be selected for coverag
 ```yaml
 dockerfile:
   enabled: true
+  checkEnv:
+    - CHECK_SKIPS_FUNCTIONAL_TEST=true
   entrypoint: [ "/bin/bash", "--", "--arg" ]
   extraBuildStages:
     - |
@@ -204,6 +206,7 @@ To ensure that the resulting Docker Image is functional, tests should be run bef
 As an additional smoke test, the compiled binaries are invoked with the `--version` argument after being copied to the final image.
 With [go-api-declarations](https://github.com/sapcc/go-api-declarations)'s [`bininfo.HandleVersionArgument` function](https://pkg.go.dev/github.com/sapcc/go-api-declarations/bininfo#HandleVersionArgument), this can be implemented in one line. If you are using Cobra or any other library to handle arguments, the [`bininfo.Version` function](https://pkg.go.dev/github.com/sapcc/go-api-declarations/bininfo#Version) is recommended instead.
 
+* `checkEnv` sets environment variables for `make check` inside the docker check build.
 * `entrypoint` allows overwriting the final entrypoint.
 * `extraBuildStages` prepends additional build stages at the top of the Dockerfile. This is useful for bringing in precompiled assets from other images, or if a non-Go compilation step is required.
 * `extraBuildPackages` installs extra Alpine packages in the Docker layer where `make install` is executed. We always install `ca-certificates`, `gcc`, `git`, `make` and `musl-dev`.

internal/core/config.go

@@ -240,6 +240,7 @@ type RenovateConfig struct {
 // DockerfileConfig appears in type Configuration.
 type DockerfileConfig struct {
 	Enabled              bool     `yaml:"enabled"`
+	CheckEnv             []string `yaml:"checkEnv"`
 	Entrypoint           []string `yaml:"entrypoint"`
 	ExtraBuildDirectives []string `yaml:"extraBuildDirectives"`
 	ExtraBuildPackages   []string `yaml:"extraBuildPackages"`

internal/dockerfile/Dockerfile.tmpl

@@ -26,6 +26,41 @@ RUN {{ if .UseBuildKit }}--mount=type=cache,target=/go/pkg/mod \
 {{ end -}}
 ################################################################################
 
+# To only build the tests run: docker build . --target test
+# We can't do `FROM builder AS test` here, as then make prepare-static-check would not be cached during interactive use when developing
+# and caching all the tools, especially golangci-lint, takes a few minutes.
+FROM {{ .DockerHubMirror }}golang:{{ .Constants.DefaultGoVersion }}-alpine{{ .Constants.DefaultAlpineImage }} AS test
+
+COPY Makefile /src/Makefile
+
+# used below by USER
+RUN addgroup -g 4200 appgroup \
+  && adduser -h /home/appuser -s /sbin/nologin -G appgroup -D -u 4200 appuser
+
+RUN apk add --no-cache --no-progress git make {{- range .ExtraTestPackages }} {{.}}{{ end }} \
+{{- if .ReuseEnabled }}
+  && pip3 install --break-system-packages reuse \
+{{- end }}
+  && make -C /src prepare-static-check
+
+
+# We only copy here because we want the "prepare-static-check" to be cacheable.
+# It is not a problem that we are overwriting the go cache from the earlier steps because we do not need to rebuild those tools.
+COPY --from=builder /go /go
+COPY --from=builder /src /src
+
+RUN {{ range $dcfg.CheckEnv }}{{ . }} {{ end }}make -C /src static-check
+
+# Some things like postgres do not like to run as root. For simplicity, just always run as an unprivileged user,
+# but for it to be able to read the go cache, we need to allow it.
+RUN chmod 777 -R /src/
+USER 4200:4200
+RUN cd /src \
+  && git config --global --add safe.directory /src \
+  && {{ range $dcfg.CheckEnv }}{{ . }} {{ end }}make build/cover.out
+
+################################################################################
+
 FROM {{ .DockerHubMirror }}alpine:{{ .Constants.DefaultAlpineImage }}
 
 {{ if not $dcfg.RunAsRoot -}}

internal/dockerfile/docker.go

@@ -12,6 +12,7 @@ import (
 	"github.com/sapcc/go-bits/must"
 
 	"github.com/sapcc/go-makefile-maker/internal/core"
+	"github.com/sapcc/go-makefile-maker/internal/golang"
 	"github.com/sapcc/go-makefile-maker/internal/util"
 )
 
@@ -22,7 +23,7 @@ var (
 	dockerignoreTemplate string
 )
 
-func RenderConfig(cfg core.Configuration) {
+func RenderConfig(cfg core.Configuration, sr golang.ScanResult) {
 	// if there is an entrypoint configured use that otherwise fallback to the first binary name
 	var entrypoint string
 	if len(cfg.Dockerfile.Entrypoint) > 0 {
@@ -70,19 +71,36 @@ func RenderConfig(cfg core.Configuration) {
 		dockerHubMirror = "keppel.eu-de-1.cloud.sap/ccloud-dockerhub-mirror/library/"
 	}
 
+	var extraTestPackages []string
+	reuseEnabled := cfg.Reuse.Enabled.UnwrapOr(true)
+	if reuseEnabled {
+		extraTestPackages = append(extraTestPackages, "py3-pip")
+	}
+	if sr.UsesPostgres {
+		extraTestPackages = append(extraTestPackages, "postgresql")
+	}
+
 	must.Succeed(util.WriteFileFromTemplate("Dockerfile", dockerfileTemplate, map[string]any{
 		"Config": cfg,
 		"Constants": map[string]any{
 			"DefaultGoVersion":   core.DefaultGoVersion,
 			"DefaultAlpineImage": core.DefaultAlpineImage,
 		},
 		"DockerHubMirror":    dockerHubMirror,
+		"CheckEnv":           cfg.Dockerfile.CheckEnv,
+		"ExtraTestPackages":  extraTestPackages,
 		"Entrypoint":         entrypoint,
+		"ReuseEnabled":       reuseEnabled,
 		"RunCommands":        strings.Join(commands, " \\\n  && "),
 		"RunVersionCommands": strings.Join(runVersionCommands, " \\\n  && "),
 		"UseBuildKit":        cfg.Dockerfile.UseBuildKit,
 	}))
+
+	ignores := cfg.Dockerfile.ExtraIgnores
+	if sr.UsesPostgres {
+		ignores = append(ignores, "/.testdb/")
+	}
 	must.Succeed(util.WriteFileFromTemplate(".dockerignore", dockerignoreTemplate, map[string]any{
-		"ExtraIgnores": cfg.Dockerfile.ExtraIgnores,
+		"ExtraIgnores": ignores,
 	}))
 }

internal/dockerfile/dockerignore.tmpl

@@ -7,13 +7,10 @@
 # TODO: uncomment when applications no longer use git to get version information
 #.git/
 /.github/
-/.gitignore
-/.golangci.yaml
 /.goreleaser.yml
 /.vscode/
 /CONTRIBUTING.md
 /Dockerfile
-/LICENSE*
 /Makefile.maker.yaml
 /README.md
 /build/

internal/makefile/makefile.go

@@ -42,6 +42,7 @@ func newMakefile(cfg core.Configuration, sr golang.ScanResult) *makefile {
 	}
 
 	isSAPCC := cfg.Metadata.IsSAPProject()
+	reuseEnabled := cfg.Reuse.Enabled.UnwrapOr(true)
 
 	///////////////////////////////////////////////////////////////////////////
 	// General
@@ -153,24 +154,26 @@ endif
 		})
 		prepareStaticRecipe = append(prepareStaticRecipe, "install-addlicense")
 
-		prepare.addRule(rule{
-			description: "Install reuse required by license-headers/check-reuse",
-			phony:       true,
-			target:      "install-reuse",
-			recipe: []string{
-				`@if ! hash reuse 2>/dev/null; then` +
-					` if ! hash pipx 2>/dev/null; then` +
-					` printf "\e[1;31m>> You are required to manually intervene to install reuse as go-makefile-maker cannot automatically resolve installing reuse on all setups.\e[0m\n";` +
-					` printf "\e[1;31m>> The preferred way for go-makefile-maker to install python tools after nix-shell is pipx which could not be found. Either install pipx using your package manager or install reuse using your package manager if at least version 6 is available.\e[0m\n";` +
-					` printf "\e[1;31m>> As your Python was likely installed by your package manager, just doing pip install --user sadly does no longer work as pip issues a warning about breaking your system. Generally running --break-system-packages with --user is safe to do but you should only run this command if you can resolve issues with it yourself: pip3 install --user --break-system-packages reuse\e[0m\n";` +
-					` else` +
-					` printf "\e[1;36m>> Installing reuse...\e[0m\n";` +
-					` pipx install reuse;` +
-					` fi;` +
-					` fi`,
-			},
-		})
-		prepareStaticRecipe = append(prepareStaticRecipe, "install-reuse")
+		if reuseEnabled {
+			prepare.addRule(rule{
+				description: "Install reuse required by license-headers/check-reuse",
+				phony:       true,
+				target:      "install-reuse",
+				recipe: []string{
+					`@if ! hash reuse 2>/dev/null; then` +
+						` if ! hash pipx 2>/dev/null; then` +
+						` printf "\e[1;31m>> You are required to manually intervene to install reuse as go-makefile-maker cannot automatically resolve installing reuse on all setups.\e[0m\n";` +
+						` printf "\e[1;31m>> The preferred way for go-makefile-maker to install python tools after nix-shell is pipx which could not be found. Either install pipx using your package manager or install reuse using your package manager if at least version 6 is available.\e[0m\n";` +
+						` printf "\e[1;31m>> As your Python was likely installed by your package manager, just doing pip install --user sadly does no longer work as pip issues a warning about breaking your system. Generally running --break-system-packages with --user is safe to do but you should only run this command if you can resolve issues with it yourself: pip3 install --user --break-system-packages reuse\e[0m\n";` +
+						` else` +
+						` printf "\e[1;36m>> Installing reuse...\e[0m\n";` +
+						` pipx install reuse;` +
+						` fi;` +
+						` fi`,
+				},
+			})
+			prepareStaticRecipe = append(prepareStaticRecipe, "install-reuse")
+		}
 	}
 	// add target for installing dependencies for `make static-check`
 	prepare.addRule(rule{
@@ -501,11 +504,16 @@ endif
 `))
 		copyright := cfg.License.Copyright.UnwrapOr("SAP SE or an SAP affiliate company")
 
+		licenseHeaderPrereqs := []string{"install-addlicense"}
+		if reuseEnabled {
+			licenseHeaderPrereqs = append(licenseHeaderPrereqs, "install-reuse")
+		}
+
 		dev.addRule(rule{
 			description:   "Add (or overwrite) license headers on all non-vendored source code files.",
 			target:        "license-headers",
 			phony:         true,
-			prerequisites: []string{"install-addlicense", "install-reuse"},
+			prerequisites: licenseHeaderPrereqs,
 			recipe: []string{
 				`@printf "\e[1;36m>> addlicense (for license headers on source code files)\e[0m\n"`,
 				// We must use gawk to use gnu awk on Darwin
@@ -539,22 +547,29 @@ endif
 				fmt.Sprintf(`@addlicense --check %s %s`, ignoreOptionsStr, allSourceFilesExpr),
 			},
 		})
-		test.addRule(rule{
-			description:   "Check reuse compliance",
-			target:        "check-reuse",
-			phony:         true,
-			prerequisites: []string{"install-reuse"},
-			recipe: []string{
-				`@printf "\e[1;36m>> reuse lint\e[0m\n"`,
-				// reuse is very verbose, so we only show the output if there are problems
-				`@if ! reuse lint -q; then reuse lint; fi`,
-			},
-		})
+		if reuseEnabled {
+			test.addRule(rule{
+				description:   "Check reuse compliance",
+				target:        "check-reuse",
+				phony:         true,
+				prerequisites: []string{"install-reuse"},
+				recipe: []string{
+					`@printf "\e[1;36m>> reuse lint\e[0m\n"`,
+					// reuse is very verbose, so we only show the output if there are problems
+					`@if ! reuse lint -q; then reuse lint; fi`,
+				},
+			})
+		}
+		// add target for static code checks
+		staticCodeChecksPrerequisites := []string{"check-addlicense"}
+		if reuseEnabled {
+			staticCodeChecksPrerequisites = append(staticCodeChecksPrerequisites, "check-reuse")
+		}
 		test.addRule(rule{
 			description:   "Run static code checks",
 			phony:         true,
 			target:        "check-license-headers",
-			prerequisites: []string{"check-addlicense", "check-reuse"},
+			prerequisites: staticCodeChecksPrerequisites,
 		})
 
 		if isGolang {

main.go

@@ -95,7 +95,7 @@ func main() {
 	// Render Dockerfile
 	if cfg.Dockerfile.Enabled {
 		logg.Debug("rendering Dockerfile")
-		dockerfile.RenderConfig(cfg)
+		dockerfile.RenderConfig(cfg, sr)
 	}
 
 	// Render golangci-lint config file