Align build steps - distinguish clean vs dev builds (#745)

- Reducing the Dockerfile dependencies allows faster dev builds
- Rename target build_code -> compile for consistency
- Remove now useless validate_go target
- Remove confgenerator from docker images (we may introduce another
  dockerfile if someone finds it useful and want it back)

Author: jotak

.github/workflows/push_image.yml

@@ -35,10 +35,10 @@ jobs:
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: build and push manifest with images
         run: |
-          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.WF_VERSION }} make images
-          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.short_sha }} OCI_BUILD_OPTS="--label quay.expires-after=2w" make images
+          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.WF_VERSION }} CLEAN_BUILD=1 make images
+          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.short_sha }} CLEAN_BUILD=1 OCI_BUILD_OPTS="--label quay.expires-after=2w" make images
           if [[ "main" == "$WF_VERSION" ]]; then
-            MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=latest make images
+            MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=latest CLEAN_BUILD=1 make images
           fi
 
   codecov:

.github/workflows/push_image_pr.yml

@@ -37,7 +37,7 @@ jobs:
       - name: get short sha
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: build and push manifest with images
-        run: OCI_BUILD_OPTS="--label quay.expires-after=2w" IMAGE_ORG=${{ env.WF_ORG }} IMAGE=${{ env.WF_REGISTRY }}/${{ env.WF_IMAGE }}:${{ env.short_sha }} make images
+        run: OCI_BUILD_OPTS="--label quay.expires-after=2w" IMAGE_ORG=${{ env.WF_ORG }} IMAGE=${{ env.WF_REGISTRY }}/${{ env.WF_IMAGE }}:${{ env.short_sha }} CLEAN_BUILD=1 make images
       - uses: actions/github-script@v6
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}

.github/workflows/release.yml

@@ -42,4 +42,4 @@ jobs:
           password: ${{ secrets.QUAY_SECRET }}
           registry: quay.io
       - name: build and push manifest with images
-        run: MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.tag }} make images
+        run: MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" IMAGE_ORG=${{ env.WF_ORG }} VERSION=${{ env.tag }} CLEAN_BUILD=1 make images

Makefile

@@ -15,17 +15,6 @@ export GOOS=linux
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= main
-BUILD_DATE := $(shell date +%Y-%m-%d\ %H:%M)
-TAG_COMMIT := $(shell git rev-list --abbrev-commit --tags --max-count=1)
-TAG := $(shell git describe --abbrev=0 --tags ${TAG_COMMIT} 2>/dev/null || true)
-BUILD_SHA := $(shell git rev-parse --short HEAD)
-BUILD_VERSION := $(TAG:v%=%)
-ifneq ($(COMMIT), $(TAG_COMMIT))
-	BUILD_VERSION := $(BUILD_VERSION)-$(BUILD_SHA)
-endif
-ifneq ($(shell git status --porcelain),)
-	BUILD_VERSION := $(BUILD_VERSION)-dirty
-endif
 
 # Go architecture and targets images to build
 GOARCH ?= amd64
@@ -39,13 +28,18 @@ IMAGE_TAG_BASE ?= quay.io/$(IMAGE_ORG)/flowlogs-pipeline
 
 # Image URL to use all building/pushing image targets
 IMAGE ?= $(IMAGE_TAG_BASE):$(VERSION)
-OCI_BUILD_OPTS ?=
 
 # Image building tool (docker / podman) - docker is preferred in CI
 OCI_BIN_PATH = $(shell which docker 2>/dev/null || which podman)
 OCI_BIN ?= $(shell basename ${OCI_BIN_PATH})
+OCI_BUILD_OPTS ?=
+
+ifneq ($(CLEAN_BUILD),)
+	BUILD_DATE := $(shell date +%Y-%m-%d\ %H:%M)
+	BUILD_SHA := $(shell git rev-parse --short HEAD)
+	LDFLAGS ?= -X 'main.buildVersion=${VERSION}-${BUILD_SHA}' -X 'main.buildDate=${BUILD_DATE}'
+endif
 
-MIN_GO_VERSION := 1.20.0
 FLP_BIN_FILE=flowlogs-pipeline
 CG_BIN_FILE=confgenerator
 NETFLOW_GENERATOR=nflow-generator
@@ -60,7 +54,7 @@ FORCE: ;
 # build a single arch target provided as argument
 define build_target
 	echo 'building image for arch $(1)'; \
-	DOCKER_BUILDKIT=1 $(OCI_BIN) buildx build --load --build-arg TARGETARCH=$(1) ${OCI_BUILD_OPTS} -t ${IMAGE}-$(1) -f contrib/docker/Dockerfile .;
+	DOCKER_BUILDKIT=1 $(OCI_BIN) buildx build --load --build-arg LDFLAGS="${LDFLAGS}" --build-arg TARGETARCH=$(1) ${OCI_BUILD_OPTS} -t ${IMAGE}-$(1) -f contrib/docker/Dockerfile .;
 endef
 
 # push a single arch target image
@@ -97,25 +91,19 @@ vendors: ## Check go vendors
 	@echo "### Checking vendors"
 	go mod tidy && go mod vendor
 
-.PHONY: validate_go
-validate_go:
-	@current_ver=$$(go version | { read _ _ v _; echo $${v#go}; }); \
-	required_ver=${MIN_GO_VERSION}; min_ver=$$(echo -e "$$current_ver\n$$required_ver" | sort -V | head -n 1); \
-	if [[ $$min_ver == $$current_ver ]]; then echo -e "\n!!! golang version > $$required_ver required !!!\n"; exit 7;fi
-
 ##@ Develop
 
-.PHONY: validate_go lint
+.PHONY: lint
 lint: $(GOLANGCI_LINT) ## Lint the code
 	$(GOLANGCI_LINT) run ./... --timeout=3m
 
-.PHONY: build_code
-build_code:
-	GOARCH=${GOARCH} go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" "${CMD_DIR}${FLP_BIN_FILE}"
-	GOARCH=${GOARCH} go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" "${CMD_DIR}${CG_BIN_FILE}"
+.PHONY: compile
+compile: ## Compile main flowlogs-pipeline and config generator
+	GOARCH=${GOARCH} go build "${CMD_DIR}${FLP_BIN_FILE}"
+	GOARCH=${GOARCH} go build "${CMD_DIR}${CG_BIN_FILE}"
 
 .PHONY: build
-build: validate_go lint build_code docs ## Build flowlogs-pipeline executable and update the docs
+build: lint compile docs ## Build flowlogs-pipeline executable and update the docs
 
 .PHONY: docs
 docs: FORCE ## Update flowlogs-pipeline documentation
@@ -131,7 +119,7 @@ clean: ## Clean
 
 TEST_OPTS := -race -coverpkg=./... -covermode=atomic -coverprofile cover.out
 .PHONY: tests-unit
-tests-unit: validate_go ## Unit tests
+tests-unit: ## Unit tests
 	# tests may rely on non-thread safe libs such as go-ipfix => no -race flag
 	go test $$(go list ./... | grep /testnorace)
 	# enabling CGO is required for -race flag
@@ -152,15 +140,15 @@ tests-fast: TEST_OPTS=
 tests-fast: tests-unit ## Fast unit tests (no race tests / coverage)
 
 .PHONY: tests-e2e
-tests-e2e: validate_go $(KIND)  ## End-to-end tests
+tests-e2e: $(KIND)  ## End-to-end tests
 	go test -p 1 -v -timeout 20m $$(go list ./... | grep  /e2e)
 
 .PHONY: tests-all
-tests-all: validate_go tests-unit tests-e2e ## All tests
+tests-all: tests-unit tests-e2e ## All tests
 
 # note: to review profile execute: go tool pprof -web /tmp/flowlogs-pipeline-cpu-profile.out (make sure graphviz is installed)
 .PHONY: benchmarks
-benchmarks: $(BENCHSTAT) validate_go ## Benchmark
+benchmarks: $(BENCHSTAT) ## Benchmark
 	go test -bench=. ./cmd/flowlogs-pipeline -o=/tmp/flowlogs-pipeline.test \
 	-cpuprofile /tmp/flowlogs-pipeline-cpu-profile.out \
 	-run=^# -count=10 -parallel=1 -cpu=1 -benchtime=100x \
@@ -187,7 +175,7 @@ image-push: ## Push MULTIARCH_TARGETS images
 .PHONY: manifest-build
 manifest-build: ## Build MULTIARCH_TARGETS manifest
 	@echo 'building manifest $(IMAGE)'
-	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${IMAGE} -f
+	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${IMAGE} -f || true
 	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${IMAGE} $(foreach target,$(MULTIARCH_TARGETS), --amend ${IMAGE}-$(target));
 
 .PHONY: manifest-push

cmd/confgenerator/main.go

@@ -34,8 +34,8 @@ import (
 )
 
 var (
-	BuildVersion       string
-	BuildDate          string
+	buildVersion       = "unknown"
+	buildDate          = "unknown"
 	cfgFile            string
 	logLevel           string
 	envPrefix          = "FLP_CONFGEN"
@@ -151,8 +151,7 @@ func main() {
 
 func run() {
 	// Initial log message
-	fmt.Printf("Starting %s:\n=====\nBuild Version: %s\nBuild Date: %s\n\n",
-		filepath.Base(os.Args[0]), BuildVersion, BuildDate)
+	fmt.Printf("Starting %s:\n=====\nBuild version: %s\nBuild date: %s\n\n", filepath.Base(os.Args[0]), buildVersion, buildDate)
 	// Dump the configuration
 	dumpConfig(&opts)
 	// creating a new configuration generator

cmd/flowlogs-pipeline/main.go

@@ -42,8 +42,8 @@ import (
 )
 
 var (
-	BuildVersion       string
-	BuildDate          string
+	buildVersion       = "unknown"
+	buildDate          = "unknown"
 	cfgFile            string
 	logLevel           string
 	envPrefix          = "FLOWLOGS-PIPELINE"
@@ -168,8 +168,7 @@ func run() {
 	)
 
 	// Initial log message
-	fmt.Printf("Starting %s:\n=====\nBuild Version: %s\nBuild Date: %s\n\n",
-		filepath.Base(os.Args[0]), BuildVersion, BuildDate)
+	fmt.Printf("Starting %s:\n=====\nBuild version: %s\nBuild date: %s\n\n", filepath.Base(os.Args[0]), buildVersion, buildDate)
 
 	// Dump configuration
 	dumpConfig(&opts)

contrib/docker/Dockerfile

@@ -2,29 +2,21 @@ ARG TARGETARCH
 FROM docker.io/library/golang:1.22 as builder
 
 ARG TARGETARCH=amd64
+ARG LDFLAGS
 WORKDIR /app
 
 # Copy source code
 COPY go.mod .
 COPY go.sum .
-COPY Makefile .
-COPY .mk/ .mk/
-COPY .bingo/ .bingo/
 COPY vendor/ vendor/
-COPY .git/ .git/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN git status --porcelain
-RUN GOARCH=$TARGETARCH make build_code
+RUN GOARCH=$TARGETARCH go build -ldflags "$LDFLAGS" -mod vendor -o flowlogs-pipeline cmd/flowlogs-pipeline/main.go
 
 # final stage
 FROM --platform=linux/$TARGETARCH registry.access.redhat.com/ubi9/ubi-minimal:9.4
 
 COPY --from=builder /app/flowlogs-pipeline /app/
-COPY --from=builder /app/confgenerator /app/
-
-# expose ports
-EXPOSE 2055
 
 ENTRYPOINT ["/app/flowlogs-pipeline"]

contrib/docker/Dockerfile.downstream

@@ -3,28 +3,23 @@ ARG COMMIT
 FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:v1.22.5-202407301806.g4c8b32d.el9 as builder
 
 ARG TARGETARCH=amd64
+ARG LDFLAGS
 WORKDIR /app
 
 # Copy source code
 COPY go.mod .
 COPY go.sum .
-COPY Makefile .
-COPY .mk/ .mk/
-COPY .bingo/ .bingo/
 COPY vendor/ vendor/
-COPY .git/ .git/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN git status --porcelain
-RUN GOARCH=$TARGETARCH make build_code
+RUN GOARCH=$TARGETARCH go build -ldflags "$LDFLAGS" -mod vendor -o flowlogs-pipeline cmd/flowlogs-pipeline/main.go
 
 # final stage
 FROM --platform=linux/$TARGETARCH registry.access.redhat.com/ubi9/ubi-minimal:9.4
 
 COPY --from=builder /app/flowlogs-pipeline /app/
 
-# expose ports
 ENTRYPOINT ["/app/flowlogs-pipeline"]
 
 LABEL com.redhat.component="network-observability-flowlogs-pipeline-container"
@@ -36,4 +31,4 @@ LABEL maintainer="[email protected]"
 LABEL io.openshift.tags="network-observability-flowlogs-pipeline"
 LABEL upstream-vcs-type="git"
 LABEL upstream-vcs-type="$COMMIT"
-LABEL description="Flow-Logs Pipeline (a.k.a. FLP) is an observability tool that consumes logs from various inputs, transform them and export logs to loki and / or time series metrics to prometheus."
+LABEL description="Flow-Logs Pipeline is an observability tool that consumes logs from various inputs, transform them and export logs to Loki and / or metrics to Prometheus."