DEVX-1334 Reworks Docker build and push (#48)

Author: rspurgeon

Dockerfile-confluenthub

@@ -14,8 +14,10 @@
 # limitations under the License.
 
 ARG CP_VERSION
+ARG BASE_PREFIX=confluentinc 
+ARG CONNECT_IMAGE=cp-kafka-connect
 
-FROM confluentinc/cp-kafka-connect:$CP_VERSION
+FROM $BASE_PREFIX/$CONNECT_IMAGE:$CP_VERSION
 
 ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
 

Dockerfile renamed to Dockerfile-local

@@ -14,8 +14,10 @@
 # limitations under the License.
 
 ARG CP_VERSION
+ARG BASE_PREFIX=confluentinc 
+ARG CONNECT_IMAGE=cp-kafka-connect
 
-FROM confluentinc/cp-kafka-connect:$CP_VERSION
+FROM $BASE_PREFIX/$CONNECT_IMAGE:$CP_VERSION
 
 ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
 

Dockerfile-operator

@@ -8,50 +8,76 @@ SHELL=/bin/bash -o pipefail
 
 check-dependency = $(if $(shell command -v $(1)),,$(error Make sure $(1) is installed))
 
-check-dependencies:
-	@#(call check-dependency,mvn)
-	@#(call check-dependency,docker)
-	@#(call check-dependency,grep)
-	@#(call check-dependency,cut)
-	@#(call check-dependency,sed)
-
 CP_VERSION ?= 5.3.1
 OPERATOR_VERSION ?= 0
 
-KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.6
+KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.7
 AGGREGATE_VERSION = $(KAFKA_CONNECT_DATAGEN_VERSION)-$(CP_VERSION)
 OPERATOR_AGGREGATE_VERSION = $(AGGREGATE_VERSION).$(OPERATOR_VERSION)
 
 KAFKA_CONNECT_DATAGEN_LOCAL_VERSION = $(shell make local-package-version)
 AGGREGATE_LOCAL_VERSION = $(KAFKA_CONNECT_DATAGEN_LOCAL_VERSION)-$(CP_VERSION)
 
+BASE_PREFIX ?= confluentinc
+PUSH_PREFIX ?= cnfldemos
+
 help:
 	@$(foreach m,$(MAKEFILE_LIST),grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(m) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-$(HELP_TAB_WIDTH)s\033[0m %s\n", $$1, $$2}';)
 
-local-package-version: check-dependencies ## Retrieves the jar version from the maven project definition
+local-package-version: ## Retrieves the jar version from the maven project definition
 	@mvn help:evaluate -Dexpression=project.version -q -DforceStdout
 
-package: check-dependencies ## Creates the assembly jar
+package: ## Creates the assembly jar
 	@mvn clean package
 
-build-docker-from-local: check-dependencies package ## Build the Docker image using the locally mvn built kafka-connect-datagen package
-	@docker build -t kafka-connect-datagen:$(AGGREGATE_LOCAL_VERSION) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_LOCAL_VERSION) --build-arg CP_VERSION=$(CP_VERSION) -f Dockerfile .
+build-docker-from-local: package ## Build the Docker image using the locally mvn built kafka-connect-datagen package
+	@docker build -t kafka-connect-datagen:$(AGGREGATE_LOCAL_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_LOCAL_VERSION) --build-arg CP_VERSION=$(CP_VERSION) -f Dockerfile-local .
+
+build-docker-from-released: ## Build a Docker image using a released version of the kafka-connect-datagen connector 
+	@docker build -t kafka-connect-datagen:$(AGGREGATE_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) -f Dockerfile-confluenthub .
+
+build-cp-server-connect-from-local: package ## Build the Docker image based on cp-server-connect from locally mvn built kafka-connect-datagen package
+	@docker build -t cp-server-connect-datagen:$(AGGREGATE_LOCAL_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_LOCAL_VERSION) --build-arg CP_VERSION=$(CP_VERSION) --build-arg CONNECT_IMAGE=cp-server-connect -f Dockerfile-local .
+
+build-cp-server-connect-from-released: ## Build a Docker image using a released version of the kafka-connect-datagen connector 
+	@docker build -t cp-server-connect-datagen:$(AGGREGATE_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) --build-arg CONNECT_IMAGE=cp-server-connect -f Dockerfile-confluenthub .
+
+# In the case of Operator based images, there is an additional REV version appended to the end of the CP Version
+# which allows Operator images to rev independently
+
+build-cp-server-connect-operator-from-local: package ## Build the Docker image based on cp-server-connect from locally mvn built kafka-connect-datagen package
+	@docker build -t cp-server-connect-operator-datagen:$(AGGREGATE_LOCAL_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_LOCAL_VERSION) --build-arg CP_VERSION=$(CP_VERSION).$(OPERATOR_VERSION) --build-arg CONNECT_IMAGE=cp-server-connect-operator -f Dockerfile-local .
+
+build-cp-server-connect-operator-from-released: ## Build a Docker image using a released version of the kafka-connect-datagen connector 
+	@docker build -t cp-server-connect-operator-datagen:$(AGGREGATE_VERSION) --build-arg BASE_PREFIX=$(BASE_PREFIX) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION).$(OPERATOR_VERSION) --build-arg CONNECT_IMAGE=cp-server-connect-operator -f Dockerfile-confluenthub .
+
+push-from-local:
+	@make --no-print-directory build-docker-from-local
+	@docker tag kafka-connect-datagen:$(AGGREGATE_LOCAL_VERSION) $(PUSH_PREFIX)/kafka-connect-datagen:$(AGGREGATE_LOCAL_VERSION)
+	@docker push $(PUSH_PREFIX)/kafka-connect-datagen:$(AGGREGATE_LOCAL_VERSION)
+
+push-from-released:
+	@make --no-print-directory build-docker-from-released
+	@docker tag kafka-connect-datagen:$(AGGREGATE_VERSION) $(PUSH_PREFIX)/kafka-connect-datagen:$(AGGREGATE_VERSION)
+	@docker push $(PUSH_PREFIX)/kafka-connect-datagen:$(AGGREGATE_VERSION)
 
-build-docker-from-released: check-dependencies ## Build a Docker image using a released version of the kafka-connect-datagen connector 
-	@docker build -t kafka-connect-datagen:$(AGGREGATE_VERSION) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) -f Dockerfile-confluenthub .
+push-cp-server-connect-from-local:
+	@make --no-print-directory build-cp-server-connect-from-local
+	@docker tag cp-server-connect-datagen:$(AGGREGATE_LOCAL_VERSION) $(PUSH_PREFIX)/cp-server-connect-datagen:$(AGGREGATE_LOCAL_VERSION)
+	@docker push $(PUSH_PREFIX)/cp-server-connect-datagen:$(AGGREGATE_LOCAL_VERSION)
 
-publish-cp-kafka-connect-confluenthub: check-dependencies ## Build the cp-kafka-connect image pulling datagen from Confluent Hub
-	@docker build -t cnfldemos/kafka-connect-datagen:$(AGGREGATE_VERSION) --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) -f Dockerfile-confluenthub .
-	@docker push cnfldemos/kafka-connect-datagen:$(AGGREGATE_VERSION)
+push-cp-server-connect-from-released:
+	@make --no-print-directory build-cp-server-connect-from-released
+	@docker tag cp-server-connect-datagen:$(AGGREGATE_VERSION) $(PUSH_PREFIX)/cp-server-connect-datagen:$(AGGREGATE_VERSION)
+	@docker push $(PUSH_PREFIX)/cp-server-connect-datagen:$(AGGREGATE_VERSION)
 
-#  The combination of CP_VERSION & OPERATOR_VERSION will determine two things:
-#    1. The version of the operator base image used in the Dockerfiles for operator (Dockerfile-operator-local & Dockerfile-operator)
-#    2. The version of the docker images _this_ repository builds and pushes to cnfldemos (cp-server-connect-operator-with-datagen)
+push-cp-server-connect-operator-from-local:
+	@make --no-print-directory build-cp-server-connect-operator-from-local
+	@docker tag cp-server-connect-operator-datagen:$(AGGREGATE_LOCAL_VERSION) $(PUSH_PREFIX)/cp-server-connect-operator-datagen:$(AGGREGATE_LOCAL_VERSION)
+	@docker push $(PUSH_PREFIX)/cp-server-connect-operator-datagen:$(AGGREGATE_LOCAL_VERSION)
 
-publish-cp-server-connect-operator-confluenthub: check-dependencies ## Build the cp-server-connect-operator image pulling datagen from Confluent Hub
-	@docker build -t cnfldemos/cp-server-connect-operator-with-datagen:$(AGGREGATE_VERSION).0 --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) --build-arg OPERATOR_VERSION=$(OPERATOR_VERSION) -f Dockerfile-operator .
-	@docker push cnfldemos/cp-server-connect-operator-with-datagen:$(OPERATOR_AGGREGATE_VERSION)
+push-cp-server-connect-operator-from-released:
+	@make --no-print-directory build-cp-server-connect-operator-from-released
+	@docker tag cp-server-connect-operator-datagen:$(AGGREGATE_VERSION) $(PUSH_PREFIX)/cp-server-connect-operator-datagen:$(AGGREGATE_VERSION)
+	@docker push $(PUSH_PREFIX)/cp-server-connect-operator-datagen:$(AGGREGATE_VERSION)
 
-publish-cp-server-connect-operator-local: check-dependencies ## Build the cp-server-connect-operator image installing datagen from the local build	
-	@docker build -t cnfldemos/cp-server-connect-operator-with-datagen:$(AGGREGATE_VERSION).0 --build-arg KAFKA_CONNECT_DATAGEN_VERSION=$(KAFKA_CONNECT_DATAGEN_VERSION) --build-arg CP_VERSION=$(CP_VERSION) --build-arg OPERATOR_VERSION=$(OPERATOR_VERSION) -f Dockerfile-operator-local .
-	@docker push cnfldemos/cp-server-connect-operator-with-datagen:$(OPERATOR_AGGREGATE_VERSION)

Dockerfile-operator-local

@@ -50,29 +50,30 @@ confluent local consume test1 --value-format avro --max-messages 5 --property pr
 
 A Docker image based on Kafka Connect with the `kafka-connect-datagen` plugin is already available in [Dockerhub](https://hub.docker.com/r/cnfldemos/kafka-connect-datagen), and it is ready for you to use.
 
-If you want to build a local copy of the Docker image with `kafka-connect-datagen`, this project provides a [Dockerfile](Dockerfile) that you can reference.
+If you want to build a local copy of the Docker image with `kafka-connect-datagen`, this project provides a [Dockerfile](Dockerfile-local) that you can reference.
 
-You can create a Docker image packaged with the locally built source by running:
+You can create a Docker image packaged with the locally built source by running (for example with the 5.3.1 version of Confluent Platform):
 ```bash
-make build-docker-from-local
+make build-docker-from-local CP_VERSION=5.3.1
 ```
 
-This will build the connector from source and create a local image with an aggregate version number.  The aggregate version number is the kafka-connect-datagen connector version number and the Confluent Platform version number separated with a `-`.   The local kafka-connect-datagen version number is defined in the `pom.xml` file, and the Confluent Platform version defined in the [Makefile](Makfile).  An example of the aggregate version number might be: `0.1.6-5.3.1`.
+This will build the connector from source and create a local image with an aggregate version number.  The aggregate version number is the kafka-connect-datagen connector version number and the Confluent Platform version number separated with a `-`.   The local kafka-connect-datagen version number is defined in the `pom.xml` file, and the Confluent Platform version defined in the [Makefile](Makfile).  An example of the aggregate version number might be: `0.1.7-5.3.1`.
 
 Alternatively, you can install the `kafka-connect-datagen` connector from [Confluent Hub](https://www.confluent.io/connector/kafka-connect-datagen/) into a Docker image by running:
 ```bash
-make build-docker-from-released
+make build-docker-from-released CP_VERSION=5.3.1
 ```
 
 The [Makefile](Makefile) contains some default variables that affect the version numbers of both the installed `kafka-connect-datagen` as well as the base Confluent Platform version.  The variables are located near the top of the [Makefile](Makefile) with the following names and current default values:
 
 ```bash
 CP_VERSION ?= 5.3.1
-KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.6
+KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.7
 ```
 These values can be overriden with variable declarations before the `make` command.  For example:
+
 ```bash
-KAFKA_CONNECT_DATAGEN_VERSION=0.1.4 make build-docker-from-released
+make build-docker-from-released KAFKA_CONNECT_DATAGEN_VERSION=0.1.6
 ```
 
 ### Run connector in Docker Compose
@@ -193,13 +194,13 @@ The [Makefile](Makefile) contains some default variables that affect the version
 
 ```bash
 CP_VERSION ?= 5.3.1
-KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.6
+KAFKA_CONNECT_DATAGEN_VERSION ?= 0.1.7
 OPERATOR_VERSION ?= 0 # Operator is a 'rev' version appended at the end of the CP version, like so: 5.3.1.0
 ```
 
 To publish the https://hub.docker.com/r/cnfldemos/kafka-connect-datagen/ image:
 ```bash
-make publish-cp-kafka-connect-confluenthub
+make push-from-released
 ```
 
 and to override the CP Version of the `kafka-connect-datagen` version you can run something similar to:
@@ -209,11 +210,7 @@ CP_VERSION=5.3.0 KAFKA_CONNECT_DATAGEN_VERSION=0.1.4 make publish-cp-kafka-conne
 
 to override the CP Version and the Operator version, which may happen if Operator releases a patch version, you could run something similar to:
 ```bash
-CP_VERSION=5.3.0 OPERATOR_VERSION=1 KAFKA_CONNECT_DATAGEN_VERSION=0.1.4 make publish-cp-server-connect-operator-confluenthub
+CP_VERSION=5.3.0 OPERATOR_VERSION=1 KAFKA_CONNECT_DATAGEN_VERSION=0.1.4 make push-cp-server-connect-operator-from-released
 ```
-which would result in a docker image tagged as: `cp-server-connect-operator-with-datagen:0.1.4-5.3.0.1`
+which would result in a docker image tagged as: `cp-server-connect-operator-datagen:0.1.4-5.3.0.1` and pushed to DockerHub
 
-To publish the https://hub.docker.com/r/cnfldemos/cp-server-connect-operator-with-datagen image:
-```bash
-make publish-cp-server-connect-operator-confluenthub
-```