NETOBSERV-258: JSON+Kafka export (#33)

* NETOBSERV-258: integration tests env & not-yet-working JSON+Kafka conversion

* working JSON conversion

* tests are passing

* fix linter and parallel tests

* documented and cleaned-up code

* reused common code in e2e tests

* Disabling kafka batch timeout

* Updated build image

Author: Mario Macias

Dockerfile

@@ -1,20 +1,11 @@
 # Build the manager binary
-FROM registry.access.redhat.com/ubi8/go-toolset:1.16.7-5 as builder
+FROM registry.access.redhat.com/ubi8/go-toolset:1.17.7 as builder
 
 ARG SW_VERSION="unknown"
 ARG GOVERSION="1.17.8"
 
 WORKDIR /opt/app-root
 
-# TEMPORARY STEPS UNTIL ubi8 releases a go1.17 image
-RUN wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz && tar -xzf go$GOVERSION.linux-amd64.tar.gz
-ENV GOROOT /opt/app-root/go
-RUN mkdir -p /opt/app-root/gopath
-ENV GOPATH /opt/app-root/gopath
-ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
-WORKDIR /opt/app-root/src
-# END OF LINES TO REMOVE
-
 # Copy the go manifests and source
 COPY .git/ .git/
 COPY bpf/ bpf/
@@ -29,9 +20,9 @@ COPY Makefile Makefile
 RUN make compile
 
 # Create final image from minimal + built binary
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5-240
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 WORKDIR /
-COPY --from=builder /opt/app-root/src/bin/netobserv-ebpf-agent .
+COPY --from=builder /opt/app-root/bin/netobserv-ebpf-agent .
 USER 65532:65532
 
 ENTRYPOINT ["/netobserv-ebpf-agent"]

Makefile

@@ -22,7 +22,7 @@ IMG_SHA = $(IMAGE_TAG_BASE):$(BUILD_SHA)
 LOCAL_GENERATOR_IMAGE ?= ebpf-generator:latest
 
 CILIUM_EBPF_VERSION := v0.8.1
-GOLANGCI_LINT_VERSION = v1.42.1
+GOLANGCI_LINT_VERSION = v1.46.2
 
 CLANG ?= clang
 CFLAGS := -O2 -g -Wall -Werror $(CFLAGS)
@@ -49,11 +49,11 @@ vendors:
 .PHONY: prereqs
 prereqs:
 	@echo "### Check if prerequisites are met, and installing missing dependencies"
-	test -f $(go env GOPATH)/bin/golangci-lint || GOFLAGS="" go install github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}
-	test -f $(go env GOPATH)/bin/bpf2go || go install github.com/cilium/ebpf/cmd/bpf2go@${CILIUM_EBPF_VERSION}
-	test -f $(go env GOPATH)/bin/protoc-gen-go || go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
-	test -f $(go env GOPATH)/bin/protoc-gen-go-grpc || go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
-	test -f $(go env GOPATH)/bin/kind || go install sigs.k8s.io/kind@latest
+	test -f $(shell go env GOPATH)/bin/golangci-lint || GOFLAGS="" go install github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}
+	test -f $(shell go env GOPATH)/bin/bpf2go || go install github.com/cilium/ebpf/cmd/bpf2go@${CILIUM_EBPF_VERSION}
+	test -f $(shell go env GOPATH)/bin/protoc-gen-go || go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+	test -f $(shell go env GOPATH)/bin/protoc-gen-go-grpc || go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+	test -f $(shell go env GOPATH)/bin/kind || go install sigs.k8s.io/kind@latest
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -130,4 +130,4 @@ tests-e2e: prereqs
 	# environments: (1) as image tagged in the local repository (2) as image archive.
 	$(OCI_BIN) build . -t localhost/ebpf-agent:test
 	$(OCI_BIN) save -o ebpf-agent.tar localhost/ebpf-agent:test
-	GOOS=$(GOOS) go test -v -mod vendor -tags e2e ./e2e/...
+	GOOS=$(GOOS) go test -p 1 -timeout 30m -v -mod vendor -tags e2e ./e2e/...

docs/config.md

@@ -2,8 +2,9 @@
 
 The following environment variables are available to configure the NetObserv eBFP Agent:
 
-* `FLOWS_TARGET_HOST` (required). Host name or IP of the target Flow collector.
-* `FLOWS_TARGET_PORT` (required). Port of the target flow collector.
+* `EXPORT` (default: `grpc`). Flows' exporter protocol. Accepted values are: `grpc` or `kafka`.
+* `FLOWS_TARGET_HOST` (required if `EXPORT` is `grpc`). Host name or IP of the target Flow collector.
+* `FLOWS_TARGET_PORT` (required if `EXPORT` is `grpc`). Port of the target flow collector.
 * `INTERFACES` (optional). Comma-separated list of the interface names from where flows will be collected. If 
   empty, the agent will use all the interfaces in the system, excepting the ones listed in
   the `EXCLUDE_INTERFACES` variable.
@@ -33,3 +34,15 @@ The following environment variables are available to configure the NetObserv eBF
     `LISTEN_POLL_PERIOD` variable.
 * `LISTEN_POLL_PERIOD` (default: `10s`). When `LISTEN_INTERFACES` value is `poll`, this duration
   string specifies the frequency in which the current network interfaces are polled.
+* `KAFKA_BROKERS` (required if `EXPORT` is `kafka`). Comma-separated list of tha addresses of the
+  brokers of the Kafka cluster that this agent is configured to send messages to.
+* `KAFKA_TOPIC`(default: `network-flows`). Name of the topic where the flows' processor will receive
+  the flows from.
+* `KAFKA_BATCH_SIZE` (default: `100`). Limit on how many messages will be buffered before being sent
+  to a Kafka partition.
+* `KAFKA_BATCH_BYTES` (default: `1048576`). Limit of the maximum size of a request in bytes before
+  being sent to a Kafka partition.
+* `KAFKA_ASYNC` (default: `true`). If `true`, the message writing process will never block. It also
+  means that errors are ignored since the caller will not receive the returned value.
+* `KAFKA_COMPRESSION` (default: `none`). Compression codec to be used to compress messages. Accepted
+  values: `none`, `gzip`, `snappy`, `lz4`, `zstd`.