Coverage with gcov and gcovr

Author: emasab

.gitignore

@@ -11,6 +11,8 @@ rdkafka*.pc
 *.dylib
 *.a
 *.d
+*.gcno
+*.gcda
 librdkafka*.lds
 core
 vgcore.*
@@ -23,10 +25,13 @@ gmon.out
 *.bz2
 *.deb
 *.rpm
+coverage/json/*
+coverage/html/*
 staging-docs
 tmp
 stats*.json
 test_report*.json
+tests/coverage.json
 cov-int
 gdbrun*.gdb
 TAGS

.semaphore/run-all-tests.yml

@@ -3,27 +3,22 @@ name: run-all-tests
 
 agent:
   machine:
-    type: s1-prod-ubuntu24-04-amd64-1
+    type: s1-prod-ubuntu24-04-arm64-1
 
 execution_time_limit:
   hours: 3
 
 global_job_config:
+  env_vars:
+    - name: TEST_COVERAGE
+      value: "True"
   prologue:
     commands:
       - checkout
       - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF'
-      - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb
-      - sudo dpkg -i rapidjson-dev.deb
       - sudo apt update
       - sudo apt remove -y needrestart
-      - sudo apt install -y valgrind
-      - python3 -m pip install -U pip
-      - python3 -m pip -V
-      - (cd tests && python3 -m pip install -r requirements.txt)
-      - ./configure --install-deps --enable-werror --enable-devel
-      - make -j all
-      - make -j -C tests build
+      - sudo apt install -y gcovr libcurl4-openssl-dev
       - sem-version java 17
 
 blocks:
@@ -35,22 +30,28 @@ blocks:
           type: s1-prod-ubuntu24-04-amd64-2
       prologue:
         commands:
-          - if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then exit 0; fi
+          - if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then
+              return 130;
+            fi
       jobs:
         - name: "PLAINTEXT cluster (x86_64)"
           env_vars:
             - name: TEST_SSL
               value: "False"
           commands:
-            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi
-            - ./tests/run-all-tests.sh 
+            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then
+                return 130;
+              fi
+            - ./tests/run-all-tests.sh x86_64_plaintext
         - name: "SSL cluster (x86_64)"
           env_vars:
             - name: TEST_SSL
               value: "True"
           commands:
-            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi
-            - ./tests/run-all-tests.sh
+            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then
+                return 130;
+              fi
+            - ./tests/run-all-tests.sh x86_64_ssl
   - name: "Run all tests (aarch64)"
     dependencies: []
     task:
@@ -59,19 +60,44 @@ blocks:
           type: s1-prod-ubuntu24-04-arm64-2
       prologue:
         commands:
-          - if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then exit 0; fi
+          - if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then
+              return 130;
+            fi
       jobs:
         - name: "PLAINTEXT cluster (aarch64)"
           env_vars:
             - name: TEST_SSL
               value: "False"
           commands:
-            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi
-            - ./tests/run-all-tests.sh
+            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then
+                return 130;
+              fi
+            - ./tests/run-all-tests.sh aarch64_plaintext
         - name: "SSL cluster (aarch64)"
           env_vars:
             - name: TEST_SSL
               value: "True"
           commands:
-            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi
-            - ./tests/run-all-tests.sh
+            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then
+                return 130;
+              fi
+            - ./tests/run-all-tests.sh aarch64_ssl
+
+after_pipeline:
+  task:
+    env_vars:
+      - name: TEST_COVERAGE
+        value: "True"
+    jobs:
+      - name: Coverage report
+        commands:
+          - if [[ "$TEST_COVERAGE" != "True" ]]; then
+              return 130;
+            fi
+          - checkout
+          - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF'
+          - sudo apt update
+          - sudo apt remove -y needrestart
+          - sudo apt install -y gcovr
+          - sem-version java 17
+          - ./tests/run-all-tests.sh coverage_report

Makefile

@@ -97,6 +97,17 @@ TAGS: .PHONY
 		cmp $@ $@.tmp || mv $@.tmp $@ ; rm -f $@.tmp ; \
 	fi)
 
+gcovr-build:
+	@(export CFLAGS="$$CFLAGS -O0 -fprofile-arcs -ftest-coverage" LDFLAGS="-lgcov --coverage" && \
+	  make libs && make -C tests build)
+
+gcovr-report:
+	@(gcovr --json tests/coverage.json --gcov-ignore-parse-errors \
+	--exclude src/nanopb/ \
+	--exclude src/cJSON\.c \
+	--exclude src/crc32c\.c \
+	--exclude src/snappy\.c ./src)
+
 coverity: Makefile.config
 	@(which cov-build >/dev/null 2>&1 || echo "Make sure coverity../bin is in your PATH")
 	@(cd src && \

mklove/Makefile.base

@@ -8,6 +8,8 @@ MKL_BLUE?=	\033[034m
 MKL_CLR_RESET?=	\033[0m
 
 DEPS=		$(OBJS:%.o=%.d)
+GCNOS=		$(OBJS:%.o=%.gcno)
+GCDAS=		$(OBJS:%.o=%.gcda)
 
 # TOPDIR is "TOPDIR/mklove/../" i.e., TOPDIR.
 # We do it with two dir calls instead of /.. to support mklove being symlinked.
@@ -316,7 +318,7 @@ doc-uninstall:
 	rmdir $$DESTDIR$(docdir) || true
 
 generic-clean:
-	rm -f $(OBJS) $(DEPS)
+	rm -f $(OBJS) $(DEPS) $(GCNOS) $(GCDAS)
 
 lib-clean: generic-clean lib-clean-pkg-config
 	rm -f $(LIBNAME)*.a $(LIBFILENAME) $(LIBFILENAMEDBG) \

src/rdkafka_range_assignor.c

@@ -967,8 +967,8 @@ static int setupRackAwareAssignment0(rd_kafka_t *rk,
             rd_kafka_metadata_get_internal(*metadata));
 
         for (i = 0; i < member_cnt; i++) {
-                char member_id[10];
-                snprintf(member_id, 10, "consumer%d", (int)(i + 1));
+                char member_id[20];
+                snprintf(member_id, 20, "consumer%d", (int)(i + 1));
                 ut_init_member_with_rack(
                     &members[i], member_id, ALL_RACKS[consumer_racks[i]],
                     subscriptions[i], subscriptions_count[i]);

src/rdkafka_sticky_assignor.c

@@ -4200,8 +4200,8 @@ setupRackAwareAssignment0(rd_kafka_t *rk,
             rd_kafka_metadata_get_internal(*metadata));
 
         for (i = 0; initialize_members && i < member_cnt; i++) {
-                char member_id[10];
-                snprintf(member_id, 10, "consumer%d", (int)(i + 1));
+                char member_id[20];
+                snprintf(member_id, 20, "consumer%d", (int)(i + 1));
                 ut_init_member_with_rack(
                     &members[i], member_id, ALL_RACKS[consumer_racks[i]],
                     subscriptions[i], subscriptions_count[i]);

tests/Makefile

@@ -124,7 +124,7 @@ rdlist.o: ../src/rdlist.c
 
 
 clean:
-	rm -f *.test $(OBJS) $(BIN)
+	rm -f *.test $(OBJS) $(BIN) $(GCNOS) $(GCDAS) coverage.json
 	$(MAKE) -C interceptor_test clean
 
 # Remove test reports, temporary test files, crash dumps, etc.

tests/run-all-tests.sh

@@ -14,6 +14,21 @@ set -e
 # - no assertions
 # - following AK and CP versions
 
+if [ "$1" = "coverage_report" ]; then
+    artifact pull workflow coverage
+    mkdir -p coverage/html
+    gcovr --gcov-ignore-parse-errors --exclude src/nanopb/ \
+        --exclude src/lz4.*\.c --exclude src/cJSON\.c \
+        --exclude src/crc32c\.c --exclude src/snappy\.c \
+        --json-add-tracefile "coverage/json/*.json" \
+        --html-details coverage/html/coverage.html \
+        --html-self-contained
+    tar -czf coverage/coverage_report_html.tar.gz coverage/html
+    artifact push workflow coverage/coverage_report_html.tar.gz \
+        --destination "coverage/coverage_report_html.tar.gz"
+    exit 0;
+fi
+
 export TEST_KAFKA_GIT_REF=${TEST_KAFKA_GIT_REF:-4.0.0}
 export TEST_CP_VERSION=${TEST_CP_VERSION:-7.9.0}
 
@@ -26,6 +41,18 @@ TEST_CONF_ARG=""
 TEST_QUICK_ARG=""
 TEST_ASSERT_ARG=""
 TEST_PARALLEL_ARG=""
+
+if [ "$TEST_COVERAGE" = "True" ]; then
+    if [ "$1" = "aarch64_plaintext" ]; then
+        export TEST_CONSUMER_GROUP_PROTOCOL="consumer"
+        TEST_SASL="PLAIN"
+    elif [ "$1" = "aarch64_ssl" ]; then
+        export TEST_CONSUMER_GROUP_PROTOCOL="consumer"
+        TEST_SASL="OAUTHBEARER"
+        TEST_TRIVUP_PARAMETERS="$TEST_TRIVUP_PARAMETERS --oidc"
+    fi
+fi
+
 if [ "$TEST_SSL" = "True" ]; then
     TEST_SSL_ARG="--ssl"
 fi
@@ -73,7 +100,38 @@ echo "kafka version: $TEST_KAFKA_VERSION"
 echo "CP version: $TEST_CP_VERSION"
 echo "configuration: $TEST_CONFIGURATION"
 echo "arguments: $TEST_ARGS"
+
+# Install requirements for running the tests
+wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb
+sudo dpkg -i rapidjson-dev.deb
+sudo apt install -y valgrind
+python3 -m pip install -U pip
+python3 -m pip -V
+(cd tests && python3 -m pip install -r requirements.txt)
+./configure --install-deps --enable-werror --enable-devel
+if [ "$TEST_COVERAGE" = "True" ]; then
+    make -j gcovr-build
+else
+    make -j
+fi
+
+# Disable exit on error
+set +e
 (cd tests && python3 -m trivup.clusters.KafkaCluster $TEST_CONFIGURATION \
 --version "$TEST_KAFKA_GIT_REF" \
 --cpversion "$TEST_CP_VERSION" \
 --cmd "python run-test-batches.py $TEST_ARGS")
+RESULT=$?
+
+if [ ! -z "$1" -a "$TEST_COVERAGE" = "True" ]; then
+    make gcovr-report
+    artifact push workflow tests/coverage.json --destination "coverage/json/coverage_$1.json"
+fi
+
+if [ $RESULT -eq 0 ]; then
+    echo "All tests passed"
+    exit 0
+else
+    echo "Some tests failed"
+    exit 1
+fi