ATLAS-5054: Introduce JaCoCo plugin and generate Code Coverage results with CI (#387)

* ATLAS-5054: Introduce JaCoCo plugin and generate Code Coverage results with CI

* ATLAS-5054: refactor coverage.sh execution in atlas-build.sh

* ATLAS-5054: add unzip command in Dockerfile.atlas-build

* ATLAS-5054: move coverage artifacts to dist directory for upload later

* ATLAS-5054: use host relative path for upload

* ATLAS-5054: debug coverage files

* ATLAS-5054: debug coverage files 2

* ATLAS-5054: debug coverage files 3

* ATLAS-5054: debug coverage files 4

* ATLAS-5054: revert all testing changes and make PR ready for review

Author: kumaab

.github/workflows/ci.yml

@@ -79,6 +79,12 @@ jobs:
             exit 1
           fi
 
+      - name: Upload Code Coverage Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage
+          path: dev-support/atlas-docker/dist/coverage/*
+
       - name: Cache downloaded archives
         if: success()
         uses: actions/cache@v4

dev-support/atlas-docker/Dockerfile.atlas-build

@@ -20,7 +20,7 @@ ARG ATLAS_BUILD_JAVA_VERSION
 ARG TARGETARCH
 
 # Install necessary packages to build Atlas
-RUN apt-get update && apt-get -y install git maven
+RUN apt-get update && apt-get -y install git maven unzip
 
 # Set environment variables
 ENV JAVA_HOME=/usr/lib/jvm/java-${ATLAS_BUILD_JAVA_VERSION}-openjdk-${TARGETARCH}

dev-support/atlas-docker/scripts/atlas-build.sh

@@ -92,3 +92,11 @@ mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz     /home/atlas/
 mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz  /home/atlas/dist/
 mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hbase-hook.tar.gz /home/atlas/dist/
 mv -f distro/target/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
+
+# Run code coverage and generate reports
+./dev-support/checks/coverage.sh
+status=$?
+
+# save coverage reports to the dist directory before container shutdown
+mv -f target/coverage /home/atlas/dist/
+exit $status

dev-support/checks/coverage.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script merges the jacoco exec files and generates a report in HTML and XML formats
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+cd "$DIR/../.." || exit 1
+
+set -ex
+
+REPORT_DIR="$DIR/../../target/coverage"
+
+mkdir -p "$REPORT_DIR"
+
+JACOCO_VERSION=$(mvn help:evaluate -Dexpression=jacoco.version -q -DforceStdout -Dscan=false)
+
+# Install jacoco cli
+mvn --non-recursive --no-transfer-progress -Dscan=false \
+  org.apache.maven.plugins:maven-dependency-plugin:copy \
+  -Dartifact=org.jacoco:org.jacoco.cli:${JACOCO_VERSION}:jar:nodeps
+
+jacoco() {
+  java -jar target/dependency/org.jacoco.cli-${JACOCO_VERSION}-nodeps.jar "$@"
+}
+
+# Merge all the jacoco.exec files
+jacoco merge $(find **/target -name jacoco.exec) --destfile "$REPORT_DIR/jacoco-all.exec"
+
+rm -rf target/coverage-classes || true
+mkdir -p target/coverage-classes
+
+# Unzip all the classes from the last build
+{ find */target/ -name 'atlas-*.jar'; find addons/**/target/*.jar -name '*-bridge-*.jar'; } | grep -v 'tests' | xargs -n1 unzip -o -q -d target/coverage-classes
+
+# get all source file paths
+src=$(find . -path '*/src/main/java' -o -path './target' -prune | sed 's/^/--sourcefiles /g' | xargs echo)
+
+# generate the reports
+jacoco report "$REPORT_DIR/jacoco-all.exec" $src --classfiles target/coverage-classes --html "$REPORT_DIR/all" --xml "$REPORT_DIR/all.xml"

pom.xml

@@ -129,6 +129,7 @@
         <ivy.version>2.5.2</ivy.version>
         <jackson.databind.version>2.12.7</jackson.databind.version>
         <jackson.version>2.12.7</jackson.version>
+        <jacoco.version>0.8.13</jacoco.version>
         <janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
         <janusgraph.version>1.0.0</janusgraph.version>
         <java.version.required>1.8</java.version.required>
@@ -905,6 +906,12 @@
                 <version>3.0.0</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>${jacoco.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>jul-to-slf4j</artifactId>
@@ -1334,7 +1341,7 @@
                     <forkCount>${surefire.forkCount}</forkCount>
                     <reuseForks>false</reuseForks>
                     <redirectTestOutputToFile>true</redirectTestOutputToFile>
-                    <argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
+                    <argLine>${argLine} -Djava.awt.headless=true -Dproject.version=${project.version}
                         -Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
                         -Xmx1024m -Djava.net.preferIPv4Stack=true ${atlas.surefire.options}</argLine>
                     <skip>${skipUTs}</skip>
@@ -1567,6 +1574,27 @@
                 <artifactId>javancss-maven-plugin</artifactId>
             </plugin>
 
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>${jacoco.version}</version>
+                <executions>
+                    <execution>
+                        <id>jacoco-initialize</id>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>jacoco-site</id>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <artifactId>maven-clean-plugin</artifactId>
                 <configuration>