fix: Adding coverage report for integration tests (#599)

* fix: Adding coverage for integration tests

* fix: Optimize integration test coverage

* fix: Optimize fix

* fix: Docker image fix

* fix: Fix codecov config file

* fix: Improvements to  codecov config

* fix: Fix codecov

* fix: Commented out upload coverage tests

* fix: Commit for re running tests

* chore: Improvements

* chore: Improvements

Author: NicoMolinaOZ

.github/codecov.yml

@@ -16,10 +16,20 @@ coverage:
           - ai
           - dev
           - properties
+      # TODO: Integration tests will be included in the future when test cases are considerable enough
+      # integration:
+      #   informational: true
+      #   target: auto
+      #   flags:
+      #     - integration
     patch:
       default:
         target: auto
         threshold: 1%
+        flags:
+          - ai
+          - dev
+          - properties
 flags:
   ai:
     paths:
@@ -29,6 +39,15 @@ flags:
     paths:
       - src/
     carryforward: true
+  properties:
+    paths:
+      - src/
+    carryforward: true
+  # TODO: Integration flag will be re-enabled when test cases are considerable enough to include
+  # integration:
+  #   paths:
+  #     - src/
+  #   carryforward: false  # Don't carry forward manual test coverage
 comment:
   layout: flags, diff, files
   require_changes: false

.github/workflows/integration-tests.yaml

@@ -32,6 +32,7 @@ jobs:
           API_KEY=${{ secrets.INTEGRATION_TESTS_API_KEY }}
           LOG_LEVEL=info
           USE_KMS=true
+          CI=true
           AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
           AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
           AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
@@ -45,5 +46,31 @@ jobs:
           mkdir -p tests/integration/config/testnet
           echo '${{ vars.INTEGRATION_CONFIG_JSON }}' | envsubst > tests/integration/config/testnet/config.json
           echo '${{ vars.INTEGRATION_REGISTRY_JSON }}' > tests/integration/config/testnet/registry.json
+      - name: Build Integration Test Image
+        run: |
+          docker build -f Dockerfile.integration --target runtime -t openzeppelin-relayer-integration:latest .
+      - name: Create coverage directories
+        run: mkdir -p ./coverage ./profraw
       - name: Run Integration Tests
         run: ./scripts/run-integration-docker.sh
+      - name: Verify coverage exists
+        if: always()
+        run: |
+          ls -lah ./coverage/
+          if [ -f ./coverage/integration-lcov.info ]; then
+            echo "✓ Coverage file found ($(wc -l < ./coverage/integration-lcov.info) lines)"
+          else
+            echo "✗ Coverage file not found!"
+            exit 1
+          fi
+      # TODO: Integration test coverage will be included in the future when test cases are considerable enough to include
+      # - name: Upload Integration Coverage to Codecov
+      #   if: always()  # Upload coverage even if tests fail
+      #   uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7  # v5.5.1
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     name: integration-coverage
+      #     files: ./coverage/integration-lcov.info
+      #     flags: integration
+      #     fail_ci_if_error: false
+      #     verbose: true

.gitignore

@@ -66,3 +66,5 @@ node_modules
 # Integration test artifacts
 test-results/
 **/*lcov.info
+profraw/
+coverage/

Dockerfile.integration

@@ -16,6 +16,11 @@ RUN apk update && apk --no-cache add \
 
 WORKDIR /app
 
+# Install llvm-tools-preview component and cargo-llvm-cov for coverage
+RUN rustup default stable && \
+    rustup component add llvm-tools-preview && \
+    cargo install cargo-llvm-cov --locked
+
 # Copy manifests
 COPY Cargo.toml Cargo.lock ./
 
@@ -28,27 +33,40 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
 COPY src ./src
 COPY tests ./tests
 
-# Build integration tests
-# Clear cached integration test artifacts to force recompilation when source changes.
-# This ensures changes to test files are always reflected in the binary while still
-# keeping dependency compilation cached for fast builds.
+# Build integration tests with coverage instrumentation
+# Using RUSTFLAGS to enable coverage instrumentation during compilation
+# This avoids the triple compilation issue (normal build, test build, coverage build)
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=/app/target \
-    rm -rf /app/target/release/deps/integration-* \
-           /app/target/release/.fingerprint/*integration* && \
+    RUSTFLAGS="-C instrument-coverage" \
     cargo test --features integration-tests --test integration --release --no-run && \
     TEST_BINARY=$(find target/release/deps -name 'integration-*' -type f -executable | head -n 1) && \
     cp "$TEST_BINARY" /app/integration-test-binary
 
 # ============================================================================
-# Runtime Stage - Minimal image with only what's needed to run tests
+# Runtime Stage - Minimal image with only what's needed to run tests and generate coverage
 # ============================================================================
-FROM cgr.dev/chainguard/wolfi-base
+FROM cgr.dev/chainguard/wolfi-base AS runtime
 
 WORKDIR /app
 
-# Copy test binary from builder
+# Copy LLVM tools and libraries from builder (architecture-agnostic with wildcards)
+# This fixes the hardcoded aarch64 issue - works on both x86_64 and aarch64
+COPY --from=builder /root/.rustup/toolchains/stable-*/lib/rustlib/*/bin/llvm-cov \
+     /usr/local/bin/llvm-cov
+COPY --from=builder /root/.rustup/toolchains/stable-*/lib/rustlib/*/bin/llvm-profdata \
+     /usr/local/bin/llvm-profdata
+
+# Copy LLVM shared libraries that llvm-cov and llvm-profdata depend on
+# Use a broad pattern to cover libLLVM.so.* and libLLVM-*.so variants.
+COPY --from=builder /root/.rustup/toolchains/stable-*/lib/libLLVM*.so* /usr/local/lib/
+COPY --from=builder /root/.rustup/toolchains/stable-*/lib/librustc_driver-*.so /usr/local/lib/
+
+# Update library path so the LLVM tools can find their dependencies
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+# Copy instrumented test binary from builder
 COPY --from=builder /app/integration-test-binary ./integration-tests
 
 # Copy SSL libraries from builder
@@ -62,9 +80,19 @@ COPY tests/integration ./tests/integration
 # Networks is mounted separately at /app/networks to avoid nested mount issues
 RUN mkdir -p /app/config /app/networks
 
-# Create directory for test results
-RUN mkdir -p /app/test-results
+# Create directories for coverage output with proper permissions
+RUN mkdir -p /app/coverage /app/profraw && chmod -R 777 /app/coverage /app/profraw
 
-# Default test command
-# Can be overridden via docker-compose or docker run
-CMD ["./integration-tests", "--nocapture", "--test-threads=1"]
+# Execute tests directly, then generate lcov report using llvm-cov
+# This approach:
+# 1. Runs the instrumented test binary (generates .profraw files)
+# 2. Merges .profraw files into .profdata
+# 3. Generates lcov report from .profdata
+# All without invoking cargo or recompiling
+CMD ["sh", "-c", "\
+  LLVM_PROFILE_FILE=/app/profraw/integration-%p-%m.profraw \
+    ./integration-tests --nocapture --test-threads=1 && \
+  llvm-profdata merge -sparse /app/profraw/*.profraw -o /app/coverage/integration.profdata && \
+  llvm-cov export --format=lcov \
+    --instr-profile=/app/coverage/integration.profdata \
+    ./integration-tests > /app/coverage/integration-lcov.info"]

docker-compose.integration.yml

@@ -80,10 +80,13 @@ services:
       retries: 15
       start_period: 10s
   integration-tests:
+    image: openzeppelin-relayer-integration:latest
     build:
       context: .
       dockerfile: Dockerfile.integration
+      target: runtime  # Use runtime stage that executes pre-built binary with coverage
     container_name: integration-tests
+    user: ${HOST_UID:-1000}:${HOST_GID:-1000}  # Run as host user for proper file permissions
     depends_on:
       relayer:
         condition: service_healthy
@@ -101,10 +104,10 @@ services:
     volumes:
       - ./tests/integration/config:/app/config:ro
       - ./config/networks:/app/networks:ro
-      # Optional: mount test results output
-      - ./test-results:/app/test-results
-    # Uses the pre-built binary from Dockerfile.integration
-    # The binary runs with --ignored --nocapture --test-threads=1 by default
+      # Mount coverage output directory
+      - ./coverage:/app/coverage
+      # Mount profraw directory for debugging raw coverage data
+      - ./profraw:/app/profraw
 networks:
   integration:
     driver: bridge

scripts/run-integration-docker.sh

@@ -51,6 +51,13 @@ set -a
 source "$PROJECT_ROOT/.env.integration"
 set +a
 
+# Export UID/GID for docker-compose (allows container to run as host user)
+export HOST_UID=$(id -u)
+export HOST_GID=$(id -g)
+
+# Detect if running in CI
+CI=${CI:-false}
+
 # MODE handling (local/testnet)
 MODE=${MODE:-local}  # Default to local
 
@@ -215,11 +222,33 @@ case "$COMMAND" in
         log_info "Starting relayer..."
         docker compose $PROFILE -f docker-compose.integration.yml up -d relayer
 
-        # Run tests
+        # Create coverage directories (container runs as host user, so default perms are fine)
+        mkdir -p "$PROJECT_ROOT/coverage" "$PROJECT_ROOT/profraw"
+
+        # Run tests - different behavior for CI vs local
         log_info "Running integration tests..."
-        docker compose $PROFILE -f docker-compose.integration.yml up --build --abort-on-container-exit integration-tests
+        if [ "$CI" = "true" ]; then
+            # CI: Use pre-built images (already built by workflow)
+            docker compose $PROFILE -f docker-compose.integration.yml up \
+                --no-build \
+                --abort-on-container-exit \
+                integration-tests
+        else
+            # Local: Build if needed
+            docker compose $PROFILE -f docker-compose.integration.yml up \
+                --build \
+                --abort-on-container-exit \
+                integration-tests
+        fi
         EXIT_CODE=$?
 
+        # Verify coverage was generated
+        if [ -f "$PROJECT_ROOT/coverage/integration-lcov.info" ]; then
+            log_success "Coverage generated: $(wc -l < "$PROJECT_ROOT/coverage/integration-lcov.info") lines"
+        else
+            log_warn "Coverage file not found at $PROJECT_ROOT/coverage/integration-lcov.info"
+        fi
+
         if [ $EXIT_CODE -eq 0 ]; then
             log_success "Integration tests passed!"
         else

tests/integration/common/registry.rs

@@ -108,7 +108,7 @@ impl TestRegistry {
     pub fn get_network(&self, network: &str) -> Result<&NetworkConfig> {
         self.networks
             .get(network)
-            .ok_or_else(|| eyre::eyre!("Network '{}' not found in registry", network))
+            .ok_or_else(|| eyre::eyre!("Network '{}' not found in registry!", network))
     }
 
     /// Get contract address for a network