Feature/run it test locally (#151)

Add comprehensive integration test automation with cleanup

- Implement integration test script for automated tool testing
- Add path normalization for cross-platform compatibility
- Integrate cleanup functionality with trap handlers
- Refactor GitHub workflow to use centralized test script
- Ensure proper restoration of original configurations

Author: andrzej-janczak

.github/workflows/it-test.yml

@@ -93,34 +93,8 @@ jobs:
         continue-on-error: true
         shell: bash
         run: |
-          # Make the script executable
-          chmod +x run-tool-tests.sh
-          
-          # Initialize failed tools file
-          rm -f /tmp/failed_tools.txt
-          touch /tmp/failed_tools.txt
-          
-          # Run tests for each tool directory
-          for tool_dir in plugins/tools/*/; do
-            tool_name=$(basename "$tool_dir")
-            if [ -d "$tool_dir/test/src" ]; then
-              echo "Running tests for $tool_name..."
-              ./run-tool-tests.sh "$tool_name" || {
-                echo "❌ Test failed for $tool_name"
-                echo "$tool_name" >> /tmp/failed_tools.txt
-              }
-            fi
-          done
-          
-          # Check if any tools failed
-          if [ -s /tmp/failed_tools.txt ] && [ "$(wc -l < /tmp/failed_tools.txt)" -gt 0 ]; then
-            echo -e "\n❌ The following tools failed their tests:"
-            cat /tmp/failed_tools.txt
-            echo "::error::Some tool tests failed. Please check the logs above for details."
-            exit 1
-          else
-            echo "✅ All tool tests passed successfully!"
-          fi
+          chmod +x integration-tests/test-tools.sh
+          ./integration-tests/test-tools.sh
 
       - name: Check test results
         if: always()

integration-tests/test-tools.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Function to cleanup all generated test files
+cleanup_all_test_files() {
+    echo "🧹 Cleaning up test files..."
+    
+    # Remove generated SARIF and sorted files from all tool test directories
+    find plugins/tools/*/test/src -name "actual.sarif" -o -name "actual.sorted.json" -o -name "expected.sorted.json" -o -name "codacy.yaml.backup" | xargs rm -f 2>/dev/null || true
+    
+    # Restore original codacy.yaml files if they were modified
+    git checkout -- plugins/tools/*/test/src/.codacy/codacy.yaml 2>/dev/null || true
+    
+    # Clean up any empty .codacy directories that were created during testing
+    find plugins/tools/*/test/src -name ".codacy" -type d -empty | xargs rmdir 2>/dev/null || true
+    
+    echo "✅ Cleanup completed"
+}
+
+# Set up trap to ensure cleanup happens even if script fails
+trap cleanup_all_test_files EXIT
+
+# Initialize failed tools file
+rm -f /tmp/failed_tools.txt
+touch /tmp/failed_tools.txt
+
+# Run tests for each tool directory
+for tool_dir in plugins/tools/*/; do
+    tool_name=$(basename "$tool_dir")
+    if [ -d "$tool_dir/test/src" ]; then
+        echo "Running tests for $tool_name..."
+        ./run-tool-tests.sh "$tool_name" || {
+            echo "❌ Test failed for $tool_name"
+            echo "$tool_name" >>/tmp/failed_tools.txt
+        }
+    fi
+done
+
+# Check if any tools failed
+if [ -s /tmp/failed_tools.txt ] && [ "$(wc -l </tmp/failed_tools.txt)" -gt 0 ]; then
+    echo -e "\n❌ The following tools failed their tests:"
+    cat /tmp/failed_tools.txt
+    echo "::error::Some tool tests failed. Please check the logs above for details."
+    exit 1
+else
+    echo "✅ All tool tests passed successfully!"
+fi

run-tool-tests.sh

@@ -3,24 +3,29 @@
 # Function to normalize paths in a file
 normalize_paths() {
   local file=$1
-  local path_prefix
+  # Get the repository root directory (5 levels up from current test/src directory)
+  local repo_root=$(cd ../../../../.. && pwd)
 
+  # Normalize absolute paths to relative ones for consistent testing
   if [[ "$OSTYPE" == "darwin"* ]]; then
-    path_prefix="/Users/runner/work/codacy-cli-v2/codacy-cli-v2/"
+    # Replace absolute paths with relative paths in URI contexts
+    sed -i '' "s#file://${repo_root}/plugins/tools/#file:///plugins/tools/#g" "$file"
+    sed -i '' "s#${repo_root}/plugins/tools/#/plugins/tools/#g" "$file"
+    # Handle CI runner paths for macOS
+    sed -i '' "s#file:///Users/runner/work/codacy-cli-v2/codacy-cli-v2/plugins/tools/#file:///plugins/tools/#g" "$file"
+    sed -i '' "s#/Users/runner/work/codacy-cli-v2/codacy-cli-v2/plugins/tools/#/plugins/tools/#g" "$file"
   else
-    path_prefix="/home/runner/work/codacy-cli-v2/codacy-cli-v2/"
-  fi
-  
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    sed -i '' "s|file://${path_prefix}|file:///|g" "$file"
-    sed -i '' "s|${path_prefix}|/|g" "$file"
-  else
-    sed -i "s|file://${path_prefix}|file:///|g" "$file"
-    sed -i "s|${path_prefix}|/|g" "$file"
+    # Replace absolute paths with relative paths in URI contexts  
+    sed -i "s#file://${repo_root}/plugins/tools/#file:///plugins/tools/#g" "$file"
+    sed -i "s#${repo_root}/plugins/tools/#/plugins/tools/#g" "$file"
+    # Handle CI runner paths for Linux
+    sed -i "s#file:///home/runner/work/codacy-cli-v2/codacy-cli-v2/plugins/tools/#file:///plugins/tools/#g" "$file"
+    sed -i "s#/home/runner/work/codacy-cli-v2/codacy-cli-v2/plugins/tools/#/plugins/tools/#g" "$file"
   fi
 }
 
 # Function to sort SARIF file
+
 sort_sarif() {
   local input=$1
   local output=$2
@@ -56,6 +61,35 @@ fi
 # Change to the tool's test directory
 cd "$TOOL_DIR" || exit 1
 
+# Store initial state for cleanup
+initial_codacy_config=""
+if [ -f .codacy/codacy.yaml ]; then
+  # Backup existing config if it exists
+  cp .codacy/codacy.yaml .codacy/codacy.yaml.backup
+  initial_codacy_config="exists"
+fi
+
+# Function to cleanup generated files
+cleanup_test_files() {
+  # Remove generated SARIF and sorted files
+  rm -f actual.sarif actual.sorted.json expected.sorted.json
+  
+  # Restore or clean up .codacy/codacy.yaml
+  if [ "$initial_codacy_config" = "exists" ] && [ -f .codacy/codacy.yaml.backup ]; then
+    # Restore original config
+    mv .codacy/codacy.yaml.backup .codacy/codacy.yaml
+  elif [ "$initial_codacy_config" != "exists" ]; then
+    # Remove generated config and directory if they didn't exist initially
+    rm -f .codacy/codacy.yaml
+    if [ -d .codacy ]; then
+      rmdir .codacy 2>/dev/null || true  # Only remove if empty
+    fi
+  fi
+}
+
+# Set trap to ensure cleanup happens even if script fails
+trap cleanup_test_files EXIT
+
 # Run analysis
 "$CLI_PATH" install
 "$CLI_PATH" analyze --tool "$TOOL_NAME" --format sarif --output actual.sarif
@@ -75,6 +109,8 @@ if ! diff expected.sorted.json actual.sorted.json; then
   echo -e "\nActual SARIF output:"
   cat actual.sorted.json
   echo "$TOOL_NAME" >> /tmp/failed_tools.txt
+  # Return to original directory before exit
+  cd ../../../../.. || exit 1
   exit 1
 else
   echo "✅ Tests passed successfully for $TOOL_NAME"