run

#!/usr/bin/env bash
set -e
shopt -s globstar

# Default values
target_dir="integration-tests"
overwrite=0
verbose=0

# Parse arguments
while [ "$#" -gt 0 ]; do
    case "$1" in
        --overwrite)
            overwrite=1
            ;;
        -v|--verbose)
            verbose=1
            ;;
        *)
            target_dir="$1"
            ;;
    esac
    shift
done


judge="$PWD"
exact_match=0
description_changed=0
minor_change=0
major_change=0

err_out="/dev/null"
if [ $verbose -gt 0 ]; then
    err_out="/dev/stderr"
fi

# Find all config.json files in target directory
for config in $target_dir/**/config.json; do
    test_dir="$(dirname "$config")"
    
    # Extract config values
    filename="$(jq -r '.evaluation.filename // .filename' "$config")"
    allow_compilation_warnings="$(jq -r '.allow_compilation_warnings == true' "$config")"
    lang="$(jq -r '.natural_language // "en"' "$config")"
    
    # Determine submission source and result file
    submission_source="$test_dir/submission.java"
    config_source="$test_dir/solution/$filename"
    if [ -f "$submission_source" ]; then
        source="$submission_source"
    elif [ -f "$config_source" ]; then
        source="$config_source"
    else
        echo "Warning: No submission.java or $filename found in $test_dir. Skipping." >&2
        continue
    fi
    
    result_file="$test_dir/result.json"

    echo -en "Testing ${test_dir} ...\t"
    
    workdir="$(mktemp -d)"
    cd "$workdir"

    # Copy workdir contents if they exist
    if [ -d "$judge/$test_dir/workdir" ]; then
        find "$judge/$test_dir/workdir" -mindepth 1 -maxdepth 1 -exec cp -r \{\} . \;
    fi

    # Run the judge
    output=$(echo '{ "resources": "'"$judge/$test_dir/evaluation/"'"
          , "judge": "'"$judge"'"
          , "natural_language": "'"$lang"'"
          , "workdir": "'"$workdir"'"
          , "allow_compilation_warnings": "'"$allow_compilation_warnings"'"
          , "filename": "'"$filename"'"
          , "time_limit": 30
          , "memory_limit": 1000000000
          , "source": "'"$judge/$source"'"
          }' \
          | (timeout -k 10s 60s "$judge/run" 2> "$err_out") \
          | jq --sort-keys 'if(.command == "append-message")
                then .message.description |= gsub("\n at [^\n]+\\([^)]+\\)"; "")
                else .
            end' 
    )

    cd "$judge"
    rm -r "$workdir"

    # Count accepted and failed in output
    accepted_output="$(echo "$output" | jq -s 'map(select(.accepted == true)) | length')"
    failed_output="$(echo "$output" | jq -s 'map(select(.accepted == false)) | length')"
    echo -en "$accepted_output accepted, $failed_output failed\t"



    if [ -f "$result_file" ]; then
        # Count accepted and failed in result
        accepted_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )"
        failed_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )"


        # First check if files are exact using diff, then check if they differ except '.description', then check they differ in accepted or failed

        if diff -c "$result_file" <(echo "$output") > "$err_out"; then
            echo "[EXACT MATCH]"
            exact_match=$((exact_match + 1))
        elif diff -c <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > "$err_out"; then
            echo "[DESCRIPTION CHANGED]"
            description_changed=$((description_changed + 1))
        elif [ "$accepted_output" -eq "$accepted_result" ] && [ "$failed_output" -eq "$failed_result" ]; then
            echo "[MINOR CHANGE] (accepted/failed unchanged)"
            minor_change=$((minor_change + 1))
        else
            echo "[MAJOR CHANGE] (accepted/failed changed, was $accepted_result/$failed_result)"
            major_change=$((major_change + 1))
        fi
    fi

    # Check or overwrite results
    if [ "$overwrite" -eq 1 ]; then
        echo "$output" > "$result_file"
        echo "[OVERWRITE]"
    fi
done

# Report results
echo
echo "=== RESULTS ==="
echo "Exact matches: $exact_match"
echo "Description changed: $description_changed"
echo "Minor changes: $minor_change"
echo "Major changes: $major_change"

# Exit with error on minor or major changes
if [ "$minor_change" -gt 0 ] || [ "$major_change" -gt 0 ]; then
    echo "FAIL. Run with --overwrite to update the results."
    exit 1
fi