test.sh

#!/usr/bin/env bash
# End-to-end test: build each CTF, run its exploit, verify submit-proof passes.
# Usage: ./test.sh [CVE-ID]   (test one CVE, or all if no argument)

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
IMAGES_DIR="$SCRIPT_DIR/images"
EXPLOITS_DIR="$SCRIPT_DIR/hints/exploits"

PASSED=()
FAILED=()
SKIPPED=()
CONTAINERS=()

cleanup() {
    for c in "${CONTAINERS[@]}"; do
        docker rm -f "$c" > /dev/null 2>&1
    done
}
trap cleanup EXIT

test_cve() {
    local cve="$1"
    local build_dir="$IMAGES_DIR/$cve"
    local tag="ctf-test-$(echo "$cve" | tr '[:upper:]' '[:lower:]')"
    local container="${tag}-test"

    echo ""
    echo "############################################################"
    echo "# Testing: $cve"
    echo "############################################################"

    # Find exploit file
    local exploit=""
    local exploit_ext=""
    for ext in py c; do
        if [ -f "$EXPLOITS_DIR/${cve}-exploit.${ext}" ]; then
            exploit="$EXPLOITS_DIR/${cve}-exploit.${ext}"
            exploit_ext="$ext"
            break
        fi
    done

    if [ -z "$exploit" ]; then
        echo "  SKIP: no exploit found"
        SKIPPED+=("$cve")
        return
    fi

    # Find manifest (optional)
    local manifest="$EXPLOITS_DIR/${cve}-exploit-manifest.json"

    # Build image
    echo "  Building image..."
    if ! docker build -q -t "$tag" "$build_dir" > /dev/null 2>&1; then
        echo "  FAIL: docker build failed"
        FAILED+=("$cve (build)")
        return
    fi

    # Remove any existing container
    docker rm -f "$container" > /dev/null 2>&1 || true

    # Start container
    echo "  Starting container..."
    docker run -d --name "$container" "$tag" > /dev/null 2>&1
    CONTAINERS+=("$container")
    sleep 2  # wait for service to initialize

    # Install required packages if manifest specifies them
    if [ -f "$manifest" ]; then
        # Extract required_packages using grep/sed (no jq dependency)
        local pkgs
        pkgs=$(grep -o '"required_packages": \[[^]]*\]' "$manifest" | sed 's/"required_packages": \[//;s/\]//;s/"//g;s/,/ /g' | tr -s ' ')
        if [ -n "$pkgs" ] && [ "$pkgs" != " " ]; then
            echo "  Installing packages: $pkgs"
            docker exec "$container" pip3 install --break-system-packages $pkgs > /dev/null 2>&1
        fi
    fi

    # Copy exploit into container
    docker cp "$exploit" "$container:/tmp/exploit.${exploit_ext}"

    # Compile if C
    if [ "$exploit_ext" = "c" ]; then
        echo "  Compiling exploit..."
        local compile_cmd="cc -o /tmp/exploit /tmp/exploit.c"
        # Use compile_command from manifest if available
        if [ -f "$manifest" ]; then
            local manifest_cc
            manifest_cc=$(grep -o '"compile_command": "[^"]*"' "$manifest" | sed 's/"compile_command": "//;s/"$//')
            if [ -n "$manifest_cc" ]; then
                compile_cmd="$manifest_cc"
            fi
        fi
        local compile_output
        compile_output=$(docker exec "$container" sh -c "$compile_cmd" 2>&1)
        if [ $? -ne 0 ]; then
            echo "  FAIL: compile failed"
            echo "  $compile_output"
            FAILED+=("$cve (compile)")
            docker rm -f "$container" > /dev/null 2>&1
            return
        fi
    fi

    # Determine run command
    local run_cmd=""
    if [ -f "$manifest" ]; then
        run_cmd=$(grep -o '"run_command": "[^"]*"' "$manifest" | sed 's/"run_command": "//;s/"$//')
    fi
    if [ -z "$run_cmd" ]; then
        if [ "$exploit_ext" = "c" ]; then
            run_cmd="/tmp/exploit"
        else
            run_cmd="python3 /tmp/exploit.${exploit_ext}"
        fi
    fi

    # Run exploit
    echo "  Running exploit: $run_cmd"
    local exploit_output
    exploit_output=$(docker exec --user ctf "$container" sh -c "$run_cmd" 2>&1)
    local exploit_rc=$?
    # Don't fail on nonzero exit — some exploits exit nonzero intentionally

    # Give a moment for async effects (file writes, crashes)
    sleep 1

    # Submit proof
    echo "  Submitting proof..."
    local proof_output
    proof_output=$(docker exec --user ctf "$container" /ctf/submit-proof 2>&1)
    local proof_rc=$?

    if [ $proof_rc -eq 0 ]; then
        echo "  PASS: $proof_output"
        PASSED+=("$cve")
    else
        echo "  FAIL: $proof_output"
        echo "  Exploit output: $(echo "$exploit_output" | head -5)"
        FAILED+=("$cve (proof)")
    fi

    # Cleanup
    docker rm -f "$container" > /dev/null 2>&1
}

# Main
if [ $# -ge 1 ]; then
    # Test specific CVE(s)
    for cve in "$@"; do
        test_cve "$cve"
    done
else
    # Test all
    for dir in "$IMAGES_DIR"/CVE-*; do
        cve=$(basename "$dir")
        test_cve "$cve"
    done
fi

echo ""
echo "============================================================"
echo "RESULTS: ${#PASSED[@]} passed, ${#FAILED[@]} failed, ${#SKIPPED[@]} skipped"
echo "============================================================"

if [ ${#PASSED[@]} -gt 0 ]; then
    echo "  Passed:"
    for cve in "${PASSED[@]}"; do
        echo "    $cve"
    done
fi

if [ ${#FAILED[@]} -gt 0 ]; then
    echo "  Failed:"
    for cve in "${FAILED[@]}"; do
        echo "    $cve"
    done
fi

if [ ${#SKIPPED[@]} -gt 0 ]; then
    echo "  Skipped:"
    for cve in "${SKIPPED[@]}"; do
        echo "    $cve"
    done
fi

[ ${#FAILED[@]} -eq 0 ]