diff --git a/tests/test_issue_232_reproduction.py b/tests/test_issue_232_reproduction.py new file mode 100644 index 00000000..44502f1b --- /dev/null +++ b/tests/test_issue_232_reproduction.py @@ -0,0 +1,322 @@ +""" +Reproduction test for GitHub Issue #232: + pdd fix incorrectly prints the name of an output code file when + no new code file was generated + +Issue URL: https://github.com/promptdriven/pdd/issues/232 + +Bug Description: + When `pdd fix` only modifies the test file (not the code file), the output + still prints both file paths: + Test file: /path/to/test_fixed.py + Code file: /path/to/code_fixed.py <-- BUG: This should NOT appear + + The file-saving logic correctly only writes files that have content, + but the output messaging doesn't follow the same logic. + +Location of bug: pdd/fix_main.py lines 418-423 +""" + +import pytest +from unittest.mock import patch, MagicMock, mock_open +import click +from io import StringIO + +from pdd import DEFAULT_STRENGTH +from pdd.fix_main import fix_main + + +@pytest.fixture +def mock_ctx(): + """Create a mock Click context with default parameters.""" + ctx = MagicMock(spec=click.Context) + ctx.obj = { + 'force': False, + 'quiet': False, # IMPORTANT: quiet=False to see output + 'strength': DEFAULT_STRENGTH, + 'temperature': 0.0, + 'verbose': False, + 'time': None, + 'context': None, + 'confirm_callback': None, + 'local': True # Force local execution + } + return ctx + + +@patch('pdd.fix_main.rprint') +@patch('pdd.fix_main.run_pytest_on_file') +@patch('pdd.fix_main.Path') +@patch('pdd.fix_main.construct_paths') +@patch('pdd.fix_main.fix_errors_from_unit_tests') +def test_issue_232_only_test_file_modified_should_not_print_code_file( + mock_fix_errors, + mock_construct_paths, + mock_path, + mock_run_pytest, + mock_rprint, + mock_ctx +): + """ + REPRODUCTION TEST for Issue #232. + + Scenario: + - pdd fix is called + - The LLM only fixes the test file (fixed_unit_test has content) + - The LLM does NOT fix the code file (fixed_code is empty) + + Expected behavior: + - Output should ONLY show: "Test file: ..." + - Output should NOT show: "Code file: ..." (since it wasn't modified) + + Current buggy behavior: + - Output shows BOTH paths, even though code file wasn't modified + """ + mock_path.return_value.exists.return_value = True + + mock_construct_paths.return_value = ( + {}, # resolved_config + { + 'prompt_file': 'Test prompt', + 'code_file': 'def add(a, b): return a + b', + 'unit_test_file': 'def test_add(): assert add(1, 2) == 4', # Wrong assertion + 'error_file': 'AssertionError: 3 != 4' + }, + { + 'output_test': '/home/james/pdd_cap/tests/test_fixed.py', + 'output_code': '/home/james/pdd_cap/code_fixed.py', # This path should NOT be printed + 'output_results': '/home/james/pdd_cap/fix_results.log' + }, + None + ) + + # LLM only fixes the test file, NOT the code file + mock_fix_errors.return_value = ( + True, # update_unit_test = True (test was fixed) + False, # update_code = False (code was NOT modified) + "def test_add(): assert add(1, 2) == 3", # Fixed test content + "", # Empty string - code was NOT modified + "Analysis: The test had wrong assertion, fixed expected value", + 0.50, + "gpt-4" + ) + + # Simulate tests passing after the fix + mock_run_pytest.return_value = (0, 0, 0, "All tests passed") + + # Execute fix_main + m_open = mock_open() + with patch('builtins.open', m_open): + success, fixed_test, fixed_code, attempts, total_cost, model_name = fix_main( + ctx=mock_ctx, + prompt_file="prompt.prompt", + code_file="code.py", + unit_test_file="test_code.py", + error_file="errors.log", + output_test=None, + output_code=None, + output_results=None, + loop=False, + verification_program=None, + max_attempts=3, + budget=5.0, + auto_submit=False + ) + + # Verify the fix was successful + assert success is True, "Fix should succeed since tests pass" + assert fixed_test == "def test_add(): assert add(1, 2) == 3" + assert fixed_code == "" # Code was NOT modified + + # Collect all rprint calls + all_rprint_calls = [str(call) for call in mock_rprint.call_args_list] + output_text = '\n'.join(all_rprint_calls) + + # ASSERTION FOR ISSUE #232: + # The code file path should NOT be printed when no code file was generated + + # First, verify the test file path WAS printed (expected behavior) + assert any('/home/james/pdd_cap/tests/test_fixed.py' in str(call) for call in mock_rprint.call_args_list), \ + f"Expected test file path to be printed. Got calls: {all_rprint_calls}" + + # Now check if the code file path was incorrectly printed (THE BUG) + code_file_printed = any('code_fixed.py' in str(call) for call in mock_rprint.call_args_list) + + # This assertion will FAIL until the bug is fixed + assert not code_file_printed, ( + "BUG #232: Code file path is being printed even though no code file was modified.\n" + f"Output included: {output_text}\n" + "Expected: Only 'Test file: ...' should be printed\n" + "Got: Both 'Test file: ...' and 'Code file: ...' were printed" + ) + + +@patch('pdd.fix_main.rprint') +@patch('pdd.fix_main.run_pytest_on_file') +@patch('pdd.fix_main.Path') +@patch('pdd.fix_main.construct_paths') +@patch('pdd.fix_main.fix_errors_from_unit_tests') +def test_issue_232_only_code_file_modified_should_not_print_test_file( + mock_fix_errors, + mock_construct_paths, + mock_path, + mock_run_pytest, + mock_rprint, + mock_ctx +): + """ + Related test for Issue #232 - opposite scenario. + + Scenario: + - pdd fix is called + - The LLM only fixes the code file (fixed_code has content) + - The LLM does NOT fix the test file (fixed_unit_test is empty) + + Expected behavior: + - Output should ONLY show: "Code file: ..." + - Output should NOT show: "Test file: ..." (since it wasn't modified) + """ + mock_path.return_value.exists.return_value = True + + mock_construct_paths.return_value = ( + {}, + { + 'prompt_file': 'Test prompt', + 'code_file': 'def divide(a, b): return a / b', # Bug: no zero check + 'unit_test_file': 'def test_divide(): assert divide(10, 2) == 5', + 'error_file': 'ZeroDivisionError when dividing by zero' + }, + { + 'output_test': '/path/to/test_fixed.py', + 'output_code': '/path/to/code_fixed.py', + 'output_results': '/path/to/fix_results.log' + }, + None + ) + + # LLM only fixes the code file, NOT the test + mock_fix_errors.return_value = ( + False, # update_unit_test = False (test was NOT modified) + True, # update_code = True (code was fixed) + "", # Empty - test was NOT modified + "def divide(a, b):\n if b == 0:\n raise ValueError('Cannot divide by zero')\n return a / b", + "Analysis: Added zero check to divide function", + 0.60, + "gpt-4" + ) + + mock_run_pytest.return_value = (0, 0, 0, "All tests passed") + + m_open = mock_open() + with patch('builtins.open', m_open): + success, fixed_test, fixed_code, attempts, total_cost, model_name = fix_main( + ctx=mock_ctx, + prompt_file="prompt.prompt", + code_file="code.py", + unit_test_file="test_code.py", + error_file="errors.log", + output_test=None, + output_code=None, + output_results=None, + loop=False, + verification_program=None, + max_attempts=3, + budget=5.0, + auto_submit=False + ) + + assert success is True + assert fixed_test == "" # Test was NOT modified + assert "raise ValueError" in fixed_code # Code was modified + + all_rprint_calls = [str(call) for call in mock_rprint.call_args_list] + + # Verify code file path WAS printed + assert any('code_fixed.py' in str(call) for call in mock_rprint.call_args_list), \ + f"Expected code file path to be printed. Got calls: {all_rprint_calls}" + + # Verify test file path was NOT printed (symmetrical bug) + test_file_printed = any('test_fixed.py' in str(call) for call in mock_rprint.call_args_list) + + assert not test_file_printed, ( + "BUG #232 (related): Test file path is being printed even though no test file was modified.\n" + f"Output included: {'\n'.join(all_rprint_calls)}\n" + "Expected: Only 'Code file: ...' should be printed" + ) + + +@patch('pdd.fix_main.rprint') +@patch('pdd.fix_main.run_pytest_on_file') +@patch('pdd.fix_main.Path') +@patch('pdd.fix_main.construct_paths') +@patch('pdd.fix_main.fix_errors_from_unit_tests') +def test_both_files_modified_should_print_both_paths( + mock_fix_errors, + mock_construct_paths, + mock_path, + mock_run_pytest, + mock_rprint, + mock_ctx +): + """ + Positive test: When both files are modified, both paths should be printed. + This should work correctly both before and after the bug fix. + """ + mock_path.return_value.exists.return_value = True + + mock_construct_paths.return_value = ( + {}, + { + 'prompt_file': 'Test prompt', + 'code_file': 'original code', + 'unit_test_file': 'original test', + 'error_file': 'Some error' + }, + { + 'output_test': '/path/to/test_fixed.py', + 'output_code': '/path/to/code_fixed.py', + 'output_results': '/path/to/fix_results.log' + }, + None + ) + + # LLM fixes BOTH files + mock_fix_errors.return_value = ( + True, # update_unit_test = True + True, # update_code = True + "fixed test content", + "fixed code content", + "Analysis: Fixed both files", + 0.75, + "gpt-4" + ) + + mock_run_pytest.return_value = (0, 0, 0, "All tests passed") + + m_open = mock_open() + with patch('builtins.open', m_open): + success, _, _, _, _, _ = fix_main( + ctx=mock_ctx, + prompt_file="prompt.prompt", + code_file="code.py", + unit_test_file="test_code.py", + error_file="errors.log", + output_test=None, + output_code=None, + output_results=None, + loop=False, + verification_program=None, + max_attempts=3, + budget=5.0, + auto_submit=False + ) + + assert success is True + + all_rprint_calls = [str(call) for call in mock_rprint.call_args_list] + + # Both paths should be printed + assert any('test_fixed.py' in str(call) for call in mock_rprint.call_args_list), \ + "Test file path should be printed when test was modified" + assert any('code_fixed.py' in str(call) for call in mock_rprint.call_args_list), \ + "Code file path should be printed when code was modified"