Skip to content

increase the min number of test cases passed #259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 2, 2025
Merged

increase the min number of test cases passed #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3109674
bump to 6.
misrasaurabh1 May 31, 2025
b5c3dfe
Merge branch 'main' into increase-the-min-test-case-passed
misrasaurabh1 May 31, 2025
792819a
Update critic.py
KRRT7 May 31, 2025
4f3b870
more tests need to pass
KRRT7 Jun 2, 2025
9080051
Merge branch 'main' into increase-the-min-test-case-passed
KRRT7 Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions codeflash/code_utils/config_consts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@
N_TESTS_TO_GENERATE = 2
TOTAL_LOOPING_TIME = 10.0 # 10 second candidate benchmarking budget
COVERAGE_THRESHOLD = 60.0
MIN_TESTCASE_PASSED_THRESHOLD = 6
8 changes: 6 additions & 2 deletions codeflash/result/critic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@

from codeflash.cli_cmds.console import logger
from codeflash.code_utils import env_utils
from codeflash.code_utils.config_consts import COVERAGE_THRESHOLD, MIN_IMPROVEMENT_THRESHOLD
from codeflash.code_utils.config_consts import (
COVERAGE_THRESHOLD,
MIN_IMPROVEMENT_THRESHOLD,
MIN_TESTCASE_PASSED_THRESHOLD,
)
from codeflash.models.models import TestType

if TYPE_CHECKING:
Expand Down Expand Up @@ -50,7 +54,7 @@ def quantity_of_tests_critic(candidate_result: OptimizedCandidateResult) -> bool
for test_type in report:
pass_count += report[test_type]["passed"]

if pass_count >= 4:
if pass_count >= MIN_TESTCASE_PASSED_THRESHOLD:
return True
# If only one test passed, check if it's a REPLAY_TEST
return bool(pass_count == 1 and report[TestType.REPLAY_TEST]["passed"] == 1)
Expand Down
13 changes: 6 additions & 7 deletions tests/test_critic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,7 @@ def test_generated_test_critic() -> None:
timed_out=False,
loop_index=1,
)

test_results = [test_1, test_2, test_3, test_7]
test_results = [test_1, test_2, test_3, test_4, test_5, test_6, test_7, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand All @@ -209,7 +208,7 @@ def test_generated_test_critic() -> None:

assert quantity_of_tests_critic(candidate_result)

test_results = [test_1, test_2, test_3, test_6, test_7]
test_results = [test_1, test_2, test_3, test_6, test_7, test_1, test_4, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand All @@ -222,7 +221,7 @@ def test_generated_test_critic() -> None:

assert quantity_of_tests_critic(candidate_result)

test_results = [test_1, test_3, test_4, test_2, test_7]
test_results = [test_1, test_3, test_4, test_2, test_7, test_1, test_6, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand All @@ -248,7 +247,7 @@ def test_generated_test_critic() -> None:

assert not quantity_of_tests_critic(candidate_result)

test_results = [test_1, test_2, test_3, test_4, test_5]
test_results = [test_1, test_2, test_3, test_4, test_5, test_1, test_1, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand Down Expand Up @@ -287,7 +286,7 @@ def test_generated_test_critic() -> None:

assert quantity_of_tests_critic(candidate_result)

test_results = [test_1, test_2, test_3, test_4, test_5]
test_results = [test_1, test_2, test_3, test_4, test_5, test_1, test_1, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand Down Expand Up @@ -328,7 +327,7 @@ def test_generated_test_critic() -> None:

assert not quantity_of_tests_critic(candidate_result)

test_results = [test_1, test_2, test_3, test_5]
test_results = [test_1, test_2, test_3, test_5, test_1, test_1, test_1, test_1]

candidate_result = OptimizedCandidateResult(
max_loop_count=5,
Expand Down
Loading