Skip to content

Commit ab43ff0

Browse files
aseembits93aseembits93
committed
ready to merge
1 parent 4da2b1f commit ab43ff0

File tree

2 files changed

+258
-18
lines changed

2 files changed

+258
-18
lines changed

‎codeflash/result/create_pr.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ def existing_tests_source_for(
109109
f"`{print_filename}::{qualified_name}`",
110110
f"{print_original_runtime}",
111111
f"{print_optimized_runtime}",
112-
f"⚠️{perf_gain}%",
112+
f"{perf_gain}%⚠️",
113113
]
114114
)
115115
elif "codeflash_concolic" in str(print_filename):
@@ -118,7 +118,7 @@ def existing_tests_source_for(
118118
f"`{print_filename}::{qualified_name}`",
119119
f"{print_original_runtime}",
120120
f"{print_optimized_runtime}",
121-
f"⚠️{perf_gain}%",
121+
f"{perf_gain}%⚠️",
122122
]
123123
)
124124
else:
@@ -127,7 +127,7 @@ def existing_tests_source_for(
127127
f"`{print_filename}::{qualified_name}`",
128128
f"{print_original_runtime}",
129129
f"{print_optimized_runtime}",
130-
f"⚠️{perf_gain}%",
130+
f"{perf_gain}%⚠️",
131131
]
132132
)
133133
elif "__replay_test_" in str(print_filename):
@@ -136,7 +136,7 @@ def existing_tests_source_for(
136136
f"`{print_filename}::{qualified_name}`",
137137
f"{print_original_runtime}",
138138
f"{print_optimized_runtime}",
139-
f"✅{perf_gain}%",
139+
f"{perf_gain}%✅",
140140
]
141141
)
142142
elif "codeflash_concolic" in str(print_filename):
@@ -145,7 +145,7 @@ def existing_tests_source_for(
145145
f"`{print_filename}::{qualified_name}`",
146146
f"{print_original_runtime}",
147147
f"{print_optimized_runtime}",
148-
f"✅{perf_gain}%",
148+
f"{perf_gain}%✅",
149149
]
150150
)
151151
else:
@@ -154,7 +154,7 @@ def existing_tests_source_for(
154154
f"`{print_filename}::{qualified_name}`",
155155
f"{print_original_runtime}",
156156
f"{print_optimized_runtime}",
157-
f"✅{perf_gain}%",
157+
f"{perf_gain}%✅",
158158
]
159159
)
160160
output_existing += tabulate( # type: ignore[no-untyped-call]

‎tests/test_existing_tests_source_for.py‎

Lines changed: 252 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
from unittest.mock import Mock
2+
import contextlib
13
import os
4+
import shutil
5+
import unittest
6+
from dataclasses import dataclass
27
from pathlib import Path
3-
from unittest.mock import Mock
4-
5-
import pytest
8+
from typing import Optional
69

710
from codeflash.result.create_pr import existing_tests_source_for
811

@@ -71,7 +74,7 @@ def test_single_test_with_improvement(self):
7174

7275
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
7376
|:------------------------------------------|:--------------|:---------------|:----------|
74-
| `test_module.py::TestClass.test_function` | 1.00ms | 500μs | ✅100% |
77+
| `test_module.py::TestClass.test_function` | 1.00ms | 500μs | 100%✅ |
7578
"""
7679

7780
assert result == expected
@@ -99,7 +102,7 @@ def test_single_test_with_regression(self):
99102

100103
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
101104
|:------------------------------------------|:--------------|:---------------|:----------|
102-
| `test_module.py::TestClass.test_function` | 500μs | 1.00ms | ⚠️-50.0% |
105+
| `test_module.py::TestClass.test_function` | 500μs | 1.00ms | -50.0%⚠️ |
103106
"""
104107

105108
assert result == expected
@@ -132,7 +135,7 @@ def test_test_without_class_name(self):
132135

133136
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
134137
|:----------------------------------|:--------------|:---------------|:----------|
135-
| `test_module.py::test_standalone` | 1.00ms | 800μs | ✅25.0% |
138+
| `test_module.py::test_standalone` | 1.00ms | 800μs | 25.0%✅ |
136139
"""
137140

138141
assert result == expected
@@ -218,8 +221,8 @@ def test_multiple_tests_sorted_output(self):
218221

219222
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
220223
|:-----------------------------------------------------|:--------------|:---------------|:----------|
221-
| `test_another.py::TestAnother.test_another_function` | 2.00ms | 1.50ms | ✅33.3% |
222-
| `test_module.py::TestClass.test_function` | 1.00ms | 800μs | ✅25.0% |
224+
| `test_another.py::TestAnother.test_another_function` | 2.00ms | 1.50ms | 33.3%✅ |
225+
| `test_module.py::TestClass.test_function` | 1.00ms | 800μs | 25.0%✅ |
223226
"""
224227

225228
assert result == expected
@@ -247,7 +250,7 @@ def test_multiple_runtimes_uses_minimum(self):
247250

248251
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
249252
|:------------------------------------------|:--------------|:---------------|:----------|
250-
| `test_module.py::TestClass.test_function` | 800μs | 500μs | ✅60.0% |
253+
| `test_module.py::TestClass.test_function` | 800μs | 500μs | 60.0%✅ |
251254
"""
252255

253256
assert result == expected
@@ -284,7 +287,7 @@ def test_complex_module_path_conversion(self):
284287

285288
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
286289
|:------------------------------------------------------------------------|:--------------|:---------------|:----------|
287-
| `integration/test_complex_module.py::TestComplex.test_complex_function` | 1.00ms | 750μs | ✅33.3% |
290+
| `integration/test_complex_module.py::TestComplex.test_complex_function` | 1.00ms | 750μs | 33.3%✅ |
288291
"""
289292

290293
assert result == expected
@@ -350,9 +353,246 @@ def test_filters_out_generated_tests(self):
350353
# Should only include the non-generated test
351354
expected = """| Test File::Test Function | Original ⏱️ | Optimized ⏱️ | Speedup |
352355
|:------------------------------------------|:--------------|:---------------|:----------|
353-
| `test_module.py::TestClass.test_function` | 1.00ms | 800μs | ✅25.0% |
356+
| `test_module.py::TestClass.test_function` | 1.00ms | 800μs | 25.0%✅ |
354357
"""
355358

356359
assert result == expected
357360

358-
361+
@dataclass(frozen=True)
362+
class MockInvocationId:
363+
"""Mocks codeflash.models.models.InvocationId"""
364+
test_module_path: str
365+
test_function_name: str
366+
test_class_name: Optional[str] = None
367+
368+
369+
@dataclass(frozen=True)
370+
class MockTestsInFile:
371+
"""Mocks a part of codeflash.models.models.FunctionCalledInTest"""
372+
test_file: Path
373+
374+
375+
@dataclass(frozen=True)
376+
class MockFunctionCalledInTest:
377+
"""Mocks codeflash.models.models.FunctionCalledInTest"""
378+
tests_in_file: MockTestsInFile
379+
380+
381+
@dataclass(frozen=True)
382+
class MockTestConfig:
383+
"""Mocks codeflash.verification.verification_utils.TestConfig"""
384+
tests_root: Path
385+
386+
387+
@contextlib.contextmanager
388+
def temp_project_dir():
389+
"""A context manager to create and chdir into a temporary project directory."""
390+
original_cwd = os.getcwd()
391+
# Use a unique name to avoid conflicts in /tmp
392+
project_root = Path(f"/tmp/test_project_{os.getpid()}").resolve()
393+
try:
394+
project_root.mkdir(exist_ok=True, parents=True)
395+
os.chdir(project_root)
396+
yield project_root
397+
finally:
398+
os.chdir(original_cwd)
399+
shutil.rmtree(project_root, ignore_errors=True)
400+
401+
402+
class ExistingTestsSourceForTests(unittest.TestCase):
403+
def setUp(self):
404+
self.func_qual_name = "my_module.my_function"
405+
# A default test_cfg for tests that don't rely on file system.
406+
self.test_cfg = MockTestConfig(tests_root=Path("/tmp/tests"))
407+
408+
def test_no_tests_for_function(self):
409+
"""Test case where no tests are found for the given function."""
410+
existing, replay, concolic = existing_tests_source_for(
411+
function_qualified_name_with_modules_from_root=self.func_qual_name,
412+
function_to_tests={},
413+
test_cfg=self.test_cfg,
414+
original_runtimes_all={},
415+
optimized_runtimes_all={},
416+
)
417+
self.assertEqual(existing, "")
418+
self.assertEqual(replay, "")
419+
self.assertEqual(concolic, "")
420+
421+
def test_no_runtime_data(self):
422+
"""Test case where tests exist but there is no runtime data."""
423+
with temp_project_dir() as project_root:
424+
tests_dir = project_root / "tests"
425+
tests_dir.mkdir(exist_ok=True)
426+
test_file_path = (tests_dir / "test_stuff.py").resolve()
427+
test_file_path.touch()
428+
429+
test_cfg = MockTestConfig(tests_root=tests_dir.resolve())
430+
function_to_tests = {
431+
self.func_qual_name: {
432+
MockFunctionCalledInTest(
433+
tests_in_file=MockTestsInFile(test_file=test_file_path)
434+
)
435+
}
436+
}
437+
existing, replay, concolic = existing_tests_source_for(
438+
function_qualified_name_with_modules_from_root=self.func_qual_name,
439+
function_to_tests=function_to_tests,
440+
test_cfg=test_cfg,
441+
original_runtimes_all={},
442+
optimized_runtimes_all={},
443+
)
444+
self.assertEqual(existing, "")
445+
self.assertEqual(replay, "")
446+
self.assertEqual(concolic, "")
447+
448+
def test_with_existing_test_speedup(self):
449+
"""Test with a single existing test that shows a speedup."""
450+
with temp_project_dir() as project_root:
451+
tests_dir = project_root / "tests"
452+
tests_dir.mkdir(exist_ok=True)
453+
test_file_path = (tests_dir / "test_existing.py").resolve()
454+
test_file_path.touch()
455+
456+
test_cfg = MockTestConfig(tests_root=tests_dir.resolve())
457+
function_to_tests = {
458+
self.func_qual_name: {
459+
MockFunctionCalledInTest(
460+
tests_in_file=MockTestsInFile(test_file=test_file_path)
461+
)
462+
}
463+
}
464+
465+
invocation_id = MockInvocationId(
466+
test_module_path="tests.test_existing",
467+
test_class_name="TestMyStuff",
468+
test_function_name="test_one",
469+
)
470+
471+
original_runtimes = {invocation_id: [200_000_000]}
472+
optimized_runtimes = {invocation_id: [100_000_000]}
473+
474+
existing, replay, concolic = existing_tests_source_for(
475+
function_qualified_name_with_modules_from_root=self.func_qual_name,
476+
function_to_tests=function_to_tests,
477+
test_cfg=test_cfg,
478+
original_runtimes_all=original_runtimes,
479+
optimized_runtimes_all=optimized_runtimes,
480+
)
481+
482+
self.assertIn("| Test File::Test Function", existing)
483+
self.assertIn("`test_existing.py::TestMyStuff.test_one`", existing)
484+
self.assertIn("200ms", existing)
485+
self.assertIn("100ms", existing)
486+
self.assertIn("100%✅", existing)
487+
self.assertEqual(replay, "")
488+
self.assertEqual(concolic, "")
489+
490+
def test_with_replay_and_concolic_tests_slowdown(self):
491+
"""Test with replay and concolic tests showing a slowdown."""
492+
with temp_project_dir() as project_root:
493+
tests_dir = project_root / "tests"
494+
tests_dir.mkdir(exist_ok=True)
495+
replay_test_path = (tests_dir / "__replay_test_abc.py").resolve()
496+
replay_test_path.touch()
497+
concolic_test_path = (tests_dir / "codeflash_concolic_xyz.py").resolve()
498+
concolic_test_path.touch()
499+
500+
test_cfg = MockTestConfig(tests_root=tests_dir.resolve())
501+
function_to_tests = {
502+
self.func_qual_name: {
503+
MockFunctionCalledInTest(
504+
tests_in_file=MockTestsInFile(test_file=replay_test_path)
505+
),
506+
MockFunctionCalledInTest(
507+
tests_in_file=MockTestsInFile(test_file=concolic_test_path)
508+
),
509+
}
510+
}
511+
512+
replay_inv_id = MockInvocationId(
513+
test_module_path="tests.__replay_test_abc",
514+
test_function_name="test_replay_one",
515+
)
516+
concolic_inv_id = MockInvocationId(
517+
test_module_path="tests.codeflash_concolic_xyz",
518+
test_function_name="test_concolic_one",
519+
)
520+
521+
original_runtimes = {
522+
replay_inv_id: [100_000_000],
523+
concolic_inv_id: [150_000_000],
524+
}
525+
optimized_runtimes = {
526+
replay_inv_id: [200_000_000],
527+
concolic_inv_id: [300_000_000],
528+
}
529+
530+
existing, replay, concolic = existing_tests_source_for(
531+
function_qualified_name_with_modules_from_root=self.func_qual_name,
532+
function_to_tests=function_to_tests,
533+
test_cfg=test_cfg,
534+
original_runtimes_all=original_runtimes,
535+
optimized_runtimes_all=optimized_runtimes,
536+
)
537+
538+
self.assertEqual(existing, "")
539+
self.assertIn("`__replay_test_abc.py::test_replay_one`", replay)
540+
self.assertIn("-50.0%⚠️", replay)
541+
self.assertIn("`codeflash_concolic_xyz.py::test_concolic_one`", concolic)
542+
self.assertIn("-50.0%⚠️", concolic)
543+
544+
def test_mixed_results_and_min_runtime(self):
545+
"""Test with mixed results and that min() of runtimes is used."""
546+
with temp_project_dir() as project_root:
547+
tests_dir = project_root / "tests"
548+
tests_dir.mkdir(exist_ok=True)
549+
existing_test_path = (tests_dir / "test_existing.py").resolve()
550+
existing_test_path.touch()
551+
replay_test_path = (tests_dir / "__replay_test_mixed.py").resolve()
552+
replay_test_path.touch()
553+
554+
test_cfg = MockTestConfig(tests_root=tests_dir.resolve())
555+
function_to_tests = {
556+
self.func_qual_name: {
557+
MockFunctionCalledInTest(
558+
tests_in_file=MockTestsInFile(test_file=existing_test_path)
559+
),
560+
MockFunctionCalledInTest(
561+
tests_in_file=MockTestsInFile(test_file=replay_test_path)
562+
),
563+
}
564+
}
565+
566+
existing_inv_id = MockInvocationId(
567+
"tests.test_existing", "test_speedup", "TestExisting"
568+
)
569+
replay_inv_id = MockInvocationId(
570+
"tests.__replay_test_mixed", "test_slowdown"
571+
)
572+
573+
original_runtimes = {
574+
existing_inv_id: [400_000_000, 500_000_000], # min is 400ms
575+
replay_inv_id: [100_000_000, 110_000_000], # min is 100ms
576+
}
577+
optimized_runtimes = {
578+
existing_inv_id: [210_000_000, 200_000_000], # min is 200ms
579+
replay_inv_id: [300_000_000, 290_000_000], # min is 290ms
580+
}
581+
582+
existing, replay, concolic = existing_tests_source_for(
583+
self.func_qual_name,
584+
function_to_tests,
585+
test_cfg,
586+
original_runtimes,
587+
optimized_runtimes,
588+
)
589+
590+
self.assertIn("`test_existing.py::TestExisting.test_speedup`", existing)
591+
self.assertIn("400ms", existing)
592+
self.assertIn("200ms", existing)
593+
self.assertIn("100%✅", existing)
594+
self.assertIn("`__replay_test_mixed.py::test_slowdown`", replay)
595+
self.assertIn("100ms", replay)
596+
self.assertIn("290ms", replay)
597+
self.assertIn("-65.5%⚠️", replay)
598+
self.assertEqual(concolic, "")

0 commit comments

Comments
 (0)