fix flaky test_simple_profiler_overhead #9462
Description
🐛 Bug
A flaky profiler test blocks our auto-merge. Bad!
To Reproduce
Test run tests/profiler/test_profiler/test_simple_profiler_overhead
or test_simple_profiler_iterable_durations
________________________ test_simple_profiler_overhead _________________________
simple_profiler = <pytorch_lightning.profiler.simple.SimpleProfiler object at 0x7f30f86b8eb8>
n_iter = 5
def test_simple_profiler_overhead(simple_profiler, n_iter=5):
"""Ensure that the profiler doesn't introduce too much overhead during training."""
for _ in range(n_iter):
with simple_profiler.profile("no-op"):
pass
durations = np.array(simple_profiler.recorded_durations["no-op"])
> assert all(durations < PROFILER_OVERHEAD_MAX_TOLERANCE)
E assert False
E + where False = all(array([3.19999992e-06, 1.90000003e-06, 1.60000002e-06, 5.33505000e-04,\n 1.90000003e-06]) < 0.0005)
/home/runner/work/pytorch-lightning/pytorch-lightning/tests/profiler/test_profiler.py:91: AssertionError
_____________ test_simple_profiler_iterable_durations[a-expected0] _____________
simple_profiler = <pytorch_lightning.profiler.simple.SimpleProfiler object at 0x13529c0a0>
action = 'a', expected = [3, 1]
@pytest.mark.parametrize(
["action", "expected"], [pytest.param("a", [3, 1]), pytest.param("b", [2]), pytest.param("c", [1])]
)
def test_simple_profiler_iterable_durations(simple_profiler, action: str, expected: list):
"""Ensure the reported durations are reasonably accurate."""
iterable = _sleep_generator(expected)
for _ in simple_profiler.profile_iterable(iterable, action):
pass
# we exclude the last item in the recorded durations since that's when StopIteration is raised
> np.testing.assert_allclose(simple_profiler.recorded_durations[action][:-1], expected, rtol=0.2)
E AssertionError:
E Not equal to tolerance rtol=0.2, atol=0
E
E Mismatch: 50%
E Max absolute difference: 0.75093935
E Max relative difference: 0.25031312
E x: array([3.750939, 1.083181])
E y: array([3, 1])
/Users/runner/work/pytorch-lightning/pytorch-lightning/tests/profiler/test_profiler.py:81: AssertionError
Expected behavior
Obvious.
Environment
Our own CI:
pytest (ubuntu-18.04, 3.6, minimal, stable)