Merge pull request #3152 from bsipocz/CI_rerun_failures

CI: rerun failed tests in a 2nd pytest run rather than immediately

Author: bsipocz

tox.ini

@@ -22,7 +22,10 @@ description = run tests
 
 setenv =
     PYTEST_ARGS = ''
-    online: PYTEST_ARGS = --remote-data=any --reruns=1 --reruns-delay 10 -m "not bigdata"
+    # We have two pytest runs for the online tests, need to suppress the failing status for the first one to be able to run the second.
+    online: PYTEST_ARGS = --remote-data=any -P sdss -m "not bigdata" --suppress-tests-failed-exit-code
+    online: PYTEST_ARGS_2 =  --remote-data=any -vv -P sdss --last-failed -m "not bigdata"
+    online: SINGLE_RUN = False
     devdeps: PIP_EXTRA_INDEX_URL =  https://pypi.anaconda.org/scientific-python-nightly-wheels/simple https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple
     # astropy doesn't yet have a 3.13 compatible release
     py313: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple
@@ -53,7 +56,7 @@ deps =
     oldestdeps-alldeps: mocpy==0.9
     oldestdeps-alldeps: regions==0.5
 
-    online: pytest-rerunfailures
+    online: pytest-custom_exit_code
 
 extras =
     test
@@ -67,6 +70,10 @@ commands =
     python -m pip freeze
     !cov: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS} {posargs}
     cov:  pytest --pyargs astroquery {toxinidir}/docs --cov astroquery --cov-config={toxinidir}/setup.cfg {env:PYTEST_ARGS} {posargs}
+    # For remote tests, we re-run the failures to filter out at least some of the flaky ones.
+    # We use a second pytest run with --last-failed as opposed to --rerun in order to rerun the
+    # failed ones at the end rather than right away.
+    online: pytest --pyargs astroquery {toxinidir}/docs {env:PYTEST_ARGS_2} {posargs}
     cov: coverage xml -o {toxinidir}/coverage.xml
 
 pip_pre =