tests: reorder to unpin pytest

Fix: #3035

Rearrange test items so that forked tests come before normal tests
within their respective modules.
Swap the last forked test with the last normal test if necessary.

Workaround to unpin pytest. See:
pytest-dev/pytest#9621,
pytest-dev/pytest-forked#67, and specifically:
pytest-dev/pytest-forked#67 (comment)

Author: rominf

requirements-devenv.txt

@@ -1,5 +1,4 @@
 -r requirements-linting.txt
 -r requirements-testing.txt
 mockupdb # required by `pymongo` tests that are enabled by `pymongo` from linter requirements
-pytest<7.0.0 # https://github.com/pytest-dev/pytest/issues/9621; see tox.ini
 pytest-asyncio

tests/conftest.py

@@ -2,6 +2,7 @@
 import os
 import socket
 import warnings
+from collections import OrderedDict
 from threading import Thread
 from contextlib import contextmanager
 from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -645,3 +646,41 @@ def __eq__(self, other):
 
     def __ne__(self, other):
         return not self.__eq__(other)
+
+
+def pytest_collection_modifyitems(config, items):
+    """
+    Rearrange test items so that forked tests come before normal tests within their respective modules.
+    Swap the last forked test with the last normal test if necessary.
+
+    Workaround to unpin pytest. See:
+    https://github.com/pytest-dev/pytest/issues/9621,
+    https://github.com/pytest-dev/pytest-forked/issues/67, and specifically:
+    https://github.com/pytest-dev/pytest-forked/issues/67#issuecomment-1964718720
+    """
+    module_states = OrderedDict()
+
+    for idx in range(len(items)):
+        item = items[idx]
+        current_module = item.module.__name__
+
+        if current_module not in module_states:
+            module_states[current_module] = {"forked": [], "normal": []}
+
+        if "forked" in item.keywords:
+            module_states[current_module]["forked"].append(idx)
+        else:
+            module_states[current_module]["normal"].append(idx)
+
+    # Swap the last forked test with the last normal test if necessary
+    for states in module_states.values():
+        if states["forked"] and states["normal"]:
+            last_forked_idx = states["forked"][-1]
+            last_normal_idx = states["normal"][-1]
+
+            if last_forked_idx > last_normal_idx:
+                # Swap the items
+                items[last_forked_idx], items[last_normal_idx] = (
+                    items[last_normal_idx],
+                    items[last_forked_idx],
+                )

tox.ini

@@ -290,19 +290,10 @@ deps =
     # === Common ===
     py3.8-common: hypothesis
     {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12,py3.13}-common: pytest-asyncio
-    # See https://github.com/pytest-dev/pytest/issues/9621
-    # and https://github.com/pytest-dev/pytest-forked/issues/67
-    # for justification of the upper bound on pytest
-    {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-common: pytest<7.0.0
-    py3.13-common: pytest
 
     # === Gevent ===
     {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-gevent: gevent>=22.10.0, <22.11.0
     {py3.12}-gevent: gevent
-    # See https://github.com/pytest-dev/pytest/issues/9621
-    # and https://github.com/pytest-dev/pytest-forked/issues/67
-    # for justification of the upper bound on pytest
-    {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-gevent: pytest<7.0.0
 
     # === Integrations ===
 
@@ -372,7 +363,6 @@ deps =
     celery-latest: Celery
 
     celery: newrelic
-    celery: pytest<7
     {py3.7}-celery: importlib-metadata<5.0
 
     # Chalice