Test infra improvements (drivendataorg#461)

* Add pytest and loosen versions

* Cache strategy

* debug code

* remove extra import

* no handle.exe on gha

* resilient cache tests

* initial sleep

* weakref finalize instead of __del__

* delay on write check

* remove debug code

* remove merge cruft

* Revert "weakref finalize instead of __del__"

This reverts commit 197a058.

* save time by reducing sleep

* gcs retry

Author: pjbull

.gitignore

@@ -59,6 +59,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+reportlog.jsonl
 
 # Translations
 *.mo

Makefile

@@ -86,5 +86,8 @@ test-debug:  ## rerun tests that failed in last run and stop with pdb at failure
 test-live-cloud:  ## run tests on live cloud backends
 	USE_LIVE_CLOUD=1 python -m pytest -vv
 
+test-time-report:
+	pytest-duration-insights explore --no-trim reportlog.jsonl
+
 perf:  ## run performance measurement suite for s3 and save results to perf-results.csv
 	python tests/performance/cli.py s3 --save-csv=perf-results.csv

pyproject.toml

@@ -65,7 +65,7 @@ ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--cov=cloudpathlib --cov-report=term --cov-report=html --cov-report=xml -n=auto"
+addopts = "--cov=cloudpathlib --cov-report=term --cov-report=html --cov-report=xml -n=auto --report-log reportlog.jsonl"
 testpaths = ["tests"]
 
 [tool.coverage.run]

requirements-dev.txt

@@ -19,9 +19,11 @@ pandas
 pillow
 psutil
 pydantic
-pytest<8
-pytest-cases>=3.7.0
+pytest
+pytest-cases
 pytest-cov
+pytest-duration-insights
+pytest-reportlog
 pytest-xdist
 python-dotenv
 pywin32; sys_platform == 'win32'

tests/test_caching.py

@@ -3,8 +3,14 @@
 from time import sleep
 from pathlib import Path
 
+from google.api_core.exceptions import TooManyRequests
 import pytest
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
 
 from cloudpathlib.enums import FileCacheMode
 from cloudpathlib.exceptions import (
@@ -433,17 +439,38 @@ def test_environment_variables_force_overwrite_to(rig: CloudProviderTestRig, tmp
             assert p.stat().st_mtime >= new_local.stat().st_mtime
 
             # would raise if not set
-            sleep(1.01)  # give time so not equal when rounded
-            p._upload_file_to_cloud(new_local)
-            assert p.stat().st_mtime > orig_cloud_mod_time  # cloud now overwritten
+            @retry(
+                retry=retry_if_exception_type((AssertionError, TooManyRequests)),
+                wait=wait_random_exponential(multiplier=0.5, max=5),
+                stop=stop_after_attempt(10),
+                reraise=True,
+            )
+            def _wait_for_cloud_newer():
+                p._upload_file_to_cloud(new_local)
+                assert p.stat().st_mtime > orig_cloud_mod_time  # cloud now overwritten
+
+            _wait_for_cloud_newer()
 
             new_also_cloud = rig.create_cloud_path("dir_0/another_cloud_file.txt")
-            sleep(1.01)  # give time so not equal when rounded
-            new_also_cloud.write_text("newer")
 
-            new_cloud_mod_time = new_also_cloud.stat().st_mtime
+            sleep(0.1)  # at least a little different
+
+            @retry(
+                retry=retry_if_exception_type(
+                    (OverwriteNewerLocalError, AssertionError, TooManyRequests)
+                ),
+                wait=wait_random_exponential(multiplier=0.5, max=5),
+                stop=stop_after_attempt(10),
+                reraise=True,
+            )
+            def _retry_write_until_old_enough():
+                new_also_cloud.write_text("newer")
+                new_cloud_mod_time = new_also_cloud.stat().st_mtime
+                assert p.stat().st_mtime < new_cloud_mod_time  # would raise if not set
+                return new_cloud_mod_time
+
+            new_cloud_mod_time = _retry_write_until_old_enough()
 
-            assert p.stat().st_mtime < new_cloud_mod_time  # would raise if not set
             p.copy(new_also_cloud)
             assert new_also_cloud.stat().st_mtime >= new_cloud_mod_time
 

tests/test_cloudpath_upload_copy.py

@@ -57,7 +57,7 @@ def test_upload_from_file(rig, upload_assets_dir):
 
     # to file, file exists
     to_upload_2 = upload_assets_dir / "upload_2.txt"
-    sleep(1.5)
+    sleep(1.1)
     to_upload_2.touch()  # make sure local is newer
     p.upload_from(to_upload_2)
     assert p.exists()
@@ -70,7 +70,7 @@ def test_upload_from_file(rig, upload_assets_dir):
 
     # to file, file exists and is newer; overwrite
     p.touch()
-    sleep(1.5)
+    sleep(1.1)
     p.upload_from(upload_assets_dir / "upload_1.txt", force_overwrite_to_cloud=True)
     assert p.exists()
     assert p.read_text() == "Hello from 1"