dev: Pytest-split test (#1144)

Author: ajay-sentry

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
   test:
     name: Test
     needs: [build]
-    uses: codecov/gha-workflows/.github/workflows/[email protected].27
+    uses: codecov/gha-workflows/.github/workflows/run-tests-split[email protected].29
     secrets: inherit
     with:
       repo: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-api' }}

.test_durations

@@ -34,10 +34,18 @@ test:
 	COVERAGE_CORE=sysmon python -m pytest --cov=./ --junitxml=junit.xml -o junit_family=legacy
 
 test.unit:
-	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml -o junit_family=legacy
+	@if [ -n "$(GROUP)" ]; then \
+		COVERAGE_CORE=sysmon python -m pytest --splits ${SPLIT} --group $(GROUP) --cov=./ -m "not integration" --cov-report=xml:unit.$(GROUP).coverage.xml --junitxml=unit.$(GROUP).junit.xml -o junit_family=legacy; \
+	else \
+		COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml -o junit_family=legacy; \
+	fi
 
 test.integration:
-	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml -o junit_family=legacy
+	@if [ -n "$(GROUP)" ]; then \
+		COVERAGE_CORE=sysmon python -m pytest --splits ${SPLIT} --group $(GROUP) --cov=./ -m "integration" --cov-report=xml:integration.$(GROUP).coverage.xml --junitxml=integration.$(GROUP).junit.xml -o junit_family=legacy; \
+	else \
+		COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml -o junit_family=legacy; \
+	fi
 
 lint:
 	make lint.install
@@ -205,10 +213,18 @@ test_env.container_check_db:
 	while ! nc -vz timescale 5432; do sleep 1; echo "waiting for timescale"; done
 
 test_env.run_unit:
-	docker-compose exec api make test.unit
+	@if [ -n "$(GROUP)" ]; then \
+		docker-compose exec api make test.unit SPLIT=${SPLIT} GROUP=${GROUP}; \
+	else \
+		docker-compose exec api make test.unit; \
+	fi
 
 test_env.run_integration:
-	#docker-compose exec api make test.integration
+	# @if [ -n "$(GROUP)" ]; then \
+	# 	docker-compose exec api make test.integration SPLIT=${SPLIT} GROUP=${GROUP}; \
+	# else \
+	# 	docker-compose exec api make test.integration; \
+	# fi
 	echo "Skipping. No Tests"
 
 test_env.check-for-migration-conflicts:

Makefile

@@ -2,6 +2,7 @@
 from datetime import datetime, timedelta
 from unittest.mock import patch
 
+import pytest
 from django.test import TransactionTestCase
 from freezegun import freeze_time
 from shared.api_archive.archive import ArchiveService
@@ -459,27 +460,28 @@ def test_with_complete_pull_request(self):
             "behindByCommit": "1089nf898as-jdf09hahs09fgh",
         }
 
+    @pytest.mark.skip(
+        reason="Skipping due to https://github.com/codecov/engineering-team/issues/3358"
+    )
     def test_compare_bundle_analysis_missing_reports(self):
+        repository = RepositoryFactory(author=self.owner)
         head = CommitFactory(
-            repository=self.repository,
+            repository=repository,
             author=self.owner,
-            commitid="5672734ij1n234918231290j12nasdfioasud0f9",
+            commitid="cool-commit-id",
             totals={"c": "78.38", "diff": [0, 0, 0, 0, 0, "14"]},
         )
         compared_to = CommitFactory(
-            repository=self.repository,
+            repository=repository,
             author=self.owner,
-            commitid="9asd78fa7as8d8fa97s8d7fgagsd8fa9asd8f77s",
+            commitid="blah",
         )
 
         my_pull = PullFactory(
-            repository=self.repository,
-            title="test-pull-request",
+            repository=repository,
             author=self.owner,
             head=head.commitid,
             compared_to=compared_to.commitid,
-            behind_by=23,
-            behind_by_commit="1089nf898as-jdf09hahs09fgh",
         )
 
         pull = self.fetch_one_pull_request(

graphql_api/tests/test_pull.py

@@ -43,6 +43,7 @@ pytest-cov
 pytest-django
 pytest-mock
 pytest>=7.2.0
+pytest-split
 python-dateutil
 python-json-logger
 python-redis-lock

requirements.in

@@ -360,6 +360,7 @@ pytest==8.1.1
     #   pytest-cov
     #   pytest-django
     #   pytest-mock
+    #   pytest-split
 pytest-asyncio==0.23.6
     # via -r requirements.in
 pytest-cov==5.0.0
@@ -368,6 +369,8 @@ pytest-django==4.8.0
     # via -r requirements.in
 pytest-mock==3.14.0
     # via -r requirements.in
+pytest-split==0.10.0
+    # via -r requirements.in
 python-dateutil==2.9.0.post0
     # via
     #   -r requirements.in