diff --git a/.github/workflows/python-tox.yaml b/.github/workflows/python-tox.yaml index 1d3d11f..785a3aa 100644 --- a/.github/workflows/python-tox.yaml +++ b/.github/workflows/python-tox.yaml @@ -39,4 +39,4 @@ jobs: pip install tox tox-gh-actions - name: Test with tox and upload coverage results - run: tox -- --codecov --codecov-token=${{ secrets.CODECOV_TOKEN }} + run: tox -- --codecov --codecov-token=${{ secrets.CODECOV_TOKEN }} --junit-xml=junit.xml -o junit_family=legacy diff --git a/pyproject.toml b/pyproject.toml index 1f79b62..87bed06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,11 +83,14 @@ legacy_tox_ini = """ [testenv] setenv = py{38,39,310,311,312,313}: COVERAGE_FILE = .coverage.{envname} - commands = pytest --cov --cov-report= {posargs:tests} + commands = + pytest --cov --cov-report= {posargs:tests} + pytest -n2 --cov --cov-report= {posargs:tests} deps = pytest coverage pytest-cov + pytest-xdist . GitPython diff --git a/src/pytest_codecov/__init__.py b/src/pytest_codecov/__init__.py index e84a826..8e6eeb5 100644 --- a/src/pytest_codecov/__init__.py +++ b/src/pytest_codecov/__init__.py @@ -85,18 +85,26 @@ def pytest_addoption(parser, pluginmanager): default=True, help='Don\'t upload coverage results on test failure' ) + group.addoption( + '--codecov-exclude-junit-xml', + action='store_false', + dest='codecov_junit_xml', + default=True, + help='Don\'t upload the junit xml file' + ) class CodecovPlugin: def upload_report(self, terminalreporter, config, cov): + option = config.option uploader = codecov.CodecovUploader( - config.option.codecov_slug, - commit=config.option.codecov_commit, - branch=config.option.codecov_branch, - token=config.option.codecov_token, + option.codecov_slug, + commit=option.codecov_commit, + branch=option.codecov_branch, + token=option.codecov_token, ) - uploader.write_network_files(git.ls_files()) + uploader.add_network_files(git.ls_files()) from coverage.misc import CoverageException try: uploader.add_coverage_report(cov) @@ -110,14 +118,21 @@ def upload_report(self, terminalreporter, config, cov): terminalreporter.line('') return - if config.option.codecov_dump: + xmlpath = option.xmlpath if option.codecov_junit_xml else None + if xmlpath and os.path.isfile(xmlpath): + uploader.add_junit_xml(xmlpath) + has_junit_xml = True + else: + has_junit_xml = False + + if option.codecov_dump: terminalreporter.section('Prepared Codecov.io payload') terminalreporter.write_line(uploader.get_payload()) return terminalreporter.section('Codecov.io upload') - if not config.option.codecov_slug: + if not option.codecov_slug: terminalreporter.write_line( 'ERROR: Failed to determine git repository slug. ' 'Cannot upload without a valid slug.', @@ -126,24 +141,35 @@ def upload_report(self, terminalreporter, config, cov): ) terminalreporter.line('') return - if not config.option.codecov_branch: + if not option.codecov_branch: terminalreporter.write_line( 'WARNING: Failed to determine git repository branch.', yellow=True, bold=True, ) - if not config.option.codecov_commit: + if not option.codecov_commit: terminalreporter.write_line( 'WARNING: Failed to determine git commit.', yellow=True, bold=True, ) + if has_junit_xml and config.getini('junit_family') != 'legacy': + terminalreporter.write_line( + 'INFO: We recommend using junit_family=legacy with Codecov.', + blue=True, + bold=True, + ) + terminalreporter.write_line( 'Environment:\n' - f'Slug: {config.option.codecov_slug}\n' - f'Branch: {config.option.codecov_branch}\n' - f'Commit: {config.option.codecov_commit}\n' + f'Slug: {option.codecov_slug}\n' + f'Branch: {option.codecov_branch}\n' + f'Commit: {option.codecov_commit}\n' ) + if has_junit_xml: + terminalreporter.write_line( + 'JUnit XML file detected and included in upload.\n' + ) try: terminalreporter.write_line('Pinging codecov API...') uploader.ping() @@ -178,6 +204,10 @@ def pytest_terminal_summary(self, terminalreporter, exitstatus, config): def pytest_configure(config): # pragma: no cover + # NOTE: Don't report codecov results on worker nodes + if hasattr(config, 'workerinput'): + return + # NOTE: if cov is missing we fail silently if config.option.codecov and config.pluginmanager.has_plugin('_cov'): config.pluginmanager.register(CodecovPlugin()) diff --git a/src/pytest_codecov/codecov.py b/src/pytest_codecov/codecov.py index e5e3732..95bdb84 100644 --- a/src/pytest_codecov/codecov.py +++ b/src/pytest_codecov/codecov.py @@ -1,7 +1,10 @@ import gzip import io +import json import requests import tempfile +import zlib +from base64 import b64encode from urllib.parse import urljoin @@ -23,25 +26,38 @@ def __init__(self, slug, commit=None, branch=None, token=None): self.commit = commit self.branch = branch self.token = token - self.store_url = None - self._buffer = io.StringIO() + self._coverage_store_url = None + self._coverage_buffer = io.StringIO() + self._test_result_store_url = None + self._test_result_files = [] - def write_network_files(self, files): - self._buffer.write( + def add_network_files(self, files): + self._coverage_buffer.write( '\n'.join(files + ['<<<<<< network']) ) def add_coverage_report(self, cov, filename='coverage.xml', **kwargs): with tempfile.NamedTemporaryFile(mode='r') as xml_report: # embed xml report - self._buffer.write(f'\n# path=./{filename}\n') + self._coverage_buffer.write(f'\n# path=./{filename}\n') cov.xml_report(outfile=xml_report.name) xml_report.seek(0) - self._buffer.write(xml_report.read()) - self._buffer.write('\n<<<<<< EOF') + self._coverage_buffer.write(xml_report.read()) + self._coverage_buffer.write('\n<<<<<< EOF') + + def add_junit_xml(self, path, filename='junit.xml'): + with open(path, 'rb') as junit_xml: + self._test_result_files.append({ + 'filename': filename, + 'format': 'base64+compressed', + 'data': b64encode( + zlib.compress(junit_xml.read()) + ).decode('ascii'), + 'labels': '', + }) def get_payload(self): - return self._buffer.getvalue() + return self._coverage_buffer.getvalue() def ping(self): if not self.slug: @@ -77,10 +93,30 @@ def ping(self): raise CodecovError( f'Invalid response from codecov API:\n{response.text}' ) - self.store_url = lines[1] + self._coverage_store_url = lines[1] + + if not self._test_result_files: + return + + headers = {} if self.token is None else { + 'Authorization': f'token {self.token}', + 'User-Agent': package() + } + data = { + 'slug': self.slug, + 'branch': self.branch or '', + 'commit': self.commit or '', + } + api_url = urljoin(self.api_endpoint, '/upload/test_results/v1') + response = requests.post(api_url, headers=headers, json=data) + if response.ok: + # TODO: Fail more loudly? + url = response.json()['raw_upload_location'] + if url.startswith(self.storage_endpoint): + self._test_result_store_url = url def upload(self): - if not self.store_url: + if not self._coverage_store_url: raise CodecovError('Need to ping API before upload.') headers = { @@ -92,10 +128,20 @@ def upload(self): payload.write(self.get_payload().encode('utf-8')) gz_payload.seek(0) response = requests.put( - self.store_url, headers=headers, data=gz_payload + self._coverage_store_url, headers=headers, data=gz_payload ) if not response.ok: raise CodecovError('Failed to upload report to storage endpoint.') - self.store_url = None # NOTE: Invalidate store url after upload + self._coverage_store_url = None + + if not self._test_result_store_url or not self._test_result_files: + return + + json_payload = json.dumps({ + 'test_results_files': self._test_result_files + }).encode('ascii') + # TODO: Fail more loudly? + requests.put(self._test_result_store_url, data=json_payload) + self._test_result_store_url = None diff --git a/tests/conftest.py b/tests/conftest.py index 1c3eb63..800b642 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,13 @@ +import importlib +import json + +from coverage.misc import CoverageException import pytest + import pytest_codecov import pytest_codecov.codecov import pytest_codecov.git -from coverage.misc import CoverageException -from importlib import reload - pytest_plugins = 'pytester' @@ -71,6 +73,9 @@ def __init__(self, text='', ok=True): self.text = text self.ok = ok + def json(self): + return json.loads(self.text) + class MockRequests: @@ -82,6 +87,10 @@ def __init__(self) -> None: def set_response(self, text, ok=True): self._response = MockResponse(text, ok=ok) + def set_responses(self, *texts): + assert texts + self._response = [MockResponse(text) for text in texts] + def pop(self): calls = self._calls self.clear() @@ -95,6 +104,12 @@ def mock_method(self, method, url, **kwargs): raise ConnectionError() self._calls.append((method.lower(), url, kwargs)) + if isinstance(self._response, list): + response = self._response.pop(0) + if not self._response: + # repeat the final response indefinitely + self._response = response + return response return self._response @@ -113,16 +128,19 @@ class DummyUploader: # TODO: Implement some basic behavior, so we can test # more exhaustively. - def __init__(self, slug, **kwargs): - self.fail_report_generation = False + def __init__(self, factory, slug, **kwargs): + self.factory = factory - def write_network_files(self, files): + def add_network_files(self, files): pass def add_coverage_report(self, cov, **kwargs): - if self.fail_report_generation: + if self.factory.fail_report_generation: raise CoverageException('test exception') + def add_junit_xml(self, path): + self.factory.junit_xml = path + def get_payload(self): return 'stub' @@ -135,12 +153,15 @@ def upload(self): class DummyUploaderFactory: - fail_report_generation = False + def __init__(self): + self.fail_report_generation = False + self.junit_xml = None def __call__(self, slug, **kwargs): - inst = DummyUploader(slug, **kwargs) - inst.fail_report_generation = self.fail_report_generation - return inst + return DummyUploader(self, slug, **kwargs) + + def clear(self): + self.junit_xml = None @pytest.fixture @@ -156,5 +177,5 @@ def dummy_uploader(monkeypatch): # NOTE: Ensure modules are reloaded when coverage.py is looking. # This means we want to avoid importing module members when # using these modules, to ensure they get reloaded as well. -reload(pytest_codecov) -reload(pytest_codecov.codecov) +importlib.reload(pytest_codecov) +importlib.reload(pytest_codecov.codecov) diff --git a/tests/test_codecov.py b/tests/test_codecov.py index 53348b9..1c3e293 100644 --- a/tests/test_codecov.py +++ b/tests/test_codecov.py @@ -15,7 +15,7 @@ def test_init(): def test_write_network_files(): uploader = CodecovUploader('seantis/pytest-codecov') - uploader.write_network_files(['foo.py']) + uploader.add_network_files(['foo.py']) assert uploader.get_payload() == ( 'foo.py\n' '<<<<<< network' @@ -24,7 +24,7 @@ def test_write_network_files(): def test_add_coverage_report(dummy_cov): uploader = CodecovUploader('seantis/pytest-codecov') - uploader.write_network_files(['foo.py']) + uploader.add_network_files(['foo.py']) uploader.add_coverage_report(dummy_cov) assert uploader.get_payload() == ( 'foo.py\n' @@ -55,11 +55,27 @@ def test_ping(dummy_cov, mock_requests): mock_requests.set_response(f'codecov.io\n{uploader.storage_endpoint}') uploader.ping() - assert uploader.store_url == uploader.storage_endpoint + assert uploader._coverage_store_url == uploader.storage_endpoint + assert uploader._test_result_store_url is None # TODO: Verify correct url/headers/params +def test_ping_junit(dummy_cov, mock_requests, tmp_path): + junit_xml = tmp_path / 'junit.xml' + junit_xml.write_text('foo') + uploader = CodecovUploader('seantis/pytest-codecov') + uploader.add_junit_xml(str(junit_xml)) + + mock_requests.set_responses( + f'codecov.io\n{uploader.storage_endpoint}', + f'{{"raw_upload_location":"{uploader.storage_endpoint}"}}' + ) + uploader.ping() + assert uploader._coverage_store_url == uploader.storage_endpoint + assert uploader._test_result_store_url == uploader.storage_endpoint + + def test_ping_no_slug(dummy_cov, mock_requests): uploader = CodecovUploader(None) with pytest.raises(CodecovError, match=r'valid slug'): @@ -80,6 +96,27 @@ def test_upload(dummy_cov, mock_requests): mock_requests.set_response('') uploader.upload() - assert uploader.store_url is None + assert uploader._coverage_store_url is None + assert uploader._test_result_store_url is None + + # TODO: Verify correct url/headers/params + + +def test_upload_junit(dummy_cov, mock_requests, tmp_path): + junit_xml = tmp_path / 'junit.xml' + junit_xml.write_text('foo') + uploader = CodecovUploader('seantis/pytest-codecov') + uploader.add_junit_xml(str(junit_xml)) + + mock_requests.set_responses( + f'codecov.io\n{uploader.storage_endpoint}', + f'{{"raw_upload_location":"{uploader.storage_endpoint}"}}' + ) + uploader.ping() + + mock_requests.set_response('') + uploader.upload() + assert uploader._coverage_store_url is None + assert uploader._test_result_store_url is None # TODO: Verify correct url/headers/params diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 291fa91..40c9cb4 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -113,6 +113,98 @@ def test_upload_report(pytester, dummy_reporter, dummy_uploader, ) in dummy_reporter.text +def test_upload_report_junit(pytester, dummy_reporter, dummy_uploader, + dummy_cov, no_gitpython, tmp_path): + + # create a junit xml + junit_xml = tmp_path / 'junit.xml' + junit_xml.write_text('foo') + config = pytester.parseconfig( + f'--junit-xml={junit_xml}', + '-o', + 'junit_family=legacy', + '--codecov', + '--codecov-token=12345678-1234-1234-1234-1234567890ab', + '--codecov-slug=foo/bar', + '--codecov-branch=master', + '--codecov-commit=deadbeef' + ) + plugin = CodecovPlugin() + plugin.upload_report(dummy_reporter, config, dummy_cov) + assert ( + 'Environment:\n' + 'Slug: foo/bar\n' + 'Branch: master\n' + 'Commit: deadbeef\n' + '\n' + 'JUnit XML file detected and included in upload.\n' + ) in dummy_reporter.text + assert ( + 'INFO: We recommend using junit_family=legacy with Codecov.' + ) not in dummy_reporter.text + assert dummy_uploader.junit_xml == str(junit_xml) + + +def test_upload_report_junit_info(pytester, dummy_reporter, dummy_uploader, + dummy_cov, no_gitpython, tmp_path): + + # create a junit xml + junit_xml = tmp_path / 'junit.xml' + junit_xml.write_text('foo') + config = pytester.parseconfig( + f'--junit-xml={junit_xml}', + '--codecov', + '--codecov-token=12345678-1234-1234-1234-1234567890ab', + '--codecov-slug=foo/bar', + '--codecov-branch=master', + '--codecov-commit=deadbeef' + ) + plugin = CodecovPlugin() + plugin.upload_report(dummy_reporter, config, dummy_cov) + assert ( + 'INFO: We recommend using junit_family=legacy with Codecov.\n' + 'Environment:\n' + 'Slug: foo/bar\n' + 'Branch: master\n' + 'Commit: deadbeef\n' + '\n' + 'JUnit XML file detected and included in upload.\n' + ) in dummy_reporter.text + assert dummy_uploader.junit_xml == str(junit_xml) + + +def test_no_upload_report_junit(pytester, dummy_reporter, dummy_uploader, + dummy_cov, no_gitpython, tmp_path): + + # create a junit xml + junit_xml = tmp_path / 'junit.xml' + junit_xml.write_text('foo') + config = pytester.parseconfig( + f'--junit-xml={junit_xml}', + '--codecov', + '--codecov-token=12345678-1234-1234-1234-1234567890ab', + '--codecov-slug=foo/bar', + '--codecov-branch=master', + '--codecov-commit=deadbeef', + '--codecov-exclude-junit-xml' + ) + plugin = CodecovPlugin() + plugin.upload_report(dummy_reporter, config, dummy_cov) + assert ( + 'Environment:\n' + 'Slug: foo/bar\n' + 'Branch: master\n' + 'Commit: deadbeef\n' + ) in dummy_reporter.text + assert ( + 'JUnit XML file detected and included in upload.' + ) not in dummy_reporter.text + assert ( + 'INFO: We recommend using junit_family=legacy with Codecov.' + ) not in dummy_reporter.text + assert dummy_uploader.junit_xml is None + + def test_upload_report_generation_failure( pytester, dummy_reporter, dummy_uploader, dummy_cov, no_gitpython