Remove report_builder feature

Author: Swatinem

api/internal/tests/views/test_compare_viewset.py

@@ -17,14 +17,11 @@
 
 class MockSerializableReport(SerializableReport):
     """
-    Stubs the 'file_reports' and 'get' methods of SerializableReport, which usually constructs
+    Stubs the 'get' method of SerializableReport, which usually constructs
     report files on the fly from information not provided by these test, like the chunks
     for example.
     """
 
-    def file_reports(self):
-        return [report_file for name, report_file in self.mocked_files.items()]
-
     def get(self, file_name):
         return self.mocked_files.get(file_name)
 

api/public/v2/tests/test_api_compare_viewset.py

@@ -78,14 +78,11 @@ def sample_report2():
 
 class MockSerializableReport(SerializableReport):
     """
-    Stubs the 'file_reports' and 'get' methods of SerializableReport, which usually constructs
+    Stubs the 'get' method of SerializableReport, which usually constructs
     report files on the fly from information not provided by these test, like the chunks
     for example.
     """
 
-    def file_reports(self):
-        return [report_file for name, report_file in self.mocked_files.items()]
-
     def get(self, file_name):
         return self.mocked_files.get(file_name)
 

codecov/settings_base.py

@@ -374,10 +374,6 @@
 
 SHELTER_SHARED_SECRET = get_config("setup", "shelter_shared_secret", default=None)
 
-# list of repo IDs that will use the new-style report builder
-# TODO: we can eventually get rid of this once it's confirmed working well for many repos
-REPORT_BUILDER_REPO_IDS = get_config("setup", "report_builder", "repo_ids", default=[])
-
 SENTRY_ENV = os.environ.get("CODECOV_ENV", False)
 SENTRY_DSN = os.environ.get("SERVICES__SENTRY__SERVER_DSN", None)
 SENTRY_DENY_LIST = DEFAULT_DENYLIST + ["_headers", "token_to_use"]

services/report.py

@@ -1,32 +1,22 @@
 import logging
-from typing import List, Optional
 
 import sentry_sdk
-from django.conf import settings
-from django.db.models import Prefetch, Q
 from django.utils.functional import cached_property
 from shared.helpers.flag import Flag
 from shared.reports.readonly import ReadOnlyReport as SharedReadOnlyReport
 from shared.reports.resources import Report
-from shared.reports.types import ReportFileSummary, ReportTotals
 from shared.storage.exceptions import FileNotInStorageError
-from shared.utils.sessions import Session, SessionType
+from shared.utils.sessions import Session
 
 from core.models import Commit
-from reports.models import AbstractTotals, CommitReport, ReportSession
 from services.archive import ArchiveService
-from utils.config import RUN_ENV
 
 log = logging.getLogger(__name__)
 
 
 class ReportMixin:
-    def file_reports(self):
-        for f in self.files:
-            yield self.get(f)
-
     @cached_property
-    def flags(self):
+    def flags(self) -> dict:
         """returns dict(:name=<Flag>)"""
         flags_dict = {}
         for session in self.sessions.values():
@@ -61,42 +51,20 @@ def build_report(chunks, files, sessions, totals, report_class=None):
 
 
 @sentry_sdk.trace
-def build_report_from_commit(commit: Commit, report_class=None):
+def build_report_from_commit(commit: Commit, report_class=None) -> Report | None:
     """
     Builds a `shared.reports.resources.Report` from a given commit.
-
-    Chunks are fetched from archive storage and the rest of the data is sourced
-    from various `reports_*` tables in the database.
     """
 
-    # TODO: this can be removed once confirmed working well on prod
-    new_report_builder_enabled = (
-        RUN_ENV == "DEV"
-        or RUN_ENV == "STAGING"
-        or RUN_ENV == "TESTING"
-        or commit.repository_id in settings.REPORT_BUILDER_REPO_IDS
-    )
-
-    with sentry_sdk.start_span(description="Fetch files/sessions/totals"):
-        commit_report = fetch_commit_report(commit)
-        if commit_report and new_report_builder_enabled:
-            files = build_files(commit_report)
-            sessions = build_sessions(commit_report)
-            try:
-                totals = build_totals(commit_report.reportleveltotals)
-            except CommitReport.reportleveltotals.RelatedObjectDoesNotExist:
-                totals = None
-        else:
-            if not commit.report:
-                return None
+    if not commit.report:
+        return None
 
-            files = commit.report["files"]
-            sessions = commit.report["sessions"]
-            totals = commit.totals
+    files = commit.report["files"]
+    sessions = commit.report["sessions"]
+    totals = commit.totals
 
     try:
-        with sentry_sdk.start_span(description="Fetch chunks"):
-            chunks = ArchiveService(commit.repository).read_chunks(commit.commitid)
+        chunks = ArchiveService(commit.repository).read_chunks(commit.commitid)
         return build_report(chunks, files, sessions, totals, report_class=report_class)
     except FileNotInStorageError:
         log.warning(
@@ -109,130 +77,7 @@ def build_report_from_commit(commit: Commit, report_class=None):
         return None
 
 
-def fetch_commit_report(commit: Commit) -> Optional[CommitReport]:
-    """
-    Fetch a single `CommitReport` for the given commit.
-    All the necessary report relations are prefetched.
-    """
-    return (
-        commit.reports.coverage_reports()
-        .filter(code=None)
-        .prefetch_related(
-            Prefetch(
-                "sessions",
-                queryset=ReportSession.objects.prefetch_related("flags").select_related(
-                    "uploadleveltotals"
-                ),
-            ),
-        )
-        .select_related("reportdetails", "reportleveltotals")
-        .first()
-    )
-
-
-def build_totals(totals: AbstractTotals) -> ReportTotals:
-    """
-    Build a `shared.reports.types.ReportTotals` instance from one of the
-    various database totals records.
-    """
-    return ReportTotals(
-        files=totals.files,
-        lines=totals.lines,
-        hits=totals.hits,
-        misses=totals.misses,
-        partials=totals.partials,
-        coverage=totals.coverage,
-        branches=totals.branches,
-        methods=totals.methods,
-    )
-
-
-def build_session(upload: ReportSession) -> Session:
-    """
-    Build a `shared.utils.sessions.Session` from a database `reports_upload` record.
-    """
-    try:
-        upload_totals = build_totals(upload.uploadleveltotals)
-    except ReportSession.uploadleveltotals.RelatedObjectDoesNotExist:
-        # upload does not have any totals - maybe the processing failed
-        # or the upload was empty?
-        upload_totals = None
-    flags = [flag.flag_name for flag in upload.flags.all()]
-
-    return Session(
-        id=upload.id,
-        totals=upload_totals,
-        time=upload.created_at.timestamp,
-        archive=upload.storage_path,
-        flags=flags,
-        provider=upload.provider,
-        build=upload.build_code,
-        job=upload.job_code,
-        url=upload.build_url,
-        state=upload.state,
-        env=upload.env,
-        name=upload.name,
-        session_type=SessionType.get_from_string(upload.upload_type),
-        session_extras=upload.upload_extras,
-    )
-
-
-def build_sessions(commit_report: CommitReport) -> dict[int, Session]:
-    """
-    Build mapping of report number -> session that can be passed to the report class.
-    Does not include CF sessions if there is also an upload session with the same
-    flag name.
-    """
-    sessions = {}
-
-    carryforward_sessions = {}
-    uploaded_flags = set()
-
-    for upload in commit_report.sessions.filter(
-        Q(state="complete") | Q(state="processed")
-    ):
-        session = build_session(upload)
-        if session.session_type == SessionType.carriedforward:
-            carryforward_sessions[upload.order_number] = session
-        else:
-            sessions[upload.order_number] = session
-            uploaded_flags |= set(session.flags)
-
-    for sid, session in carryforward_sessions.items():
-        # we only ever expect 1 flag for CF sessions
-        overlapping_flags = uploaded_flags & set(session.flags)
-
-        if len(overlapping_flags) == 0:
-            # we can include this CF session since there are no direct uploads
-            # with the same flag name
-            sessions[sid] = session
-
-    return sessions
-
-
-def build_files(commit_report: CommitReport) -> dict[str, ReportFileSummary]:
-    """
-    Construct a files dictionary in a format compatible with `shared.reports.resources.Report`
-    from data in the `reports_reportdetails.files_array` column in the database.
-    """
-    try:
-        report_details = commit_report.reportdetails
-    except CommitReport.reportdetails.RelatedObjectDoesNotExist:
-        # we don't expect this but something could have gone wrong in the worker
-        # we can't really recover here
-        return {}
-
-    return {
-        file["filename"]: ReportFileSummary(
-            file_index=file["file_index"],
-            file_totals=ReportTotals(*file["file_totals"]),
-            diff_totals=file["diff_totals"],
-        )
-        for file in report_details.files_array
-    }
-
-
-def files_belonging_to_flags(commit_report: Report, flags: List[str]) -> List[str]:
+def files_belonging_to_flags(commit_report: Report, flags: list[str]) -> list[str]:
     sessions_for_specific_flags = sessions_with_specific_flags(
         commit_report=commit_report, flags=flags
     )
@@ -244,7 +89,7 @@ def files_belonging_to_flags(commit_report: Report, flags: List[str]) -> List[st
 
 
 def sessions_with_specific_flags(
-    commit_report: Report, flags: List[str]
+    commit_report: Report, flags: list[str]
 ) -> dict[int, Session]:
     sessions = [
         (sid, session)
@@ -254,7 +99,7 @@ def sessions_with_specific_flags(
     return dict(sessions)
 
 
-def files_in_sessions(commit_report: Report, session_ids: List[int]) -> List[str]:
+def files_in_sessions(commit_report: Report, session_ids: list[int]) -> list[str]:
     files, session_ids = [], set(session_ids)
     for file in commit_report:
         found = False

services/tests/test_report.py

@@ -39,6 +39,11 @@ def flags_report():
     return report
 
 
+def sorted_files(report: Report) -> list[ReportFile]:
+    files = [report.get(file) for file in report.files]
+    return sorted(files, key=lambda x: x.name)
+
+
 class ReportServiceTest(TestCase):
     def test_report_generator(self):
         data = {
@@ -116,7 +121,7 @@ def test_build_report_from_commit(self, read_chunks_mock):
         res = build_report_from_commit(commit)
         assert len(res._chunks) == 3
         assert len(res.files) == 3
-        file_1, file_2, file_3 = sorted(res.file_reports(), key=lambda x: x.name)
+        file_1, file_2, file_3 = sorted_files(res)
         assert file_1.name == "awesome/__init__.py"
         assert tuple(file_1.totals) == (0, 10, 8, 2, 0, "80.00000", 0, 0, 0, 0, 0, 0, 0)
         assert file_2.name == "tests/__init__.py"
@@ -180,7 +185,7 @@ def test_build_report_from_commit_null_session_totals(self, read_chunks_mock):
         res = build_report_from_commit(commit)
         assert len(res._chunks) == 3
         assert len(res.files) == 3
-        file_1, file_2, file_3 = sorted(res.file_reports(), key=lambda x: x.name)
+        file_1, file_2, file_3 = sorted_files(res)
         assert file_1.name == "awesome/__init__.py"
         assert tuple(file_1.totals) == (0, 10, 8, 2, 0, "80.00000", 0, 0, 0, 0, 0, 0, 0)
         assert file_2.name == "tests/__init__.py"
@@ -213,7 +218,7 @@ def test_build_report_from_commit_with_flags(self, read_chunks_mock):
         res = report.flags["integrations"].report
         assert len(res.report._chunks) == 3
         assert len(res.files) == 3
-        file_1, file_2, file_3 = sorted(res.file_reports(), key=lambda x: x.name)
+        file_1, file_2, file_3 = sorted_files(res)
         assert file_1.name == "awesome/__init__.py"
         assert tuple(file_1.totals) == (0, 10, 1, 9, 0, "10.00000", 0, 0, 0, 0, 0, 0, 0)
         assert file_2.name == "tests/__init__.py"
@@ -245,7 +250,7 @@ def test_build_report_from_commit_with_non_carried_forward_flags(
         res = report.flags["integrations"].report
         assert len(res.report._chunks) == 3
         assert len(res.files) == 3
-        file_1, file_2, file_3 = sorted(res.file_reports(), key=lambda x: x.name)
+        file_1, file_2, file_3 = sorted_files(res)
         assert file_1.name == "awesome/__init__.py"
         assert tuple(file_1.totals) == (0, 10, 1, 9, 0, "10.00000", 0, 0, 0, 0, 0, 0, 0)
         assert file_2.name == "tests/__init__.py"
@@ -330,7 +335,7 @@ def test_build_report_from_commit_fallback(self, read_chunks_mock):
 
         assert len(res._chunks) == 3
         assert len(res.files) == 3
-        file_1, file_2, file_3 = sorted(res.file_reports(), key=lambda x: x.name)
+        file_1, file_2, file_3 = sorted_files(res)
         assert file_1.name == "awesome/__init__.py"
         assert tuple(file_1.totals) == (0, 10, 8, 2, 0, "80.00000", 0, 0, 0, 0, 0, 0, 0)
         assert file_2.name == "tests/__init__.py"