From 6f738c1005e98e96f4f5dd1dbeb57ae7f413c998 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Tue, 29 Oct 2024 13:26:56 -0400 Subject: [PATCH 01/10] refactor: test results GQL endpoints this refactor simplifies the tests from test results aggregates and flake aggregates and adds some typing to the generate_test_results function. It also changes the arguments passed to the generate_test_results function so handling them is simpler and changes some things regarding error handling in that function. --- graphql_api/tests/test_test_result.py | 315 +----------------- .../tests/test_test_results_headers.py | 124 +------ .../flake_aggregates/flake_aggregates.py | 27 +- .../types/test_analytics/test_analytics.py | 112 +++---- .../test_results_aggregates.py | 37 +- utils/test_results.py | 305 +++++++++++------ utils/tests/unit/test_cursor.py | 3 +- utils/tests/unit/test_search_base_query.py | 13 +- 8 files changed, 305 insertions(+), 631 deletions(-) diff --git a/graphql_api/tests/test_test_result.py b/graphql_api/tests/test_test_result.py index 0970030084..93e974e4fa 100644 --- a/graphql_api/tests/test_test_result.py +++ b/graphql_api/tests/test_test_result.py @@ -50,34 +50,6 @@ def setUp(self): flaky_fail_count=1, ) - def test_fetch_test_result_name(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { - name - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][ - 0 - ]["node"]["name"] == self.test.name.replace("\x1f", " ") - def test_fetch_test_result_name_with_computed_name(self) -> None: self.test.computed_name = "Computed Name" self.test.save() @@ -92,281 +64,14 @@ def test_fetch_test_result_name_with_computed_name(self) -> None: edges { node { name - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["name"] - == self.test.computed_name - ) - - def test_fetch_test_result_updated_at(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { updatedAt - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["updatedAt"] - == datetime.now(UTC).isoformat() - ) - - def test_fetch_test_result_commits_failed(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { commitsFailed - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["commitsFailed"] - == 3 - ) - - def test_fetch_test_result_failure_rate(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { failureRate - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["failureRate"] - == 0.75 - ) - - def test_fetch_test_result_last_duration(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { lastDuration - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["lastDuration"] - == 1.0 - ) - - def test_fetch_test_result_avg_duration(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { avgDuration - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][ - 0 - ]["node"]["avgDuration"] == (5.6 / 3) - - def test_fetch_test_result_total_fail_count(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { totalFailCount - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["totalFailCount"] - == 9 - ) - - def test_fetch_test_result_total_skip_count(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { totalSkipCount - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["totalSkipCount"] - == 6 - ) - - def test_fetch_test_result_total_pass_count(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { totalPassCount - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["totalPassCount"] - == 3 - ) - - def test_fetch_test_result_total_flaky_fail_count(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { totalFlakyFailCount } } @@ -381,9 +86,17 @@ def test_fetch_test_result_total_flaky_fail_count(self) -> None: result = self.gql_request(query, owner=self.owner) assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][0][ - "node" - ]["totalFlakyFailCount"] - == 2 - ) + assert result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][ + 0 + ]["node"] == { + "name": self.test.computed_name, + "updatedAt": datetime.now(UTC).isoformat(), + "commitsFailed": 3, + "failureRate": 0.75, + "lastDuration": 1.0, + "avgDuration": (5.6 / 3), + "totalFailCount": 9, + "totalSkipCount": 6, + "totalPassCount": 3, + "totalFlakyFailCount": 2, + } diff --git a/graphql_api/tests/test_test_results_headers.py b/graphql_api/tests/test_test_results_headers.py index ceeb62a76a..41adc4155a 100644 --- a/graphql_api/tests/test_test_results_headers.py +++ b/graphql_api/tests/test_test_results_headers.py @@ -29,7 +29,7 @@ def setUp(self): branch="main", ) - def test_fetch_test_result_total_runtime(self) -> None: + def test_fetch_test_result_aggregates(self) -> None: query = """ query { owner(username: "%s") { @@ -38,119 +38,15 @@ def test_fetch_test_result_total_runtime(self) -> None: testAnalytics { testResultsAggregates { totalDuration - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"][ - "totalDuration" - ] - == 465.0 - ) - - def test_fetch_test_result_slowest_tests_runtime(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResultsAggregates { slowestTestsDuration - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"][ - "slowestTestsDuration" - ] - == 30.0 - ) - - def test_fetch_test_result_failed_tests(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResultsAggregates { totalFails - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"][ - "totalFails" - ] - == 30 - ) - - def test_fetch_test_result_skipped_tests(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResultsAggregates { totalSkips - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"][ - "totalSkips" - ] - == 30 - ) - - def test_fetch_test_result_slow_tests(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResultsAggregates { totalSlowTests } } } } - } + } } """ % (self.owner.username, self.repository.name) @@ -158,8 +54,16 @@ def test_fetch_test_result_slow_tests(self) -> None: assert "errors" not in result assert ( - result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"][ - "totalSlowTests" - ] - == 1 + result["owner"]["repository"]["testAnalytics"] is not None + and result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"] + is not None ) + assert result["owner"]["repository"]["testAnalytics"][ + "testResultsAggregates" + ] == { + "totalDuration": 465.0, + "slowestTestsDuration": 30.0, + "totalFails": 30, + "totalSkips": 30, + "totalSlowTests": 1, + } diff --git a/graphql_api/types/flake_aggregates/flake_aggregates.py b/graphql_api/types/flake_aggregates/flake_aggregates.py index 0bb536fff5..9efd0c9607 100644 --- a/graphql_api/types/flake_aggregates/flake_aggregates.py +++ b/graphql_api/types/flake_aggregates/flake_aggregates.py @@ -1,37 +1,30 @@ -from typing import TypedDict - from ariadne import ObjectType from graphql import GraphQLResolveInfo -flake_aggregates_bindable = ObjectType("FlakeAggregates") - +from utils.test_results import FlakeAggregates -class FlakeAggregate(TypedDict): - flake_count: int - flake_count_percent_change: float | None - flake_rate: float - flake_rate_percent_change: float | None +flake_aggregates_bindable = ObjectType("FlakeAggregates") @flake_aggregates_bindable.field("flakeCount") -def resolve_flake_count(obj: FlakeAggregate, _: GraphQLResolveInfo) -> int: - return obj["flake_count"] +def resolve_flake_count(obj: FlakeAggregates, _: GraphQLResolveInfo) -> int: + return obj.flake_count @flake_aggregates_bindable.field("flakeCountPercentChange") def resolve_flake_count_percent_change( - obj: FlakeAggregate, _: GraphQLResolveInfo + obj: FlakeAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("flake_count_percent_change") + return obj.flake_count_percent_change @flake_aggregates_bindable.field("flakeRate") -def resolve_flake_rate(obj: FlakeAggregate, _: GraphQLResolveInfo) -> float: - return obj["flake_rate"] +def resolve_flake_rate(obj: FlakeAggregates, _: GraphQLResolveInfo) -> float: + return obj.flake_rate @flake_aggregates_bindable.field("flakeRatePercentChange") def resolve_flake_rate_percent_change( - obj: FlakeAggregate, _: GraphQLResolveInfo + obj: FlakeAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("flake_rate_percent_change") + return obj.flake_rate_percent_change diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index 7625d3225a..089bbf4d3e 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -1,15 +1,21 @@ import logging -from datetime import timedelta +from typing import Any, TypedDict from ariadne import ObjectType from graphql.type.definition import GraphQLResolveInfo from codecov.db import sync_to_async from core.models import Repository -from graphql_api.types.enums import OrderingDirection, TestResultsFilterParameter +from graphql_api.types.enums import ( + OrderingDirection, + TestResultsFilterParameter, + TestResultsOrderingParameter, +) from graphql_api.types.enums.enum_types import MeasurementInterval from utils.test_results import ( - GENERATE_TEST_RESULT_PARAM, + FlakeAggregates, + TestResultConnection, + TestResultsAggregates, generate_flake_aggregates, generate_test_results, generate_test_results_aggregates, @@ -19,6 +25,21 @@ log = logging.getLogger(__name__) + +class TestResultsOrdering(TypedDict): + parameter: TestResultsOrderingParameter + direction: OrderingDirection + + +class TestResultsFilters(TypedDict): + parameter: TestResultsFilterParameter | None + interval: MeasurementInterval + branch: str | None + test_suites: list[str] | None + flags: list[str] | None + term: str | None + + # Bindings for GraphQL types test_analytics_bindable: ObjectType = ObjectType("TestAnalytics") @@ -27,37 +48,32 @@ async def resolve_test_results( repository: Repository, info: GraphQLResolveInfo, - ordering=None, - filters=None, + ordering: TestResultsOrdering | None = None, + filters: TestResultsFilters | None = None, first: int | None = None, after: str | None = None, last: int | None = None, before: str | None = None, -): - parameter = ( - convert_test_results_filter_parameter(filters.get("parameter")) - if filters - else None - ) - interval = ( - convert_interval_to_timedelta(filters.get("interval")) - if filters - else timedelta(days=30) - ) - +) -> TestResultConnection: queryset = await sync_to_async(generate_test_results)( - ordering=ordering.get("parameter").value if ordering else "avg_duration", - ordering_direction=( - ordering.get("direction").name if ordering else OrderingDirection.DESC.name - ), + ordering=ordering.get("parameter", TestResultsOrderingParameter.AVG_DURATION) + if ordering + else TestResultsOrderingParameter.AVG_DURATION, + ordering_direction=ordering.get("direction", OrderingDirection.DESC) + if ordering + else OrderingDirection.DESC, repoid=repository.repoid, - interval=interval, + measurement_interval=filters.get( + "interval", MeasurementInterval.INTERVAL_30_DAY + ) + if filters + else MeasurementInterval.INTERVAL_30_DAY, first=first, after=after, last=last, before=before, branch=filters.get("branch") if filters else None, - parameter=parameter, + parameter=filters.get("parameter") if filters else None, testsuites=filters.get("test_suites") if filters else None, flags=filters.get("flags") if filters else None, term=filters.get("term") if filters else None, @@ -71,10 +87,10 @@ async def resolve_test_results_aggregates( repository: Repository, info: GraphQLResolveInfo, interval: MeasurementInterval | None = None, - **_, -): + **_: Any, +) -> TestResultsAggregates | None: return await sync_to_async(generate_test_results_aggregates)( - repoid=repository.repoid, interval=convert_interval_to_timedelta(interval) + repoid=repository.repoid, interval=interval.value if interval else 30 ) @@ -83,52 +99,22 @@ async def resolve_flake_aggregates( repository: Repository, info: GraphQLResolveInfo, interval: MeasurementInterval | None = None, - **_, -): + **_: Any, +) -> FlakeAggregates | None: return await sync_to_async(generate_flake_aggregates)( - repoid=repository.repoid, interval=convert_interval_to_timedelta(interval) + repoid=repository.repoid, interval=interval.value if interval else 30 ) @test_analytics_bindable.field("testSuites") async def resolve_test_suites( - repository: Repository, info: GraphQLResolveInfo, term: str | None = None, **_ -): + repository: Repository, info: GraphQLResolveInfo, term: str | None = None, **_: Any +) -> list[str]: return await sync_to_async(get_test_suites)(repository.repoid, term) @test_analytics_bindable.field("flags") async def resolve_flags( - repository: Repository, info: GraphQLResolveInfo, term: str | None = None, **_ -): + repository: Repository, info: GraphQLResolveInfo, term: str | None = None, **_: Any +) -> list[str]: return await sync_to_async(get_flags)(repository.repoid, term) - - -def convert_interval_to_timedelta(interval: MeasurementInterval | None) -> timedelta: - if interval is None: - return timedelta(days=30) - - match interval: - case MeasurementInterval.INTERVAL_1_DAY: - return timedelta(days=1) - case MeasurementInterval.INTERVAL_7_DAY: - return timedelta(days=7) - case MeasurementInterval.INTERVAL_30_DAY: - return timedelta(days=30) - - -def convert_test_results_filter_parameter( - parameter: TestResultsFilterParameter | None, -) -> GENERATE_TEST_RESULT_PARAM | None: - if parameter is None: - return None - - match parameter: - case TestResultsFilterParameter.FLAKY_TESTS: - return GENERATE_TEST_RESULT_PARAM.FLAKY - case TestResultsFilterParameter.FAILED_TESTS: - return GENERATE_TEST_RESULT_PARAM.FAILED - case TestResultsFilterParameter.SLOWEST_TESTS: - return GENERATE_TEST_RESULT_PARAM.SLOWEST - case TestResultsFilterParameter.SKIPPED_TESTS: - return GENERATE_TEST_RESULT_PARAM.SKIPPED diff --git a/graphql_api/types/test_results_aggregates/test_results_aggregates.py b/graphql_api/types/test_results_aggregates/test_results_aggregates.py index 93b425f94e..7920af95a0 100644 --- a/graphql_api/types/test_results_aggregates/test_results_aggregates.py +++ b/graphql_api/types/test_results_aggregates/test_results_aggregates.py @@ -1,81 +1,68 @@ -from typing import TypedDict - from ariadne import ObjectType from graphql import GraphQLResolveInfo -test_results_aggregates_bindable = ObjectType("TestResultsAggregates") - +from utils.test_results import TestResultsAggregates -class TestResultsAggregates(TypedDict): - total_duration: float - total_duration_percent_change: float | None - slowest_tests_duration: float - slowest_tests_duration_percent_change: float | None - total_slow_tests: int - total_slow_tests_percent_change: float | None - fails: int - fails_percent_change: float | None - skips: int - skips_percent_change: float | None +test_results_aggregates_bindable = ObjectType("TestResultsAggregates") @test_results_aggregates_bindable.field("totalDuration") def resolve_total_duration(obj: TestResultsAggregates, _: GraphQLResolveInfo) -> float: - return obj["total_duration"] + return obj.total_duration @test_results_aggregates_bindable.field("totalDurationPercentChange") def resolve_total_duration_percent_change( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("total_duration_percent_change") + return obj.total_duration_percent_change @test_results_aggregates_bindable.field("slowestTestsDuration") def resolve_slowest_tests_duration( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float: - return obj["slowest_tests_duration"] + return obj.slowest_tests_duration @test_results_aggregates_bindable.field("slowestTestsDurationPercentChange") def resolve_slowest_tests_duration_percent_change( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("slowest_tests_duration_percent_change") + return obj.slowest_tests_duration_percent_change @test_results_aggregates_bindable.field("totalSlowTests") def resolve_total_slow_tests(obj: TestResultsAggregates, _: GraphQLResolveInfo) -> int: - return obj["total_slow_tests"] + return obj.total_slow_tests @test_results_aggregates_bindable.field("totalSlowTestsPercentChange") def resolve_total_slow_tests_percent_change( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("total_slow_tests_percent_change") + return obj.total_slow_tests_percent_change @test_results_aggregates_bindable.field("totalFails") def resolve_total_fails(obj: TestResultsAggregates, _: GraphQLResolveInfo) -> int: - return obj["fails"] + return obj.fails @test_results_aggregates_bindable.field("totalFailsPercentChange") def resolve_total_fails_percent_change( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("fails_percent_change") + return obj.fails_percent_change @test_results_aggregates_bindable.field("totalSkips") def resolve_total_skips(obj: TestResultsAggregates, _: GraphQLResolveInfo) -> int: - return obj["skips"] + return obj.skips @test_results_aggregates_bindable.field("totalSkipsPercentChange") def resolve_total_skips_percent_change( obj: TestResultsAggregates, _: GraphQLResolveInfo ) -> float | None: - return obj.get("skips_percent_change") + return obj.skips_percent_change diff --git a/utils/test_results.py b/utils/test_results.py index e2108091b8..4f0e484992 100644 --- a/utils/test_results.py +++ b/utils/test_results.py @@ -22,6 +22,12 @@ ) from codecov.commands.exceptions import ValidationError +from graphql_api.types.enums import ( + OrderingDirection, + TestResultsFilterParameter, + TestResultsOrderingParameter, +) +from graphql_api.types.enums.enum_types import MeasurementInterval thirty_days_ago = dt.datetime.now(dt.UTC) - dt.timedelta(days=30) @@ -36,13 +42,6 @@ def slow_test_threshold(total_tests: int) -> int: return min(max(slow_tests_to_return, 1), 100) -class GENERATE_TEST_RESULT_PARAM: - FLAKY = "flaky" - FAILED = "failed" - SLOWEST = "slowest" - SKIPPED = "skipped" - - @dataclass class TestResultsQuery: query: str @@ -67,7 +66,29 @@ class TestResultsRow: @dataclass -class Connection: +class TestResultsAggregates: + total_duration: float + total_duration_percent_change: float | None + slowest_tests_duration: float + slowest_tests_duration_percent_change: float | None + total_slow_tests: int + total_slow_tests_percent_change: float | None + fails: int + fails_percent_change: float | None + skips: int + skips_percent_change: float | None + + +@dataclass +class FlakeAggregates: + flake_count: int + flake_count_percent_change: float | None + flake_rate: float + flake_rate_percent_change: float | None + + +@dataclass +class TestResultConnection: edges: list[dict[str, str | TestResultsRow]] page_info: dict total_count: int @@ -96,55 +117,46 @@ def decode_cursor(value: str | None) -> CursorValue | None: ) -def encode_cursor(row: TestResultsRow, ordering: str) -> str: +def encode_cursor(row: TestResultsRow, ordering: TestResultsOrderingParameter) -> str: return b64encode( - DELIMITER.join([str(getattr(row, ordering)), str(row.name)]).encode("utf-8") + DELIMITER.join([str(getattr(row, ordering.value)), str(row.name)]).encode( + "utf-8" + ) ).decode("ascii") def validate( - interval_num_days: int, - ordering: str, - ordering_direction: str, + interval: int, + ordering: TestResultsOrderingParameter, + ordering_direction: OrderingDirection, after: str | None, before: str | None, first: int | None, last: int | None, -) -> ValidationError | None: - if interval_num_days not in {1, 7, 30}: - return ValidationError(f"Invalid interval: {interval_num_days}") - - if ordering_direction not in {"ASC", "DESC"}: - return ValidationError(f"Invalid ordering direction: {ordering_direction}") - - if ordering not in { - "name", - "computed_name", - "avg_duration", - "failure_rate", - "flake_rate", - "commits_where_fail", - "last_duration", - "updated_at", - }: - return ValidationError(f"Invalid ordering field: {ordering}") +) -> None: + if interval not in {1, 7, 30}: + raise ValidationError(f"Invalid interval: {interval}") + + if not isinstance(ordering_direction, OrderingDirection): + raise ValidationError(f"Invalid ordering direction: {ordering_direction}") + + if not isinstance(ordering, TestResultsOrderingParameter): + raise ValidationError(f"Invalid ordering field: {ordering}") if first is not None and last is not None: - return ValidationError("First and last can not be used at the same time") + raise ValidationError("First and last can not be used at the same time") if after is not None and before is not None: - return ValidationError("After and before can not be used at the same time") - - return None + raise ValidationError("After and before can not be used at the same time") def generate_base_query( repoid: int, - ordering: str, - ordering_direction: str, + ordering: TestResultsOrderingParameter, + ordering_direction: OrderingDirection, should_reverse: bool, branch: str | None, - interval_num_days: int, + interval: int, testsuites: list[str] | None = None, term: str | None = None, test_ids: set[str] | None = None, @@ -152,13 +164,19 @@ def generate_base_query( term_filter = f"%{term}%" if term else None if should_reverse: - ordering_direction = "DESC" if ordering_direction == "ASC" else "ASC" + ordering_direction = ( + OrderingDirection.DESC + if ordering_direction == OrderingDirection.ASC + else OrderingDirection.ASC + ) - order_by = f"with_cursor.{ordering} {ordering_direction}, with_cursor.name" + order_by = ( + f"with_cursor.{ordering.value} {ordering_direction.name}, with_cursor.name" + ) params: dict[str, int | str | tuple[str, ...] | None] = { "repoid": repoid, - "interval": f"{interval_num_days} days", + "interval": f"{interval} days", "branch": branch, "test_ids": convert_tuple_else_none(test_ids), "testsuites": convert_tuple_else_none(testsuites), @@ -242,7 +260,7 @@ def generate_base_query( def search_base_query( rows: list[TestResultsRow], - ordering: str, + ordering: TestResultsOrderingParameter, cursor: CursorValue | None, descending: bool = False, ) -> list[TestResultsRow]: @@ -266,11 +284,13 @@ def search_base_query( if not cursor: return rows + print(f"descending: {descending}") + def compare(row: TestResultsRow) -> int: # -1 means row value is to the left of the cursor value (search to the right) # 0 means row value is equal to cursor value # 1 means row value is to the right of the cursor value (search to the left) - row_value = getattr(row, ordering) + row_value = getattr(row, ordering.value) row_value_str = str(row_value) cursor_value_str = cursor.ordered_value row_is_greater = row_value_str > cursor_value_str @@ -312,20 +332,20 @@ def get_relevant_totals( def generate_test_results( - ordering: str, - ordering_direction: str, + ordering: TestResultsOrderingParameter, + ordering_direction: OrderingDirection, repoid: int, - interval: dt.timedelta, + measurement_interval: MeasurementInterval, first: int | None = None, after: str | None = None, last: int | None = None, before: str | None = None, branch: str | None = None, - parameter: GENERATE_TEST_RESULT_PARAM | None = None, + parameter: TestResultsFilterParameter | None = None, testsuites: list[str] | None = None, flags: defaultdict[str, str] | None = None, term: str | None = None, -) -> Connection | ValidationError: +) -> TestResultConnection: """ Function that retrieves aggregated information about all tests in a given repository, for a given time range, optionally filtered by branch name. The fields it calculates are: the test failure rate, commits where this test failed, last duration and average duration of the test. @@ -342,14 +362,10 @@ def generate_test_results( :returns: queryset object containing list of dictionaries of results """ - interval_num_days = interval.days - - if validation_error := validate( - interval_num_days, ordering, ordering_direction, after, before, first, last - ): - return validation_error + interval = measurement_interval.value + validate(interval, ordering, ordering_direction, after, before, first, last) - since = dt.datetime.now(dt.UTC) - interval + since = dt.datetime.now(dt.UTC) - dt.timedelta(days=interval) test_ids: set[str] | None = None @@ -365,14 +381,14 @@ def generate_test_results( flag__flag_name__in=flags ) - filtered_test_ids = set([bridge.test_id for bridge in bridges]) + filtered_test_ids = set([bridge.test_id for bridge in bridges]) # type: ignore test_ids = test_ids & filtered_test_ids if test_ids else filtered_test_ids if parameter is not None: totals = get_relevant_totals(repoid, branch, since) match parameter: - case GENERATE_TEST_RESULT_PARAM.FLAKY: + case TestResultsFilterParameter.FLAKY_TESTS: flaky_test_ids = ( totals.values("test") .annotate(flaky_fail_count_sum=Sum("flaky_fail_count")) @@ -384,7 +400,7 @@ def generate_test_results( test_ids = ( test_ids & flaky_test_id_set if test_ids else flaky_test_id_set ) - case GENERATE_TEST_RESULT_PARAM.FAILED: + case TestResultsFilterParameter.FAILED_TESTS: failed_test_ids = ( totals.values("test") .annotate(fail_count_sum=Sum("fail_count")) @@ -396,7 +412,7 @@ def generate_test_results( test_ids = ( test_ids & failed_test_id_set if test_ids else failed_test_id_set ) - case GENERATE_TEST_RESULT_PARAM.SKIPPED: + case TestResultsFilterParameter.SKIPPED_TESTS: skipped_test_ids = ( totals.values("test") .annotate( @@ -412,7 +428,7 @@ def generate_test_results( test_ids = ( test_ids & skipped_test_id_set if test_ids else skipped_test_id_set ) - case GENERATE_TEST_RESULT_PARAM.SLOWEST: + case TestResultsFilterParameter.SLOWEST_TESTS: num_tests = totals.distinct("test_id").count() slowest_test_ids = ( @@ -441,7 +457,7 @@ def generate_test_results( ordering_direction=ordering_direction, should_reverse=should_reverse, branch=branch, - interval_num_days=interval_num_days, + interval=interval, testsuites=testsuites, term=term, test_ids=test_ids, @@ -459,13 +475,15 @@ def generate_test_results( page_size: int = first or last or 20 cursor_value = decode_cursor(after) if after else decode_cursor(before) - descending = ordering_direction == "DESC" + print(f"cursor_value: {cursor_value}") + descending = ordering_direction == OrderingDirection.DESC search_rows = search_base_query( rows, ordering, cursor_value, descending=descending, ) + print(f"search_rows: {search_rows}") page: list[dict[str, str | TestResultsRow]] = [ {"cursor": encode_cursor(row, ordering), "node": row} @@ -473,7 +491,7 @@ def generate_test_results( if i < page_size ] - return Connection( + return TestResultConnection( edges=page, total_count=len(rows), page_info={ @@ -485,35 +503,103 @@ def generate_test_results( ) -def percent_diff( - current_value: int | float, past_value: int | float -) -> int | float | None: +def percent_diff(current_value: int | float, past_value: int | float) -> float | None: if past_value == 0: return None return round((current_value - past_value) / past_value * 100, 5) -def get_percent_change( - fields: list[str], - curr_numbers: dict[str, int | float], - past_numbers: dict[str, int | float], -) -> dict[str, int | float | None]: - percent_change_fields = {} +@dataclass +class TestResultsAggregateNumbers: + total_duration: float + slowest_tests_duration: float + skips: int + fails: int + total_slow_tests: int + + +@dataclass +class FlakeAggregateNumbers: + flake_count: int + flake_rate: float - percent_change_fields = { - f"{field}_percent_change": percent_diff( - curr_numbers[field], past_numbers[field] + +def test_results_aggregates_from_numbers( + curr_numbers: TestResultsAggregateNumbers | None, + past_numbers: TestResultsAggregateNumbers | None, +) -> TestResultsAggregates | None: + if curr_numbers is None: + return None + if past_numbers is None: + return TestResultsAggregates( + total_duration=curr_numbers.total_duration, + total_duration_percent_change=None, + slowest_tests_duration=curr_numbers.slowest_tests_duration, + slowest_tests_duration_percent_change=None, + total_slow_tests=curr_numbers.total_slow_tests, + total_slow_tests_percent_change=None, + fails=curr_numbers.fails, + fails_percent_change=None, + skips=curr_numbers.skips, + skips_percent_change=None, ) - for field in fields - if past_numbers.get(field) - } + else: + return TestResultsAggregates( + total_duration=curr_numbers.total_duration, + total_duration_percent_change=percent_diff( + curr_numbers.total_duration, + past_numbers.total_duration, + ), + slowest_tests_duration=curr_numbers.slowest_tests_duration, + slowest_tests_duration_percent_change=percent_diff( + curr_numbers.slowest_tests_duration, + past_numbers.slowest_tests_duration, + ), + skips=curr_numbers.skips, + skips_percent_change=percent_diff( + curr_numbers.skips, + past_numbers.skips, + ), + fails=curr_numbers.fails, + fails_percent_change=percent_diff( + curr_numbers.fails, + past_numbers.fails, + ), + total_slow_tests=curr_numbers.total_slow_tests, + total_slow_tests_percent_change=percent_diff( + curr_numbers.total_slow_tests, + past_numbers.total_slow_tests, + ), + ) + + +def flake_aggregates_from_numbers( + curr_numbers: FlakeAggregateNumbers | None, + past_numbers: FlakeAggregateNumbers | None, +) -> FlakeAggregates | None: + if curr_numbers is None: + return None - return percent_change_fields + return FlakeAggregates( + flake_count=curr_numbers.flake_count, + flake_count_percent_change=percent_diff( + curr_numbers.flake_count, past_numbers.flake_count + ) + if past_numbers + else None, + flake_rate=curr_numbers.flake_rate, + flake_rate_percent_change=percent_diff( + curr_numbers.flake_rate, + past_numbers.flake_rate, + ) + if past_numbers + else None, + ) def get_test_results_aggregate_numbers( repo: Repository, since: dt.datetime, until: dt.datetime | None = None -) -> dict[str, float | int]: +) -> TestResultsAggregateNumbers | None: totals = DailyTestRollup.objects.filter( repoid=repo.repoid, date__gte=since, branch=repo.branch ) @@ -554,37 +640,44 @@ def get_test_results_aggregate_numbers( total_slow_tests=Value(slow_test_threshold(num_tests)), ) - return test_headers[0] if len(test_headers) > 0 else {} + if len(test_headers) == 0: + return None + else: + headers = test_headers[0] + return TestResultsAggregateNumbers( + total_duration=headers["total_duration"] or 0.0, + slowest_tests_duration=headers["slowest_tests_duration"] or 0.0, + skips=headers["skips"] or 0, + fails=headers["fails"] or 0, + total_slow_tests=headers["total_slow_tests"] or 0, + ) def generate_test_results_aggregates( - repoid: int, interval: dt.timedelta = dt.timedelta(days=30) -) -> dict[str, float | int | None] | None: + repoid: int, interval: int +) -> TestResultsAggregates | None: repo = Repository.objects.get(repoid=repoid) - since = dt.datetime.now(dt.UTC) - interval + since = dt.datetime.now(dt.UTC) - dt.timedelta(days=interval) curr_numbers = get_test_results_aggregate_numbers(repo, since) - double_time_ago = since - interval + double_time_ago = since - dt.timedelta(days=interval) past_numbers = get_test_results_aggregate_numbers(repo, double_time_ago, since) - return curr_numbers | get_percent_change( - [ - "total_duration", - "slowest_tests_duration", - "skips", - "fails", - "total_slow_tests", - ], - curr_numbers, - past_numbers, + aggregates_with_percentage: TestResultsAggregates | None = ( + test_results_aggregates_from_numbers( + curr_numbers, + past_numbers, + ) ) + return aggregates_with_percentage + def get_flake_aggregate_numbers( repo: Repository, since: dt.datetime, until: dt.datetime | None = None -) -> dict[str, int | float]: +) -> FlakeAggregateNumbers: if until is None: flakes = Flake.objects.filter( Q(repository_id=repo.repoid) @@ -601,7 +694,7 @@ def get_flake_aggregate_numbers( flake_count = flakes.count() - test_ids = [flake.test_id for flake in flakes] + test_ids = [flake.test_id for flake in flakes] # type: ignore test_rollups = DailyTestRollup.objects.filter( repoid=repo.repoid, @@ -613,7 +706,7 @@ def get_flake_aggregate_numbers( test_rollups = test_rollups.filter(date__lt=until.date()) if len(test_rollups) == 0: - return {"flake_count": 0, "flake_rate": 0} + return FlakeAggregateNumbers(flake_count=0, flake_rate=0.0) numerator = 0 denominator = 0 @@ -626,26 +719,20 @@ def get_flake_aggregate_numbers( else: flake_rate = numerator / denominator - return {"flake_count": flake_count, "flake_rate": flake_rate} + return FlakeAggregateNumbers(flake_count=flake_count, flake_rate=flake_rate) -def generate_flake_aggregates( - repoid: int, interval: dt.timedelta = dt.timedelta(days=30) -) -> dict[str, int | float | None]: +def generate_flake_aggregates(repoid: int, interval: int) -> FlakeAggregates | None: repo = Repository.objects.get(repoid=repoid) - since = dt.datetime.today() - interval + since = dt.datetime.today() - dt.timedelta(days=interval) curr_numbers = get_flake_aggregate_numbers(repo, since) - double_time_ago = since - interval + double_time_ago = since - dt.timedelta(days=interval) past_numbers = get_flake_aggregate_numbers(repo, double_time_ago, since) - return curr_numbers | get_percent_change( - ["flake_count", "flake_rate"], - curr_numbers, - past_numbers, - ) + return flake_aggregates_from_numbers(curr_numbers, past_numbers) def get_test_suites(repoid: int, term: str | None = None) -> list[str]: diff --git a/utils/tests/unit/test_cursor.py b/utils/tests/unit/test_cursor.py index 69e8b23254..f5a30d322c 100644 --- a/utils/tests/unit/test_cursor.py +++ b/utils/tests/unit/test_cursor.py @@ -1,5 +1,6 @@ from datetime import datetime +from graphql_api.types.enums.enums import TestResultsOrderingParameter from utils.test_results import CursorValue, TestResultsRow, decode_cursor, encode_cursor @@ -18,7 +19,7 @@ def test_cursor(): total_skip_count=1, total_pass_count=1, ) - cursor = encode_cursor(row, "updated_at") + cursor = encode_cursor(row, TestResultsOrderingParameter.UPDATED_AT) assert cursor == "MjAyNC0wMS0wMSAwMDowMDowMCswMDowMHx0ZXN0" decoded_cursor = decode_cursor(cursor) assert decoded_cursor == CursorValue(str(row.updated_at), "test") diff --git a/utils/tests/unit/test_search_base_query.py b/utils/tests/unit/test_search_base_query.py index 0387c0df7a..0b466f2006 100644 --- a/utils/tests/unit/test_search_base_query.py +++ b/utils/tests/unit/test_search_base_query.py @@ -1,5 +1,6 @@ from datetime import datetime +from graphql_api.types.enums.enums import TestResultsOrderingParameter from utils.test_results import CursorValue, TestResultsRow, search_base_query @@ -22,14 +23,14 @@ def row_factory(name: str, failure_rate: float): def test_search_base_query_cursor_val_none(): rows = [row_factory(str(i), float(i) * 0.1) for i in range(10)] - res = search_base_query(rows, "failure_rate", None) + res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, None) assert res == rows def test_search_base_query_with_existing_cursor(): rows = [row_factory(str(i), float(i) * 0.1) for i in range(10)] cursor = CursorValue(name="5", ordered_value="0.5") - res = search_base_query(rows, "failure_rate", cursor) + res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) assert res == rows[6:] @@ -39,7 +40,7 @@ def test_search_base_query_with_missing_cursor_high_name_low_failure_rate(): # here's where the cursor is pointing at rows = [row_factory(str(i), float(i) * 0.1) for i in range(3)] cursor = CursorValue(name="111111", ordered_value="0.05") - res = search_base_query(rows, "failure_rate", cursor) + res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) assert res == rows[1:] @@ -49,7 +50,7 @@ def test_search_base_query_with_missing_cursor_low_name_high_failure_rate(): # here's where the cursor is pointing at rows = [row_factory(str(i), float(i) * 0.1) for i in range(3)] cursor = CursorValue(name="0", ordered_value="0.15") - res = search_base_query(rows, "failure_rate", cursor) + res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) assert res == rows[-1:] @@ -59,5 +60,7 @@ def test_search_base_query_descending(): # here's where the cursor is pointing at rows = [row_factory(str(i), float(i) * 0.1) for i in range(2, -1, -1)] cursor = CursorValue(name="0", ordered_value="0.15") - res = search_base_query(rows, "failure_rate", cursor, descending=True) + res = search_base_query( + rows, TestResultsOrderingParameter.FAILURE_RATE, cursor, descending=True + ) assert res == rows[1:] From ac6a7648dfbd7bdc1c7040f5521816726caaf2c3 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 1 Nov 2024 10:30:39 -0400 Subject: [PATCH 02/10] feat: use cached results from GCS in TA GQL we want the GQL endpoints to consume the cached query results from GCS and then do some filtering and ordering on top of that, and also do some extra caching of those results in redis. this commit also contains a bunch of reorganization and refactoring of the GQL code --- graphql_api/tests/test_flake_aggregates.py | 86 - graphql_api/tests/test_test_analytics.py | 1608 +++++------------ graphql_api/tests/test_test_result.py | 102 -- .../tests/test_test_results_headers.py | 69 - .../flake_aggregates/flake_aggregates.py | 60 +- .../types/test_analytics/test_analytics.py | 288 ++- .../types/test_results/test_results.py | 2 +- .../test_results_aggregates.py | 87 +- requirements.in | 1 + requirements.txt | 4 +- utils/test_results.py | 793 +------- utils/tests/unit/test_cursor.py | 25 - utils/tests/unit/test_search_base_query.py | 66 - utils/tests/unit/test_slow_test_threshold.py | 23 - 14 files changed, 958 insertions(+), 2256 deletions(-) delete mode 100644 graphql_api/tests/test_flake_aggregates.py delete mode 100644 graphql_api/tests/test_test_result.py delete mode 100644 graphql_api/tests/test_test_results_headers.py delete mode 100644 utils/tests/unit/test_cursor.py delete mode 100644 utils/tests/unit/test_search_base_query.py delete mode 100644 utils/tests/unit/test_slow_test_threshold.py diff --git a/graphql_api/tests/test_flake_aggregates.py b/graphql_api/tests/test_flake_aggregates.py deleted file mode 100644 index b6f6fbe6c9..0000000000 --- a/graphql_api/tests/test_flake_aggregates.py +++ /dev/null @@ -1,86 +0,0 @@ -from datetime import date, datetime, timedelta - -from django.test import TransactionTestCase -from freezegun import freeze_time -from shared.django_apps.core.tests.factories import OwnerFactory, RepositoryFactory -from shared.django_apps.reports.tests.factories import FlakeFactory - -from reports.tests.factories import DailyTestRollupFactory, TestFactory - -from .helper import GraphQLTestHelper - - -@freeze_time(datetime.now().isoformat()) -class TestResultTestCase(GraphQLTestHelper, TransactionTestCase): - def setUp(self): - self.owner = OwnerFactory(username="randomOwner") - self.repository = RepositoryFactory(author=self.owner, branch="main") - - test = TestFactory(repository=self.repository) - for i in range(0, 30): - _ = FlakeFactory( - repository=self.repository, - test=test, - end_date=datetime.now() - timedelta(days=i), - ) - _ = DailyTestRollupFactory( - test=test, - date=date.today() - timedelta(days=i), - avg_duration_seconds=float(i), - latest_run=datetime.now() - timedelta(days=i), - fail_count=1, - skip_count=1, - pass_count=1, - flaky_fail_count=1 if i % 5 == 0 else 0, - branch="main", - ) - - for i in range(30, 60): - if i % 2 == 0: - _ = FlakeFactory( - repository=self.repository, - test=test, - start_date=datetime.now() - timedelta(days=i + 1), - end_date=datetime.now() - timedelta(days=i), - ) - _ = DailyTestRollupFactory( - test=test, - date=date.today() - timedelta(days=i), - avg_duration_seconds=float(i), - latest_run=datetime.now() - timedelta(days=i), - fail_count=3, - skip_count=1, - pass_count=1, - flaky_fail_count=3 if i % 5 == 0 else 0, - branch="main", - ) - - def test_fetch_test_result_total_runtime(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - flakeAggregates { - flakeRate - flakeCount - flakeRatePercentChange - flakeCountPercentChange - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert result["owner"]["repository"]["testAnalytics"]["flakeAggregates"] == { - "flakeRate": 0.140625, - "flakeCount": 31, - "flakeRatePercentChange": 31.25, - "flakeCountPercentChange": 106.66667, - } diff --git a/graphql_api/tests/test_test_analytics.py b/graphql_api/tests/test_test_analytics.py index ecce9f7b7c..78e6cc9618 100644 --- a/graphql_api/tests/test_test_analytics.py +++ b/graphql_api/tests/test_test_analytics.py @@ -1,1179 +1,537 @@ import datetime from base64 import b64encode -from django.test import TransactionTestCase -from shared.django_apps.core.tests.factories import OwnerFactory, RepositoryFactory -from shared.django_apps.reports.tests.factories import FlakeFactory - -from reports.tests.factories import ( - DailyTestRollupFactory, - RepositoryFlagFactory, - TestFactory, - TestFlagBridgeFactory, +import polars as pl +import pytest +from shared.django_apps.codecov_auth.tests.factories import OwnerFactory +from shared.django_apps.core.tests.factories import RepositoryFactory +from shared.storage.exceptions import BucketAlreadyExistsError + +from graphql_api.types.enums import ( + OrderingDirection, + TestResultsOrderingParameter, +) +from graphql_api.types.enums.enum_types import MeasurementInterval +from graphql_api.types.test_analytics.test_analytics import ( + TestResultConnection, + TestResultsRow, + generate_test_results, + get_results, ) +from services.redis_configuration import get_redis_connection +from services.storage import StorageService from .helper import GraphQLTestHelper - -def base64_encode_string(x: str) -> str: - return b64encode(x.encode()).decode("utf-8") - - -class TestAnalyticsTestCase(GraphQLTestHelper, TransactionTestCase): - def setUp(self) -> None: - self.owner = OwnerFactory(username="codecov-user") - self.repo = RepositoryFactory( - author=self.owner, name="testRepoName", active=True - ) - - query_builder = """ - query TestAnalytics($name: String!){ - me { - owner { - repository(name: $name) { - __typename - ... on Repository { - testAnalytics { - %s - } - } - ... on ResolverError { - message - } - } - } - } - } - """ - - def fetch_test_analytics(self, name, fields=None): - data = self.gql_request( - self.query_builder % fields, - owner=self.owner, - variables={"name": name}, - ) - return data["me"]["owner"]["repository"]["testAnalytics"] - - def test_repository_test_analytics_typename(self): - response = self.gql_request( - """ - query($owner: String!, $repo: String!) { - owner(username: $owner) { - repository(name: $repo) { - ... on Repository { +base_gql_query = """ + query { + owner(username: "%s") { + repository(name: "%s") { + ... on Repository { testAnalytics { - __typename + %s } - } - ... on ResolverError { - message - } } - } } - """, - owner=self.owner, - variables={"owner": self.owner.username, "repo": self.repo.name}, - ) - - assert ( - response["owner"]["repository"]["testAnalytics"]["__typename"] - == "TestAnalytics" - ) - - def test_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - - _ = DailyTestRollupFactory(test=test) - res = self.fetch_test_analytics( - repo.name, """testResults { edges { node { name } } }""" - ) - - assert res["testResults"] == {"edges": [{"node": {"name": test.name}}]} - - def test_test_results_no_tests(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - res = self.fetch_test_analytics( - repo.name, """testResults { edges { node { name } } }""" - ) - assert res["testResults"] == {"edges": []} - - def test_branch_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="feature", - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { branch: "main"}) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test.name}}]} - - def test_interval_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.datetime.now() - datetime.timedelta(days=7), - repoid=repo.repoid, - branch="main", - ) - _ = DailyTestRollupFactory( - test=test2, - date=datetime.datetime.now(), - repoid=repo.repoid, - branch="feature", - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { interval: INTERVAL_1_DAY }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test2.name}}]} - - def test_flaky_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = FlakeFactory(test=test2, repository=repo, end_date=None) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - flaky_fail_count=0, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="feature", - flaky_fail_count=1000, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { parameter: FLAKY_TESTS }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test2.name}}]} - - def test_failed_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - fail_count=0, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="feature", - fail_count=1000, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { parameter: FAILED_TESTS }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test2.name}}]} - - def test_skipped_filter_on_test_results(self) -> None: - # note - this test guards against division by zero errors for the failure/flake rate - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - skip_count=10, - pass_count=10, - fail_count=10, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="feature", - skip_count=1000, - pass_count=0, - fail_count=0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { parameter: SKIPPED_TESTS }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test2.name}}]} - - def test_slowest_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=0.1, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=20.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { parameter: SLOWEST_TESTS }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test2.name}}]} - - def test_flags_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - - repo_flag = RepositoryFlagFactory(repository=repo, flag_name="hello_world") - - _ = TestFlagBridgeFactory(flag=repo_flag, test=test) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=0.1, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=20.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { flags: ["hello_world"] }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test.name}}]} - - def test_testsuites_filter_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo, testsuite="hello") - test2 = TestFactory(repository=repo, testsuite="world") - - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=0.1, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=20.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { test_suites: ["hello"] }) { edges { node { name } } }""", - ) - assert res["testResults"] == {"edges": [{"node": {"name": test.name}}]} - - def test_commits_failed_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - commits_where_fail=["1"], - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - commits_where_fail=["2"], - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - commits_where_fail=["3"], - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: COMMITS_WHERE_FAIL, direction: ASC }) { edges { node { name commitsFailed } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "commitsFailed": 1}}, - {"node": {"name": test.name, "commitsFailed": 2}}, - ] } + } +""" + +row_1 = { + "name": "test1", + "testsuite": "testsuite1", + "flags": ["flag1"], + "test_id": "test_id1", + "failure_rate": 0.1, + "flake_rate": 0.0, + "updated_at": datetime.datetime(2024, 1, 1), + "avg_duration": 100.0, + "total_fail_count": 1, + "total_flaky_fail_count": 0, + "total_pass_count": 1, + "total_skip_count": 1, + "commits_where_fail": 1, + "last_duration": 100.0, +} + + +row_2 = { + "name": "test2", + "testsuite": "testsuite2", + "flags": ["flag2"], + "test_id": "test_id2", + "failure_rate": 0.2, + "flake_rate": 0.3, + "updated_at": datetime.datetime(2024, 1, 2), + "avg_duration": 200.0, + "total_fail_count": 2, + "total_flaky_fail_count": 2, + "total_pass_count": 2, + "total_skip_count": 2, + "commits_where_fail": 2, + "last_duration": 200.0, +} + + +def row_to_camel_case(row: dict) -> dict: + return { + "commitsFailed" + if key == "commits_where_fail" + else "".join( + part.capitalize() if i > 0 else part.lower() + for i, part in enumerate(key.split("_")) + ): value.isoformat() if key == "updated_at" else value + for key, value in row.items() + if key not in ("test_id", "testsuite", "flags") + } - def test_desc_commits_failed_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - commits_where_fail=["1"], - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - commits_where_fail=["2"], - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - commits_where_fail=["3"], - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: COMMITS_WHERE_FAIL, direction: DESC }) { edges { node { name commitsFailed } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "commitsFailed": 2}}, - {"node": {"name": test_2.name, "commitsFailed": 1}}, - ] - } - - def test_last_duration_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - last_duration_seconds=1.0, - latest_run=datetime.datetime.now() - datetime.timedelta(days=1), - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - last_duration_seconds=2.0, - latest_run=datetime.datetime.now(), - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - last_duration_seconds=3.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: LAST_DURATION, direction: ASC }) { edges { node { name lastDuration } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "lastDuration": 2.0}}, - {"node": {"name": test_2.name, "lastDuration": 3.0}}, - ] - } - - def test_desc_last_duration_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - last_duration_seconds=1.0, - latest_run=datetime.datetime.now() - datetime.timedelta(days=1), - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - last_duration_seconds=2.0, - latest_run=datetime.datetime.now(), - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - last_duration_seconds=3.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: LAST_DURATION, direction: DESC }) { edges { node { name lastDuration } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "lastDuration": 3.0}}, - {"node": {"name": test.name, "lastDuration": 2.0}}, - ] - } - - def test_avg_duration_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - avg_duration_seconds=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - avg_duration_seconds=2, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - avg_duration_seconds=3, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: AVG_DURATION, direction: ASC }) { edges { node { name avgDuration } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "avgDuration": 1.5}}, - {"node": {"name": test_2.name, "avgDuration": 3}}, - ] - } - - def test_desc_avg_duration_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - avg_duration_seconds=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - avg_duration_seconds=2, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - avg_duration_seconds=3, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: AVG_DURATION, direction: DESC }) { edges { node { name avgDuration } } }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "avgDuration": 3}}, - {"node": {"name": test.name, "avgDuration": 1.5}}, - ] - } - - def test_failure_rate_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: ASC }) { edges { node { name failureRate } } }""", - ) - - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "failureRate": 0.2}}, - {"node": {"name": test_2.name, "failureRate": 0.6}}, - ] - } - def test_desc_failure_rate_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: DESC }) { edges { node { name failureRate } } }""", - ) +test_results_table = pl.DataFrame( + [row_1, row_2], +) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "failureRate": 0.6}}, - {"node": {"name": test.name, "failureRate": 0.2}}, - ] - } - def test_desc_failure_rate_ordering_on_test_results_with_after(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - ) +def base64_encode_string(x: str) -> str: + return b64encode(x.encode()).decode("utf-8") - test_3 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_3, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=1, - fail_count=4, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: DESC }, first: 1) { edges { node { name failureRate } }, pageInfo { hasNextPage, hasPreviousPage, startCursor, endCursor }, totalCount }""", - ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_3.name, "failureRate": 0.8}}, +@pytest.fixture(autouse=True) +def repository(mocker, transactional_db): + owner = OwnerFactory(username="codecov-user") + repo = RepositoryFactory(author=owner, name="testRepoName", active=True) + + return repo + + +@pytest.fixture +def store_in_redis(repository): + redis = get_redis_connection() + redis.set( + f"test_results:{repository.repoid}:{repository.branch}:30", + test_results_table.write_ipc(None).getvalue(), + ) + + yield + + redis.delete( + f"test_results:{repository.repoid}:{repository.branch}:30", + ) + + +@pytest.fixture +def store_in_storage(repository): + storage = StorageService() + try: + storage.create_root_storage("codecov") + except BucketAlreadyExistsError: + pass + + storage.write_file( + "codecov", + f"test_results/rollups/{repository.repoid}/{repository.branch}/30", + test_results_table.write_ipc(None).getvalue(), + ) + + yield + + storage.delete_file( + "codecov", + f"test_results/rollups/{repository.repoid}/{repository.branch}/30", + ) + + +class TestAnalyticsTestCase( + GraphQLTestHelper, +): + def test_get_test_results( + self, transactional_db, repository, store_in_redis, store_in_storage + ): + results = get_results(repository.repoid, repository.branch, 30) + assert results is not None + assert results.equals(test_results_table) + + def test_get_test_results_no_storage(self, transactional_db, repository): + with pytest.raises(FileNotFoundError): + get_results(repository.repoid, repository.branch, 30) + + def test_get_test_results_no_redis( + self, transactional_db, repository, store_in_storage + ): + results = get_results(repository.repoid, repository.branch, 30) + assert results is not None + assert results.equals(test_results_table) + + def test_test_results(self, transactional_db, repository, store_in_redis): + test_results = generate_test_results( + repoid=repository.repoid, + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.DESC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + ) + assert test_results is not None + assert test_results == TestResultConnection( + total_count=2, + edges=[ + { + "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", + "node": TestResultsRow(**row_2), + }, + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": TestResultsRow(**row_1), + }, ], - "pageInfo": { - "endCursor": base64_encode_string(f"0.8|{test_3.name}"), - "hasNextPage": True, - "hasPreviousPage": False, - "startCursor": base64_encode_string(f"0.8|{test_3.name}"), + page_info={ + "has_next_page": False, + "has_previous_page": False, + "start_cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", + "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", }, - "totalCount": 3, - } - - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: DESC }, first: 1, after: "%s") { edges { node { name failureRate } }, pageInfo { hasNextPage, hasPreviousPage, startCursor, endCursor }, totalCount }""" - % res["testResults"]["pageInfo"]["endCursor"], ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "failureRate": 0.6}}, + def test_test_results_asc(self, transactional_db, repository, store_in_redis): + test_results = generate_test_results( + repoid=repository.repoid, + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.ASC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + ) + assert test_results is not None + assert test_results == TestResultConnection( + total_count=2, + edges=[ + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": TestResultsRow(**row_1), + }, + { + "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", + "node": TestResultsRow(**row_2), + }, ], - "pageInfo": { - "endCursor": base64_encode_string(f"0.6|{test_2.name}"), - "hasNextPage": True, - "hasPreviousPage": False, - "startCursor": base64_encode_string(f"0.6|{test_2.name}"), + page_info={ + "has_next_page": False, + "has_previous_page": False, + "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "end_cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", }, - "totalCount": 3, - } - - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: DESC }, first: 1, after: "%s") { edges { node { name failureRate } }, pageInfo { hasNextPage, hasPreviousPage, startCursor, endCursor }, totalCount }""" - % res["testResults"]["pageInfo"]["endCursor"], ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "failureRate": 0.2}}, + @pytest.mark.parametrize( + "first, after, before, last, has_next_page, has_previous_page, rows", + [ + (1, None, None, None, True, False, [row_2]), + ( + 1, + base64_encode_string(f"{row_2['updated_at']}|{row_2['name']}"), + None, + None, + False, + False, + [row_1], + ), + (None, None, None, 1, False, True, [row_1]), + ( + None, + None, + base64_encode_string(f"{row_1['updated_at']}|{row_1['name']}"), + 1, + False, + False, + [row_2], + ), + ], + ) + def test_test_results_pagination( + self, + first, + after, + before, + last, + has_next_page, + has_previous_page, + rows, + repository, + store_in_redis, + ): + test_results = generate_test_results( + repoid=repository.repoid, + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.DESC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + first=first, + after=after, + before=before, + last=last, + ) + assert test_results == TestResultConnection( + total_count=2, + edges=[ + { + "cursor": base64_encode_string( + f"{row['updated_at']}|{row['name']}" + ), + "node": TestResultsRow(**row), + } + for row in rows ], - "pageInfo": { - "endCursor": base64_encode_string(f"0.2|{test.name}"), - "hasNextPage": False, - "hasPreviousPage": False, - "startCursor": base64_encode_string(f"0.2|{test.name}"), + page_info={ + "has_next_page": has_next_page, + "has_previous_page": has_previous_page, + "start_cursor": base64_encode_string( + f"{rows[0]['updated_at']}|{rows[0]['name']}" + ) + if after + else base64_encode_string( + f"{rows[-1]['updated_at']}|{rows[-1]['name']}" + ), + "end_cursor": base64_encode_string( + f"{rows[-1]['updated_at']}|{rows[-1]['name']}" + ) + if before + else base64_encode_string(f"{rows[0]['updated_at']}|{rows[0]['name']}"), }, - "totalCount": 3, - } - - def test_flake_rate_filtering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - flaky_fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - flaky_fail_count=0, ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - flaky_fail_count=1, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: ASC }) { edges { node { name flakeRate } } }""", - ) - - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "flakeRate": 0.2}}, - {"node": {"name": test_2.name, "flakeRate": 0.2}}, - ] - } - def test_flake_rate_filtering_by_term(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo, name="hello") - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - flaky_fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - flaky_fail_count=0, - ) - test_2 = TestFactory(repository=repo, name="world") - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - flaky_fail_count=1, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(filters: { term: "hello" }) { edges { node { name failureRate } } }""", + @pytest.mark.parametrize( + "first, after, before, last, has_next_page, has_previous_page, rows", + [ + (1, None, None, None, True, False, [row_1]), + ( + 1, + base64_encode_string(f"{row_1['updated_at']}|{row_1['name']}"), + None, + None, + False, + False, + [row_2], + ), + (None, None, None, 1, False, True, [row_2]), + ( + None, + None, + base64_encode_string(f"{row_2['updated_at']}|{row_2['name']}"), + 1, + False, + False, + [row_1], + ), + ], + ) + def test_test_results_pagination_asc( + self, + first, + after, + before, + last, + has_next_page, + has_previous_page, + rows, + repository, + store_in_redis, + ): + test_results = generate_test_results( + repoid=repository.repoid, + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.ASC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + first=first, + after=after, + before=before, + last=last, + ) + assert test_results == TestResultConnection( + total_count=2, + edges=[ + { + "cursor": base64_encode_string( + f"{row['updated_at']}|{row['name']}" + ), + "node": TestResultsRow(**row), + } + for row in rows + ], + page_info={ + "has_next_page": has_next_page, + "has_previous_page": has_previous_page, + "start_cursor": base64_encode_string( + f"{rows[0]['updated_at']}|{rows[0]['name']}" + ) + if after + else base64_encode_string( + f"{rows[-1]['updated_at']}|{rows[-1]['name']}" + ), + "end_cursor": base64_encode_string( + f"{rows[-1]['updated_at']}|{rows[-1]['name']}" + ) + if before + else base64_encode_string(f"{rows[0]['updated_at']}|{rows[0]['name']}"), + }, ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test.name, "failureRate": 0.2}}, - ] - } - - def test_desc_flake_rate_ordering_on_test_results(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today() - datetime.timedelta(days=1), - repoid=repo.repoid, - pass_count=1, - fail_count=1, - flaky_fail_count=1, - ) - _ = DailyTestRollupFactory( - test=test, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=3, - fail_count=0, - flaky_fail_count=0, - ) - test_2 = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test_2, - date=datetime.date.today(), - repoid=repo.repoid, - pass_count=2, - fail_count=3, - flaky_fail_count=1, - ) - res = self.fetch_test_analytics( - repo.name, - """testResults(ordering: { parameter: FAILURE_RATE, direction: DESC }) { edges { node { name flakeRate } } }""", + def test_test_analytics_term_filter(self, repository, store_in_redis): + test_results = generate_test_results( + repoid=repository.repoid, + term="test1", + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.DESC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + ) + assert test_results is not None + assert test_results == TestResultConnection( + total_count=1, + edges=[ + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": TestResultsRow(**row_1), + }, + ], + page_info={ + "has_next_page": False, + "has_previous_page": False, + "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + }, ) - assert res["testResults"] == { - "edges": [ - {"node": {"name": test_2.name, "flakeRate": 0.2}}, - {"node": {"name": test.name, "flakeRate": 0.2}}, - ] - } - - def test_test_results_aggregates(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" + def test_test_analytics_testsuite_filter(self, repository, store_in_redis): + test_results = generate_test_results( + repoid=repository.repoid, + testsuites=["testsuite1"], + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.DESC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + ) + assert test_results is not None + assert test_results == TestResultConnection( + total_count=1, + edges=[ + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": TestResultsRow(**row_1), + }, + ], + page_info={ + "has_next_page": False, + "has_previous_page": False, + "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + }, ) - for i in range(0, 30): - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - fail_count=1 if i % 3 == 0 else 0, - skip_count=1 if i % 6 == 0 else 0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - - for i in range(30, 60): - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - fail_count=1 if i % 6 == 0 else 0, - skip_count=1 if i % 3 == 0 else 0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=(i)), - ) - res = self.fetch_test_analytics( - repo.name, - """testResultsAggregates { totalDuration, slowestTestsDuration, totalFails, totalSkips, totalSlowTests, totalDurationPercentChange, slowestTestsDurationPercentChange, totalFailsPercentChange, totalSkipsPercentChange, totalSlowTestsPercentChange }""", + def test_test_analytics_flag_filter(self, repository, store_in_redis): + test_results = generate_test_results( + repoid=repository.repoid, + flags=["flag1"], + ordering=TestResultsOrderingParameter.UPDATED_AT, + ordering_direction=OrderingDirection.DESC, + measurement_interval=MeasurementInterval.INTERVAL_30_DAY, + ) + assert test_results is not None + assert test_results == TestResultConnection( + total_count=1, + edges=[ + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": TestResultsRow(**row_1), + }, + ], + page_info={ + "has_next_page": False, + "has_previous_page": False, + "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + }, ) - assert res["testResultsAggregates"] == { - "totalDuration": 630.0, - "totalDurationPercentChange": -57.57576, - "slowestTestsDuration": 60.0, - "slowestTestsDurationPercentChange": 1.69492, - "totalFails": 11, - "totalFailsPercentChange": 175.0, - "totalSkips": 6, - "totalSkipsPercentChange": -33.33333, - "totalSlowTests": 1, - "totalSlowTestsPercentChange": 0.0, - } - def test_test_results_aggregates_no_history(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" + def test_gql_query(self, repository, store_in_redis): + query = base_gql_query % ( + repository.author.username, + repository.name, + """ + testResults(ordering: { parameter: UPDATED_AT, direction: DESC } ) { + totalCount + edges { + cursor + node { + name + failureRate + flakeRate + updatedAt + avgDuration + totalFailCount + totalFlakyFailCount + totalPassCount + totalSkipCount + commitsFailed + lastDuration + } + } + } + """, ) - for i in range(0, 30): - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - fail_count=1 if i % 3 == 0 else 0, - skip_count=1 if i % 6 == 0 else 0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) + result = self.gql_request(query, owner=repository.author) - res = self.fetch_test_analytics( - repo.name, - """testResultsAggregates { totalDuration, slowestTestsDuration, totalFails, totalSkips, totalSlowTests, totalDurationPercentChange, slowestTestsDurationPercentChange, totalFailsPercentChange, totalSkipsPercentChange, totalSlowTestsPercentChange }""", - ) - - assert res["testResultsAggregates"] == { - "totalDuration": 570.0, - "totalDurationPercentChange": None, - "slowestTestsDuration": 29.0, - "slowestTestsDurationPercentChange": None, - "totalFails": 10, - "totalFailsPercentChange": None, - "totalSkips": 5, - "totalSkipsPercentChange": None, - "totalSlowTests": 1, - "totalSlowTestsPercentChange": None, - } + assert ( + result["owner"]["repository"]["testAnalytics"]["testResults"]["totalCount"] + == 2 + ) + assert result["owner"]["repository"]["testAnalytics"]["testResults"][ + "edges" + ] == [ + { + "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", + "node": row_to_camel_case(row_2), + }, + { + "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "node": row_to_camel_case(row_1), + }, + ] - def test_test_results_aggregates_no_history_7_days(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" + def test_gql_query_aggregates(self, repository, store_in_redis): + query = base_gql_query % ( + repository.author.username, + repository.name, + """ + testResultsAggregates { + totalDuration + slowestTestsDuration + totalFails + totalSkips + totalSlowTests + } + """, ) - for i in range(0, 7): - test = TestFactory(repository=repo) - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - fail_count=1 if i % 3 == 0 else 0, - skip_count=1 if i % 6 == 0 else 0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - - res = self.fetch_test_analytics( - repo.name, - """testResultsAggregates(interval: INTERVAL_7_DAY) { totalDuration, slowestTestsDuration, totalFails, totalSkips, totalSlowTests, totalDurationPercentChange, slowestTestsDurationPercentChange, totalFailsPercentChange, totalSkipsPercentChange, totalSlowTestsPercentChange }""", - ) + result = self.gql_request(query, owner=repository.author) - assert res["testResultsAggregates"] == { - "totalDuration": 30.0, - "totalDurationPercentChange": None, - "slowestTestsDuration": 12.0, - "slowestTestsDurationPercentChange": None, + assert result["owner"]["repository"]["testAnalytics"][ + "testResultsAggregates" + ] == { + "totalDuration": 1000.0, + "slowestTestsDuration": 800.0, "totalFails": 3, - "totalFailsPercentChange": None, - "totalSkips": 2, - "totalSkipsPercentChange": None, + "totalSkips": 3, "totalSlowTests": 1, - "totalSlowTestsPercentChange": None, - } - - def test_flake_aggregates(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" - ) - - test = TestFactory(repository=repo) - - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=1), - end_date=None, - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=90), - end_date=None, - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=90), - end_date=datetime.datetime.now() - datetime.timedelta(days=30), - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=90), - end_date=datetime.datetime.now() - datetime.timedelta(days=59), - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=90), - end_date=datetime.datetime.now() - datetime.timedelta(days=61), - ) - - for i in range(0, 30): - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - flaky_fail_count=1 if i % 6 == 0 else 0, - fail_count=1 if i % 3 == 0 else 0, - skip_count=0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - for i in range(30, 60): - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - flaky_fail_count=5 if i % 3 == 0 else 0, - fail_count=5 if i % 3 == 0 else 0, - skip_count=0, - pass_count=5, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - - res = self.fetch_test_analytics( - repo.name, - """flakeAggregates { flakeCount, flakeRate, flakeCountPercentChange, flakeRatePercentChange }""", - ) - - assert res["flakeAggregates"] == { - "flakeCount": 3, - "flakeRate": 0.2, - "flakeCountPercentChange": 0.0, - "flakeRatePercentChange": -15.55556, } - def test_flake_aggregates_no_history(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" - ) - - test = TestFactory(repository=repo) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=1), - end_date=None, - ) - - for i in range(0, 30): - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - flaky_fail_count=1 if i % 3 == 0 else 0, - fail_count=1 if i % 3 == 0 else 0, - skip_count=0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - - res = self.fetch_test_analytics( - repo.name, - """flakeAggregates { flakeCount, flakeRate, flakeCountPercentChange, flakeRatePercentChange }""", - ) - - assert res["flakeAggregates"] == { - "flakeCount": 1, - "flakeRate": 0.25, - "flakeCountPercentChange": None, - "flakeRatePercentChange": None, - } - - def test_flake_aggregates_7_days(self) -> None: - repo = RepositoryFactory( - author=self.owner, active=True, private=True, branch="main" - ) - - test = TestFactory(repository=repo) - - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=1), - end_date=None, - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=11), - end_date=datetime.datetime.now() - datetime.timedelta(days=8), - ) - _ = FlakeFactory( - repository=repo, - test=test, - start_date=datetime.datetime.now() - datetime.timedelta(days=30), - end_date=datetime.datetime.now() - datetime.timedelta(days=10), + def test_gql_query_flake_aggregates(self, repository, store_in_redis): + query = base_gql_query % ( + repository.author.username, + repository.name, + """ + flakeAggregates { + flakeRate + flakeCount + } + """, ) - for i in range(0, 7): - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - flaky_fail_count=1 if i % 7 == 0 else 0, - fail_count=1 if i % 7 == 0 else 0, - skip_count=0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) - for i in range(7, 14): - _ = DailyTestRollupFactory( - test=test, - repoid=repo.repoid, - branch="main", - flaky_fail_count=1 if i % 3 == 0 else 0, - fail_count=1 if i % 3 == 0 else 0, - skip_count=0, - pass_count=1, - avg_duration_seconds=float(i), - last_duration_seconds=float(i), - date=datetime.date.today() - datetime.timedelta(days=i), - ) + result = self.gql_request(query, owner=repository.author) - res = self.fetch_test_analytics( - repo.name, - """flakeAggregates(interval: INTERVAL_7_DAY) { flakeCount, flakeRate, flakeCountPercentChange, flakeRatePercentChange }""", - ) - - assert res["flakeAggregates"] == { + assert result["owner"]["repository"]["testAnalytics"]["flakeAggregates"] == { + "flakeRate": 1 / 3, "flakeCount": 1, - "flakeRate": 0.1111111111111111, - "flakeCountPercentChange": -50.0, - "flakeRatePercentChange": -55.55556, } - - def test_test_suites(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo, testsuite="hello_world") - test2 = TestFactory(repository=repo, testsuite="goodbye_world") - - repo_flag = RepositoryFlagFactory(repository=repo, flag_name="hello_world") - - _ = TestFlagBridgeFactory(flag=repo_flag, test=test) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=0.1, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=20.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testSuites(term: "hello")""", - ) - assert res["testSuites"] == ["hello_world"] - - def test_test_suites_no_term(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo, testsuite="hello_world") - test2 = TestFactory(repository=repo, testsuite="goodbye_world") - - repo_flag = RepositoryFlagFactory(repository=repo, flag_name="hello_world") - - _ = TestFlagBridgeFactory(flag=repo_flag, test=test) - _ = DailyTestRollupFactory( - test=test, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=0.1, - ) - _ = DailyTestRollupFactory( - test=test2, - created_at=datetime.datetime.now(), - repoid=repo.repoid, - branch="main", - avg_duration_seconds=20.0, - ) - res = self.fetch_test_analytics( - repo.name, - """testSuites""", - ) - assert sorted(res["testSuites"]) == ["goodbye_world", "hello_world"] - - def test_flags(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - - repo_flag = RepositoryFlagFactory(repository=repo, flag_name="hello_world") - repo_flag2 = RepositoryFlagFactory(repository=repo, flag_name="goodbye_world") - - _ = TestFlagBridgeFactory(flag=repo_flag, test=test) - _ = TestFlagBridgeFactory(flag=repo_flag2, test=test2) - - res = self.fetch_test_analytics( - repo.name, - """flags(term: "hello")""", - ) - assert res["flags"] == ["hello_world"] - - def test_flags_no_term(self) -> None: - repo = RepositoryFactory(author=self.owner, active=True, private=True) - test = TestFactory(repository=repo) - test2 = TestFactory(repository=repo) - - repo_flag = RepositoryFlagFactory(repository=repo, flag_name="hello_world") - repo_flag2 = RepositoryFlagFactory(repository=repo, flag_name="goodbye_world") - - _ = TestFlagBridgeFactory(flag=repo_flag, test=test) - _ = TestFlagBridgeFactory(flag=repo_flag2, test=test2) - - res = self.fetch_test_analytics( - repo.name, - """flags""", - ) - assert sorted(res["flags"]) == ["goodbye_world", "hello_world"] diff --git a/graphql_api/tests/test_test_result.py b/graphql_api/tests/test_test_result.py deleted file mode 100644 index 93e974e4fa..0000000000 --- a/graphql_api/tests/test_test_result.py +++ /dev/null @@ -1,102 +0,0 @@ -from datetime import UTC, date, datetime, timedelta - -from django.test import TransactionTestCase -from freezegun import freeze_time -from shared.django_apps.core.tests.factories import OwnerFactory, RepositoryFactory - -from reports.tests.factories import ( - DailyTestRollupFactory, - TestFactory, -) - -from .helper import GraphQLTestHelper - - -@freeze_time(datetime.now().isoformat()) -class TestResultTestCase(GraphQLTestHelper, TransactionTestCase): - def setUp(self): - self.owner = OwnerFactory(username="randomOwner") - self.repository = RepositoryFactory( - author=self.owner, - ) - self.test = TestFactory( - name="Test\x1fName", - repository=self.repository, - ) - - _ = DailyTestRollupFactory( - test=self.test, - commits_where_fail=["123"], - date=date.today() - timedelta(days=2), - avg_duration_seconds=0.6, - latest_run=datetime.now() - timedelta(days=2), - flaky_fail_count=0, - ) - _ = DailyTestRollupFactory( - test=self.test, - commits_where_fail=["123", "456"], - date=datetime.now() - timedelta(days=1), - avg_duration_seconds=2, - latest_run=datetime.now() - timedelta(days=1), - flaky_fail_count=1, - ) - _ = DailyTestRollupFactory( - test=self.test, - commits_where_fail=["123", "789"], - date=date.today(), - last_duration_seconds=5.0, - avg_duration_seconds=3, - latest_run=datetime.now(), - flaky_fail_count=1, - ) - - def test_fetch_test_result_name_with_computed_name(self) -> None: - self.test.computed_name = "Computed Name" - self.test.save() - - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResults { - edges { - node { - name - updatedAt - commitsFailed - failureRate - lastDuration - avgDuration - totalFailCount - totalSkipCount - totalPassCount - totalFlakyFailCount - } - } - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert result["owner"]["repository"]["testAnalytics"]["testResults"]["edges"][ - 0 - ]["node"] == { - "name": self.test.computed_name, - "updatedAt": datetime.now(UTC).isoformat(), - "commitsFailed": 3, - "failureRate": 0.75, - "lastDuration": 1.0, - "avgDuration": (5.6 / 3), - "totalFailCount": 9, - "totalSkipCount": 6, - "totalPassCount": 3, - "totalFlakyFailCount": 2, - } diff --git a/graphql_api/tests/test_test_results_headers.py b/graphql_api/tests/test_test_results_headers.py deleted file mode 100644 index 41adc4155a..0000000000 --- a/graphql_api/tests/test_test_results_headers.py +++ /dev/null @@ -1,69 +0,0 @@ -from datetime import date, datetime, timedelta - -from django.test import TransactionTestCase -from freezegun import freeze_time -from shared.django_apps.core.tests.factories import OwnerFactory, RepositoryFactory - -from reports.tests.factories import DailyTestRollupFactory, TestFactory - -from .helper import GraphQLTestHelper - - -@freeze_time(datetime.now().isoformat()) -class TestResultTestCase(GraphQLTestHelper, TransactionTestCase): - def setUp(self): - self.owner = OwnerFactory(username="randomOwner") - self.repository = RepositoryFactory(author=self.owner, branch="main") - - for i in range(1, 31): - test = TestFactory(repository=self.repository) - - _ = DailyTestRollupFactory( - test=test, - date=date.today() - timedelta(days=i), - avg_duration_seconds=float(i), - latest_run=datetime.now() - timedelta(days=i), - fail_count=1, - skip_count=1, - pass_count=0, - branch="main", - ) - - def test_fetch_test_result_aggregates(self) -> None: - query = """ - query { - owner(username: "%s") { - repository(name: "%s") { - ... on Repository { - testAnalytics { - testResultsAggregates { - totalDuration - slowestTestsDuration - totalFails - totalSkips - totalSlowTests - } - } - } - } - } - } - """ % (self.owner.username, self.repository.name) - - result = self.gql_request(query, owner=self.owner) - - assert "errors" not in result - assert ( - result["owner"]["repository"]["testAnalytics"] is not None - and result["owner"]["repository"]["testAnalytics"]["testResultsAggregates"] - is not None - ) - assert result["owner"]["repository"]["testAnalytics"][ - "testResultsAggregates" - ] == { - "totalDuration": 465.0, - "slowestTestsDuration": 30.0, - "totalFails": 30, - "totalSkips": 30, - "totalSlowTests": 1, - } diff --git a/graphql_api/types/flake_aggregates/flake_aggregates.py b/graphql_api/types/flake_aggregates/flake_aggregates.py index 9efd0c9607..0bb600f9d4 100644 --- a/graphql_api/types/flake_aggregates/flake_aggregates.py +++ b/graphql_api/types/flake_aggregates/flake_aggregates.py @@ -1,7 +1,65 @@ +from dataclasses import dataclass + +import polars as pl from ariadne import ObjectType from graphql import GraphQLResolveInfo +from shared.django_apps.core.models import Repository + +from utils.test_results import get_results + + +@dataclass +class FlakeAggregates: + flake_count: int + flake_rate: float + flake_count_percent_change: float | None = None + flake_rate_percent_change: float | None = None + + +def calculate_flake_aggregates(table: pl.DataFrame) -> pl.DataFrame: + return table.select( + (pl.col("total_flaky_fail_count") > 0).sum().alias("flake_count"), + ( + pl.col("total_flaky_fail_count").sum() + / (pl.col("total_fail_count").sum() + pl.col("total_pass_count").sum()) + ).alias("flake_rate"), + ) + + +def flake_aggregates_from_table(table: pl.DataFrame) -> FlakeAggregates: + aggregates = calculate_flake_aggregates(table).row(0, named=True) + return FlakeAggregates(**aggregates) + + +def flake_aggregates_with_percentage( + curr_results: pl.DataFrame, + past_results: pl.DataFrame, +) -> FlakeAggregates: + curr_aggregates = calculate_flake_aggregates(curr_results) + past_aggregates = calculate_flake_aggregates(past_results) + + merged_results: pl.DataFrame = pl.concat([past_aggregates, curr_aggregates]) + + merged_results = merged_results.with_columns( + pl.all().pct_change().name.suffix("_percent_change") + ) + aggregates = merged_results.row(0, named=True) + + return FlakeAggregates(**aggregates) + + +def generate_flake_aggregates(repoid: int, interval: int) -> FlakeAggregates | None: + repo = Repository.objects.get(repoid=repoid) + + curr_results = get_results(repo.repoid, repo.branch, interval) + if curr_results is None: + return None + past_results = get_results(repo.repoid, repo.branch, interval * 2, interval) + if past_results is None: + return flake_aggregates_from_table(curr_results) + else: + return flake_aggregates_with_percentage(curr_results, past_results) -from utils.test_results import FlakeAggregates flake_aggregates_bindable = ObjectType("FlakeAggregates") diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index 089bbf4d3e..a327e701c8 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -1,31 +1,305 @@ +import datetime as dt import logging +from base64 import b64decode, b64encode +from collections import defaultdict +from dataclasses import dataclass from typing import Any, TypedDict +import polars as pl from ariadne import ObjectType from graphql.type.definition import GraphQLResolveInfo +from shared.django_apps.core.models import Repository +from codecov.commands.exceptions import ValidationError from codecov.db import sync_to_async -from core.models import Repository from graphql_api.types.enums import ( OrderingDirection, TestResultsFilterParameter, TestResultsOrderingParameter, ) from graphql_api.types.enums.enum_types import MeasurementInterval -from utils.test_results import ( +from graphql_api.types.flake_aggregates.flake_aggregates import ( FlakeAggregates, - TestResultConnection, - TestResultsAggregates, generate_flake_aggregates, - generate_test_results, +) +from graphql_api.types.test_results_aggregates.test_results_aggregates import ( + TestResultsAggregates, generate_test_results_aggregates, - get_flags, - get_test_suites, ) +from utils.test_results import get_results log = logging.getLogger(__name__) +@dataclass +class TestResultsRow: + # the order here must match the order of the fields in the query + name: str + test_id: str + testsuite: str | None + flags: list[str] + failure_rate: float + flake_rate: float + updated_at: dt.datetime + avg_duration: float + total_fail_count: int + total_flaky_fail_count: int + total_pass_count: int + total_skip_count: int + commits_where_fail: int + last_duration: float + + +@dataclass +class TestResultConnection: + edges: list[dict[str, str | TestResultsRow]] + page_info: dict + total_count: int + + +DELIMITER = "|" + + +@dataclass +class CursorValue: + ordered_value: float | int | dt.datetime | str + name: str + + +def decode_cursor( + value: str | None, ordering: TestResultsOrderingParameter +) -> CursorValue | None: + if value is None: + return None + + split_cursor = b64decode(value.encode("ascii")).decode("utf-8").split(DELIMITER) + ordered_value: str = split_cursor[0] + name: str = split_cursor[1] + match ordering: + case ( + TestResultsOrderingParameter.AVG_DURATION + | TestResultsOrderingParameter.FLAKE_RATE + | TestResultsOrderingParameter.FAILURE_RATE + | TestResultsOrderingParameter.LAST_DURATION + ): + return CursorValue(ordered_value=float(ordered_value), name=name) + case TestResultsOrderingParameter.COMMITS_WHERE_FAIL: + return CursorValue(ordered_value=int(ordered_value), name=name) + case TestResultsOrderingParameter.UPDATED_AT: + return CursorValue( + ordered_value=dt.datetime.fromisoformat(ordered_value), name=name + ) + + raise ValueError(f"Invalid ordering field: {ordering}") + + +def encode_cursor(row: TestResultsRow, ordering: TestResultsOrderingParameter) -> str: + return b64encode( + DELIMITER.join([str(getattr(row, ordering.value)), str(row.name)]).encode( + "utf-8" + ) + ).decode("ascii") + + +def validate( + interval: int, + ordering: TestResultsOrderingParameter, + ordering_direction: OrderingDirection, + after: str | None, + before: str | None, + first: int | None, + last: int | None, +) -> None: + if interval not in {1, 7, 30}: + raise ValidationError(f"Invalid interval: {interval}") + + if not isinstance(ordering_direction, OrderingDirection): + raise ValidationError(f"Invalid ordering direction: {ordering_direction}") + + if not isinstance(ordering, TestResultsOrderingParameter): + raise ValidationError(f"Invalid ordering field: {ordering}") + + if first is not None and last is not None: + raise ValidationError("First and last can not be used at the same time") + + if after is not None and before is not None: + raise ValidationError("After and before can not be used at the same time") + + +def generate_test_results( + ordering: TestResultsOrderingParameter, + ordering_direction: OrderingDirection, + repoid: int, + measurement_interval: MeasurementInterval, + *, + first: int | None = None, + after: str | None = None, + last: int | None = None, + before: str | None = None, + branch: str | None = None, + parameter: TestResultsFilterParameter | None = None, + testsuites: list[str] | None = None, + flags: list[str] | None = None, + term: str | None = None, +) -> TestResultConnection: + """ + Function that retrieves aggregated information about all tests in a given repository, for a given time range, optionally filtered by branch name. + The fields it calculates are: the test failure rate, commits where this test failed, last duration and average duration of the test. + + :param repoid: repoid of the repository we want to calculate aggregates for + :param branch: optional name of the branch we want to filter on, if this is provided the aggregates calculated will only take into account + test instances generated on that branch. By default branches will not be filtered and test instances on all branches wil be taken into + account. + :param interval: timedelta for filtering test instances used to calculate the aggregates by time, the test instances used will be + those with a created at larger than now - interval. + :param testsuites: optional list of testsuite names to filter by, this is done via a union + :param flags: optional list of flag names to filter by, this is done via a union so if a user specifies multiple flags, we get all tests with any + of the flags, not tests that have all of the flags + :returns: queryset object containing list of dictionaries of results + + """ + repo = Repository.objects.get(repoid=repoid) + if branch is None: + branch = repo.branch + interval = measurement_interval.value + validate(interval, ordering, ordering_direction, after, before, first, last) + + table = get_results(repoid, branch, interval) + + if table is None: + return TestResultConnection( + edges=[], + total_count=0, + page_info={}, + ) + + if term: + table = table.filter(pl.col("name").str.starts_with(term)) + + if testsuites: + table = table.filter( + pl.col("testsuite").is_not_null() & pl.col("testsuite").is_in(testsuites) + ) + + if flags: + table = table.filter( + pl.col("flags").list.eval(pl.element().is_in(flags)).list.any() + ) + + match parameter: + case TestResultsFilterParameter.FAILED_TESTS: + table = table.filter(pl.col("total_fail_count") > 0) + case TestResultsFilterParameter.FLAKY_TESTS: + table = table.filter(pl.col("total_flaky_fail_count") > 0) + case TestResultsFilterParameter.SKIPPED_TESTS: + table = table.filter( + pl.col("total_skip_count") > 0 & pl.col("total_pass_count") == 0 + ) + case TestResultsFilterParameter.SLOWEST_TESTS: + table = table.filter( + pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95) + ).top_k(100, by=pl.col("avg_duration")) + + total_count = table.height + + def ordering_expression(cursor_value: CursorValue, is_forward: bool) -> pl.Expr: + if is_forward: + ordering_expression = ( + pl.col(ordering.value) > cursor_value.ordered_value + ) | ( + (pl.col(ordering.value) == cursor_value.ordered_value) + & (pl.col("name") > cursor_value.name) + ) + else: + ordering_expression = ( + pl.col(ordering.value) < cursor_value.ordered_value + ) | ( + (pl.col(ordering.value) == cursor_value.ordered_value) + & (pl.col("name") > cursor_value.name) + ) + return ordering_expression + + if after: + if ordering_direction == OrderingDirection.ASC: + is_forward = True + else: + is_forward = False + + cursor_value = decode_cursor(after, ordering) + if cursor_value: + table = table.filter(ordering_expression(cursor_value, is_forward)) + elif before: + if ordering_direction == OrderingDirection.DESC: + is_forward = True + else: + is_forward = False + + cursor_value = decode_cursor(before, ordering) + if cursor_value: + table = table.filter(ordering_expression(cursor_value, is_forward)) + + table = table.sort( + [ordering.value, "name"], + descending=[ordering_direction == OrderingDirection.DESC, False], + ) + + if first: + page_elements = table.slice(0, first) + elif last: + page_elements = table.reverse().slice(0, last) + else: + page_elements = table + + rows = [TestResultsRow(**row) for row in page_elements.rows(named=True)] + + page: list[dict[str, str | TestResultsRow]] = [ + {"cursor": encode_cursor(row, ordering), "node": row} for row in rows + ] + + return TestResultConnection( + edges=page, + total_count=total_count, + page_info={ + "has_next_page": True if first and len(table) > first else False, + "has_previous_page": True if last and len(table) > last else False, + "start_cursor": page[0]["cursor"] if page else None, + "end_cursor": page[-1]["cursor"] if page else None, + }, + ) + + +def get_test_suites( + repoid: int, term: str | None = None, interval: int = 30 +) -> list[str]: + repo = Repository.objects.get(repoid=repoid) + + table = get_results(repoid, repo.branch, interval) + if table is None: + return [] + + testsuites = table.select(pl.col("testsuite")).unique() + + if term: + testsuites = testsuites.filter(pl.col("testsuite").str.starts_with(term)) + + return testsuites.to_series().to_list() + + +def get_flags(repoid: int, term: str | None = None, interval: int = 30) -> list[str]: + repo = Repository.objects.get(repoid=repoid) + + table = get_results(repoid, repo.branch, interval) + if table is None: + return [] + + flags = table.select(pl.col("flags")).unique() + + if term: + flags = flags.filter(pl.col("flags").str.starts_with(term)) + + return flags.to_series().to_list() + + class TestResultsOrdering(TypedDict): parameter: TestResultsOrderingParameter direction: OrderingDirection diff --git a/graphql_api/types/test_results/test_results.py b/graphql_api/types/test_results/test_results.py index 01ae47385a..28d839f10a 100644 --- a/graphql_api/types/test_results/test_results.py +++ b/graphql_api/types/test_results/test_results.py @@ -3,7 +3,7 @@ from ariadne import ObjectType from graphql import GraphQLResolveInfo -from utils.test_results import TestResultsRow +from graphql_api.types.test_analytics.test_analytics import TestResultsRow test_result_bindable = ObjectType("TestResult") diff --git a/graphql_api/types/test_results_aggregates/test_results_aggregates.py b/graphql_api/types/test_results_aggregates/test_results_aggregates.py index 7920af95a0..54b5baeb3d 100644 --- a/graphql_api/types/test_results_aggregates/test_results_aggregates.py +++ b/graphql_api/types/test_results_aggregates/test_results_aggregates.py @@ -1,7 +1,92 @@ +from dataclasses import dataclass + +import polars as pl from ariadne import ObjectType from graphql import GraphQLResolveInfo +from shared.django_apps.core.models import Repository + +from utils.test_results import get_results + + +@dataclass +class TestResultsAggregates: + total_duration: float + slowest_tests_duration: float + total_slow_tests: int + fails: int + skips: int + total_duration_percent_change: float | None = None + slowest_tests_duration_percent_change: float | None = None + total_slow_tests_percent_change: float | None = None + fails_percent_change: float | None = None + skips_percent_change: float | None = None + + +def calculate_aggregates(table: pl.DataFrame) -> pl.DataFrame: + return table.select( + ( + pl.col("avg_duration") + * (pl.col("total_pass_count") + pl.col("total_fail_count")) + ) + .sum() + .alias("total_duration"), + ( + pl.when(pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95)) + .then( + pl.col("avg_duration") + * (pl.col("total_pass_count") + pl.col("total_fail_count")) + ) + .otherwise(0) + .top_k(100) + .sum() + .alias("slowest_tests_duration") + ), + (pl.col("total_skip_count").sum()).alias("skips"), + (pl.col("total_fail_count").sum()).alias("fails"), + ((pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95)).sum()).alias( + "total_slow_tests" + ), + ) + + +def test_results_aggregates_from_table( + table: pl.DataFrame, +) -> TestResultsAggregates: + aggregates = calculate_aggregates(table).row(0, named=True) + return TestResultsAggregates(**aggregates) + + +def test_results_aggregates_with_percentage( + curr_results: pl.DataFrame, + past_results: pl.DataFrame, +) -> TestResultsAggregates: + curr_aggregates = calculate_aggregates(curr_results) + past_aggregates = calculate_aggregates(past_results) + + merged_results: pl.DataFrame = pl.concat([past_aggregates, curr_aggregates]) + + merged_results = merged_results.with_columns( + pl.all().pct_change().name.suffix("_percent_change") + ) + aggregates = merged_results.row(0, named=True) + + return TestResultsAggregates(**aggregates) + + +def generate_test_results_aggregates( + repoid: int, interval: int +) -> TestResultsAggregates | None: + repo = Repository.objects.get(repoid=repoid) + + curr_results = get_results(repo.repoid, repo.branch, interval) + if curr_results is None: + return None + past_results = get_results(repo.repoid, repo.branch, interval * 2, interval) + if past_results is None: + return test_results_aggregates_from_table(curr_results) + else: + return test_results_aggregates_with_percentage(curr_results, past_results) -from utils.test_results import TestResultsAggregates test_results_aggregates_bindable = ObjectType("TestResultsAggregates") diff --git a/requirements.in b/requirements.in index c8b46a8bfc..e94129f8fc 100644 --- a/requirements.in +++ b/requirements.in @@ -29,6 +29,7 @@ minio opentelemetry-instrumentation-django>=0.45b0 opentelemetry-sdk>=1.24.0 opentracing +polars pre-commit psycopg2 PyJWT diff --git a/requirements.txt b/requirements.txt index 8994de8fad..1d7bf29611 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile # aiodataloader==0.4.0 # via -r requirements.in @@ -307,6 +307,8 @@ packaging==24.1 # pytest pluggy==1.5.0 # via pytest +polars==1.12.0 + # via -r requirements.in pre-commit==2.11.1 # via -r requirements.in prometheus-client==0.17.1 diff --git a/utils/test_results.py b/utils/test_results.py index 4f0e484992..90757bd540 100644 --- a/utils/test_results.py +++ b/utils/test_results.py @@ -1,769 +1,64 @@ -import datetime as dt -from base64 import b64decode, b64encode -from collections import defaultdict -from dataclasses import dataclass -from math import floor +import polars as pl +from shared.storage.exceptions import FileNotInStorageError -from django.db import connection -from django.db.models import ( - Avg, - F, - Q, - QuerySet, - Sum, - Value, -) -from shared.django_apps.core.models import Repository -from shared.django_apps.reports.models import ( - DailyTestRollup, - Flake, - Test, - TestFlagBridge, -) +from services.redis_configuration import get_redis_connection +from services.storage import StorageService -from codecov.commands.exceptions import ValidationError -from graphql_api.types.enums import ( - OrderingDirection, - TestResultsFilterParameter, - TestResultsOrderingParameter, -) -from graphql_api.types.enums.enum_types import MeasurementInterval -thirty_days_ago = dt.datetime.now(dt.UTC) - dt.timedelta(days=30) - -SLOW_TEST_PERCENTILE = 95 - -DELIMITER = "|" - - -def slow_test_threshold(total_tests: int) -> int: - percentile = (100 - SLOW_TEST_PERCENTILE) / 100 - slow_tests_to_return = floor(percentile * total_tests) - return min(max(slow_tests_to_return, 1), 100) - - -@dataclass -class TestResultsQuery: - query: str - params: dict[str, int | str | tuple[str, ...]] - - -@dataclass -class TestResultsRow: - # the order here must match the order of the fields in the query - name: str - test_id: str - failure_rate: float - flake_rate: float - updated_at: dt.datetime - avg_duration: float - total_fail_count: int - total_flaky_fail_count: int - total_pass_count: int - total_skip_count: int - commits_where_fail: int - last_duration: float - - -@dataclass -class TestResultsAggregates: - total_duration: float - total_duration_percent_change: float | None - slowest_tests_duration: float - slowest_tests_duration_percent_change: float | None - total_slow_tests: int - total_slow_tests_percent_change: float | None - fails: int - fails_percent_change: float | None - skips: int - skips_percent_change: float | None - - -@dataclass -class FlakeAggregates: - flake_count: int - flake_count_percent_change: float | None - flake_rate: float - flake_rate_percent_change: float | None - - -@dataclass -class TestResultConnection: - edges: list[dict[str, str | TestResultsRow]] - page_info: dict - total_count: int - - -def convert_tuple_else_none( - value: set[str] | list[str] | None, -) -> tuple[str, ...] | None: - return tuple(value) if value else None - - -@dataclass -class CursorValue: - ordered_value: str - name: str - - -def decode_cursor(value: str | None) -> CursorValue | None: - if value is None: - return None - - split_cursor = b64decode(value.encode("ascii")).decode("utf-8").split(DELIMITER) - return CursorValue( - ordered_value=split_cursor[0], - name=split_cursor[1], - ) - - -def encode_cursor(row: TestResultsRow, ordering: TestResultsOrderingParameter) -> str: - return b64encode( - DELIMITER.join([str(getattr(row, ordering.value)), str(row.name)]).encode( - "utf-8" - ) - ).decode("ascii") - - -def validate( - interval: int, - ordering: TestResultsOrderingParameter, - ordering_direction: OrderingDirection, - after: str | None, - before: str | None, - first: int | None, - last: int | None, -) -> None: - if interval not in {1, 7, 30}: - raise ValidationError(f"Invalid interval: {interval}") - - if not isinstance(ordering_direction, OrderingDirection): - raise ValidationError(f"Invalid ordering direction: {ordering_direction}") - - if not isinstance(ordering, TestResultsOrderingParameter): - raise ValidationError(f"Invalid ordering field: {ordering}") - - if first is not None and last is not None: - raise ValidationError("First and last can not be used at the same time") - - if after is not None and before is not None: - raise ValidationError("After and before can not be used at the same time") - - -def generate_base_query( +def get_redis_key( repoid: int, - ordering: TestResultsOrderingParameter, - ordering_direction: OrderingDirection, - should_reverse: bool, - branch: str | None, - interval: int, - testsuites: list[str] | None = None, - term: str | None = None, - test_ids: set[str] | None = None, -) -> TestResultsQuery: - term_filter = f"%{term}%" if term else None - - if should_reverse: - ordering_direction = ( - OrderingDirection.DESC - if ordering_direction == OrderingDirection.ASC - else OrderingDirection.ASC - ) - - order_by = ( - f"with_cursor.{ordering.value} {ordering_direction.name}, with_cursor.name" - ) - - params: dict[str, int | str | tuple[str, ...] | None] = { - "repoid": repoid, - "interval": f"{interval} days", - "branch": branch, - "test_ids": convert_tuple_else_none(test_ids), - "testsuites": convert_tuple_else_none(testsuites), - "term": term_filter, - } - - filtered_params: dict[str, int | str | tuple[str, ...]] = { - k: v for k, v in params.items() if v is not None - } - - base_query = f""" -with -base_cte as ( - select rd.* - from reports_dailytestrollups rd - { "join reports_test rt on rt.id = rd.test_id" if testsuites or term else ""} - where - rd.repoid = %(repoid)s - and rd.date >= current_date - interval %(interval)s - { "and rd.branch = %(branch)s" if branch else ""} - { "and rd.test_id in %(test_ids)s" if test_ids else ""} - { "and rt.testsuite in %(testsuites)s" if testsuites else ""} - { "and rt.name like %(term)s" if term else ""} -), -failure_rate_cte as ( - select - test_id, - CASE - WHEN SUM(pass_count) + SUM(fail_count) = 0 THEN 0 - ELSE SUM(fail_count)::float / (SUM(pass_count) + SUM(fail_count)) - END as failure_rate, - CASE - WHEN SUM(pass_count) + SUM(fail_count) = 0 THEN 0 - ELSE SUM(flaky_fail_count)::float / (SUM(pass_count) + SUM(fail_count)) - END as flake_rate, - MAX(latest_run) as updated_at, - AVG(avg_duration_seconds) AS avg_duration, - SUM(fail_count) as total_fail_count, - SUM(flaky_fail_count) as total_flaky_fail_count, - SUM(pass_count) as total_pass_count, - SUM(skip_count) as total_skip_count - from base_cte - group by test_id -), -commits_where_fail_cte as ( - select test_id, array_length((array_agg(distinct unnested_cwf)), 1) as failed_commits_count from ( - select test_id, commits_where_fail as cwf - from base_cte - where array_length(commits_where_fail,1) > 0 - ) as tests_with_commits_that_failed, unnest(cwf) as unnested_cwf group by test_id -), -last_duration_cte as ( - select base_cte.test_id, last_duration_seconds from base_cte - join ( - select - test_id, - max(created_at) as created_at - from base_cte - group by test_id - ) as latest_rollups - on base_cte.created_at = latest_rollups.created_at -) - -select * from ( - select - COALESCE(rt.computed_name, rt.name) as name, - results.* - from - ( - select failure_rate_cte.*, coalesce(commits_where_fail_cte.failed_commits_count, 0) as commits_where_fail, last_duration_cte.last_duration_seconds as last_duration - from failure_rate_cte - full outer join commits_where_fail_cte using (test_id) - full outer join last_duration_cte using (test_id) - ) as results join reports_test rt on results.test_id = rt.id -) as with_cursor -order by {order_by} -""" - - return TestResultsQuery(query=base_query, params=filtered_params) - - -def search_base_query( - rows: list[TestResultsRow], - ordering: TestResultsOrderingParameter, - cursor: CursorValue | None, - descending: bool = False, -) -> list[TestResultsRow]: - """ - The reason we have to do this filtering in the application logic is because we need to get the total count of rows that - match from the base query, but we only want to return the rows from after the cursor, so to avoid doing multiple SQL queries - to get the total count of rows that match and then filtering in the database we do the filtering here. - - This is a binary search to find the cursor based on the ordering field. - - The base query we get back is not filtered, we need to filter the rows in the application logic here - so we decode the cursor, which is a value for the ordering field (based on the OrderingParameter) and - a value for the name field. - - The list of rows we get back from the base query is ordered by the ordering field, then by name, so we - can do a binary search to find the value corresponding to the cursor. - - When we find the value corresponding to the cursor we return the rows starting from there, and then we filter - by the page size after we call this function. - """ - if not cursor: - return rows - - print(f"descending: {descending}") - - def compare(row: TestResultsRow) -> int: - # -1 means row value is to the left of the cursor value (search to the right) - # 0 means row value is equal to cursor value - # 1 means row value is to the right of the cursor value (search to the left) - row_value = getattr(row, ordering.value) - row_value_str = str(row_value) - cursor_value_str = cursor.ordered_value - row_is_greater = row_value_str > cursor_value_str - row_is_less = row_value_str < cursor_value_str - if descending: - return row_is_less - row_is_greater - else: - return row_is_greater - row_is_less - - left, right = 0, len(rows) - 1 - while left <= right: - mid = (left + right) // 2 - comparison = compare(rows[mid]) - - if comparison == 0: - if rows[mid].name == cursor.name: - return rows[mid + 1 :] - elif rows[mid].name < cursor.name: - left = mid + 1 - else: - right = mid - 1 - elif comparison < 0: - left = mid + 1 - else: - right = mid - 1 - - return rows[left:] - - -def get_relevant_totals( - repoid: int, branch: str | None, since: dt.datetime -) -> QuerySet: - if branch: - return DailyTestRollup.objects.filter( - repoid=repoid, date__gt=since, branch=branch - ) + branch: str, + interval_start: int, + interval_end: int | None = None, +) -> str: + if interval_end is None: + return f"test_results:{repoid}:{branch}:{interval_start}" else: - return DailyTestRollup.objects.filter(repoid=repoid, date__gt=since) - - -def generate_test_results( - ordering: TestResultsOrderingParameter, - ordering_direction: OrderingDirection, - repoid: int, - measurement_interval: MeasurementInterval, - first: int | None = None, - after: str | None = None, - last: int | None = None, - before: str | None = None, - branch: str | None = None, - parameter: TestResultsFilterParameter | None = None, - testsuites: list[str] | None = None, - flags: defaultdict[str, str] | None = None, - term: str | None = None, -) -> TestResultConnection: - """ - Function that retrieves aggregated information about all tests in a given repository, for a given time range, optionally filtered by branch name. - The fields it calculates are: the test failure rate, commits where this test failed, last duration and average duration of the test. - - :param repoid: repoid of the repository we want to calculate aggregates for - :param branch: optional name of the branch we want to filter on, if this is provided the aggregates calculated will only take into account - test instances generated on that branch. By default branches will not be filtered and test instances on all branches wil be taken into - account. - :param interval: timedelta for filtering test instances used to calculate the aggregates by time, the test instances used will be - those with a created at larger than now - interval. - :param testsuites: optional list of testsuite names to filter by, this is done via a union - :param flags: optional list of flag names to filter by, this is done via a union so if a user specifies multiple flags, we get all tests with any - of the flags, not tests that have all of the flags - :returns: queryset object containing list of dictionaries of results - - """ - interval = measurement_interval.value - validate(interval, ordering, ordering_direction, after, before, first, last) - - since = dt.datetime.now(dt.UTC) - dt.timedelta(days=interval) - - test_ids: set[str] | None = None - - if term is not None: - totals = get_relevant_totals(repoid, branch, since) - - totals = totals.filter(test__name__icontains=term).values("test_id") - - test_ids = set([test["test_id"] for test in totals]) + return f"test_results:{repoid}:{branch}:{interval_start}:{interval_end}" - if flags is not None: - bridges = TestFlagBridge.objects.select_related("flag").filter( - flag__flag_name__in=flags - ) - filtered_test_ids = set([bridge.test_id for bridge in bridges]) # type: ignore - - test_ids = test_ids & filtered_test_ids if test_ids else filtered_test_ids - - if parameter is not None: - totals = get_relevant_totals(repoid, branch, since) - match parameter: - case TestResultsFilterParameter.FLAKY_TESTS: - flaky_test_ids = ( - totals.values("test") - .annotate(flaky_fail_count_sum=Sum("flaky_fail_count")) - .filter(flaky_fail_count_sum__gt=0) - .values("test_id") - ) - flaky_test_id_set = {test["test_id"] for test in flaky_test_ids} - - test_ids = ( - test_ids & flaky_test_id_set if test_ids else flaky_test_id_set - ) - case TestResultsFilterParameter.FAILED_TESTS: - failed_test_ids = ( - totals.values("test") - .annotate(fail_count_sum=Sum("fail_count")) - .filter(fail_count_sum__gt=0) - .values("test_id") - ) - failed_test_id_set = {test["test_id"] for test in failed_test_ids} - - test_ids = ( - test_ids & failed_test_id_set if test_ids else failed_test_id_set - ) - case TestResultsFilterParameter.SKIPPED_TESTS: - skipped_test_ids = ( - totals.values("test") - .annotate( - skip_count_sum=Sum("skip_count"), - fail_count_sum=Sum("fail_count"), - pass_count_sum=Sum("pass_count"), - ) - .filter(skip_count_sum__gt=0, fail_count_sum=0, pass_count_sum=0) - .values("test_id") - ) - skipped_test_id_set = {test["test_id"] for test in skipped_test_ids} - - test_ids = ( - test_ids & skipped_test_id_set if test_ids else skipped_test_id_set - ) - case TestResultsFilterParameter.SLOWEST_TESTS: - num_tests = totals.distinct("test_id").count() - - slowest_test_ids = ( - totals.values("test") - .annotate( - runtime=Avg("avg_duration_seconds") - * (Sum("pass_count") + Sum("fail_count")) - ) - .order_by("-runtime") - .values("test_id")[0 : slow_test_threshold(num_tests)] - ) - slowest_test_id_set = {test["test_id"] for test in slowest_test_ids} - - test_ids = ( - test_ids & slowest_test_id_set if test_ids else slowest_test_id_set - ) - - if not first and not last: - first = 20 - - should_reverse = False if first else True - - query = generate_base_query( - repoid=repoid, - ordering=ordering, - ordering_direction=ordering_direction, - should_reverse=should_reverse, - branch=branch, - interval=interval, - testsuites=testsuites, - term=term, - test_ids=test_ids, - ) - - with connection.cursor() as cursor: - cursor.execute( - query.query, - query.params, - ) - aggregation_of_test_results = cursor.fetchall() - - rows = [TestResultsRow(*row) for row in aggregation_of_test_results] - - page_size: int = first or last or 20 - - cursor_value = decode_cursor(after) if after else decode_cursor(before) - print(f"cursor_value: {cursor_value}") - descending = ordering_direction == OrderingDirection.DESC - search_rows = search_base_query( - rows, - ordering, - cursor_value, - descending=descending, - ) - print(f"search_rows: {search_rows}") - - page: list[dict[str, str | TestResultsRow]] = [ - {"cursor": encode_cursor(row, ordering), "node": row} - for i, row in enumerate(search_rows) - if i < page_size - ] - - return TestResultConnection( - edges=page, - total_count=len(rows), - page_info={ - "has_next_page": True if first and len(search_rows) > first else False, - "has_previous_page": True if last and len(search_rows) > last else False, - "start_cursor": page[0]["cursor"] if page else None, - "end_cursor": page[-1]["cursor"] if page else None, - }, - ) - - -def percent_diff(current_value: int | float, past_value: int | float) -> float | None: - if past_value == 0: - return None - return round((current_value - past_value) / past_value * 100, 5) - - -@dataclass -class TestResultsAggregateNumbers: - total_duration: float - slowest_tests_duration: float - skips: int - fails: int - total_slow_tests: int - - -@dataclass -class FlakeAggregateNumbers: - flake_count: int - flake_rate: float - - -def test_results_aggregates_from_numbers( - curr_numbers: TestResultsAggregateNumbers | None, - past_numbers: TestResultsAggregateNumbers | None, -) -> TestResultsAggregates | None: - if curr_numbers is None: - return None - if past_numbers is None: - return TestResultsAggregates( - total_duration=curr_numbers.total_duration, - total_duration_percent_change=None, - slowest_tests_duration=curr_numbers.slowest_tests_duration, - slowest_tests_duration_percent_change=None, - total_slow_tests=curr_numbers.total_slow_tests, - total_slow_tests_percent_change=None, - fails=curr_numbers.fails, - fails_percent_change=None, - skips=curr_numbers.skips, - skips_percent_change=None, - ) +def get_storage_key( + repoid: int, branch: str, interval_start: int, interval_end: int | None = None +) -> str: + if interval_end is None: + return f"test_results/rollups/{repoid}/{branch}/{interval_start}" else: - return TestResultsAggregates( - total_duration=curr_numbers.total_duration, - total_duration_percent_change=percent_diff( - curr_numbers.total_duration, - past_numbers.total_duration, - ), - slowest_tests_duration=curr_numbers.slowest_tests_duration, - slowest_tests_duration_percent_change=percent_diff( - curr_numbers.slowest_tests_duration, - past_numbers.slowest_tests_duration, - ), - skips=curr_numbers.skips, - skips_percent_change=percent_diff( - curr_numbers.skips, - past_numbers.skips, - ), - fails=curr_numbers.fails, - fails_percent_change=percent_diff( - curr_numbers.fails, - past_numbers.fails, - ), - total_slow_tests=curr_numbers.total_slow_tests, - total_slow_tests_percent_change=percent_diff( - curr_numbers.total_slow_tests, - past_numbers.total_slow_tests, - ), - ) - - -def flake_aggregates_from_numbers( - curr_numbers: FlakeAggregateNumbers | None, - past_numbers: FlakeAggregateNumbers | None, -) -> FlakeAggregates | None: - if curr_numbers is None: - return None + return f"test_results/rollups/{repoid}/{branch}/{interval_start}_{interval_end}" - return FlakeAggregates( - flake_count=curr_numbers.flake_count, - flake_count_percent_change=percent_diff( - curr_numbers.flake_count, past_numbers.flake_count - ) - if past_numbers - else None, - flake_rate=curr_numbers.flake_rate, - flake_rate_percent_change=percent_diff( - curr_numbers.flake_rate, - past_numbers.flake_rate, - ) - if past_numbers - else None, - ) +def get_results( + repoid: int, + branch: str, + interval_start: int, + interval_end: int | None = None, +) -> pl.DataFrame | None: + serialized_table = None -def get_test_results_aggregate_numbers( - repo: Repository, since: dt.datetime, until: dt.datetime | None = None -) -> TestResultsAggregateNumbers | None: - totals = DailyTestRollup.objects.filter( - repoid=repo.repoid, date__gte=since, branch=repo.branch - ) + redis_conn = get_redis_connection() + redis_key = get_redis_key(repoid, branch, interval_start, interval_end) - if until: - totals = totals.filter(date__lt=until) + redis_result = redis_conn.get(redis_key) - num_tests = totals.distinct("test_id").count() + if redis_result is not None: + serialized_table = redis_result - slowest_test_ids = ( - totals.values("test") - .annotate( - runtime=Sum(F("avg_duration_seconds") * (F("pass_count") + F("fail_count"))) - / Sum(F("pass_count") + F("fail_count")) - ) - .order_by("-runtime") - .values("test_id")[0 : slow_test_threshold(num_tests)] - ) + if serialized_table is None: + storage_service = StorageService() + storage_key = get_storage_key(repoid, branch, interval_start, interval_end) - slowest_tests_duration = ( - totals.filter(test_id__in=slowest_test_ids) - .values("repoid") - .annotate( - slowest_tests=Sum( - F("avg_duration_seconds") * (F("pass_count") + F("fail_count")) + try: + serialized_table = storage_service.read_file( + bucket_name="codecov", path=storage_key ) - ) - .values("slowest_tests") - ) - - test_headers = totals.values("repoid").annotate( - total_duration=Sum( - F("avg_duration_seconds") * (F("pass_count") + F("fail_count")) - ), - slowest_tests_duration=slowest_tests_duration, - skips=Sum("skip_count"), - fails=Sum("fail_count"), - total_slow_tests=Value(slow_test_threshold(num_tests)), - ) + except FileNotInStorageError as e: + if interval_end is not None: + return None + else: + raise FileNotFoundError(f"File not found in archive: {e}") - if len(test_headers) == 0: + if serialized_table is None: return None - else: - headers = test_headers[0] - return TestResultsAggregateNumbers( - total_duration=headers["total_duration"] or 0.0, - slowest_tests_duration=headers["slowest_tests_duration"] or 0.0, - skips=headers["skips"] or 0, - fails=headers["fails"] or 0, - total_slow_tests=headers["total_slow_tests"] or 0, - ) - - -def generate_test_results_aggregates( - repoid: int, interval: int -) -> TestResultsAggregates | None: - repo = Repository.objects.get(repoid=repoid) - since = dt.datetime.now(dt.UTC) - dt.timedelta(days=interval) - - curr_numbers = get_test_results_aggregate_numbers(repo, since) - - double_time_ago = since - dt.timedelta(days=interval) - - past_numbers = get_test_results_aggregate_numbers(repo, double_time_ago, since) - - aggregates_with_percentage: TestResultsAggregates | None = ( - test_results_aggregates_from_numbers( - curr_numbers, - past_numbers, - ) - ) - - return aggregates_with_percentage - - -def get_flake_aggregate_numbers( - repo: Repository, since: dt.datetime, until: dt.datetime | None = None -) -> FlakeAggregateNumbers: - if until is None: - flakes = Flake.objects.filter( - Q(repository_id=repo.repoid) - & (Q(end_date__isnull=True) | Q(end_date__date__gte=since.date())) - ) - else: - flakes = Flake.objects.filter( - Q(repository_id=repo.repoid) - & ( - Q(start_date__date__lt=until.date()) - & (Q(end_date__date__gte=since.date()) | Q(end_date__isnull=True)) - ) - ) - - flake_count = flakes.count() - - test_ids = [flake.test_id for flake in flakes] # type: ignore - - test_rollups = DailyTestRollup.objects.filter( - repoid=repo.repoid, - date__gte=since.date(), - branch=repo.branch, - test_id__in=test_ids, - ) - if until: - test_rollups = test_rollups.filter(date__lt=until.date()) - - if len(test_rollups) == 0: - return FlakeAggregateNumbers(flake_count=0, flake_rate=0.0) - numerator = 0 - denominator = 0 - for test_rollup in test_rollups: - numerator += test_rollup.flaky_fail_count - denominator += test_rollup.fail_count + test_rollup.pass_count + table = pl.read_ipc(serialized_table) - if denominator == 0: - flake_rate = 0.0 - else: - flake_rate = numerator / denominator - - return FlakeAggregateNumbers(flake_count=flake_count, flake_rate=flake_rate) - - -def generate_flake_aggregates(repoid: int, interval: int) -> FlakeAggregates | None: - repo = Repository.objects.get(repoid=repoid) - since = dt.datetime.today() - dt.timedelta(days=interval) - - curr_numbers = get_flake_aggregate_numbers(repo, since) - - double_time_ago = since - dt.timedelta(days=interval) - - past_numbers = get_flake_aggregate_numbers(repo, double_time_ago, since) - - return flake_aggregates_from_numbers(curr_numbers, past_numbers) - - -def get_test_suites(repoid: int, term: str | None = None) -> list[str]: - if term: - return list( - Test.objects.filter(repository_id=repoid, testsuite__icontains=term) - .values_list("testsuite", flat=True) - .distinct() - ) - else: - return list( - Test.objects.filter(repository_id=repoid) - .values_list("testsuite", flat=True) - .distinct() - ) - - -def get_flags(repoid: int, term: str | None = None) -> list[str]: - if term: - return list( - TestFlagBridge.objects.filter( - test__repository_id=repoid, flag__flag_name__icontains=term - ) - .select_related("flag") - .values_list("flag__flag_name", flat=True) - .distinct() - ) - else: - return list( - TestFlagBridge.objects.filter(test__repository_id=repoid) - .select_related("flag") - .values_list("flag__flag_name", flat=True) - .distinct() - ) + return table diff --git a/utils/tests/unit/test_cursor.py b/utils/tests/unit/test_cursor.py deleted file mode 100644 index f5a30d322c..0000000000 --- a/utils/tests/unit/test_cursor.py +++ /dev/null @@ -1,25 +0,0 @@ -from datetime import datetime - -from graphql_api.types.enums.enums import TestResultsOrderingParameter -from utils.test_results import CursorValue, TestResultsRow, decode_cursor, encode_cursor - - -def test_cursor(): - row = TestResultsRow( - test_id="test", - name="test", - updated_at=datetime.fromisoformat("2024-01-01T00:00:00Z"), - commits_where_fail=1, - failure_rate=0.5, - avg_duration=100, - last_duration=100, - flake_rate=0.1, - total_fail_count=1, - total_flaky_fail_count=1, - total_skip_count=1, - total_pass_count=1, - ) - cursor = encode_cursor(row, TestResultsOrderingParameter.UPDATED_AT) - assert cursor == "MjAyNC0wMS0wMSAwMDowMDowMCswMDowMHx0ZXN0" - decoded_cursor = decode_cursor(cursor) - assert decoded_cursor == CursorValue(str(row.updated_at), "test") diff --git a/utils/tests/unit/test_search_base_query.py b/utils/tests/unit/test_search_base_query.py deleted file mode 100644 index 0b466f2006..0000000000 --- a/utils/tests/unit/test_search_base_query.py +++ /dev/null @@ -1,66 +0,0 @@ -from datetime import datetime - -from graphql_api.types.enums.enums import TestResultsOrderingParameter -from utils.test_results import CursorValue, TestResultsRow, search_base_query - - -def row_factory(name: str, failure_rate: float): - return TestResultsRow( - test_id=name, - name=name, - failure_rate=failure_rate, - flake_rate=0.0, - updated_at=datetime.now(), - avg_duration=0.0, - total_fail_count=0, - total_flaky_fail_count=0, - total_pass_count=0, - total_skip_count=0, - commits_where_fail=0, - last_duration=0.0, - ) - - -def test_search_base_query_cursor_val_none(): - rows = [row_factory(str(i), float(i) * 0.1) for i in range(10)] - res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, None) - assert res == rows - - -def test_search_base_query_with_existing_cursor(): - rows = [row_factory(str(i), float(i) * 0.1) for i in range(10)] - cursor = CursorValue(name="5", ordered_value="0.5") - res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) - assert res == rows[6:] - - -def test_search_base_query_with_missing_cursor_high_name_low_failure_rate(): - # [(0, "0.0"), (1, "0.1"), (2, "0.2")] - # ^ - # here's where the cursor is pointing at - rows = [row_factory(str(i), float(i) * 0.1) for i in range(3)] - cursor = CursorValue(name="111111", ordered_value="0.05") - res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) - assert res == rows[1:] - - -def test_search_base_query_with_missing_cursor_low_name_high_failure_rate(): - # [(0, "0.0"), (1, "0.1"), (2, "0.2")] - # ^ - # here's where the cursor is pointing at - rows = [row_factory(str(i), float(i) * 0.1) for i in range(3)] - cursor = CursorValue(name="0", ordered_value="0.15") - res = search_base_query(rows, TestResultsOrderingParameter.FAILURE_RATE, cursor) - assert res == rows[-1:] - - -def test_search_base_query_descending(): - # [(2, "0.2"), (1, "0.1"), (0, "0.0")] - # ^ - # here's where the cursor is pointing at - rows = [row_factory(str(i), float(i) * 0.1) for i in range(2, -1, -1)] - cursor = CursorValue(name="0", ordered_value="0.15") - res = search_base_query( - rows, TestResultsOrderingParameter.FAILURE_RATE, cursor, descending=True - ) - assert res == rows[1:] diff --git a/utils/tests/unit/test_slow_test_threshold.py b/utils/tests/unit/test_slow_test_threshold.py deleted file mode 100644 index 93ac4445fe..0000000000 --- a/utils/tests/unit/test_slow_test_threshold.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest - -from utils.test_results import slow_test_threshold - - -@pytest.mark.parametrize( - "total_tests, expected_threshold", - [ - (0, 1), - (1, 1), - (10, 1), - (100, 5), - (1000, 50), - (10000, 100), - (1000000, 100), - (20, 1), - (50, 2), - (200, 10), - (2000, 100), - ], -) -def test_slow_test_threshold(total_tests, expected_threshold): - assert slow_test_threshold(total_tests) == expected_threshold From 6bb4d17e48b57a1e6f94200493bac9b3512e7103 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 1 Nov 2024 13:40:18 -0400 Subject: [PATCH 03/10] deps: update shared --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d7bf29611..8552165474 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile +# pip-compile requirements.in # aiodataloader==0.4.0 # via -r requirements.in From ea0d9a6566d7c907c5cf1d50ba7e291bcbd1fa27 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 1 Nov 2024 13:40:32 -0400 Subject: [PATCH 04/10] feat: queue task to cache GCS results in redis when we miss the redis cache but find the test rollup results in GCS we want to queue up a task to cache it in redis so subsequent calls to the GQL endpoint for the same repo get to hit redis instead of GCS --- graphql_api/tests/test_test_analytics.py | 8 ++- services/task/task.py | 6 ++ utils/test_results.py | 72 +++++++++++++----------- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/graphql_api/tests/test_test_analytics.py b/graphql_api/tests/test_test_analytics.py index 78e6cc9618..3fd3f18c03 100644 --- a/graphql_api/tests/test_test_analytics.py +++ b/graphql_api/tests/test_test_analytics.py @@ -151,16 +151,18 @@ def test_get_test_results( assert results.equals(test_results_table) def test_get_test_results_no_storage(self, transactional_db, repository): - with pytest.raises(FileNotFoundError): - get_results(repository.repoid, repository.branch, 30) + assert get_results(repository.repoid, repository.branch, 30) is None def test_get_test_results_no_redis( - self, transactional_db, repository, store_in_storage + self, mocker, transactional_db, repository, store_in_storage ): + m = mocker.patch("services.task.TaskService.cache_test_results_redis") results = get_results(repository.repoid, repository.branch, 30) assert results is not None assert results.equals(test_results_table) + m.assert_called_once_with(repository.repoid, repository.branch) + def test_test_results(self, transactional_db, repository, store_in_redis): test_results = generate_test_results( repoid=repository.repoid, diff --git a/services/task/task.py b/services/task/task.py index eef79ffb09..f9a0e19c86 100644 --- a/services/task/task.py +++ b/services/task/task.py @@ -424,3 +424,9 @@ def delete_component_measurements(self, repoid: int, component_id: str) -> None: measurement_id=component_id, ), ).apply_async() + + def cache_test_results_redis(self, repoid: int, branch: str) -> None: + self._create_signature( + celery_config.cache_test_rollups_redis_task_name, + kwargs=dict(repoid=repoid, branch=branch), + ).apply_async() diff --git a/utils/test_results.py b/utils/test_results.py index 90757bd540..ffee47d397 100644 --- a/utils/test_results.py +++ b/utils/test_results.py @@ -3,27 +3,32 @@ from services.redis_configuration import get_redis_connection from services.storage import StorageService +from services.task import TaskService -def get_redis_key( +def redis_key( repoid: int, branch: str, interval_start: int, interval_end: int | None = None, ) -> str: - if interval_end is None: - return f"test_results:{repoid}:{branch}:{interval_start}" - else: - return f"test_results:{repoid}:{branch}:{interval_start}:{interval_end}" + key = f"test_results:{repoid}:{branch}:{interval_start}" + if interval_end is not None: + key = f"{key}:{interval_end}" -def get_storage_key( + return key + + +def storage_key( repoid: int, branch: str, interval_start: int, interval_end: int | None = None ) -> str: - if interval_end is None: - return f"test_results/rollups/{repoid}/{branch}/{interval_start}" - else: - return f"test_results/rollups/{repoid}/{branch}/{interval_start}_{interval_end}" + key = f"test_results/rollups/{repoid}/{branch}/{interval_start}" + + if interval_end is not None: + key = f"{key}_{interval_end}" + + return key def get_results( @@ -32,33 +37,34 @@ def get_results( interval_start: int, interval_end: int | None = None, ) -> pl.DataFrame | None: - serialized_table = None - + """ + try redis + if redis is empty + try storage + if storage is empty + return None + else + cache to redis + deserialize + """ + # try redis redis_conn = get_redis_connection() - redis_key = get_redis_key(repoid, branch, interval_start, interval_end) + key = redis_key(repoid, branch, interval_start, interval_end) + result: bytes | None = redis_conn.get(key) - redis_result = redis_conn.get(redis_key) - - if redis_result is not None: - serialized_table = redis_result - - if serialized_table is None: + if result is None: + # try storage storage_service = StorageService() - storage_key = get_storage_key(repoid, branch, interval_start, interval_end) - + key = storage_key(repoid, branch, interval_start, interval_end) try: - serialized_table = storage_service.read_file( - bucket_name="codecov", path=storage_key - ) - except FileNotInStorageError as e: - if interval_end is not None: - return None - else: - raise FileNotFoundError(f"File not found in archive: {e}") - - if serialized_table is None: - return None + result = storage_service.read_file(bucket_name="codecov", path=key) + # cache to redis + TaskService().cache_test_results_redis(repoid, branch) + except FileNotInStorageError: + # give up + return None - table = pl.read_ipc(serialized_table) + # deserialize + table = pl.read_ipc(result) return table From e3dd9a74073b6a47c3632a16b890d581ba31f182 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Thu, 7 Nov 2024 16:27:40 -0500 Subject: [PATCH 05/10] fix: tests and pin polars requirement --- conftest.py | 1 + graphql_api/tests/test_test_analytics.py | 52 +++++++++++++++++------- requirements.in | 2 +- requirements.txt | 2 +- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index a9c3902a5e..748f315dbe 100644 --- a/conftest.py +++ b/conftest.py @@ -5,6 +5,7 @@ import vcr from django.conf import settings from shared.reports.resources import Report, ReportFile, ReportLine +from shared.storage.memory import MemoryStorageService from shared.utils.sessions import Session # we need to enable this in the test environment since we're often creating diff --git a/graphql_api/tests/test_test_analytics.py b/graphql_api/tests/test_test_analytics.py index 3fd3f18c03..abbc41a5a0 100644 --- a/graphql_api/tests/test_test_analytics.py +++ b/graphql_api/tests/test_test_analytics.py @@ -6,6 +6,7 @@ from shared.django_apps.codecov_auth.tests.factories import OwnerFactory from shared.django_apps.core.tests.factories import RepositoryFactory from shared.storage.exceptions import BucketAlreadyExistsError +from shared.storage.memory import MemoryStorageService from graphql_api.types.enums import ( OrderingDirection, @@ -23,6 +24,15 @@ from .helper import GraphQLTestHelper + +@pytest.fixture +def mock_storage(mocker): + m = mocker.patch("utils.test_results.StorageService") + storage_server = MemoryStorageService({}) + m.return_value = storage_server + yield storage_server + + base_gql_query = """ query { owner(username: "%s") { @@ -119,14 +129,13 @@ def store_in_redis(repository): @pytest.fixture -def store_in_storage(repository): - storage = StorageService() +def store_in_storage(repository, mock_storage): try: - storage.create_root_storage("codecov") + mock_storage.create_root_storage("codecov") except BucketAlreadyExistsError: pass - storage.write_file( + mock_storage.write_file( "codecov", f"test_results/rollups/{repository.repoid}/{repository.branch}/30", test_results_table.write_ipc(None).getvalue(), @@ -134,7 +143,7 @@ def store_in_storage(repository): yield - storage.delete_file( + mock_storage.delete_file( "codecov", f"test_results/rollups/{repository.repoid}/{repository.branch}/30", ) @@ -144,17 +153,24 @@ class TestAnalyticsTestCase( GraphQLTestHelper, ): def test_get_test_results( - self, transactional_db, repository, store_in_redis, store_in_storage + self, + transactional_db, + repository, + store_in_redis, + store_in_storage, + mock_storage, ): results = get_results(repository.repoid, repository.branch, 30) assert results is not None assert results.equals(test_results_table) - def test_get_test_results_no_storage(self, transactional_db, repository): + def test_get_test_results_no_storage( + self, transactional_db, repository, mock_storage + ): assert get_results(repository.repoid, repository.branch, 30) is None def test_get_test_results_no_redis( - self, mocker, transactional_db, repository, store_in_storage + self, mocker, transactional_db, repository, store_in_storage, mock_storage ): m = mocker.patch("services.task.TaskService.cache_test_results_redis") results = get_results(repository.repoid, repository.branch, 30) @@ -163,7 +179,9 @@ def test_get_test_results_no_redis( m.assert_called_once_with(repository.repoid, repository.branch) - def test_test_results(self, transactional_db, repository, store_in_redis): + def test_test_results( + self, transactional_db, repository, store_in_redis, mock_storage + ): test_results = generate_test_results( repoid=repository.repoid, ordering=TestResultsOrderingParameter.UPDATED_AT, @@ -191,7 +209,9 @@ def test_test_results(self, transactional_db, repository, store_in_redis): }, ) - def test_test_results_asc(self, transactional_db, repository, store_in_redis): + def test_test_results_asc( + self, transactional_db, repository, store_in_redis, mock_storage + ): test_results = generate_test_results( repoid=repository.repoid, ordering=TestResultsOrderingParameter.UPDATED_AT, @@ -255,6 +275,7 @@ def test_test_results_pagination( rows, repository, store_in_redis, + mock_storage, ): test_results = generate_test_results( repoid=repository.repoid, @@ -331,6 +352,7 @@ def test_test_results_pagination_asc( rows, repository, store_in_redis, + mock_storage, ): test_results = generate_test_results( repoid=repository.repoid, @@ -371,7 +393,7 @@ def test_test_results_pagination_asc( }, ) - def test_test_analytics_term_filter(self, repository, store_in_redis): + def test_test_analytics_term_filter(self, repository, store_in_redis, mock_storage): test_results = generate_test_results( repoid=repository.repoid, term="test1", @@ -421,7 +443,7 @@ def test_test_analytics_testsuite_filter(self, repository, store_in_redis): }, ) - def test_test_analytics_flag_filter(self, repository, store_in_redis): + def test_test_analytics_flag_filter(self, repository, store_in_redis, mock_storage): test_results = generate_test_results( repoid=repository.repoid, flags=["flag1"], @@ -446,7 +468,7 @@ def test_test_analytics_flag_filter(self, repository, store_in_redis): }, ) - def test_gql_query(self, repository, store_in_redis): + def test_gql_query(self, repository, store_in_redis, mock_storage): query = base_gql_query % ( repository.author.username, repository.name, @@ -492,7 +514,7 @@ def test_gql_query(self, repository, store_in_redis): }, ] - def test_gql_query_aggregates(self, repository, store_in_redis): + def test_gql_query_aggregates(self, repository, store_in_redis, mock_storage): query = base_gql_query % ( repository.author.username, repository.name, @@ -519,7 +541,7 @@ def test_gql_query_aggregates(self, repository, store_in_redis): "totalSlowTests": 1, } - def test_gql_query_flake_aggregates(self, repository, store_in_redis): + def test_gql_query_flake_aggregates(self, repository, store_in_redis, mock_storage): query = base_gql_query % ( repository.author.username, repository.name, diff --git a/requirements.in b/requirements.in index e94129f8fc..0078bcb986 100644 --- a/requirements.in +++ b/requirements.in @@ -29,7 +29,7 @@ minio opentelemetry-instrumentation-django>=0.45b0 opentelemetry-sdk>=1.24.0 opentracing -polars +polars==1.12.0 pre-commit psycopg2 PyJWT diff --git a/requirements.txt b/requirements.txt index 8552165474..1d7bf29611 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile # aiodataloader==0.4.0 # via -r requirements.in From 0efd853dabab2710665fcc6196886d5a967f03eb Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Thu, 7 Nov 2024 16:52:48 -0500 Subject: [PATCH 06/10] chore: make lint --- conftest.py | 1 - graphql_api/tests/test_test_analytics.py | 1 - graphql_api/types/test_analytics/test_analytics.py | 1 - 3 files changed, 3 deletions(-) diff --git a/conftest.py b/conftest.py index 748f315dbe..a9c3902a5e 100644 --- a/conftest.py +++ b/conftest.py @@ -5,7 +5,6 @@ import vcr from django.conf import settings from shared.reports.resources import Report, ReportFile, ReportLine -from shared.storage.memory import MemoryStorageService from shared.utils.sessions import Session # we need to enable this in the test environment since we're often creating diff --git a/graphql_api/tests/test_test_analytics.py b/graphql_api/tests/test_test_analytics.py index abbc41a5a0..ae82a9ad28 100644 --- a/graphql_api/tests/test_test_analytics.py +++ b/graphql_api/tests/test_test_analytics.py @@ -20,7 +20,6 @@ get_results, ) from services.redis_configuration import get_redis_connection -from services.storage import StorageService from .helper import GraphQLTestHelper diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index a327e701c8..2cb8bcad3b 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -1,7 +1,6 @@ import datetime as dt import logging from base64 import b64decode, b64encode -from collections import defaultdict from dataclasses import dataclass from typing import Any, TypedDict From ae546edfd426fbb7e68c94cc9ea930533a82cbc1 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 8 Nov 2024 10:46:21 -0500 Subject: [PATCH 07/10] fix: address feedback --- .../flake_aggregates/flake_aggregates.py | 11 +++- .../types/test_analytics/test_analytics.py | 56 +++++++++++-------- .../test_results_aggregates.py | 19 +++++-- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/graphql_api/types/flake_aggregates/flake_aggregates.py b/graphql_api/types/flake_aggregates/flake_aggregates.py index 0bb600f9d4..b3feeb38a9 100644 --- a/graphql_api/types/flake_aggregates/flake_aggregates.py +++ b/graphql_api/types/flake_aggregates/flake_aggregates.py @@ -5,6 +5,7 @@ from graphql import GraphQLResolveInfo from shared.django_apps.core.models import Repository +from graphql_api.types.enums.enum_types import MeasurementInterval from utils.test_results import get_results @@ -48,13 +49,17 @@ def flake_aggregates_with_percentage( return FlakeAggregates(**aggregates) -def generate_flake_aggregates(repoid: int, interval: int) -> FlakeAggregates | None: +def generate_flake_aggregates( + repoid: int, interval: MeasurementInterval +) -> FlakeAggregates | None: repo = Repository.objects.get(repoid=repoid) - curr_results = get_results(repo.repoid, repo.branch, interval) + curr_results = get_results(repo.repoid, repo.branch, interval.value) if curr_results is None: return None - past_results = get_results(repo.repoid, repo.branch, interval * 2, interval) + past_results = get_results( + repo.repoid, repo.branch, interval.value * 2, interval.value + ) if past_results is None: return flake_aggregates_from_table(curr_results) else: diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index 2cb8bcad3b..48cd9ed094 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -29,6 +29,10 @@ log = logging.getLogger(__name__) +INTERVAL_30_DAY = 30 +INTERVAL_7_DAY = 7 +INTERVAL_1_DAY = 1 + @dataclass class TestResultsRow: @@ -109,7 +113,7 @@ def validate( first: int | None, last: int | None, ) -> None: - if interval not in {1, 7, 30}: + if interval not in {INTERVAL_1_DAY, INTERVAL_7_DAY, INTERVAL_30_DAY}: raise ValidationError(f"Invalid interval: {interval}") if not isinstance(ordering_direction, OrderingDirection): @@ -125,6 +129,22 @@ def validate( raise ValidationError("After and before can not be used at the same time") +def ordering_expression( + ordering: TestResultsOrderingParameter, cursor_value: CursorValue, is_forward: bool +) -> pl.Expr: + if is_forward: + ordering_expression = (pl.col(ordering.value) > cursor_value.ordered_value) | ( + (pl.col(ordering.value) == cursor_value.ordered_value) + & (pl.col("name") > cursor_value.name) + ) + else: + ordering_expression = (pl.col(ordering.value) < cursor_value.ordered_value) | ( + (pl.col(ordering.value) == cursor_value.ordered_value) + & (pl.col("name") > cursor_value.name) + ) + return ordering_expression + + def generate_test_results( ordering: TestResultsOrderingParameter, ordering_direction: OrderingDirection, @@ -182,7 +202,8 @@ def generate_test_results( if flags: table = table.filter( - pl.col("flags").list.eval(pl.element().is_in(flags)).list.any() + pl.col("flags").is_not_null() + & pl.col("flags").list.eval(pl.element().is_in(flags)).list.any() ) match parameter: @@ -201,23 +222,6 @@ def generate_test_results( total_count = table.height - def ordering_expression(cursor_value: CursorValue, is_forward: bool) -> pl.Expr: - if is_forward: - ordering_expression = ( - pl.col(ordering.value) > cursor_value.ordered_value - ) | ( - (pl.col(ordering.value) == cursor_value.ordered_value) - & (pl.col("name") > cursor_value.name) - ) - else: - ordering_expression = ( - pl.col(ordering.value) < cursor_value.ordered_value - ) | ( - (pl.col(ordering.value) == cursor_value.ordered_value) - & (pl.col("name") > cursor_value.name) - ) - return ordering_expression - if after: if ordering_direction == OrderingDirection.ASC: is_forward = True @@ -226,7 +230,9 @@ def ordering_expression(cursor_value: CursorValue, is_forward: bool) -> pl.Expr: cursor_value = decode_cursor(after, ordering) if cursor_value: - table = table.filter(ordering_expression(cursor_value, is_forward)) + table = table.filter( + ordering_expression(ordering, cursor_value, is_forward) + ) elif before: if ordering_direction == OrderingDirection.DESC: is_forward = True @@ -235,7 +241,9 @@ def ordering_expression(cursor_value: CursorValue, is_forward: bool) -> pl.Expr: cursor_value = decode_cursor(before, ordering) if cursor_value: - table = table.filter(ordering_expression(cursor_value, is_forward)) + table = table.filter( + ordering_expression(ordering, cursor_value, is_forward) + ) table = table.sort( [ordering.value, "name"], @@ -363,7 +371,8 @@ async def resolve_test_results_aggregates( **_: Any, ) -> TestResultsAggregates | None: return await sync_to_async(generate_test_results_aggregates)( - repoid=repository.repoid, interval=interval.value if interval else 30 + repoid=repository.repoid, + interval=interval if interval else MeasurementInterval.INTERVAL_30_DAY, ) @@ -375,7 +384,8 @@ async def resolve_flake_aggregates( **_: Any, ) -> FlakeAggregates | None: return await sync_to_async(generate_flake_aggregates)( - repoid=repository.repoid, interval=interval.value if interval else 30 + repoid=repository.repoid, + interval=interval if interval else MeasurementInterval.INTERVAL_30_DAY, ) diff --git a/graphql_api/types/test_results_aggregates/test_results_aggregates.py b/graphql_api/types/test_results_aggregates/test_results_aggregates.py index 54b5baeb3d..190a2d5979 100644 --- a/graphql_api/types/test_results_aggregates/test_results_aggregates.py +++ b/graphql_api/types/test_results_aggregates/test_results_aggregates.py @@ -6,6 +6,7 @@ from shared.django_apps.core.models import Repository from utils.test_results import get_results +from graphql_api.types.enums.enum_types import MeasurementInterval @dataclass @@ -43,9 +44,11 @@ def calculate_aggregates(table: pl.DataFrame) -> pl.DataFrame: ), (pl.col("total_skip_count").sum()).alias("skips"), (pl.col("total_fail_count").sum()).alias("fails"), - ((pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95)).sum()).alias( - "total_slow_tests" - ), + ( + (pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95)) + .top_k(100) + .sum() + ).alias("total_slow_tests"), ) @@ -65,6 +68,8 @@ def test_results_aggregates_with_percentage( merged_results: pl.DataFrame = pl.concat([past_aggregates, curr_aggregates]) + # with_columns upserts the new columns, so if the name already exists it get overwritten + # otherwise it's just added merged_results = merged_results.with_columns( pl.all().pct_change().name.suffix("_percent_change") ) @@ -74,14 +79,16 @@ def test_results_aggregates_with_percentage( def generate_test_results_aggregates( - repoid: int, interval: int + repoid: int, interval: MeasurementInterval ) -> TestResultsAggregates | None: repo = Repository.objects.get(repoid=repoid) - curr_results = get_results(repo.repoid, repo.branch, interval) + curr_results = get_results(repo.repoid, repo.branch, interval.value) if curr_results is None: return None - past_results = get_results(repo.repoid, repo.branch, interval * 2, interval) + past_results = get_results( + repo.repoid, repo.branch, interval.value * 2, interval.value + ) if past_results is None: return test_results_aggregates_from_table(curr_results) else: From 9d99d796bb4d4ef2550c6075e642ae0022683c46 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 8 Nov 2024 11:22:04 -0500 Subject: [PATCH 08/10] chore: make lint --- .../types/test_results_aggregates/test_results_aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphql_api/types/test_results_aggregates/test_results_aggregates.py b/graphql_api/types/test_results_aggregates/test_results_aggregates.py index 190a2d5979..ef0f77e4a6 100644 --- a/graphql_api/types/test_results_aggregates/test_results_aggregates.py +++ b/graphql_api/types/test_results_aggregates/test_results_aggregates.py @@ -5,8 +5,8 @@ from graphql import GraphQLResolveInfo from shared.django_apps.core.models import Repository -from utils.test_results import get_results from graphql_api.types.enums.enum_types import MeasurementInterval +from utils.test_results import get_results @dataclass From 6b91d863a979e270c29d1edcb0845b03d97fa229 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Fri, 8 Nov 2024 18:31:36 -0500 Subject: [PATCH 09/10] fix: ordering of tests in general this commit refactors the code and tests related to ordering direction if we map ordering direction asc = 1 and desc = 0 and first = 1 and last = 0 then the ordering direction is an xor of those two variables ordering direction determines whether we're filtering by greater than or lesser than this commit also fixes a bug in the specification of slow tests --- graphql_api/tests/test_test_analytics.py | 346 +++++++++++------- .../types/test_analytics/test_analytics.py | 31 +- .../test_results_aggregates.py | 4 +- 3 files changed, 220 insertions(+), 161 deletions(-) diff --git a/graphql_api/tests/test_test_analytics.py b/graphql_api/tests/test_test_analytics.py index ae82a9ad28..04a1cb5cde 100644 --- a/graphql_api/tests/test_test_analytics.py +++ b/graphql_api/tests/test_test_analytics.py @@ -1,5 +1,6 @@ import datetime from base64 import b64encode +from typing import Any import polars as pl import pytest @@ -16,6 +17,7 @@ from graphql_api.types.test_analytics.test_analytics import ( TestResultConnection, TestResultsRow, + encode_cursor, generate_test_results, get_results, ) @@ -24,6 +26,29 @@ from .helper import GraphQLTestHelper +class RowFactory: + idx = 0 + + def __call__(self, updated_at: datetime.datetime) -> dict[str, Any]: + RowFactory.idx += 1 + return { + "name": f"test{RowFactory.idx}", + "testsuite": f"testsuite{RowFactory.idx}", + "flags": [f"flag{RowFactory.idx}"], + "test_id": f"test_id{RowFactory.idx}", + "failure_rate": 0.1, + "flake_rate": 0.0, + "updated_at": updated_at, + "avg_duration": 100.0, + "total_fail_count": 1, + "total_flaky_fail_count": 1 if RowFactory.idx == 1 else 0, + "total_pass_count": 1, + "total_skip_count": 1, + "commits_where_fail": 1, + "last_duration": 100.0, + } + + @pytest.fixture def mock_storage(mocker): m = mocker.patch("utils.test_results.StorageService") @@ -46,40 +71,8 @@ def mock_storage(mocker): } """ -row_1 = { - "name": "test1", - "testsuite": "testsuite1", - "flags": ["flag1"], - "test_id": "test_id1", - "failure_rate": 0.1, - "flake_rate": 0.0, - "updated_at": datetime.datetime(2024, 1, 1), - "avg_duration": 100.0, - "total_fail_count": 1, - "total_flaky_fail_count": 0, - "total_pass_count": 1, - "total_skip_count": 1, - "commits_where_fail": 1, - "last_duration": 100.0, -} - - -row_2 = { - "name": "test2", - "testsuite": "testsuite2", - "flags": ["flag2"], - "test_id": "test_id2", - "failure_rate": 0.2, - "flake_rate": 0.3, - "updated_at": datetime.datetime(2024, 1, 2), - "avg_duration": 200.0, - "total_fail_count": 2, - "total_flaky_fail_count": 2, - "total_pass_count": 2, - "total_skip_count": 2, - "commits_where_fail": 2, - "last_duration": 200.0, -} + +rows = [RowFactory()(datetime.datetime(2024, 1, 1 + i)) for i in range(5)] def row_to_camel_case(row: dict) -> dict: @@ -95,15 +88,17 @@ def row_to_camel_case(row: dict) -> dict: } -test_results_table = pl.DataFrame( - [row_1, row_2], -) +test_results_table = pl.DataFrame(rows) def base64_encode_string(x: str) -> str: return b64encode(x.encode()).decode("utf-8") +def cursor(row: dict) -> str: + return encode_cursor(TestResultsRow(**row), TestResultsOrderingParameter.UPDATED_AT) + + @pytest.fixture(autouse=True) def repository(mocker, transactional_db): owner = OwnerFactory(username="codecov-user") @@ -189,22 +184,19 @@ def test_test_results( ) assert test_results is not None assert test_results == TestResultConnection( - total_count=2, + total_count=5, edges=[ { - "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", - "node": TestResultsRow(**row_2), - }, - { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": TestResultsRow(**row_1), - }, + "cursor": cursor(row), + "node": TestResultsRow(**row), + } + for row in reversed(rows) ], page_info={ "has_next_page": False, "has_previous_page": False, - "start_cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", - "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "start_cursor": cursor(rows[4]), + "end_cursor": cursor(rows[0]), }, ) @@ -219,47 +211,96 @@ def test_test_results_asc( ) assert test_results is not None assert test_results == TestResultConnection( - total_count=2, + total_count=5, edges=[ { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": TestResultsRow(**row_1), - }, - { - "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", - "node": TestResultsRow(**row_2), - }, + "cursor": cursor(row), + "node": TestResultsRow(**row), + } + for row in rows ], page_info={ "has_next_page": False, "has_previous_page": False, - "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "end_cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", + "start_cursor": cursor(rows[0]), + "end_cursor": cursor(rows[4]), }, ) @pytest.mark.parametrize( - "first, after, before, last, has_next_page, has_previous_page, rows", + "first, after, last, before, has_next_page, has_previous_page, start_cursor, end_cursor, expected_rows", [ - (1, None, None, None, True, False, [row_2]), - ( + pytest.param( 1, - base64_encode_string(f"{row_2['updated_at']}|{row_2['name']}"), None, None, + None, + True, + False, + cursor(rows[4]), + cursor(rows[4]), + [rows[4]], + id="first_1", + ), + pytest.param( + 1, + cursor(rows[4]), + None, + None, + True, + False, + cursor(rows[3]), + cursor(rows[3]), + [rows[3]], + id="first_1_after", + ), + pytest.param( + 1, + cursor(rows[1]), + None, + None, + False, + False, + cursor(rows[0]), + cursor(rows[0]), + [rows[0]], + id="first_1_after_no_next", + ), + pytest.param( + None, + None, + 1, + None, False, + True, + cursor(rows[0]), + cursor(rows[0]), + [rows[0]], + id="last_1", + ), + pytest.param( + None, + None, + 1, + cursor(rows[0]), False, - [row_1], + True, + cursor(rows[1]), + cursor(rows[1]), + [rows[1]], + id="last_1_before", ), - (None, None, None, 1, False, True, [row_1]), - ( + pytest.param( None, None, - base64_encode_string(f"{row_1['updated_at']}|{row_1['name']}"), 1, + cursor(rows[3]), False, False, - [row_2], + cursor(rows[4]), + cursor(rows[4]), + [rows[4]], + id="last_1_before_no_previous", ), ], ) @@ -271,7 +312,9 @@ def test_test_results_pagination( last, has_next_page, has_previous_page, - rows, + expected_rows, + start_cursor, + end_cursor, repository, store_in_redis, mock_storage, @@ -287,56 +330,96 @@ def test_test_results_pagination( last=last, ) assert test_results == TestResultConnection( - total_count=2, + total_count=5, edges=[ { - "cursor": base64_encode_string( - f"{row['updated_at']}|{row['name']}" - ), + "cursor": cursor(row), "node": TestResultsRow(**row), } - for row in rows + for row in expected_rows ], page_info={ "has_next_page": has_next_page, "has_previous_page": has_previous_page, - "start_cursor": base64_encode_string( - f"{rows[0]['updated_at']}|{rows[0]['name']}" - ) - if after - else base64_encode_string( - f"{rows[-1]['updated_at']}|{rows[-1]['name']}" - ), - "end_cursor": base64_encode_string( - f"{rows[-1]['updated_at']}|{rows[-1]['name']}" - ) - if before - else base64_encode_string(f"{rows[0]['updated_at']}|{rows[0]['name']}"), + "start_cursor": start_cursor, + "end_cursor": end_cursor, }, ) @pytest.mark.parametrize( - "first, after, before, last, has_next_page, has_previous_page, rows", + "first, after, last, before, has_next_page, has_previous_page, start_cursor, end_cursor, expected_rows", [ - (1, None, None, None, True, False, [row_1]), - ( + pytest.param( + 1, + None, + None, + None, + True, + False, + cursor(rows[0]), + cursor(rows[0]), + [rows[0]], + id="first_1", + ), + pytest.param( + 1, + cursor(rows[0]), + None, + None, + True, + False, + cursor(rows[1]), + cursor(rows[1]), + [rows[1]], + id="first_1_after", + ), + pytest.param( 1, - base64_encode_string(f"{row_1['updated_at']}|{row_1['name']}"), + cursor(rows[3]), None, None, False, False, - [row_2], + cursor(rows[4]), + cursor(rows[4]), + [rows[4]], + id="first_1_after_no_next", ), - (None, None, None, 1, False, True, [row_2]), - ( + pytest.param( None, None, - base64_encode_string(f"{row_2['updated_at']}|{row_2['name']}"), 1, + None, False, + True, + cursor(rows[4]), + cursor(rows[4]), + [rows[4]], + id="last_1", + ), + pytest.param( + None, + None, + 1, + cursor(rows[4]), False, - [row_1], + True, + cursor(rows[3]), + cursor(rows[3]), + [rows[3]], + id="last_1_before", + ), + pytest.param( + None, + None, + 1, + cursor(rows[1]), + False, + False, + cursor(rows[0]), + cursor(rows[0]), + [rows[0]], + id="last_1_before_no_previous", ), ], ) @@ -348,7 +431,9 @@ def test_test_results_pagination_asc( last, has_next_page, has_previous_page, - rows, + expected_rows, + start_cursor, + end_cursor, repository, store_in_redis, mock_storage, @@ -364,38 +449,26 @@ def test_test_results_pagination_asc( last=last, ) assert test_results == TestResultConnection( - total_count=2, + total_count=5, edges=[ { - "cursor": base64_encode_string( - f"{row['updated_at']}|{row['name']}" - ), + "cursor": cursor(row), "node": TestResultsRow(**row), } - for row in rows + for row in expected_rows ], page_info={ "has_next_page": has_next_page, "has_previous_page": has_previous_page, - "start_cursor": base64_encode_string( - f"{rows[0]['updated_at']}|{rows[0]['name']}" - ) - if after - else base64_encode_string( - f"{rows[-1]['updated_at']}|{rows[-1]['name']}" - ), - "end_cursor": base64_encode_string( - f"{rows[-1]['updated_at']}|{rows[-1]['name']}" - ) - if before - else base64_encode_string(f"{rows[0]['updated_at']}|{rows[0]['name']}"), + "start_cursor": start_cursor, + "end_cursor": end_cursor, }, ) def test_test_analytics_term_filter(self, repository, store_in_redis, mock_storage): test_results = generate_test_results( repoid=repository.repoid, - term="test1", + term=rows[0]["name"], ordering=TestResultsOrderingParameter.UPDATED_AT, ordering_direction=OrderingDirection.DESC, measurement_interval=MeasurementInterval.INTERVAL_30_DAY, @@ -405,22 +478,22 @@ def test_test_analytics_term_filter(self, repository, store_in_redis, mock_stora total_count=1, edges=[ { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": TestResultsRow(**row_1), + "cursor": cursor(rows[0]), + "node": TestResultsRow(**rows[0]), }, ], page_info={ "has_next_page": False, "has_previous_page": False, - "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "start_cursor": cursor(rows[0]), + "end_cursor": cursor(rows[0]), }, ) def test_test_analytics_testsuite_filter(self, repository, store_in_redis): test_results = generate_test_results( repoid=repository.repoid, - testsuites=["testsuite1"], + testsuites=[rows[0]["testsuite"]], ordering=TestResultsOrderingParameter.UPDATED_AT, ordering_direction=OrderingDirection.DESC, measurement_interval=MeasurementInterval.INTERVAL_30_DAY, @@ -430,22 +503,22 @@ def test_test_analytics_testsuite_filter(self, repository, store_in_redis): total_count=1, edges=[ { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": TestResultsRow(**row_1), + "cursor": cursor(rows[0]), + "node": TestResultsRow(**rows[0]), }, ], page_info={ "has_next_page": False, "has_previous_page": False, - "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "start_cursor": cursor(rows[0]), + "end_cursor": cursor(rows[0]), }, ) def test_test_analytics_flag_filter(self, repository, store_in_redis, mock_storage): test_results = generate_test_results( repoid=repository.repoid, - flags=["flag1"], + flags=[rows[0]["flags"][0]], ordering=TestResultsOrderingParameter.UPDATED_AT, ordering_direction=OrderingDirection.DESC, measurement_interval=MeasurementInterval.INTERVAL_30_DAY, @@ -455,15 +528,15 @@ def test_test_analytics_flag_filter(self, repository, store_in_redis, mock_stora total_count=1, edges=[ { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": TestResultsRow(**row_1), + "cursor": cursor(rows[0]), + "node": TestResultsRow(**rows[0]), }, ], page_info={ "has_next_page": False, "has_previous_page": False, - "start_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "end_cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", + "start_cursor": cursor(rows[0]), + "end_cursor": cursor(rows[0]), }, ) @@ -498,19 +571,16 @@ def test_gql_query(self, repository, store_in_redis, mock_storage): assert ( result["owner"]["repository"]["testAnalytics"]["testResults"]["totalCount"] - == 2 + == 5 ) assert result["owner"]["repository"]["testAnalytics"]["testResults"][ "edges" ] == [ { - "cursor": "MjAyNC0wMS0wMiAwMDowMDowMHx0ZXN0Mg==", - "node": row_to_camel_case(row_2), - }, - { - "cursor": "MjAyNC0wMS0wMSAwMDowMDowMHx0ZXN0MQ==", - "node": row_to_camel_case(row_1), - }, + "cursor": cursor(row), + "node": row_to_camel_case(row), + } + for row in reversed(rows) ] def test_gql_query_aggregates(self, repository, store_in_redis, mock_storage): @@ -534,9 +604,9 @@ def test_gql_query_aggregates(self, repository, store_in_redis, mock_storage): "testResultsAggregates" ] == { "totalDuration": 1000.0, - "slowestTestsDuration": 800.0, - "totalFails": 3, - "totalSkips": 3, + "slowestTestsDuration": 200.0, + "totalFails": 5, + "totalSkips": 5, "totalSlowTests": 1, } @@ -555,6 +625,6 @@ def test_gql_query_flake_aggregates(self, repository, store_in_redis, mock_stora result = self.gql_request(query, owner=repository.author) assert result["owner"]["repository"]["testAnalytics"]["flakeAggregates"] == { - "flakeRate": 1 / 3, + "flakeRate": 0.1, "flakeCount": 1, } diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index 48cd9ed094..02321b9bb2 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -218,31 +218,20 @@ def generate_test_results( case TestResultsFilterParameter.SLOWEST_TESTS: table = table.filter( pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95) - ).top_k(100, by=pl.col("avg_duration")) + ).top_k(min(100, max(table.height // 20, 1)), by=pl.col("avg_duration")) total_count = table.height - if after: - if ordering_direction == OrderingDirection.ASC: - is_forward = True - else: - is_forward = False - - cursor_value = decode_cursor(after, ordering) - if cursor_value: - table = table.filter( - ordering_expression(ordering, cursor_value, is_forward) - ) - elif before: - if ordering_direction == OrderingDirection.DESC: - is_forward = True - else: - is_forward = False - - cursor_value = decode_cursor(before, ordering) + if after or before: + comparison_direction = (ordering_direction == OrderingDirection.ASC) == ( + bool(first) + ) + cursor_value = ( + decode_cursor(after, ordering) if after else decode_cursor(before, ordering) + ) if cursor_value: table = table.filter( - ordering_expression(ordering, cursor_value, is_forward) + ordering_expression(ordering, cursor_value, comparison_direction) ) table = table.sort( @@ -253,7 +242,7 @@ def generate_test_results( if first: page_elements = table.slice(0, first) elif last: - page_elements = table.reverse().slice(0, last) + page_elements = table.slice(-last, last) else: page_elements = table diff --git a/graphql_api/types/test_results_aggregates/test_results_aggregates.py b/graphql_api/types/test_results_aggregates/test_results_aggregates.py index ef0f77e4a6..cfaf6add72 100644 --- a/graphql_api/types/test_results_aggregates/test_results_aggregates.py +++ b/graphql_api/types/test_results_aggregates/test_results_aggregates.py @@ -38,7 +38,7 @@ def calculate_aggregates(table: pl.DataFrame) -> pl.DataFrame: * (pl.col("total_pass_count") + pl.col("total_fail_count")) ) .otherwise(0) - .top_k(100) + .top_k(min(100, max(table.height // 20, 1))) .sum() .alias("slowest_tests_duration") ), @@ -46,7 +46,7 @@ def calculate_aggregates(table: pl.DataFrame) -> pl.DataFrame: (pl.col("total_fail_count").sum()).alias("fails"), ( (pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95)) - .top_k(100) + .top_k(min(100, max(table.height // 20, 1))) .sum() ).alias("total_slow_tests"), ) From b2ff3496b57fb58ac616b380129e6b2fee02e511 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Mon, 11 Nov 2024 11:56:03 -0500 Subject: [PATCH 10/10] fix: add comment and use after --- graphql_api/types/test_analytics/test_analytics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/graphql_api/types/test_analytics/test_analytics.py b/graphql_api/types/test_analytics/test_analytics.py index 02321b9bb2..16ff6f1fe2 100644 --- a/graphql_api/types/test_analytics/test_analytics.py +++ b/graphql_api/types/test_analytics/test_analytics.py @@ -218,13 +218,15 @@ def generate_test_results( case TestResultsFilterParameter.SLOWEST_TESTS: table = table.filter( pl.col("avg_duration") >= pl.col("avg_duration").quantile(0.95) - ).top_k(min(100, max(table.height // 20, 1)), by=pl.col("avg_duration")) + ).top_k( + min(100, max(table.height // 20, 1)), by=pl.col("avg_duration") + ) # the top k operation here is to make sure we don't show too many slowest tests in the case of a low sample size total_count = table.height if after or before: comparison_direction = (ordering_direction == OrderingDirection.ASC) == ( - bool(first) + bool(after) ) cursor_value = ( decode_cursor(after, ordering) if after else decode_cursor(before, ordering)