Merge pull request #3155 from athenianco/test-health-metrics

[DEV-5569] Add unit tests for internal account data health metrics

Author: vmarkovtsev

server/athenian/api/precompute/accounts.py

@@ -185,7 +185,7 @@ async def precompute_reposet(
     time_to: datetime,
     no_time_from: datetime,
     time_from: datetime,
-) -> None:
+) -> DataHealthMetrics:
     """
     Execute precomputations for a single reposet.
 
@@ -212,7 +212,7 @@ async def precompute_reposet(
     except Exception as e:
         log.error("prolog %d: %s: %s", reposet.owner_id, type(e).__name__, e)
         sentry_sdk.capture_exception(e)
-        return
+        return health_metrics
     deref_items, missing = prefixer.dereference_repositories(
         reposet_items_to_refs(reposet.items), return_missing=True, logger=log,
     )
@@ -446,6 +446,7 @@ async def report_precompute_success():
             )
     finally:
         await wait_deferred()
+    return health_metrics
 
 
 async def ensure_teams(

server/athenian/api/precompute/store_external_health.py

@@ -1,4 +1,5 @@
 import argparse
+import asyncio
 from datetime import datetime, timezone
 import logging
 from typing import Any
@@ -17,7 +18,7 @@
 async def _record_performance_metrics(
     prometheus_endpoint: str,
     rdb: Database,
-):
+) -> None:
     log = logging.getLogger(f"{metadata.__package__}._record_performance_metrics")
     for pct in ("50", "95"):
         inserted = []
@@ -62,13 +63,10 @@ async def _record_performance_metrics(
 
 async def _record_inconsistency_metrics(
     prometheus_endpoint: str,
-    sdb: Database,
+    acc_id_map_task: asyncio.Task,
     rdb: Database,
-):
+) -> None:
     log = logging.getLogger(f"{metadata.__package__}._record_inconsistency_metrics")
-    acc_id_map = dict(
-        await sdb.fetch_all(select(AccountGitHubAccount.id, AccountGitHubAccount.account_id)),
-    )
     inserted = []
     for attempt in range(3):
         async with aiohttp.ClientSession() as session:
@@ -87,7 +85,10 @@ async def _record_inconsistency_metrics(
                         await response.text(),
                     )
                     continue
-                for obj in (await response.json())["data"]["result"]:
+                objs = (await response.json())["data"]["result"]
+                await acc_id_map_task
+                acc_id_map = acc_id_map_task.result()
+                for obj in objs:
                     try:
                         account = acc_id_map[int(obj["metric"]["acc_id"])]
                     except KeyError:
@@ -105,9 +106,30 @@ async def _record_inconsistency_metrics(
         await rdb.execute_many(insert(HealthMetric), inserted)
 
 
+async def _record_pending_fetch_metrics(
+    acc_id_map_task: asyncio.Task,
+    mdb: Database,
+    rdb: Database,
+) -> None:
+    log = logging.getLogger(f"{metadata.__package__}._record_pending_fetch_metrics")
+    inserted = []
+    # df_pending_prs, df_pending_commits = await gather()
+    if inserted:
+        log.info("inserting %d data inconsistency records", len(inserted))
+        await rdb.execute_many(insert(HealthMetric), inserted)
+
+
+async def _fetch_acc_id_map(sdb: Database) -> dict[int, int]:
+    return dict(
+        await sdb.fetch_all(select(AccountGitHubAccount.id, AccountGitHubAccount.account_id)),
+    )
+
+
 async def main(context: PrecomputeContext, args: argparse.Namespace) -> Any:
     """Fill missing commit references in the deployed components."""
+    acc_id_map_task = asyncio.create_task(_fetch_acc_id_map(context.sdb), name="_fetch_acc_id_map")
     await gather(
         _record_performance_metrics(args.prometheus, context.rdb),
-        _record_inconsistency_metrics(args.prometheus, context.sdb, context.rdb),
+        _record_inconsistency_metrics(args.prometheus, acc_id_map_task, context.rdb),
+        _record_pending_fetch_metrics(acc_id_map_task, context.mdb, context.rdb),
     )

server/tests/controllers/miners/github/test_branches.py

@@ -2,14 +2,17 @@
 from sqlalchemy import delete, insert, select, update
 
 from athenian.api.defer import wait_deferred, with_defer
+from athenian.api.internal.miners.github.branches import BranchMinerMetrics
 from athenian.api.models.metadata.github import Branch
 
 
 async def test_extract_branches_zero(mdb, branch_miner, prefixer):
+    metrics = BranchMinerMetrics.empty()
     branches, defaults = await branch_miner.extract_branches(
-        ["src-d/gitbase"], prefixer, (6366825,), mdb, None,
+        ["src-d/gitbase"], prefixer, (6366825,), mdb, None, metrics=metrics,
     )
     assert branches.empty
+    assert metrics.empty_count == 1
     assert defaults == {"src-d/gitbase": "master"}
     branches, defaults = await branch_miner.extract_branches(
         ["src-d/gitbase"], prefixer, (6366825,), mdb, None,
@@ -48,6 +51,7 @@ async def test_extract_branches_cache(mdb, cache, branch_miner, prefixer):
 
 async def test_extract_branches_main(mdb_rw, branch_miner, prefixer):
     mdb = mdb_rw
+    metrics = BranchMinerMetrics.empty()
     await mdb.execute(
         update(Branch)
         .where(Branch.branch_name == "master")
@@ -60,7 +64,7 @@ async def test_extract_branches_main(mdb_rw, branch_miner, prefixer):
     )
     try:
         _, defaults = await branch_miner.extract_branches(
-            ["src-d/go-git"], prefixer, (6366825,), mdb, None,
+            ["src-d/go-git"], prefixer, (6366825,), mdb, None, metrics=metrics,
         )
         assert defaults == {"src-d/go-git": "main", "src-d/юникод": "вадим"}
     finally:
@@ -74,6 +78,8 @@ async def test_extract_branches_main(mdb_rw, branch_miner, prefixer):
                 },
             ),
         )
+    assert metrics.count == 5
+    assert metrics.no_default == 1
 
 
 async def test_extract_branches_max_date(mdb_rw, branch_miner, prefixer):

server/tests/controllers/miners/github/test_commit.py

@@ -8,6 +8,7 @@
 from athenian.api.defer import wait_deferred, with_defer
 from athenian.api.internal.miners.github.branches import BranchMiner
 from athenian.api.internal.miners.github.commit import (
+    CommitDAGMetrics,
     FilterCommitsProperty,
     _fetch_commit_history_dag,
     extract_commits,
@@ -253,6 +254,7 @@ async def test__fetch_commit_history_dag_inconsistent(mdb_rw, dag):
             },
         ),
     )
+    metrics = CommitDAGMetrics.empty()
     try:
         consistent, _, newhashes, newvertexes, newedges = await _fetch_commit_history_dag(
             True,
@@ -264,9 +266,12 @@ async def test__fetch_commit_history_dag_inconsistent(mdb_rw, dag):
             "src-d/go-git",
             (6366825,),
             mdb_rw,
+            metrics=metrics,
         )
     finally:
         await mdb_rw.execute(delete(NodeCommitParent).where(NodeCommitParent.parent_id == 100500))
     assert not consistent
     assert len(newhashes) == len(hashes)
     assert b"1" * 40 not in newhashes
+    assert metrics.corrupted == {"src-d/go-git"}
+    assert metrics.orphaned == {"src-d/go-git"}

server/tests/controllers/miners/github/test_release.py

@@ -15,6 +15,7 @@
 from athenian.api.defer import wait_deferred, with_defer
 from athenian.api.internal.miners.filters import JIRAFilter, LabelFilter
 from athenian.api.internal.miners.github.commit import (
+    CommitDAGMetrics,
     _empty_dag,
     _fetch_commit_history_dag,
     fetch_repository_commits,
@@ -26,7 +27,10 @@
     mark_dag_parents,
     partition_dag,
 )
-from athenian.api.internal.miners.github.release_load import group_repos_by_release_match
+from athenian.api.internal.miners.github.release_load import (
+    MineReleaseMetrics,
+    group_repos_by_release_match,
+)
 from athenian.api.internal.miners.github.release_mine import (
     mine_releases,
     mine_releases_by_name,
@@ -847,6 +851,7 @@ async def test_load_releases_events_unresolved(
             .explode(with_primary_keys=True),
         ),
     )
+    metrics = MineReleaseMetrics.empty()
     releases, matched_bys = await release_loader.load_releases(
         ["src-d/go-git"],
         branches,
@@ -863,9 +868,12 @@ async def test_load_releases_events_unresolved(
         rdb,
         None,
         index=Release.node_id.name,
+        metrics=metrics,
     )
     assert releases.empty
     assert matched_bys == {"src-d/go-git": ReleaseMatch.event}
+    assert metrics.count_by_event == 0
+    assert metrics.unresolved == 1
 
 
 @pytest.fixture(scope="module")
@@ -981,6 +989,7 @@ async def test__fetch_repository_commits_initial_commit(mdb, pdb, prune, heads_d
 @with_defer
 @freeze_time("2015-04-05")
 async def test__fetch_repository_commits_orphan_skip(mdb, pdb, prune, heads_df1):
+    metrics = CommitDAGMetrics.empty()
     dags = await fetch_repository_commits(
         {
             "src-d/go-git": (
@@ -1000,7 +1009,10 @@ async def test__fetch_repository_commits_orphan_skip(mdb, pdb, prune, heads_df1)
         mdb,
         pdb,
         None,
+        metrics=metrics,
     )
+    if prune:
+        assert metrics.pristine == {"src-d/go-git"}
     hashes, vertexes, edges = dags["src-d/go-git"][1]
     if prune:
         assert len(hashes) == 0
@@ -1553,6 +1565,7 @@ async def test_precomputed_smoke(
     ):
         time_from = datetime(year=2015, month=1, day=1, tzinfo=timezone.utc)
         time_to = datetime(year=2020, month=12, day=1, tzinfo=timezone.utc)
+        metrics = MineReleaseMetrics.empty()
         kwargs = self._kwargs(
             time_from=time_from,
             time_to=time_to,
@@ -1561,9 +1574,11 @@ async def test_precomputed_smoke(
             mdb=mdb,
             pdb=pdb,
             rdb=rdb,
+            metrics=metrics,
         )
         await mine_releases(**kwargs)
         await wait_deferred()
+        assert metrics.count_by_tag == 53
         kwargs["with_deployments"] = False
         releases, avatars, _, _ = await mine_releases(**kwargs)
         assert len(releases) == 53

server/tests/controllers/test_filter_controller.py

@@ -2696,6 +2696,7 @@ async def test_filter_labels_nasty_input(client, headers, account, repos, status
     assert response.status == status, response_body
 
 
+@pytest.mark.flaky(reruns=3)
 async def test_get_releases_smoke(client, headers):
     body = {
         "account": 1,

server/tests/internal/features/test_entries.py

@@ -13,6 +13,7 @@
 from athenian.api.internal.features.entries import (
     MetricEntriesCalculator,
     MetricsLineRequest,
+    MinePullRequestMetrics,
     PRFactsCalculator,
     TeamSpecificFilters,
     make_calculator,
@@ -64,9 +65,14 @@ async def test_gaetano_bug(
         calculator = PRFactsCalculator(
             1, meta_ids, mdb, pdb, rdb, **self._init_kwargs(), cache=None,
         )
+        metrics = MinePullRequestMetrics.empty()
 
         base_kwargs = await self._kwargs(meta_ids, mdb, sdb)
+        base_kwargs["metrics"] = metrics
         facts = await calculator(time_from=dt(2017, 8, 10), time_to=dt(2017, 9, 1), **base_kwargs)
+        assert metrics == MinePullRequestMetrics(
+            count=33, done_count=29, merged_count=3, open_count=1,
+        )
         last_review = facts[facts.node_id == 163078].last_review.values[0]
 
         await calculator(time_from=dt(2017, 8, 10), time_to=dt(2017, 8, 20), **base_kwargs)

server/tests/internal/miners/github/test_deployment.py

@@ -15,6 +15,7 @@
 from athenian.api.internal.jira import JIRAConfig
 from athenian.api.internal.miners.filters import JIRAFilter, LabelFilter
 from athenian.api.internal.miners.github.deployment import (
+    MineDeploymentsMetrics,
     deployment_facts_extract_mentioned_people,
     hide_outlier_first_deployments,
     mine_deployments,
@@ -59,6 +60,7 @@ async def test_mine_deployments_from_scratch(
 ):
     time_from = datetime(2015, 1, 1, tzinfo=timezone.utc)
     time_to = datetime(2020, 1, 1, tzinfo=timezone.utc)
+    metrics = MineDeploymentsMetrics.empty()
     await mine_releases(
         ["src-d/go-git"],
         {},
@@ -82,6 +84,7 @@ async def test_mine_deployments_from_scratch(
         with_deployments=False,
     )
     await wait_deferred()
+
     deps = await mine_deployments(
         ["src-d/go-git"],
         {},
@@ -105,12 +108,15 @@ async def test_mine_deployments_from_scratch(
         pdb,
         rdb,
         cache,
+        metrics=metrics,
     )
     _validate_deployments(deps, 9, True)
     deployment_facts_extract_mentioned_people(deps)
     await wait_deferred()
     commits = await pdb.fetch_all(select([GitHubCommitDeployment]))
     assert len(commits) == 4684
+    assert metrics.count == 18
+    assert metrics.unresolved == 0
 
     # test the cache
     deps = await mine_deployments(
@@ -140,6 +146,73 @@ async def test_mine_deployments_from_scratch(
     _validate_deployments(deps, 9, True)
 
 
+@with_defer
+async def test_mine_deployments_unresolved(
+    sample_deployments,
+    release_match_setting_tag_or_branch,
+    branches,
+    default_branches,
+    prefixer,
+    mdb,
+    pdb,
+    rdb,
+    cache,
+):
+    time_from = datetime(2015, 1, 1, tzinfo=timezone.utc)
+    time_to = datetime(2020, 1, 1, tzinfo=timezone.utc)
+    metrics = MineDeploymentsMetrics.empty()
+
+    await rdb.execute(
+        insert(DeploymentNotification).values(
+            account_id=1,
+            name="whatever",
+            conclusion="SUCCESS",
+            environment="production",
+            started_at=datetime(2019, 11, 2, tzinfo=timezone.utc),
+            finished_at=datetime(2019, 11, 2, 0, 10, tzinfo=timezone.utc),
+            created_at=datetime.now(timezone.utc),
+            updated_at=datetime.now(timezone.utc),
+        ),
+    )
+    await rdb.execute(
+        insert(DeployedComponent).values(
+            account_id=1,
+            deployment_name="whatever",
+            repository_node_id=40550,
+            reference="not exists",
+            created_at=datetime.now(timezone.utc),
+        ),
+    )
+
+    await mine_deployments(
+        ["src-d/go-git"],
+        {},
+        time_from,
+        time_to,
+        ["production", "staging"],
+        [],
+        {},
+        {},
+        LabelFilter.empty(),
+        JIRAFilter.empty(),
+        release_match_setting_tag_or_branch,
+        LogicalRepositorySettings.empty(),
+        branches,
+        default_branches,
+        prefixer,
+        1,
+        None,
+        (6366825,),
+        mdb,
+        pdb,
+        rdb,
+        cache,
+        metrics=metrics,
+    )
+    assert metrics.count == 18
+    assert metrics.unresolved == 1
+
+
 @with_defer
 async def test_mine_deployments_addons_cache(
     sample_deployments,