[DEV-5349] Fix missing JIRA in unfresh PR facts

Author: vmarkovtsev

server/athenian/api/internal/features/github/pull_request_filter.py

@@ -89,6 +89,7 @@
     Deployment,
     JIRAEntityToFetch,
     Label,
+    LoadedJIRADetails,
     MinedPullRequest,
     PRParticipants,
     PullRequestEvent,
@@ -1293,6 +1294,10 @@ async def noop():
     for k, v in unreleased.items():
         if k not in facts:
             facts[k] = v
+    empty_jira = LoadedJIRADetails.empty()
+    for v in facts.values():
+        if v.jira is None:
+            v.jira = empty_jira
     dfs, _, _ = await PullRequestMiner.mine_by_ids(
         prs_df,
         unreleased,

server/athenian/api/internal/features/github/unfresh_pull_request_metrics.py

@@ -1,6 +1,7 @@
 import asyncio
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
+from itertools import chain
 import logging
 from typing import Dict, KeysView, List, Optional, Set, Tuple, Type, Union
 
@@ -206,6 +207,7 @@ async def dummy_inactive_prs():
         merged_prs = unreleased_prs.index.take(np.flatnonzero(merged_mask)).union(
             inactive_merged_prs,
         )
+        del unreleased_prs
         tasks = [
             cls.open_prs_facts_loader.load_open_pull_request_facts_unfresh(
                 open_prs,
@@ -254,10 +256,10 @@ async def dummy_inactive_prs():
         if pr_jira_mapper is not None:
             unreleased_jira_map = unreleased_jira_map[0]
             empty_jira = LoadedJIRADetails.empty()
-            for node_id, repo in zip(
-                unreleased_prs.index.get_level_values(0).values,
-                unreleased_prs.index.get_level_values(1).values,
-            ):
+            # it's not enough to iterate over unreleased_prs.index
+            # we can catch a not yet precomputed pair (logical repository, node_id)
+            # which has (physical repository, node_id) precomputed and existing in the facts
+            for node_id, repo in chain(open_facts.keys(), merged_facts.keys()):
                 try:
                     jira = unreleased_jira_map[node_id]
                 except KeyError:

server/athenian/api/internal/miners/github/precomputed_prs/done_prs.py

@@ -3,19 +3,7 @@
 from itertools import chain
 import logging
 import pickle
-from typing import (
-    Any,
-    Callable,
-    Collection,
-    Dict,
-    Iterable,
-    KeysView,
-    List,
-    Mapping,
-    Optional,
-    Set,
-    Tuple,
-)
+from typing import Any, Callable, Collection, Iterable, KeysView, Mapping, Optional
 
 import aiomcache
 import morcilla
@@ -87,11 +75,11 @@ async def load_precomputed_done_candidates(
         time_from: datetime,
         time_to: datetime,
         repos: Collection[str],
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         release_settings: ReleaseSettings,
         account: int,
         pdb: morcilla.Database,
-    ) -> Tuple[Set[int], Dict[str, List[int]]]:
+    ) -> tuple[set[int], dict[str, list[int]]]:
         """
         Load the set of done PR identifiers and specifically ambiguous PR node IDs.
 
@@ -134,13 +122,13 @@ async def load_precomputed_done_facts_filters(
         repos: Collection[str],
         participants: PRParticipants,
         labels: LabelFilter,
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         exclude_inactive: bool,
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
-    ) -> Tuple[PullRequestFactsMap, Dict[str, List[int]]]:
+    ) -> tuple[PullRequestFactsMap, dict[str, list[int]]]:
         """
         Fetch precomputed done PR facts.
 
@@ -188,13 +176,13 @@ async def load_precomputed_done_facts_filters(
     async def load_precomputed_done_facts_all(
         cls,
         repos: Collection[str],
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
         extra: Iterable[InstrumentedAttribute] = (),
-    ) -> Tuple[PullRequestFactsMap, Dict[str, Mapping[str, Any]]]:
+    ) -> tuple[PullRequestFactsMap, dict[str, Mapping[str, Any]]]:
         """
         Fetch all the precomputed done PR facts we have.
 
@@ -240,13 +228,13 @@ async def load_precomputed_done_timestamp_filters(
         repos: Collection[str],
         participants: PRParticipants,
         labels: LabelFilter,
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         exclude_inactive: bool,
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
-    ) -> Tuple[Dict[PullRequestID, datetime], Dict[str, List[int]]]:
+    ) -> tuple[dict[PullRequestID, datetime], dict[str, list[int]]]:
         """
         Fetch precomputed done PR "pr_done_at" timestamps.
 
@@ -278,13 +266,13 @@ async def load_precomputed_done_timestamp_filters(
     @sentry_span
     async def load_precomputed_done_facts_reponums(
         cls,
-        repos: Dict[str, Set[int]],
-        default_branches: Dict[str, str],
+        repos: dict[str, set[int]],
+        default_branches: dict[str, str],
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
-    ) -> Tuple[PullRequestFactsMap, Dict[str, List[int]]]:
+    ) -> tuple[PullRequestFactsMap, dict[str, list[int]]]:
         """
         Load PullRequestFacts belonging to released or rejected PRs from the precomputed DB.
 
@@ -380,13 +368,13 @@ async def load_precomputed_done_facts_reponums(
     async def load_precomputed_done_facts_ids(
         cls,
         node_ids: Iterable[int],
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
         panic_on_missing_repositories: bool = True,
-    ) -> Tuple[PullRequestFactsMap, Dict[str, List[int]]]:
+    ) -> tuple[PullRequestFactsMap, dict[str, list[int]]]:
         """
         Load PullRequestFacts belonging to released or rejected PRs from the precomputed DB.
 
@@ -464,8 +452,8 @@ async def load_precomputed_pr_releases(
         cls,
         prs: pd.DataFrame,
         time_to: datetime,
-        matched_bys: Dict[str, ReleaseMatch],
-        default_branches: Dict[str, str],
+        matched_bys: dict[str, ReleaseMatch],
+        default_branches: dict[str, str],
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
@@ -566,19 +554,19 @@ async def load_precomputed_pr_releases(
     @sentry_span
     async def _load_precomputed_done_filters(
         cls,
-        columns: List[InstrumentedAttribute],
+        columns: list[InstrumentedAttribute],
         time_from: Optional[datetime],
         time_to: Optional[datetime],
         repos: Collection[str],
         participants: PRParticipants,
         labels: LabelFilter,
-        default_branches: Dict[str, str],
+        default_branches: dict[str, str],
         exclude_inactive: bool,
         release_settings: ReleaseSettings,
         prefixer: Prefixer,
         account: int,
         pdb: morcilla.Database,
-    ) -> Tuple[Dict[PullRequestID, Mapping[str, Any]], Dict[str, List[int]]]:
+    ) -> tuple[dict[PullRequestID, Mapping[str, Any]], dict[str, list[int]]]:
         """
         Load some data belonging to released or rejected PRs from the precomputed DB.
 
@@ -685,8 +673,8 @@ def or_items():
     @classmethod
     async def _compose_query_filters_undeployed(
         cls,
-        selected: Set[InstrumentedAttribute],
-        or_items: Callable[[], List[ClauseElement]],
+        selected: set[InstrumentedAttribute],
+        or_items: Callable[[], list[ClauseElement]],
         time_from: Optional[datetime],
         time_to: Optional[datetime],
         participants: PRParticipants,
@@ -695,7 +683,7 @@ async def _compose_query_filters_undeployed(
         prefixer: Prefixer,
         account: int,
         postgres: bool,
-    ) -> Tuple[List[Select], Set[datetime]]:
+    ) -> tuple[list[Select], set[datetime]]:
         ghprt = GitHubDonePullRequestFacts
         filters = cls._create_common_filters(time_from, time_to, None, account)
         selected = selected.copy()
@@ -734,8 +722,8 @@ async def _compose_query_filters_undeployed(
     @classmethod
     async def _compose_query_filters_deployed(
         cls,
-        selected: Set[InstrumentedAttribute],
-        or_items: Callable[[], List[ClauseElement]],
+        selected: set[InstrumentedAttribute],
+        or_items: Callable[[], list[ClauseElement]],
         time_from: Optional[datetime],
         time_to: Optional[datetime],
         participants: PRParticipants,
@@ -744,7 +732,7 @@ async def _compose_query_filters_deployed(
         prefixer: Prefixer,
         account: int,
         postgres: bool,
-    ) -> List[Select]:
+    ) -> list[Select]:
         ghprt = GitHubDonePullRequestFacts
         filters = cls._create_common_filters(None, time_to, None, account)
         selected = selected.copy()
@@ -780,7 +768,7 @@ def _create_common_filters(
         time_to: Optional[datetime],
         repos: Optional[Collection[str]],
         account: int,
-    ) -> List[ClauseElement]:
+    ) -> list[ClauseElement]:
         assert isinstance(time_from, (datetime, type(None)))
         assert isinstance(time_to, (datetime, type(None)))
         ghprt = GitHubDonePullRequestFacts
@@ -799,9 +787,9 @@ def _create_common_filters(
     @classmethod
     def _post_process_ambiguous_done_prs(
         cls,
-        result: Dict[Tuple[int, str], Mapping[str, Any]],
-        ambiguous: Dict[ReleaseMatch, Dict[Tuple[int, str], Mapping[str, Any]]],
-    ) -> Tuple[Dict[Tuple[int, str], Mapping[str, Any]], Dict[str, List[int]]]:
+        result: dict[tuple[int, str], Mapping[str, Any]],
+        ambiguous: dict[ReleaseMatch, dict[tuple[int, str], Mapping[str, Any]]],
+    ) -> tuple[dict[tuple[int, str], Mapping[str, Any]], dict[str, list[int]]]:
         """Figure out what to do with uncertain `tag_or_branch` release matches."""
         result.update(ambiguous[ReleaseMatch.tag.name])
         # we've found PRs released by tag belonging to these repos.
@@ -821,7 +809,7 @@ async def _build_participants_filters(
         cls,
         participants: PRParticipants,
         filters: list,
-        selected: Set[InstrumentedAttribute],
+        selected: set[InstrumentedAttribute],
         postgres: bool,
         prefixer: Prefixer,
     ) -> None:
@@ -872,7 +860,7 @@ async def _build_participants_conditions(
         cls,
         participants: PRParticipants,
         prefixer: Prefixer,
-    ) -> Tuple[list, list]:
+    ) -> tuple[list, list]:
         user_login_to_node_get = prefixer.user_login_to_node.get
 
         def _build_conditions(roles):
@@ -945,7 +933,7 @@ async def store_precomputed_done_facts(
     prs: Iterable[MinedPullRequest],
     pr_facts: Iterable[Optional[PullRequestFacts]],
     time_to: datetime,
-    default_branches: Dict[str, str],
+    default_branches: dict[str, str],
     release_settings: ReleaseSettings,
     account: int,
     pdb: morcilla.Database,
@@ -1053,7 +1041,7 @@ async def delete_force_push_dropped_prs(
     repos: Iterable[str],
     branches: pd.DataFrame,
     account: int,
-    meta_ids: Tuple[int, ...],
+    meta_ids: tuple[int, ...],
     mdb: Database,
     pdb: Database,
     cache: Optional[aiomcache.Client],

server/athenian/api/typing_utils.py

@@ -660,12 +660,19 @@ def __init__(self, first_item: NumpyStruct, length: Optional[int]):
         direct_columns = []
         dtype = [(f, v[0]) for f, v in first_item.dtype.fields.items()]
         direct_nested_fields = first_item.nested_dtypes
+        side_fields = {}
+        try:
+            side_fields |= first_item.Optional.__annotations__
+            side_fields |= first_item.Virtual.__annotations__
+        except AttributeError:
+            pass
         coerced_datas = [first_item.coerced_data]
         indirect_columns = {}
 
         for k, v in first_item.items():
             if k not in first_item.dtype.names or k in direct_nested_fields:
-                if dataclasses.is_dataclass(v):
+                if dataclasses.is_dataclass(side_fields.get(k)):
+                    assert v is not None, f"{type(first_item).__name__}.{k} may not be None"
                     for subfield, subvalue in dataclass_asdict(v).items():
                         full_k = f"{k}_{subfield}"
                         indirect_columns.setdefault(k, []).append((full_k, subfield))

server/tests/testutils/factory/miners.py

@@ -7,7 +7,7 @@
 import numpy as np
 import pandas as pd
 
-from athenian.api.internal.miners.types import PullRequestFacts
+from athenian.api.internal.miners.types import LoadedJIRADetails, PullRequestFacts
 
 _faker = faker.Faker()
 _dt_between = partial(_faker.date_time_between, tzinfo=timezone.utc)
@@ -19,6 +19,7 @@ class Meta:
 
     force_push_dropped = False
     release_ignored = False
+    jira = LoadedJIRADetails.empty()
 
     @factory.lazy_attribute
     def created(self) -> pd.Timestamp: