feat: enrich github provenance evidence and review metrics

haasonsaas · haasonsaas · commit a6d607577a4b · 2025-10-16T16:33:36.000-07:00
diff --git a/README.md b/README.md
@@ -83,6 +83,7 @@ Copy `.env.example` to `.env` and adjust values locally if you prefer dotenv-sty
 | `PROVENANCE_GITHUB_BASE_URL` | GitHub enterprise base URL (optional) | *(unset)* |
 | `PROVENANCE_GITHUB_AGENT_LABEL_PREFIX` | PR label prefix used to infer agent IDs | `agent:` |
 | `PROVENANCE_GITHUB_CACHE_TTL_SECONDS` | Cache TTL (seconds) for GitHub metadata lookups | `300` |
+| `PROVENANCE_GITHUB_AGENT_MAP` | JSON map of GitHub logins/keywords to agent IDs | `{}` |
 
 ## Detection with Semgrep
 
@@ -147,6 +148,7 @@ Example ingestion payload:
 
 - `/v1/analytics/summary` now supports additional metrics: `code_volume`, `code_churn_rate`, and `avg_line_complexity` in addition to `risk_rate` and `provenance_coverage`.
 - `/v1/analytics/agents/behavior` returns composite snapshots (volume, churn rate, heuristic complexity, and top vulnerability categories per agent) to power comparison dashboards.
+- Review-focused metrics (`review_comments`, `unique_reviewers`) leverage GitHub PR data when credentials are supplied.
 - Use `PROVENANCE_ANALYTICS_DEFAULT_WINDOW` or query parameters such as `?time_window=14d` to track longer horizons and compare agents.
 
 ## Telemetry Export
diff --git a/app/core/config.py b/app/core/config.py
@@ -40,6 +40,7 @@ class Settings(BaseSettings):
     github_base_url: str | None = None
     github_agent_label_prefix: str = "agent:"
     github_cache_ttl_seconds: int = 300
+    github_agent_map: dict[str, str] = Field(default_factory=dict)
 
     model_config = SettingsConfigDict(env_prefix="provenance_", env_file=".env", extra="ignore")
 
diff --git a/app/dependencies.py b/app/dependencies.py
@@ -55,6 +55,7 @@ def get_github_resolver() -> GitHubProvenanceResolver | None:
         base_url=settings.github_base_url,
         agent_label_prefix=settings.github_agent_label_prefix,
         cache_ttl_seconds=settings.github_cache_ttl_seconds,
+        agent_map=settings.github_agent_map,
     )
 
 
diff --git a/app/provenance/github_resolver.py b/app/provenance/github_resolver.py
@@ -23,6 +23,7 @@ def __init__(
         base_url: str | None = None,
         agent_label_prefix: str = "agent:",
         cache_ttl_seconds: int = 300,
+        agent_map: dict[str, str] | None = None,
     ) -> None:
         self._agent_label_prefix = agent_label_prefix.lower()
         auth = Token(token)
@@ -31,6 +32,7 @@ def __init__(
         else:
             self._client = Github(auth=auth)
         self._cache_ttl = max(cache_ttl_seconds, 30)
+        self._agent_map = {k.lower(): v for k, v in (agent_map or {}).items()}
         self._commit_cache: dict[tuple[str, str], tuple[float, Optional[Commit.Commit]]] = {}
         self._label_cache: dict[tuple[str, int], tuple[float, list[str]]] = {}
         self._comment_cache: dict[tuple[str, int], tuple[float, list[str]]] = {}
@@ -42,17 +44,31 @@ def resolve_agent(
         repo_full_name: str,
         pr_number: str | None,
         commit_sha: str | None,
-    ) -> tuple[Optional[str], Optional[str]]:
+    ) -> tuple[Optional[str], Optional[str], dict]:
         agent_id: Optional[str] = None
         session_id: Optional[str] = None
+        evidence: dict = {}
 
         if commit_sha:
-            agent_id, session_id = self._from_commit(repo_full_name, commit_sha)
+            agent_id, session_id, commit_evidence = self._from_commit(repo_full_name, commit_sha)
+            evidence.setdefault("sources", []).append(commit_evidence)
         if not agent_id and pr_number:
-            agent_id = self._from_pr_labels(repo_full_name, int(pr_number))
+            label_agent, label_evidence = self._from_pr_labels(repo_full_name, int(pr_number))
+            if label_agent:
+                agent_id = label_agent
+            evidence.setdefault("sources", []).append(label_evidence)
         if not agent_id and pr_number:
-            agent_id = self._from_pr_discussion(repo_full_name, int(pr_number))
-        return agent_id, session_id
+            discussion_agent, discussion_evidence = self._from_pr_discussion(repo_full_name, int(pr_number))
+            if discussion_agent:
+                agent_id = discussion_agent
+            evidence.setdefault("sources", []).append(discussion_evidence)
+        if not agent_id and pr_number:
+            body_agent, body_evidence = self._from_pr_body(repo_full_name, int(pr_number))
+            if body_agent:
+                agent_id = body_agent
+            evidence.setdefault("sources", []).append(body_evidence)
+        evidence["agent_id"] = agent_id
+        return agent_id, session_id, evidence
 
     def review_stats(self, repo_full_name: str, pr_number: int) -> dict[str, int] | None:
         comments = self._fetch_pr_comments(repo_full_name, pr_number)
@@ -82,23 +98,24 @@ def _fetch_commit(self, repo_full_name: str, sha: str) -> Optional[Commit.Commit
         self._commit_cache[key] = (now + self._cache_ttl, commit)
         return commit
 
-    def _from_commit(self, repo_full_name: str, sha: str) -> tuple[Optional[str], Optional[str]]:
+    def _from_commit(self, repo_full_name: str, sha: str) -> tuple[Optional[str], Optional[str], dict]:
         commit = self._fetch_commit(repo_full_name, sha)
         if not commit:
-            return None, None
+            return None, None, {"source": "commit", "reason": "not_found"}
         message = commit.commit.message or ""
         for line in message.splitlines():
             match = AGENT_TRAILER_PATTERN.match(line.strip())
             if match:
-                return match.group("agent"), None
+                return match.group("agent"), None, {"source": "commit_trailer", "line": line.strip()}
         for line in message.splitlines():
             match = CO_AUTHOR_PATTERN.match(line.strip())
             if match and "copilot" in match.group("author").lower():
-                return "github-copilot", None
+                return "github-copilot", None, {"source": "co_author", "value": match.group("author")}
         author_login = getattr(commit.author, "login", "") or ""
         if author_login:
-            return author_login, None
-        return None, None
+            mapped = self._agent_map.get(author_login.lower())
+            return mapped or author_login, None, {"source": "commit_author", "value": author_login}
+        return None, None, {"source": "commit", "reason": "no_author"}
 
     def _fetch_pr_labels(self, repo_full_name: str, pr_number: int) -> list[str]:
         key = (repo_full_name, pr_number)
@@ -115,12 +132,16 @@ def _fetch_pr_labels(self, repo_full_name: str, pr_number: int) -> list[str]:
         self._label_cache[key] = (now + self._cache_ttl, labels)
         return labels
 
-    def _from_pr_labels(self, repo_full_name: str, pr_number: int) -> Optional[str]:
-        for label in self._fetch_pr_labels(repo_full_name, pr_number):
+    def _from_pr_labels(self, repo_full_name: str, pr_number: int) -> tuple[Optional[str], dict]:
+        labels = self._fetch_pr_labels(repo_full_name, pr_number)
+        for label in labels:
             lower = label.lower()
             if lower.startswith(self._agent_label_prefix):
-                return label.split(":", 1)[-1].strip()
-        return None
+                return label.split(":", 1)[-1].strip(), {"source": "label", "label": label}
+            mapped = self._agent_map.get(lower)
+            if mapped:
+                return mapped, {"source": "label_map", "label": label}
+        return None, {"source": "label", "labels": labels}
 
     def _fetch_pr_comments(self, repo_full_name: str, pr_number: int) -> list[str]:
         key = (repo_full_name, pr_number)
@@ -168,18 +189,38 @@ def _fetch_review_events(self, repo_full_name: str, pr_number: int) -> int:
         self._review_event_cache[key] = (now + self._cache_ttl, events)
         return events
 
-    def _from_pr_discussion(self, repo_full_name: str, pr_number: int) -> Optional[str]:
+    def _from_pr_discussion(self, repo_full_name: str, pr_number: int) -> tuple[Optional[str], dict]:
         for body in self._fetch_pr_comments(repo_full_name, pr_number):
             for line in body.splitlines():
                 match = AGENT_TRAILER_PATTERN.match(line.strip())
                 if match:
-                    return match.group("agent")
+                    return match.group("agent"), {"source": "comment", "line": line.strip()}
         for author in self._fetch_review_authors(repo_full_name, pr_number):
             lower = author.lower()
             if "copilot" in lower:
-                return "github-copilot"
+                return "github-copilot", {"source": "reviewer", "value": author}
+            mapped = self._agent_map.get(lower)
+            if mapped:
+                return mapped, {"source": "reviewer_map", "value": author}
             if any(key in lower for key in ("claude", "gemini", "gpt", "bard")):
-                return lower
+                return lower, {"source": "reviewer_heuristic", "value": author}
             if lower.endswith("-bot"):
-                return lower
-        return None
+                return lower, {"source": "reviewer_bot", "value": author}
+        return None, {"source": "discussion", "reason": "no_match"}
+
+    def _from_pr_body(self, repo_full_name: str, pr_number: int) -> tuple[Optional[str], dict]:
+        try:
+            repo = self._client.get_repo(repo_full_name)
+            pr = repo.get_pull(pr_number)
+            body = pr.body or ""
+        except GithubException:
+            return None, {"source": "body", "reason": "error"}
+        for line in body.splitlines():
+            match = AGENT_TRAILER_PATTERN.match(line.strip())
+            if match:
+                return match.group("agent"), {"source": "body", "line": line.strip()}
+        lower_body = body.lower()
+        for key, mapped in self._agent_map.items():
+            if key in lower_body:
+                return mapped, {"source": "body_map", "value": key}
+        return None, {"source": "body", "reason": "no_match"}
diff --git a/app/services/analysis.py b/app/services/analysis.py
@@ -141,7 +141,7 @@ def _map_changed_line(
             provenance_marker=payload.attribution.provenance_marker,
         )
         if not attribution.agent.agent_id:
-            agent_id, session_id = self._resolve_agent(
+            agent_id, session_id, evidence = self._resolve_agent(
                 repo=request.repo,
                 pr_number=request.pr_number,
                 commit_sha=attribution.commit_sha,
@@ -150,6 +150,8 @@ def _map_changed_line(
                 attribution.agent.agent_id = agent_id
             if session_id:
                 attribution.agent_session_id = session_id
+            if evidence:
+                attribution.provenance_marker = str(evidence)
         return ChangedLine(
             analysis_id=analysis_id,
             repo_id=request.repo,
@@ -172,9 +174,9 @@ def _resolve_agent(
         repo: str,
         pr_number: str,
         commit_sha: str | None,
-    ) -> tuple[str | None, str | None]:
+    ) -> tuple[str | None, str | None, dict]:
         if not self._github_resolver:
-            return None, None
+            return None, None, {}
         return self._github_resolver.resolve_agent(repo, pr_number, commit_sha)
 
     def list_findings(self, analysis_id: str) -> list[Finding]:
diff --git a/tests/provenance/test_github_resolver.py b/tests/provenance/test_github_resolver.py
@@ -18,25 +18,28 @@ def test_resolver_extracts_agent_from_commit_trailer(monkeypatch):
         "_fetch_commit",
         lambda self, repo, sha: StubCommit("Fix bug\nAgent-ID: claude-3-opus"),
     )
-    agent, session = resolver.resolve_agent("acme/repo", "42", "abc123")
+    agent, session, evidence = resolver.resolve_agent("acme/repo", "42", "abc123")
     assert agent == "claude-3-opus"
     assert session is None
+    assert evidence["agent_id"] == "claude-3-opus"
 
 
 def test_resolver_uses_coauthor(monkeypatch):
     resolver = GitHubProvenanceResolver(token="token")
     message = "Refactor\nCo-authored-by: GitHub Copilot <copilot@example.com>"
     monkeypatch.setattr(GitHubProvenanceResolver, "_fetch_commit", lambda self, repo, sha: StubCommit(message))
-    agent, _ = resolver.resolve_agent("acme/repo", None, "def456")
+    agent, _, evidence = resolver.resolve_agent("acme/repo", None, "def456")
     assert agent == "github-copilot"
+    assert evidence["agent_id"] == "github-copilot"
 
 
 def test_resolver_falls_back_to_pr_labels(monkeypatch):
     resolver = GitHubProvenanceResolver(token="token", agent_label_prefix="agent:")
     monkeypatch.setattr(GitHubProvenanceResolver, "_fetch_commit", lambda self, repo, sha: None)
     monkeypatch.setattr(GitHubProvenanceResolver, "_fetch_pr_labels", lambda self, repo, pr: ["Agent: gemini-pro"])
-    agent, _ = resolver.resolve_agent("acme/repo", "77", None)
+    agent, _, evidence = resolver.resolve_agent("acme/repo", "77", None)
     assert agent == "gemini-pro"
+    assert evidence["agent_id"] == "gemini-pro"
 
 
 def test_resolver_uses_pr_comments(monkeypatch):
@@ -48,8 +51,9 @@ def test_resolver_uses_pr_comments(monkeypatch):
         "_fetch_pr_comments",
         lambda self, repo, pr: ["LGTM\nAgent-ID: gemma-7b"],
     )
-    agent, _ = resolver.resolve_agent("acme/repo", "77", None)
+    agent, _, evidence = resolver.resolve_agent("acme/repo", "77", None)
     assert agent == "gemma-7b"
+    assert evidence["agent_id"] == "gemma-7b"
 
 
 def test_review_stats(monkeypatch):

Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,7 @@ def get_github_resolver() -> GitHubProvenanceResolver \| None:`
`55`	`55`	`base_url=settings.github_base_url,`
`56`	`56`	`agent_label_prefix=settings.github_agent_label_prefix,`
`57`	`57`	`cache_ttl_seconds=settings.github_cache_ttl_seconds,`
	`58`	`+ agent_map=settings.github_agent_map,`
`58`	`59`	`)`
`59`	`60`
`60`	`61`