feat[extractors-users]: add personal repo download toggle (#22)

* feat[extractors-users]: allow toggling downloading personal repos

* fix[extractors-users]: download full user data

* chore: update version

* fix: import formatting

Author: jbristow

.github/workflows/release.yaml

@@ -2,9 +2,9 @@ name: Release
 
 # Trigger the workflow on a release event. 
 # This will trigger the workflow when a release is published - so draft releases will not trigger the workflow.
-on: 
+on:
   release:
-    types: [published]
+    types: [ published ]
 
 jobs:
   publish:
@@ -18,8 +18,8 @@ jobs:
     permissions:
       id-token: write
       attestations: write
-  
-    # Use the Github Actions Environment to isolate the workflow from the rest of the repository. 
+
+    # Use the GitHub Actions Environment to isolate the workflow from the rest of the repository.
     # See here: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment
     environment:
       name: pypi
@@ -35,9 +35,9 @@ jobs:
 
       # Set up Python 3.12 to build the package.
       # Python version here does not really matter as long as it works with 
-      # poetry because its simply building the package. We've confirmed functionality
+      # poetry because it's simply building the package. We've confirmed functionality
       # with CI testing before this step.
-      - name: Set up Python 3.12 
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
@@ -68,7 +68,7 @@ jobs:
           name: python-package-distribution
           path: dist/
 
-      # We are then going to sign the package using Github's Attest Build Provenance action.
+      # We are then going to sign the package using GitHub's Attest Build Provenance action.
       # This action will sign the package and upload the signature to the GitHub release.
       # This is to ensure that the package is verified and trusted by the user.
       - uses: actions/attest-build-provenance@v2

nodestream_github/client/githubclient.py

@@ -373,6 +373,7 @@ async def fetch_full_org(self, org_login: str) -> types.GithubOrg | None:
             return await self._get_item(f"orgs/{org_login}")
         except httpx.HTTPError as e:
             _fetch_problem(f"full organization info for {org_login}", e)
+            return None
 
     async def fetch_repos_for_user(
         self,
@@ -542,6 +543,7 @@ async def fetch_team(self, *, org_login: str, slug: str) -> types.GithubTeam | N
             return await self._get_item(f"orgs/{org_login}/teams/{slug}")
         except httpx.HTTPError as e:
             _fetch_problem(f"full team info for {org_login}/{slug}", e)
+            return None
 
     async def fetch_members_for_team(
         self,
@@ -591,3 +593,15 @@ async def fetch_repos_for_team(
                 yield repo
         except httpx.HTTPError as e:
             _fetch_problem(f"repos for team {org_login}/{slug}", e)
+
+    async def fetch_user(self, *, username: str) -> types.GithubUser | None:
+        """
+        Provides publicly available information about someone with a GitHub account.
+
+        https://docs.github.com/en/[email protected]/rest/users/users?apiVersion=2022-11-28#get-a-user
+        """
+        try:
+            return await self._get_item(f"users/{username}")
+        except httpx.HTTPError as e:
+            _fetch_problem(f"full user info for {username}", e)
+            return None

nodestream_github/teams.py

@@ -57,7 +57,7 @@ async def _fetch_members(self, team: GithubTeam) -> AsyncGenerator[SimplifiedUse
 
     async def _fetch_team(
         self, login: str, team_summary: GithubTeamSummary
-    ) -> GithubTeam:
+    ) -> GithubTeam | None:
         team = await self.client.fetch_team(org_login=login, slug=team_summary["slug"])
         if not team:
             return None

nodestream_github/types/github.py

@@ -3,21 +3,22 @@
 JSONType: TypeAlias = (
     dict[str, "JSONType"] | list["JSONType"] | str | int | float | bool | None
 )
+JSONObject: TypeAlias = dict[str, JSONType]
 
-GithubOrgSummary: TypeAlias = JSONType
-GithubOrg: TypeAlias = JSONType
-GithubRepo: TypeAlias = JSONType
-GithubUser: TypeAlias = JSONType
-Webhook: TypeAlias = JSONType
-GithubTeam: TypeAlias = JSONType
-GithubTeamSummary: TypeAlias = JSONType
-GithubAuditLog: TypeAlias = JSONType
+GithubOrgSummary: TypeAlias = JSONObject
+GithubOrg: TypeAlias = JSONObject
+GithubRepo: TypeAlias = JSONObject
+GithubUser: TypeAlias = JSONObject
+Webhook: TypeAlias = JSONObject
+GithubTeam: TypeAlias = JSONObject
+GithubTeamSummary: TypeAlias = JSONObject
+GithubAuditLog: TypeAlias = JSONObject
 
-LanguageRecord: TypeAlias = JSONType
-OrgRecord: TypeAlias = JSONType
-RepositoryRecord: TypeAlias = JSONType
-TeamRecord: TypeAlias = JSONType
-UserRecord: TypeAlias = JSONType
+LanguageRecord: TypeAlias = JSONObject
+OrgRecord: TypeAlias = JSONObject
+RepositoryRecord: TypeAlias = JSONObject
+TeamRecord: TypeAlias = JSONObject
+UserRecord: TypeAlias = JSONObject
 
-SimplifiedRepo: TypeAlias = JSONType
-SimplifiedUser: TypeAlias = JSONType
+SimplifiedRepo: TypeAlias = JSONObject
+SimplifiedUser: TypeAlias = JSONObject

nodestream_github/users.py

@@ -13,26 +13,35 @@
 from .client import GithubRestApiClient
 from .interpretations.relationship.repository import simplify_repo
 from .logging import get_plugin_logger
-from .types import UserRecord
+from .types import SimplifiedUser, UserRecord
 from .types.enums import UserRepoType
 
 logger = get_plugin_logger(__name__)
 
 
 class GithubUserExtractor(Extractor):
-    def __init__(self, **github_client_kwargs: Any):
+    def __init__(self, *, include_repos: bool = True, **github_client_kwargs: Any):
+        self.include_repos = include_repos is True  # handle None
         self.client = GithubRestApiClient(**github_client_kwargs)
 
     async def extract_records(self) -> AsyncGenerator[UserRecord]:
         """Scrapes the GitHub REST api for all users and converts them to records."""
-        async for user in self.client.fetch_all_users():
-            login = user["login"]
-            user["repositories"] = [
-                simplify_repo(repo)
-                async for repo in self.client.fetch_repos_for_user(
-                    user_login=login,
-                    repo_type=UserRepoType.OWNER,
-                )
-            ]
+        async for user_short in self.client.fetch_all_users():
+            login = user_short["login"]
+            user = await self.client.fetch_user(username=login)
+            if user is None:
+                continue
+            if self.include_repos:
+                logger.debug("including repos for %s", user)
+                user["repositories"] = await self._user_repos(login=login)
             logger.debug("yielded GithubUser{login=%s}", login)
             yield user
+
+    async def _user_repos(self, *, login: str) -> list[SimplifiedUser]:
+        return [
+            simplify_repo(repo)
+            async for repo in self.client.fetch_repos_for_user(
+                user_login=login,
+                repo_type=UserRepoType.OWNER,
+            )
+        ]