Limit repo collaborators to outside and direct affiliations. (#7)

jbristow · web-flow · commit 9080f0d3a28c · 2025-02-05T10:33:31.000-08:00
* fix: fix collaborators to only return direct and outside contributors

* chore: update and bump version

* fix: add affiliation to yaml
diff --git a/nodestream_github/client/githubclient.py b/nodestream_github/client/githubclient.py
@@ -390,9 +390,7 @@ async def fetch_webhooks_for_repo(
             _fetch_problem(f"webhooks for repo {owner_login}/{repo_name}", e)
 
     async def fetch_collaborators_for_repo(
-        self,
-        owner_login: str,
-        repo_name: str,
+        self, owner_login: str, repo_name: str, affiliation: str
     ) -> AsyncGenerator[types.GithubUser]:
         """Try to get collaborator data for this repo.
 
@@ -415,7 +413,8 @@ async def fetch_collaborators_for_repo(
         """
         try:
             async for collab_resp in self._get_paginated(
-                f"repos/{owner_login}/{repo_name}/collaborators"
+                f"repos/{owner_login}/{repo_name}/collaborators",
+                params={"affiliation": affiliation},
             ):
                 yield collab_resp
 
diff --git a/nodestream_github/github_repos.yaml b/nodestream_github/github_repos.yaml
@@ -84,6 +84,7 @@
       key_normalization:
         do_lowercase_strings: false
       relationship_properties:
+        "affiliation": !jmespath "affiliation"
         "permission": !jmespath "role_name"
 
     - type: relationship
diff --git a/nodestream_github/interpretations/relationship/user.py b/nodestream_github/interpretations/relationship/user.py
@@ -13,11 +13,19 @@
 _USER_KEYS_TO_PRESERVE = ["id", "login", "node_id", "role", "role_name"]
 
 
-def simplify_user(user: GithubUser) -> SimplifiedUser:
+def simplify_user(
+    user: GithubUser,
+    *,
+    affiliation: str | None = None,
+) -> SimplifiedUser:
     """Simplify user data.
 
     Allows us to only keep a consistent minimum for relationship data."""
-    return {k: user[k] for k in _USER_KEYS_TO_PRESERVE if k in user}
+    output = {k: user[k] for k in _USER_KEYS_TO_PRESERVE if k in user}
+
+    if affiliation:
+        output["affiliation"] = affiliation
+    return output
 
 
 class UserRelationshipInterpretation(
diff --git a/nodestream_github/repos.py b/nodestream_github/repos.py
@@ -103,12 +103,21 @@ async def _extract_repo(self, repo: GithubRepo) -> RepositoryRecord:
                 owner["login"], repo["name"]
             )
         ]
-        repo["collaborators"] = [
-            simplify_user(user)
-            async for user in self.client.fetch_collaborators_for_repo(
-                owner["login"], repo["name"]
-            )
-        ]
+        repo["collaborators"] = []
+
+        async for user in self.client.fetch_collaborators_for_repo(
+            owner["login"],
+            repo["name"],
+            "direct",
+        ):
+            repo["collaborators"].append(simplify_user(user, affiliation="direct"))
+        async for user in self.client.fetch_collaborators_for_repo(
+            owner["login"],
+            repo["name"],
+            "outside",
+        ):
+            repo["collaborators"].append(simplify_user(user, affiliation="outside"))
+
         logger.debug("yielded GithubRepo{full_name=%s}", repo["full_name"])
         return repo
 
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nodestream-plugin-github"
-version = "0.13.1-beta.5"
+version = "0.13.1-beta.6"
 description = ""
 authors = ["Jon Bristow <jonathan_bristow@intuit.com>"]
 packages = [
diff --git a/tests/mocks/githubrest.py b/tests/mocks/githubrest.py
@@ -131,10 +131,10 @@ def get_webhooks_for_repo(self, owner_login: str, repo_name: str, **kwargs: any)
         )
 
     def get_collaborators_for_repo(
-        self, owner_login: str, repo_name: str, **kwargs: any
+        self, owner_login: str, repo_name: str, affiliation: str, **kwargs: any
     ) -> None:
         self.add_response(
-            url=f"{self.base_url}/repos/{owner_login}/{repo_name}/collaborators?per_page={self.per_page}",
+            url=f"{self.base_url}/repos/{owner_login}/{repo_name}/collaborators?per_page={self.per_page}&affiliation={affiliation}",
             **kwargs,
         )
 
diff --git a/tests/test_repos.py b/tests/test_repos.py
@@ -3,10 +3,12 @@
 from nodestream_github import GithubReposExtractor
 from tests.data.orgs import GITHUB_ORG_SUMMARY
 from tests.data.repos import HELLO_WORLD_REPO, repo
-from tests.data.users import OCTOCAT_USER, TURBO_USER
+from tests.data.users import OCTOCAT_USER, TURBO_USER, user
 from tests.data.webhooks import HELLO_WORLD_WEBHOOK
 from tests.mocks.githubrest import DEFAULT_HOSTNAME, DEFAULT_PER_PAGE, GithubHttpxMock
 
+TEST_USER = user(user_login="bweaver", user_id=3)
+
 
 @pytest.fixture
 def repo_client() -> GithubReposExtractor:
@@ -35,13 +37,30 @@ async def test_pull_org_repos(gh_rest_mock: GithubHttpxMock):
     gh_rest_mock.get_repos_for_org("github", "public", json=[HELLO_WORLD_REPO])
     gh_rest_mock.get_repos_for_org("github", "private", json=[HELLO_WORLD_REPO])
     gh_rest_mock.get_languages_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        json=[],
+        is_reusable=True,
     )
     gh_rest_mock.get_webhooks_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        json=[],
+        is_reusable=True,
+    )
+    gh_rest_mock.get_collaborators_for_repo(
+        "octocat",
+        "Hello-World",
+        affiliation="direct",
+        json=[],
+        is_reusable=True,
     )
     gh_rest_mock.get_collaborators_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        affiliation="outside",
+        json=[],
+        is_reusable=True,
     )
 
     assert len([record async for record in extractor.extract_records()]) == 2
@@ -62,13 +81,30 @@ async def test_pull_user_repos(gh_rest_mock: GithubHttpxMock):
     gh_rest_mock.get_repos_for_user("octocat", "public", json=[HELLO_WORLD_REPO])
     gh_rest_mock.get_repos_for_user("octocat", "private", json=[HELLO_WORLD_REPO])
     gh_rest_mock.get_languages_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        json=[],
+        is_reusable=True,
     )
     gh_rest_mock.get_webhooks_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        json=[],
+        is_reusable=True,
+    )
+    gh_rest_mock.get_collaborators_for_repo(
+        "octocat",
+        "Hello-World",
+        affiliation="direct",
+        json=[],
+        is_reusable=True,
     )
     gh_rest_mock.get_collaborators_for_repo(
-        "octocat", "Hello-World", json=[], is_reusable=True
+        "octocat",
+        "Hello-World",
+        affiliation="outside",
+        json=[],
+        is_reusable=True,
     )
 
     assert len([record async for record in extractor.extract_records()]) == 2
@@ -94,8 +130,15 @@ async def test_extract_records(
     gh_rest_mock.get_collaborators_for_repo(
         owner_login="octocat",
         repo_name="Hello-World",
+        affiliation="direct",
         json=[TURBO_USER | {"role_name": "write"}],
     )
+    gh_rest_mock.get_collaborators_for_repo(
+        owner_login="octocat",
+        repo_name="Hello-World",
+        affiliation="outside",
+        json=[TEST_USER | {"role_name": "read"}],
+    )
     gh_rest_mock.get_languages_for_repo(
         owner_login="github",
         repo_name="Hello-Moon",
@@ -109,8 +152,15 @@ async def test_extract_records(
     gh_rest_mock.get_collaborators_for_repo(
         owner_login="github",
         repo_name="Hello-Moon",
+        affiliation="direct",
         json=[TURBO_USER],
     )
+    gh_rest_mock.get_collaborators_for_repo(
+        owner_login="github",
+        repo_name="Hello-Moon",
+        affiliation="outside",
+        json=[TEST_USER],
+    )
     assert [record async for record in repo_client.extract_records()] == [
         {
             "archive_url": (
@@ -125,12 +175,22 @@ async def test_extract_records(
                 "https://HOSTNAME/repos/octocat/Hello-World/branches{/branch}"
             ),
             "clone_url": "https://github.com/octocat/Hello-World.git",
-            "collaborators": [{
-                "id": 2,
-                "login": "turbo",
-                "node_id": "MDQ6VXNlcjI=",
-                "role_name": "write",
-            }],
+            "collaborators": [
+                {
+                    "affiliation": "direct",
+                    "id": 2,
+                    "login": "turbo",
+                    "node_id": "MDQ6VXNlcjI=",
+                    "role_name": "write",
+                },
+                {
+                    "affiliation": "outside",
+                    "id": 3,
+                    "login": "bweaver",
+                    "node_id": "MDQ6VXNlcjM=",
+                    "role_name": "read",
+                },
+            ],
             "collaborators_url": "https://HOSTNAME/repos/octocat/Hello-World/collaborators{/collaborator}",
             "comments_url": (
                 "https://HOSTNAME/repos/octocat/Hello-World/comments{/number}"
@@ -280,7 +340,20 @@ async def test_extract_records(
                 "https://HOSTNAME/repos/github/Hello-Moon/branches{/branch}"
             ),
             "clone_url": "https://github.com/github/Hello-Moon.git",
-            "collaborators": [{"id": 2, "login": "turbo", "node_id": "MDQ6VXNlcjI="}],
+            "collaborators": [
+                {
+                    "affiliation": "direct",
+                    "id": 2,
+                    "login": "turbo",
+                    "node_id": "MDQ6VXNlcjI=",
+                },
+                {
+                    "affiliation": "outside",
+                    "id": 3,
+                    "login": "bweaver",
+                    "node_id": "MDQ6VXNlcjM=",
+                },
+            ],
             "collaborators_url": (
                 "https://HOSTNAME/repos/github/Hello-Moon/collaborators{/collaborator}"
             ),

Original file line number	Diff line number	Diff line change
`@@ -131,10 +131,10 @@ def get_webhooks_for_repo(self, owner_login: str, repo_name: str, **kwargs: any)`
`131`	`131`	`)`
`132`	`132`
`133`	`133`	`def get_collaborators_for_repo(`
`134`		`- self, owner_login: str, repo_name: str, **kwargs: any`
	`134`	`+ self, owner_login: str, repo_name: str, affiliation: str, **kwargs: any`
`135`	`135`	`) -> None:`
`136`	`136`	`self.add_response(`
`137`		`- url=f"{self.base_url}/repos/{owner_login}/{repo_name}/collaborators?per_page={self.per_page}",`
	`137`	`+ url=f"{self.base_url}/repos/{owner_login}/{repo_name}/collaborators?per_page={self.per_page}&affiliation={affiliation}",`
`138`	`138`	`**kwargs,`
`139`	`139`	`)`
`140`	`140`