fix: data model adjustment (#2)

jbristow · web-flow · commit f3f0530d0c38 · 2025-01-27T15:13:31.000-08:00
* fix: do not normalize keys

* fix: correct data model

* fix: tests and lint

* fix: more data model cleaning

* fix: lint/test
diff --git a/nodestream_github/github_organizations.yaml b/nodestream_github/github_organizations.yaml
@@ -8,6 +8,8 @@
     interpretations:
     - type: source_node
       node_type: GithubOrg
+      key_normalization:
+        do_lowercase_strings: false
       key:
         node_id: !jmespath 'node_id'
       additional_indexes:
@@ -69,17 +71,18 @@
       relationship_type: IS_MEMBER
       iterate_on: !jmespath 'members[*]'
       outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       relationship_properties:
         role: !jmespath 'role'
 
     - type: github-repo-relationship
       relationship_type: IN_ORGANIZATION
       iterate_on: !jmespath 'repositories[*]'
       outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       relationship_properties:
-          triage: !jmespath 'permissions.triage'
-          maintain: !jmespath 'permissions.maintain'
-          admin: !jmespath 'permissions.admin'
-          push: !jmespath 'permissions.push'
-          pull: !jmespath 'permissions.pull'
+        permission: !jmespath 'permission'
+
 
diff --git a/nodestream_github/github_repos.yaml b/nodestream_github/github_repos.yaml
@@ -12,6 +12,8 @@
       additional_indexes:
       - full_name
       - html_url
+      key_normalization:
+        do_lowercase_strings: false
       key:
         node_id: !jmespath 'node_id'
     - type: properties
@@ -53,6 +55,8 @@
     - type: relationship
       node_type: GithubOrg
       relationship_type: IN_ORGANIZATION
+      key_normalization:
+        do_lowercase_strings: false
       node_key:
         node_id: !jmespath 'org_owner.node_id'
       node_properties:
@@ -65,6 +69,8 @@
       node_type: GithubUser
       relationship_type: IS_OWNER
       outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       node_key:
         node_id: !jmespath 'user_owner.node_id'
       node_properties:
@@ -75,16 +81,16 @@
       relationship_type: IS_COLLABORATOR
       iterate_on: !jmespath 'collaborators[*]'
       outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       relationship_properties:
-        triage: !jmespath 'permissions.triage'
-        maintain: !jmespath 'permissions.maintain'
-        admin: !jmespath 'permissions.admin'
-        push: !jmespath 'permissions.push'
-        pull: !jmespath 'permissions.pull'
+        "permission": !jmespath "role_name"
 
     - type: relationship
       node_type: ProgrammingLanguage
       relationship_type: HAS_LANGUAGE
+      key_normalization:
+        do_lowercase_strings: true
       node_key:
         name: !jmespath 'name'
       relationship_properties:
@@ -96,6 +102,8 @@
       iterate_on: !jmespath 'webhooks[*]'
       relationship_type: WEBHOOK_ON
       outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       relationship_properties:
         events: !jmespath "events"
       node_key:
diff --git a/nodestream_github/github_teams.yaml b/nodestream_github/github_teams.yaml
@@ -8,6 +8,8 @@
     interpretations:
     - type: source_node
       node_type: GithubTeam
+      key_normalization:
+        do_lowercase_strings: false
       key:
         node_id: !jmespath 'node_id'
       additional_indexes:
@@ -34,24 +36,26 @@
     - type: relationship
       node_type: GithubOrg
       relationship_type: IN_ORGANIZATION
+      key_normalization:
+        do_lowercase_strings: false
       node_key:
         node_id: !jmespath 'organization.node_id'
       outbound: true
 
     - type: github-user-relationship
       relationship_type: IS_MEMBER
       iterate_on: !jmespath 'members[*]'
+      key_normalization:
+        do_lowercase_strings: false
       outbound: false
       relationship_properties:
         role: !jmespath 'role'
 
     - type: github-repo-relationship
       relationship_type: IN_TEAM
       iterate_on: !jmespath 'repos[*]'
-      outbound: false
+      key_normalization:
+        do_lowercase_strings: false
       relationship_properties:
-          triage: !jmespath 'permissions.triage'
-          maintain: !jmespath 'permissions.maintain'
-          admin: !jmespath 'permissions.admin'
-          push: !jmespath 'permissions.push'
-          pull: !jmespath 'permissions.pull'
+        permission: !jmespath 'permission'
+      outbound: false
diff --git a/nodestream_github/github_users.yaml b/nodestream_github/github_users.yaml
@@ -8,6 +8,8 @@
     interpretations:
       - type: source_node
         node_type: GithubUser
+        key_normalization:
+          do_lowercase_strings: false
         key:
           node_id: !jmespath 'node_id'
         additional_indexes:
@@ -46,9 +48,5 @@
         iterate_on: !jmespath 'repositories[*]'
         relationship_type: IS_COLLABORATOR
         outbound: true
-        relationship_properties:
-          triage: !jmespath 'permissions.triage'
-          maintain: !jmespath 'permissions.maintain'
-          admin: !jmespath 'permissions.admin'
-          push: !jmespath 'permissions.push'
-          pull: !jmespath 'permissions.pull'
+        key_normalization:
+          do_lowercase_strings: false
diff --git a/nodestream_github/interpretations/relationship/repository.py b/nodestream_github/interpretations/relationship/repository.py
@@ -10,14 +10,18 @@
 
 from nodestream_github.types import GithubRepo, SimplifiedRepo
 
-_REPO_KEYS_TO_PRESERVE = ["id", "node_id", "name", "full_name", "url", "permissions"]
+_REPO_KEYS_TO_PRESERVE = ["id", "node_id", "name", "full_name", "url", "permission"]
 
 
-def simplify_repo(repo: GithubRepo) -> SimplifiedRepo:
+def simplify_repo(repo: GithubRepo, *, permission: str | None = None) -> SimplifiedRepo:
     """Simplify repo data.
 
     Allows us to only keep a consistent minimum for relationship data."""
-    return {k: repo[k] for k in _REPO_KEYS_TO_PRESERVE if k in repo}
+    output = {k: repo[k] for k in _REPO_KEYS_TO_PRESERVE if k in repo}
+
+    if permission:
+        output["permission"] = permission
+    return output
 
 
 class RepositoryRelationshipInterpretation(
diff --git a/nodestream_github/interpretations/relationship/user.py b/nodestream_github/interpretations/relationship/user.py
@@ -10,7 +10,7 @@
 
 from nodestream_github.types import GithubUser, SimplifiedUser
 
-_USER_KEYS_TO_PRESERVE = ["id", "login", "node_id", "role", "permissions"]
+_USER_KEYS_TO_PRESERVE = ["id", "login", "node_id", "role", "role_name"]
 
 
 def simplify_user(user: GithubUser) -> SimplifiedUser:
diff --git a/nodestream_github/orgs.py b/nodestream_github/orgs.py
@@ -37,7 +37,10 @@ async def _extract_organization(self, login: str) -> OrgRecord | None:
         full_org["members"] = [user async for user in self._fetch_all_members(login)]
 
         full_org["repositories"] = [
-            simplify_repo(repo) async for repo in self.client.fetch_repos_for_org(login)
+            simplify_repo(
+                repo, permission=full_org.get("default_repository_permission")
+            )
+            async for repo in self.client.fetch_repos_for_org(login)
         ]
         return full_org
 
diff --git a/nodestream_github/teams.py b/nodestream_github/teams.py
@@ -59,7 +59,7 @@ async def _fetch_team(
             simplify_user(member) async for member in self._fetch_members(team)
         ]
         team["repos"] = [
-            simplify_repo(repo)
+            simplify_repo(repo, permission=team.get("permission"))
             async for repo in self.client.fetch_repos_for_team(login, team["slug"])
         ]
         return team
diff --git a/tests/data/repos.py b/tests/data/repos.py
@@ -132,7 +132,6 @@ def repo(
         "pushed_at": "2011-01-26T19:06:43Z",
         "created_at": "2011-01-26T19:01:12Z",
         "updated_at": "2011-01-26T19:14:43Z",
-        "permissions": {"admin": False, "push": False, "pull": True},
         "security_and_analysis": {
             "advanced_security": {"status": "enabled"},
             "secret_scanning": {"status": "enabled"},
diff --git a/tests/interpretations/relationship/test_repository.py b/tests/interpretations/relationship/test_repository.py
@@ -31,9 +31,9 @@ def test_simplify_repo():
     assert simplify_repo(additional_keys) == _TEST_EXPECTATION
 
 
-def test_simplify_repo_keep_perms():
-    test_input = _TEST_EXPECTATION | {"permissions": {"admin": True}}
-    assert simplify_repo(test_input) == test_input
+def test_simplify_repo_set_perm():
+    expected = _TEST_EXPECTATION | {"permission": "test"}
+    assert simplify_repo(_TEST_EXPECTATION, permission="test") == expected
 
 
 def test_repo_relationship(context: ProviderContext):
diff --git a/tests/interpretations/relationship/test_user.py b/tests/interpretations/relationship/test_user.py
@@ -29,9 +29,8 @@ def test_simplify_user():
     assert simplify_user(additional_keys) == _TEST_EXPECTATION
 
 
-def test_simplify_user_keep_perms():
-    test_input = _TEST_EXPECTATION | {"permissions": {"admin": True}}
-    assert simplify_user(test_input) == test_input
+def test_simplify_user_identity():
+    assert simplify_user(_TEST_EXPECTATION) == _TEST_EXPECTATION
 
 
 def test_user_relationship(context: ProviderContext):
diff --git a/tests/test_orgs.py b/tests/test_orgs.py
@@ -146,7 +146,7 @@ async def test_orgs_continue_through_org_member_status_fail(
                 "id": 1296269,
                 "name": "Hello-World",
                 "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
-                "permissions": {"admin": False, "pull": True, "push": False},
+                "permission": "read",
                 "url": "https://HOSTNAME/repos/octocat/Hello-World",
             }],
         }
@@ -261,7 +261,7 @@ async def test_get_orgs(
                 "id": 1296269,
                 "name": "Hello-World",
                 "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
-                "permissions": {"admin": False, "pull": True, "push": False},
+                "permission": "read",
                 "url": "https://HOSTNAME/repos/octocat/Hello-World",
             }],
         }
diff --git a/tests/test_repos.py b/tests/test_repos.py
@@ -94,7 +94,7 @@ async def test_extract_records(
     gh_rest_mock.get_collaborators_for_repo(
         owner_login="octocat",
         repo_name="Hello-World",
-        json=[TURBO_USER],
+        json=[TURBO_USER | {"role_name": "write"}],
     )
     gh_rest_mock.get_languages_for_repo(
         owner_login="github",
@@ -125,7 +125,12 @@ async def test_extract_records(
                 "https://HOSTNAME/repos/octocat/Hello-World/branches{/branch}"
             ),
             "clone_url": "https://github.com/octocat/Hello-World.git",
-            "collaborators": [{"id": 2, "login": "turbo", "node_id": "MDQ6VXNlcjI="}],
+            "collaborators": [{
+                "id": 2,
+                "login": "turbo",
+                "node_id": "MDQ6VXNlcjI=",
+                "role_name": "write",
+            }],
             "collaborators_url": "https://HOSTNAME/repos/octocat/Hello-World/collaborators{/collaborator}",
             "comments_url": (
                 "https://HOSTNAME/repos/octocat/Hello-World/comments{/number}"
@@ -211,7 +216,6 @@ async def test_extract_records(
                 "type": "User",
                 "url": "https://HOSTNAME/users/octocat",
             },
-            "permissions": {"admin": False, "pull": True, "push": False},
             "private": False,
             "pulls_url": "https://HOSTNAME/repos/octocat/Hello-World/pulls{/number}",
             "pushed_at": "2011-01-26T19:06:43Z",
@@ -338,7 +342,6 @@ async def test_extract_records(
             "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
             "notifications_url": "https://HOSTNAME/repos/github/Hello-Moon/notifications{?since,all,participating}",
             "open_issues_count": 0,
-            "permissions": {"admin": False, "pull": True, "push": False},
             "private": False,
             "pulls_url": "https://HOSTNAME/repos/github/Hello-Moon/pulls{/number}",
             "pushed_at": "2011-01-26T19:06:43Z",
diff --git a/tests/test_teams.py b/tests/test_teams.py
@@ -147,7 +147,7 @@ async def test_extract_records(
             "id": 1296269,
             "name": "Hello-World",
             "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
-            "permissions": {"admin": False, "pull": True, "push": False},
+            "permission": "admin",
             "url": "https://HOSTNAME/repos/octocat/Hello-World",
         }],
         "repos_count": 10,
diff --git a/tests/test_users.py b/tests/test_users.py
@@ -44,7 +44,6 @@ async def test_github_user_extractor(
                 "id": 1296269,
                 "name": "Hello-World",
                 "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
-                "permissions": {"admin": False, "pull": True, "push": False},
                 "url": "https://HOSTNAME/repos/octocat/Hello-World",
             }]
         }

Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ async def _fetch_team(`
`59`	`59`	`simplify_user(member) async for member in self._fetch_members(team)`
`60`	`60`	`]`
`61`	`61`	`team["repos"] = [`
`62`		`- simplify_repo(repo)`
	`62`	`+ simplify_repo(repo, permission=team.get("permission"))`
`63`	`63`	`async for repo in self.client.fetch_repos_for_team(login, team["slug"])`
`64`	`64`	`]`
`65`	`65`	`return team`