fixes for multiple token support changes.

Author: cyberc001

GitHubRepoAPI.py

@@ -12,36 +12,41 @@
     Comment,
     Invite,
 )
-from github import Github
 
 
 class GitHubRepoAPI(IRepositoryAPI):
 
     def __init__(self, client):
         self.client = client
 
+    def get_user_data(self, user) -> User:
+        return User(login=user.login, username=user.name, email=user.email, html_url=user.html_url, node_id=user.node_id, type=user.type, bio=user.bio, site_admin=user.site_admin, _id=user.id)
+
+
     def get_repository(self, id: str) -> Repository | None:
         try:
             repo = self.client.get_repo(id)
-            return Repository(_id=repo.full_name, name=repo.name, url=repo.html_url)
+            return Repository(_id=repo.full_name, name=repo.name, url=repo.html_url, default_branch=Branch(name=repo.default_branch, last_commit=None), owner=User(login=repo.owner.login,username=repo.owner.name,email=repo.owner.email,html_url=repo.owner.html_url))
         except Exception as e:
             logging.error(f"Failed to get repository {id} from GitHub: {e}")
             return None
 
     def get_collaborator_permission(self, repo: Repository, user: User) -> str:
         return self.client.get_repo(repo._id).get_collaborator_permission(user.login)
 
-    def get_commits(self, repo: Repository) -> list[Commit]:
+    def get_commits(self, repo: Repository, files: bool = True) -> list[Commit]:
         try:
             commits = self.client.get_repo(repo._id).get_commits()
             return [
                 Commit(
                     _id=c.sha,
                     message=c.commit.message,
-                    author=User(
-                        login=c.author.login,username=c.author.name,email=c.author.email
-                    ),
+                    author=self.get_user_data(c.author),
                     date=c.commit.author.date,
+                    files=[
+                        f.filename
+                            for f in c.files
+                    ] if files else None
                 )
                 for c in commits
             ]
@@ -68,8 +73,17 @@ def get_issues(self, repo: Repository) -> list[Issue]:
                 Issue(
                     _id=i.number,
                     title=i.title,
-                    author=Contributor(i.user.login, i.user.email or ""),
                     state=i.state,
+                    created_at=i.created_at,
+                    closed_at=i.closed_at,
+                    closed_by=self.get_user_data(i.closed_by) if i.closed_by else None,
+                    body=i.body,
+                    user=self.get_user_data(i.user),
+                    labels= [
+                        l.name
+                        for l in i.labels
+                    ],
+                    milestone=i.milestone.title if i.milestone else None
                 )
                 for i in issues
             ]
@@ -84,8 +98,24 @@ def get_pull_requests(self, repo: Repository) -> list[PullRequest]:
                 PullRequest(
                     _id=p.number,
                     title=p.title,
-                    author=Contributor(p.user.login, p.user.email or ""),
+                    author=self.get_user_data(p.user),
                     state=p.state,
+                    created_at=p.created_at,
+                    head_label=p.head.label,
+                    base_label=p.base.label,
+                    head_ref=p.head.ref,
+                    base_ref=p.base.ref,
+                    merged_by=self.get_user_data(p.merged_by) if p.merged_by else None,
+                    files=[
+                        f.filename
+                            for f in p.get_files()
+                    ],
+                    issue_url=p.issue_url,
+                    labels= [
+                        l.name
+                        for l in p.labels
+                    ],
+                    milestone=p.milestone.title if p.milestone else None
                 )
                 for p in pulls
             ]
@@ -128,15 +158,47 @@ def get_branches(self, repo: Repository) -> list[Branch]:
             return []
 
     def get_wiki_pages(self, repo: Repository) -> list[WikiPage]:
-        pass
+        return
+
+    def get_forks(self, repo: Repository) -> list[Repository]:
+        repo_client = self.client.get_repo(repo._id)
+        result = []
+        for r in repo_client.get_forks():
+            result.append(Repository(_id=repo.full_name, name=repo.name, url=repo.html_url))
+        return result
+
+    def get_comments(self, repo, obj) -> list[Comment]:
+        result = []
+        if type(obj) == Issue:
+            # TODO оптимизировать
+            issues = self.client.get_repo(repo._id).get_issues(state='all')
+            issue = None
+            for i in issues:
+                if i.number == obj._id:
+                    issue = i
+                    break
+            for c in issue.get_comments():
+                result.append(Comment(body=c.body,created_at=c.created_at,author=self.get_user_data(c.user)))
+        elif type(obj) == PullRequest:
+            # TODO оптимизировать
+            pulls = self.client.get_repo(repo._id).get_pulls(state='all')
+            pull = None
+            for p in pulls:
+                if p.number == obj._id:
+                    pull = p
+                    break
+            for c in pull.get_comments():
+                result.append(Comment(body=c.body,created_at=c.created_at,author=self.get_user_data(c.user.login)))
+
+        return result
 
     def get_invites(self, repo: Repository) -> list[Invite]:
         try:
             invites = self.client.get_repo(repo._id).get_pending_invitations()
             return [
                 Invite(
                     _id=i._id,
-                    invitee=User(login=i.invitee.login, username=i.invitee.name, email=i.invitee.email),
+                    invitee=self.get_user_data(i.invitee),
                     created_at=i.created_at,
                     html_url=i.html_url
                 )
@@ -148,7 +210,6 @@ def get_invites(self, repo: Repository) -> list[Invite]:
             )
             return []
 
-
 # Точка входа для тестирования
 if __name__ == "__main__":
     # Создайте клиент GitHub (используйте ваш токен)

commits_parser.py

@@ -1,7 +1,8 @@
 from utils import logger
 import pytz
 from time import sleep
-from github import Github, Repository, GithubException, PullRequest
+#from github import Github, Repository, GithubException, PullRequest
+from interface_wrapper import IRepositoryAPI, Repository
 
 EMPTY_FIELD = 'Empty field'
 TIMEDELTA = 0.05
@@ -18,7 +19,7 @@
 )
 
 
-def log_repository_commits(repository: Repository, csv_name, start, finish, branch):
+def log_repository_commits(client: IRepositoryAPI, repository: Repository, csv_name, start, finish, branch):
     branches = []
     match branch:
         case 'all':
@@ -31,45 +32,41 @@ def log_repository_commits(repository: Repository, csv_name, start, finish, bran
 
     for branch in branches:
         print(f'Processing branch {branch}')
-        # TODO add support of since and until in https://pygithub.readthedocs.io/en/stable/github_objects/Repository.html#github.Repository.Repository.get_commits
-        for commit in repository.get_commits(sha=branch):
+        commits = client.get_commits(repository)
+        for commit in commits:
             if (
-                commit.commit.author.date.astimezone(pytz.timezone(TIMEZONE)) < start
-                or commit.commit.author.date.astimezone(pytz.timezone(TIMEZONE))
-                > finish
+                commit.date.astimezone(pytz.timezone(TIMEZONE)) < start
+                or commit.date.astimezone(pytz.timezone(TIMEZONE)) > finish
             ):
                 continue
-            if commit.commit is not None:
-                nvl = lambda val: val or EMPTY_FIELD
-                commit_data = [
-                    repository.full_name,
-                    commit.commit.author.name,
-                    nvl(commit.author.login if commit.author else None),
-                    nvl(commit.commit.author.email),
-                    commit.commit.author.date,
-                    '; '.join([file.filename for file in commit.files]),
-                    commit.commit.sha,
-                    branch,
-                ]
-                info = dict(zip(FIELDNAMES, commit_data))
+            commit_data = [
+                repository.name,
+                commit.author.username,
+                commit.author.email or EMPTY_FIELD,
+                commit.date,
+                '; '.join([file for file in commit.files]),
+                commit._id,
+                branch,
+            ]
+            info = dict(zip(FIELDNAMES, commit_data))
 
-                logger.log_to_csv(csv_name, FIELDNAMES, info)
-                logger.log_to_stdout(info)
+            logger.log_to_csv(csv_name, FIELDNAMES, info)
+            logger.log_to_stdout(info)
 
-                sleep(TIMEDELTA)
+            sleep(TIMEDELTA)
 
 
-def log_commits(working_repos, csv_name, start, finish, branch, fork_flag):
+def log_commits(client: IRepositoryAPI, working_repos, csv_name, start, finish, branch, fork_flag):
     logger.log_to_csv(csv_name, FIELDNAMES)
 
     for repo, token in working_repos:
         try:
-            logger.log_title(repo.full_name)
-            log_repository_commits(repo, csv_name, start, finish, branch)
+            logger.log_title(repo.name)
+            log_repository_commits(client, repo, csv_name, start, finish, branch)
             if fork_flag:
-                for forked_repo in repo.get_forks():
+                for forked_repo in client.get_forks(repo):
                     logger.log_title("FORKED:", forked_repo.full_name)
-                    log_repository_commits(forked_repo, csv_name, start, finish, branch)
+                    log_repository_commits(client, forked_repo, csv_name, start, finish, branch)
                     sleep(TIMEDELTA)
             sleep(TIMEDELTA)
         except Exception as e:

contributors_parser.py

@@ -22,24 +22,23 @@
 )
 
 
-def log_repository_contributors(repository: Repository, csv_name: str):
-    contributors_stats = get_contributors_stats(repository)
+def log_repository_contributors(client: IRepositoryAPI, repository: Repository, csv_name: str):
+    contributors_stats = get_contributors_stats(client, repository)
 
     nvl = lambda val: val or EMPTY_FIELD
 
     for contributor_stat in contributors_stats.values():
         contributor = contributor_stat["contributor_object"]
-        contributor_permissions = client.get_collaborator_permission(repository, User(login=contributor,username="",email=""))
+        contributor_permissions = client.get_collaborator_permission(repository, contributor)
 
         info_tmp = {
             'repository name': repository.name,
             'login': contributor.login,
-            'name': nvl(contributor.name),
+            'name': nvl(contributor.username),
             'email': nvl(contributor_stat['email']),
             'url': contributor.html_url,
-            'permissions': nvl(contributor_permissons),
+            'permissions': nvl(contributor_permissions),
             'total_commits': contributor_stat['total_commits'],
-            'id': contributor.id,
             'node_id': contributor.node_id,
             'type': contributor.type,
             'bio': nvl(contributor.bio),
@@ -53,15 +52,15 @@ def log_repository_contributors(repository: Repository, csv_name: str):
 
 def get_contributors_stats(client: IRepositoryAPI, repository: Repository) -> dict:
     contributors_stats = dict()
-    commits = client.get_commits(repository)
+    commits = client.get_commits(repository, False)
 
-    for commit in repository.get_commits():
+    for commit in commits:
         contributor = commit.author
 
         if not contributor.login in contributors_stats:
             contributors_stats[contributor.login] = {
                 'total_commits': 0,
-                'email': commit.author.email,
+                'email': contributor.email,
                 'contributor_object': contributor,
             }
 
@@ -72,7 +71,7 @@ def get_contributors_stats(client: IRepositoryAPI, repository: Repository) -> di
     return contributors_stats
 
 
-def log_contributors(working_repos: Generator, csv_name: str, fork_flag: bool):
+def log_contributors(client: IRepositoryAPI, working_repos: Generator, csv_name: str, fork_flag: bool):
     logger.log_to_csv(csv_name, FIELDNAMES)
 
     for repo, token in working_repos:
@@ -81,10 +80,11 @@ def log_contributors(working_repos: Generator, csv_name: str, fork_flag: bool):
             log_repository_contributors(client, repo, csv_name)
 
             if fork_flag:
-                for forked_repo in repo.get_forks():
-                    logger.log_title("FORKED:", forked_repo.full_name)
-                    log_repository_contributors(forked_repo, csv_name)
+                for forked_repo in client.get_forks(repo):
+                    logger.log_title("FORKED:", forked_repo.name)
+                    log_repository_contributors(client, forked_repo, csv_name)
                     sleep(TIMEDELTA)
+
         except Exception as e:
             print(e)
             exit(1)

git_logger.py

@@ -1,4 +1,5 @@
 from github import Github, GithubException, PullRequest
+from interface_wrapper import IRepositoryAPI, RepositoryFactory, Repository, User, Branch
 from time import sleep
 
 TIMEDELTA = 0.05
@@ -32,7 +33,8 @@ def __init__(self, tokens: list[str]):
 
     def _init_clients(self, tokens: list[str]) -> list[dict]:
         clients = [{"client": login(token), "token": token} for token in tokens]
-        # нужно ли нам рейзить ошибку в случае 403, или просто временно пропускать эти токены?
+        for c in clients:
+            c["api"] = RepositoryFactory.create_api("github", c["client"])
 
         return clients
 
@@ -74,10 +76,12 @@ def get_next_repo(clients: GithubClients, repositories):
             exit(1)
         else:
             print(cur_client['token'])
-            yield repo, cur_client['token']
+            yield Repository(_id=repo.full_name,name=repo.name,url=repo.html_url, default_branch=Branch(name=repo.default_branch, last_commit=None), owner=cur_client['api'].get_user_data(repo.owner)), cur_client['token']
 
 
 def get_assignee_story(github_object):
+    # TODO
+    return ""
     assignee_result = ""
     events = (
         github_object.get_issue_events()