multiple-tokens support implemented

Author: PeeachPie

git_logger.py

@@ -18,13 +18,55 @@ def login(token):
         return client
 
 
-def get_next_repo(client: Github, repositories):
+def get_tokens_from_file(tokens_path: str) -> list[str]:
+    with open(tokens_path, 'r') as file:
+        tokens = [token for token in file.read().split('\n') if token]
+
+    return tokens
+
+
+class GithubClients:
+    def __init__(self, tokens: list[str]):
+        self.clients = self._init_clients(tokens)
+        self.cur_client = None
+
+    def _init_clients(self, tokens: list[str]) -> list[dict]:
+        clients = [{"client": login(token), "token": token} for token in tokens]
+        # нужно ли нам рейзить ошибку в случае 403, или просто временно пропускать эти токены?
+
+        return clients
+
+    def get_next_client(self) -> Github:
+        client = None
+        max_remaining_limit = -1
+
+        for client_tmp in self.clients:
+            (remaining_limit, limit) = client_tmp["client"].rate_limiting
+
+            # можно добавить вывод износа токена
+            # можно дополнительно проверять на 403 и временно пропускать эти токены,
+            # либо завести константу "минимальный коэффициент износа" и не трогать "изношенные" токены
+
+            if remaining_limit > max_remaining_limit:
+                client = client_tmp
+                max_remaining_limit = remaining_limit
+
+        if client is None:
+            raise Exception("No github-clients available")
+
+        self.cur_client = client
+        print(client, max_remaining_limit)
+        return client["client"]
+
+
+def get_next_repo(clients: GithubClients, repositories):
     with open(repositories, 'r') as file:
         list_repos = [x for x in file.read().split('\n') if x]
     print(list_repos)
     for repo_name in list_repos:
         try:
-            repo = client.get_repo(repo_name)
+            cur_client = clients.get_next_client()
+            repo = cur_client.get_repo(repo_name)
         except GithubException as err:
             print(f'Github: Connect: error {err.data}')
             print(f'Github: Connect: failed to load repository "{repo_name}"')

issues_parser.py

@@ -3,7 +3,7 @@
 import requests
 import json
 from time import sleep
-from git_logger import get_assignee_story
+from git_logger import get_assignee_story, GithubClients
 from github import Github, Repository, GithubException, PullRequest
 
 EMPTY_FIELD = 'Empty field'
@@ -172,17 +172,27 @@ def log_repository_issues(repository: Repository, csv_name, token, start, finish
         sleep(TIMEDELTA)
 
 
-def log_issues(client: Github, working_repo, csv_name, token, start, finish, fork_flag):
+def log_issues(
+    clients: GithubClients, working_repo, csv_name, start, finish, fork_flag
+):
     logger.log_to_csv(csv_name, FIELDNAMES)
 
     for repo in working_repo:
         try:
             logger.log_title(repo.full_name)
-            log_repository_issues(repo, csv_name, token, start, finish)
+            log_repository_issues(
+                repo, csv_name, clients.cur_client["token"], start, finish
+            )
             if fork_flag:
                 for forked_repo in repo.get_forks():
                     logger.log_title("FORKED:", forked_repo.full_name)
-                    log_repository_issues(forked_repo, csv_name, token, start, finish)
+                    log_repository_issues(
+                        forked_repo,
+                        csv_name,
+                        clients.cur_client["token"],
+                        start,
+                        finish,
+                    )
                     sleep(TIMEDELTA)
             sleep(TIMEDELTA)
         except Exception as e:

main.py

@@ -31,9 +31,11 @@ def parse_args():
         help="export table to google sheets",
         action="store_true",
     )
-    parser.add_argument(
-        '-t', '--token', type=str, required=True, help='token github account'
-    )
+
+    token = parser.add_mutually_exclusive_group(required=True)
+    token.add_argument('-t', '--token', type=str, help='token github account')
+    token.add_argument('--tokens', type=str, help='path to your tokens')
+
     parser.add_argument(
         '-l',
         '--list',
@@ -126,48 +128,52 @@ def parse_time(datetime_str):
 
 def main():
     args = parse_args()
-    token = args.token
+
+    if args.token:
+        tokens = [args.token]
+    else:
+        tokens = git_logger.get_tokens_from_file(args.tokens)
+
     repositories = args.list
     csv_name = args.out
     path_drepo = args.download_repos
     fork_flag = args.forks_include
     log_pr_comments = args.pr_comments
 
     try:
-        client = git_logger.login(token=token)
+        clients = git_logger.GithubClients(tokens)
     except Exception as e:
         print(e)
     else:
-        working_repos = git_logger.get_next_repo(client, repositories)
+        working_repos = git_logger.get_next_repo(clients, repositories)
         start = parse_time(args.start.split('-'))
         finish = parse_time(args.finish.split('-'))
 
         if args.commits:
             commits_parser.log_commits(
-                client, working_repos, csv_name, start, finish, args.branch, fork_flag
+                clients, working_repos, csv_name, start, finish, args.branch, fork_flag
             )
         if args.pull_requests:
             pull_requests_parser.log_pull_requests(
-                client,
+                clients,
                 working_repos,
                 csv_name,
-                token,
                 start,
                 finish,
                 fork_flag,
                 log_pr_comments,
             )
         if args.issues:
             issues_parser.log_issues(
-                client, working_repos, csv_name, token, start, finish, fork_flag
+                clients, working_repos, csv_name, start, finish, fork_flag
             )
         if args.invites:
-            invites_parser.log_invitations(client, working_repos, csv_name)
+            invites_parser.log_invitations(clients, working_repos, csv_name)
         if args.wikis:
-            wikipars.wikiparser(client, repositories, path_drepo, csv_name)
+            wikipars.wikiparser(clients, repositories, path_drepo, csv_name)
         if args.contributors:
             contributors_parser.log_contributors(
-                client, working_repos, csv_name, fork_flag
+                clients, working_repos, csv_name, fork_flag
             )
         if args.export_google_sheets:
             export_sheets.write_data_to_table(

pull_requests_parser.py

@@ -3,7 +3,7 @@
 import requests
 import json
 from time import sleep
-from git_logger import get_assignee_story
+from git_logger import get_assignee_story, GithubClients
 from github import Github, Repository, GithubException, PullRequest
 
 EMPTY_FIELD = 'Empty field'
@@ -161,10 +161,9 @@ def log_repositories_pr(
 
 
 def log_pull_requests(
-    client: Github,
+    clients: GithubClients,
     working_repos,
     csv_name,
-    token,
     start,
     finish,
     fork_flag,
@@ -175,12 +174,19 @@ def log_pull_requests(
     for repo in working_repos:
         try:
             logger.log_title(repo.full_name)
-            log_repositories_pr(repo, csv_name, token, start, finish)
+            log_repositories_pr(
+                repo, csv_name, clients.cur_client["token"], start, finish
+            )
             if fork_flag:
                 for forked_repo in repo.get_forks():
                     logger.log_title("FORKED:", forked_repo.full_name)
                     log_repositories_pr(
-                        forked_repo, csv_name, token, start, finish, log_comments
+                        forked_repo,
+                        csv_name,
+                        clients.cur_client["token"],
+                        start,
+                        finish,
+                        log_comments,
                     )
                     sleep(TIMEDELTA)
             sleep(TIMEDELTA)