Skip to content

Commit dbc6a75

Browse files
AxelPCGAxelPCG
committed
Enhance error handling and logging in GitHub API queries; improve repository statistics calculations and SVG template layout.
1 parent 2186e13 commit dbc6a75

File tree

2 files changed

+128
-52
lines changed

2 files changed

+128
-52
lines changed

‎github_stats.py‎

Lines changed: 125 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ async def query_rest(self, path: str, params: Optional[Dict] = None) -> Dict:
7373
:return: deserialized REST JSON output
7474
"""
7575

76-
for _ in range(60):
76+
for attempt in range(60):
7777
headers = {
7878
"Authorization": f"token {self.access_token}",
7979
}
@@ -89,33 +89,49 @@ async def query_rest(self, path: str, params: Optional[Dict] = None) -> Dict:
8989
params=tuple(params.items()),
9090
)
9191
if r_async.status == 202:
92-
# print(f"{path} returned 202. Retrying...")
93-
print(f"A path returned 202. Retrying...")
92+
print(f"Request to {path} returned 202 (processing). Retrying in 2s... (attempt {attempt + 1}/60)")
9493
await asyncio.sleep(2)
9594
continue
95+
elif r_async.status == 403:
96+
print(f"Request to {path} returned 403 (rate limit). Retrying in 5s... (attempt {attempt + 1}/60)")
97+
await asyncio.sleep(5)
98+
continue
99+
elif r_async.status == 404:
100+
print(f"Request to {path} returned 404 (not found). Skipping...")
101+
return dict()
96102

97103
result = await r_async.json()
98104
if result is not None:
99105
return result
100-
except Exception:
101-
print("aiohttp failed for rest query")
106+
except Exception as e:
107+
print(f"aiohttp failed for rest query to {path}: {e}")
102108
# Fall back on non-async requests
103-
async with self.semaphore:
104-
r_requests = requests.get(
105-
f"https://api.github.com/{path}",
106-
headers=headers,
107-
params=tuple(params.items()),
108-
)
109-
if r_requests.status_code == 202:
110-
print(f"A path returned 202. Retrying...")
111-
await asyncio.sleep(2)
112-
continue
113-
elif r_requests.status_code == 200:
114-
result_json = r_requests.json()
115-
if result_json is not None:
116-
return result_json
117-
# print(f"There were too many 202s. Data for {path} will be incomplete.")
118-
print("There were too many 202s. Data for this repository will be incomplete.")
109+
try:
110+
async with self.semaphore:
111+
r_requests = requests.get(
112+
f"https://api.github.com/{path}",
113+
headers=headers,
114+
params=tuple(params.items()),
115+
)
116+
if r_requests.status_code == 202:
117+
print(f"Fallback request to {path} returned 202. Retrying in 2s... (attempt {attempt + 1}/60)")
118+
await asyncio.sleep(2)
119+
continue
120+
elif r_requests.status_code == 403:
121+
print(f"Fallback request to {path} returned 403. Retrying in 5s... (attempt {attempt + 1}/60)")
122+
await asyncio.sleep(5)
123+
continue
124+
elif r_requests.status_code == 404:
125+
print(f"Fallback request to {path} returned 404. Skipping...")
126+
return dict()
127+
elif r_requests.status_code == 200:
128+
result_json = r_requests.json()
129+
if result_json is not None:
130+
return result_json
131+
except Exception as e2:
132+
print(f"Both aiohttp and requests failed for {path}: {e2}")
133+
134+
print(f"Too many retries for {path}. Data will be incomplete.")
119135
return dict()
120136

121137
@staticmethod
@@ -376,10 +392,18 @@ async def get_stats(self) -> None:
376392
self._repos = set()
377393

378394
exclude_langs_lower = {x.lower() for x in self._exclude_langs}
395+
print(f"Fetching stats for user: {self.username}")
396+
print(f"Excluding repositories: {self._exclude_repos}")
397+
print(f"Excluding languages: {self._exclude_langs}")
398+
print(f"Ignore forked repos: {self._ignore_forked_repos}")
379399

380400
next_owned = None
381401
next_contrib = None
402+
page_count = 0
382403
while True:
404+
page_count += 1
405+
print(f"Fetching page {page_count}...")
406+
383407
raw_results = await self.queries.query(
384408
Queries.repos_overview(
385409
owned_cursor=next_owned, contrib_cursor=next_contrib
@@ -408,31 +432,36 @@ async def get_stats(self) -> None:
408432
if not self._ignore_forked_repos:
409433
repos += contrib_repos.get("nodes", [])
410434

435+
processed_repos = 0
411436
for repo in repos:
412437
if repo is None:
413438
continue
414439
name = repo.get("nameWithOwner")
415440
if name in self._repos or name in self._exclude_repos:
416441
continue
417442
self._repos.add(name)
418-
# self._stargazers += repo.get("stargazers").get("totalCount", 0)
419-
# self._forks += repo.get("forkCount", 0)
443+
processed_repos += 1
444+
445+
# Corrigir: descomentar e ajustar a contagem de stars e forks
446+
self._stargazers += repo.get("stargazers", {}).get("totalCount", 0)
447+
self._forks += repo.get("forkCount", 0)
420448

421449
for lang in repo.get("languages", {}).get("edges", []):
422-
name = lang.get("node", {}).get("name", "Other")
423-
languages = await self.languages
424-
if name.lower() in exclude_langs_lower:
450+
lang_name = lang.get("node", {}).get("name", "Other")
451+
if lang_name.lower() in exclude_langs_lower:
425452
continue
426-
if name in languages:
427-
languages[name]["size"] += lang.get("size", 0)
428-
languages[name]["occurrences"] += 1
453+
if lang_name in self._languages:
454+
self._languages[lang_name]["size"] += lang.get("size", 0)
455+
self._languages[lang_name]["occurrences"] += 1
429456
else:
430-
languages[name] = {
457+
self._languages[lang_name] = {
431458
"size": lang.get("size", 0),
432459
"occurrences": 1,
433460
"color": lang.get("node", {}).get("color"),
434461
}
435462

463+
print(f"Processed {processed_repos} repositories on page {page_count}")
464+
436465
if owned_repos.get("pageInfo", {}).get(
437466
"hasNextPage", False
438467
) or contrib_repos.get("pageInfo", {}).get("hasNextPage", False):
@@ -445,6 +474,11 @@ async def get_stats(self) -> None:
445474
else:
446475
break
447476

477+
print(f"Total repositories found: {len(self._repos)}")
478+
print(f"Total stars: {self._stargazers}")
479+
print(f"Total forks: {self._forks}")
480+
print(f"Languages found: {len(self._languages)}")
481+
448482
langs_total = sum([v.get("size", 0) for v in self._languages.values()])
449483
for k, v in self._languages.items():
450484
v["prop"] = 100 * (v.get("size", 0) / langs_total) if langs_total > 0 else 0
@@ -467,7 +501,7 @@ async def stargazers(self) -> int:
467501
"""
468502
if self._stargazers is not None:
469503
return self._stargazers
470-
await self.get_summary_stats()
504+
await self.get_stats()
471505
assert self._stargazers is not None
472506
return self._stargazers
473507

@@ -478,7 +512,7 @@ async def forks(self) -> int:
478512
"""
479513
if self._forks is not None:
480514
return self._forks
481-
await self.get_summary_stats()
515+
await self.get_stats()
482516
assert self._forks is not None
483517
return self._forks
484518

@@ -552,22 +586,42 @@ async def lines_changed(self) -> Tuple[int, int]:
552586
return self._lines_changed
553587
additions = 0
554588
deletions = 0
555-
for repo in await self.repos:
556-
r = await self.queries.query_rest(f"/repos/{repo}/stats/contributors")
557-
for author_obj in r:
558-
# Handle malformed response from the API by skipping this repo
559-
if not isinstance(author_obj, dict) or not isinstance(
560-
author_obj.get("author", {}), dict
561-
):
562-
continue
563-
author = author_obj.get("author", {}).get("login", "")
564-
if author.lower() != self.username.lower():
589+
repos = await self.repos
590+
print(f"Calculating lines changed for {len(repos)} repositories...")
591+
592+
for i, repo in enumerate(repos):
593+
try:
594+
print(f"Processing repository {i+1}/{len(repos)}: {repo}")
595+
r = await self.queries.query_rest(f"/repos/{repo}/stats/contributors")
596+
597+
if not r or not isinstance(r, list):
598+
print(f"Invalid response for {repo}: {type(r)}")
565599
continue
600+
601+
for author_obj in r:
602+
# Handle malformed response from the API by skipping this repo
603+
if not isinstance(author_obj, dict) or not isinstance(
604+
author_obj.get("author", {}), dict
605+
):
606+
continue
607+
author = author_obj.get("author", {}).get("login", "")
608+
if author.lower() != self.username.lower():
609+
continue
566610

567-
for week in author_obj.get("weeks", []):
568-
additions += week.get("a", 0)
569-
deletions += week.get("d", 0)
570-
611+
weeks = author_obj.get("weeks", [])
612+
if not isinstance(weeks, list):
613+
continue
614+
615+
for week in weeks:
616+
if isinstance(week, dict):
617+
additions += week.get("a", 0)
618+
deletions += week.get("d", 0)
619+
620+
except Exception as e:
621+
print(f"Error processing {repo}: {e}")
622+
continue
623+
624+
print(f"Total lines: +{additions}, -{deletions}")
571625
self._lines_changed = (additions, deletions)
572626
return self._lines_changed
573627

@@ -581,11 +635,31 @@ async def views(self) -> int:
581635
return self._views
582636

583637
total = 0
584-
for repo in await self.repos:
585-
r = await self.queries.query_rest(f"/repos/{repo}/traffic/views")
586-
for view in r.get("views", []):
587-
total += view.get("count", 0)
588-
638+
repos = await self.repos
639+
print(f"Calculating views for {len(repos)} repositories...")
640+
641+
for i, repo in enumerate(repos):
642+
try:
643+
print(f"Processing views for repository {i+1}/{len(repos)}: {repo}")
644+
r = await self.queries.query_rest(f"/repos/{repo}/traffic/views")
645+
646+
if not r or not isinstance(r, dict):
647+
print(f"Invalid response for {repo}: {type(r)}")
648+
continue
649+
650+
views_data = r.get("views", [])
651+
if not isinstance(views_data, list):
652+
continue
653+
654+
for view in views_data:
655+
if isinstance(view, dict):
656+
total += view.get("count", 0)
657+
658+
except Exception as e:
659+
print(f"Error processing views for {repo}: {e}")
660+
continue
661+
662+
print(f"Total views (last 14 days): {total}")
589663
self._views = total
590664
return total
591665

‎templates/overview.svg‎

Lines changed: 3 additions & 1 deletion
Loading

0 commit comments

Comments
 (0)