Skip to content

Commit b817617

Browse files
jasonrandrewsjasonrandrews
committed
Python script to generate a monthly report or multi-month report of Learning Paths published from the GitHub project.
1 parent 59f4931 commit b817617

File tree

2 files changed

+358
-0
lines changed

2 files changed

+358
-0
lines changed

‎tools/generate_monthly_report.py‎

Lines changed: 356 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,356 @@
1+
import requests
2+
import datetime
3+
import os
4+
import json
5+
from bs4 import BeautifulSoup
6+
7+
# GitHub API settings
8+
GITHUB_API_URL = "https://api.github.com/graphql"
9+
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
10+
if not GITHUB_TOKEN:
11+
# Fallback for GitHub Actions: use secrets.GITHUB_TOKEN if available
12+
GITHUB_TOKEN = os.getenv("ACTIONS_GITHUB_TOKEN")
13+
14+
# GitHub project board settings
15+
ORGANIZATION = "ArmDeveloperEcosystem"
16+
PROJECT_NUMBER = 4
17+
18+
# Update the GraphQL query to handle all possible types of ProjectV2ItemFieldValue
19+
GRAPHQL_QUERY = """
20+
query ($org: String!, $projectNumber: Int!) {
21+
organization(login: $org) {
22+
projectV2(number: $projectNumber) {
23+
title
24+
items(first: 100) {
25+
nodes {
26+
content {
27+
... on Issue {
28+
title
29+
url
30+
labels(first: 10) {
31+
nodes {
32+
name
33+
}
34+
}
35+
}
36+
... on PullRequest {
37+
title
38+
url
39+
labels(first: 10) {
40+
nodes {
41+
name
42+
}
43+
}
44+
}
45+
}
46+
fieldValues(first: 10) {
47+
nodes {
48+
... on ProjectV2ItemFieldSingleSelectValue {
49+
field {
50+
... on ProjectV2SingleSelectField { name }
51+
}
52+
name
53+
}
54+
... on ProjectV2ItemFieldTextValue {
55+
field {
56+
... on ProjectV2FieldCommon { name }
57+
}
58+
text
59+
}
60+
... on ProjectV2ItemFieldNumberValue {
61+
field {
62+
... on ProjectV2FieldCommon { name }
63+
}
64+
number
65+
}
66+
... on ProjectV2ItemFieldDateValue {
67+
field {
68+
... on ProjectV2FieldCommon { name }
69+
}
70+
date
71+
}
72+
}
73+
}
74+
}
75+
}
76+
}
77+
}
78+
}
79+
"""
80+
81+
def run_graphql_query(query, variables):
82+
"""
83+
Executes a GraphQL query against the GitHub API.
84+
"""
85+
if not GITHUB_TOKEN:
86+
raise Exception("GitHub token not found. Please set the GITHUB_TOKEN environment variable.")
87+
headers = {
88+
"Authorization": f"Bearer {GITHUB_TOKEN}",
89+
"Content-Type": "application/json",
90+
}
91+
response = requests.post(
92+
GITHUB_API_URL,
93+
json={"query": query, "variables": variables},
94+
headers=headers,
95+
)
96+
if response.status_code != 200:
97+
raise Exception(f"Query failed with status code {response.status_code}: {response.text}")
98+
99+
if "errors" in response.json():
100+
raise Exception(f"GraphQL query returned errors: {response.json()['errors']}")
101+
102+
return response.json()
103+
104+
# Add a function to fetch items in the Done column
105+
def fetch_done_items(month_filter=None, month_range=None):
106+
variables = {
107+
"org": ORGANIZATION,
108+
"projectNumber": PROJECT_NUMBER,
109+
}
110+
data = run_graphql_query(GRAPHQL_QUERY, variables)
111+
project = data["data"]["organization"]["projectV2"]
112+
done_items = []
113+
114+
def in_month_range(publish_date):
115+
if not publish_date or not month_range:
116+
return False
117+
try:
118+
pub_dt = datetime.datetime.strptime(publish_date, "%Y-%m-%d")
119+
return month_range[0] <= pub_dt <= month_range[1]
120+
except Exception:
121+
return False
122+
123+
for item in project["items"]["nodes"]:
124+
published_url = None
125+
start_date = None
126+
publish_date = None
127+
acm_label = False
128+
# Check for ACM label in content labels
129+
content = item.get("content", {})
130+
labels = []
131+
if content and "labels" in content and content["labels"] and "nodes" in content["labels"]:
132+
labels = [label["name"] for label in content["labels"]["nodes"] if "name" in label]
133+
if "ACM" in labels:
134+
acm_label = True
135+
# Check for ACM label in each node's fieldValues
136+
for field in item["fieldValues"]["nodes"]:
137+
if (
138+
"field" in field and field["field"]
139+
and field["field"].get("name") == "Status"
140+
and (field.get("name") == "Done" or field.get("value") == "Done")
141+
):
142+
# Find the Published URL, Start Date, Publish Date, and ACM fields
143+
for f in item["fieldValues"]["nodes"]:
144+
if (
145+
"field" in f and f["field"]
146+
and f["field"].get("name") == "Published URL"
147+
and f.get("text")
148+
):
149+
published_url = f["text"]
150+
if (
151+
"field" in f and f["field"]
152+
and f["field"].get("name") == "Start Date"
153+
and f.get("date")
154+
):
155+
start_date = f["date"]
156+
if (
157+
"field" in f and f["field"]
158+
and f["field"].get("name") == "Publish Date"
159+
and f.get("date")
160+
):
161+
publish_date = f["date"]
162+
# Filter by month or month_range if set
163+
if month_range:
164+
if not in_month_range(publish_date):
165+
continue
166+
elif month_filter:
167+
if not publish_date or not publish_date.startswith(month_filter):
168+
continue
169+
done_items.append({
170+
"title": item["content"]["title"],
171+
"published_url": published_url,
172+
"start_date": start_date,
173+
"publish_date": publish_date,
174+
"acm_label": acm_label
175+
})
176+
177+
return done_items
178+
179+
# Update the generate_report function to print Done items
180+
def generate_report(month_filter=None, month_range=None):
181+
variables = {
182+
"org": ORGANIZATION,
183+
"projectNumber": PROJECT_NUMBER,
184+
}
185+
data = run_graphql_query(GRAPHQL_QUERY, variables)
186+
project = data["data"]["organization"]["projectV2"]
187+
188+
# Generate the report
189+
if month_range:
190+
report_date = f"{month_range[0].strftime('%B %Y')} - {month_range[1].strftime('%B %Y')}"
191+
elif month_filter:
192+
report_month = datetime.datetime.strptime(month_filter, "%Y-%m")
193+
report_date = report_month.strftime("%B %Y")
194+
else:
195+
today = datetime.date.today()
196+
report_date = today.strftime("%B %Y")
197+
month_filter = today.strftime("%Y-%m")
198+
199+
print(f"## Learning Path Monthly Report for {report_date}\n")
200+
201+
# Planned Learning Paths Table
202+
print("## Planned Learning Paths\n| Title | ACM | Created Date |")
203+
print("|-------|-----|--------------|")
204+
open_issues = fetch_open_issues()
205+
planned_count = 0
206+
for issue in open_issues:
207+
acm_label = "ACM" if "ACM" in [label["name"] for label in issue.get("labels", [])] else ""
208+
created_date = datetime.datetime.strptime(issue.get("created_at", ""), "%Y-%m-%dT%H:%M:%SZ").strftime("%B %d, %Y")
209+
print(f"| [{issue['title']}]({issue['html_url']}) | {acm_label} | {created_date} |")
210+
planned_count += 1
211+
print(f"\nTotal planned learning paths: {planned_count}\n")
212+
213+
# Fetch and print Done items for the given month or range
214+
done_items = fetch_done_items(month_filter=month_filter, month_range=month_range)
215+
print("\n\n## Published Learning Paths\n| Title | Published URL | Start Date | Publish Date | Time to Publish (days) | ACM |")
216+
print("|-------|--------------|------------|-------------|----------------------|-----|")
217+
published_count = 0
218+
time_to_publish_values = []
219+
for item in done_items:
220+
html_title = get_html_title(item['published_url']) if item['published_url'] else ''
221+
if html_title and html_title.endswith(' | Arm Learning Paths'):
222+
html_title = html_title[:-len(' | Arm Learning Paths')]
223+
title_link = f"[{html_title}]({item['published_url']})" if item['published_url'] else html_title
224+
# Format start and publish dates
225+
formatted_start_date = ''
226+
formatted_publish_date = ''
227+
if item['start_date']:
228+
try:
229+
formatted_start_date = datetime.datetime.strptime(item['start_date'], "%Y-%m-%d").strftime("%B %d, %Y")
230+
except Exception:
231+
formatted_start_date = item['start_date']
232+
if item['publish_date']:
233+
try:
234+
formatted_publish_date = datetime.datetime.strptime(item['publish_date'], "%Y-%m-%d").strftime("%B %d, %Y")
235+
except Exception:
236+
formatted_publish_date = item['publish_date']
237+
# Calculate time to publish in days
238+
time_to_publish = ''
239+
if item['start_date'] and item['publish_date']:
240+
try:
241+
start_dt = datetime.datetime.strptime(item['start_date'], "%Y-%m-%d")
242+
publish_dt = datetime.datetime.strptime(item['publish_date'], "%Y-%m-%d")
243+
time_to_publish = (publish_dt - start_dt).days
244+
time_to_publish_values.append(time_to_publish)
245+
except Exception:
246+
time_to_publish = ''
247+
acm_col = "ACM" if item.get('acm_label') else ""
248+
print(f"| {title_link} | {html_title} | {formatted_start_date} | {formatted_publish_date} | {time_to_publish} | {acm_col} |")
249+
published_count += 1
250+
251+
print("\n| Statistic | Value |\n|-----------|-------|")
252+
print(f"| Number of Learning Paths published | {published_count} |")
253+
acm_count = sum(1 for item in done_items if item.get('acm_label'))
254+
print(f"| Number of ACM Learning Paths published | {acm_count} |")
255+
if time_to_publish_values:
256+
avg_time = sum(time_to_publish_values) / len(time_to_publish_values)
257+
max_time = max(time_to_publish_values)
258+
print(f"| Average time to publish (days) | {avg_time:.1f} |")
259+
print(f"| Longest time to publish (days) | {max_time} |")
260+
else:
261+
print(f"| Average time to publish (days) | N/A |")
262+
print(f"| Longest time to publish (days) | N/A |")
263+
print("")
264+
265+
print(f"\n_Report generated on {datetime.datetime.now().astimezone().strftime('%B %d, %Y at %H:%M:%S %Z')}_\n")
266+
267+
def fetch_open_issues():
268+
url = "https://api.github.com/repos/ArmDeveloperEcosystem/roadmap/issues"
269+
headers = {
270+
"Authorization": f"Bearer {GITHUB_TOKEN}",
271+
"Accept": "application/vnd.github.v3+json",
272+
}
273+
response = requests.get(url, headers=headers)
274+
if response.status_code != 200:
275+
raise Exception(f"Failed to fetch issues: {response.status_code} {response.text}")
276+
277+
issues = response.json()
278+
open_issues = [issue for issue in issues if issue.get("state") == "open"]
279+
return open_issues
280+
281+
def get_html_title(url):
282+
try:
283+
response = requests.get(url)
284+
response.raise_for_status()
285+
soup = BeautifulSoup(response.text, 'html.parser')
286+
title_tag = soup.find('title')
287+
if title_tag is not None and title_tag.string:
288+
return title_tag.string.strip()
289+
else:
290+
return None
291+
except Exception as e:
292+
print(f"Error: {e}")
293+
return None
294+
295+
if __name__ == "__main__":
296+
import sys
297+
import argparse
298+
import contextlib
299+
300+
parser = argparse.ArgumentParser(description="Generate Learning Path monthly report.")
301+
parser.add_argument("--month", type=str, help="Month to generate report for (format: YYYY-MM). Defaults to current month.")
302+
parser.add_argument("--month-range", nargs=2, metavar=('START', 'END'), help="Range of months to generate report for (format: YYYY-MM YYYY-MM).")
303+
args = parser.parse_args()
304+
305+
month_filter = None
306+
month_range = None
307+
308+
if args.month_range:
309+
try:
310+
start = datetime.datetime.strptime(args.month_range[0], "%Y-%m")
311+
end = datetime.datetime.strptime(args.month_range[1], "%Y-%m")
312+
# Use the first day of start month and last day of end month
313+
start_dt = start.replace(day=1)
314+
# Get last day of end month
315+
next_month = (end.replace(day=28) + datetime.timedelta(days=4)).replace(day=1)
316+
end_dt = next_month - datetime.timedelta(days=1)
317+
if start_dt > end_dt:
318+
print("Start month must be before or equal to end month.")
319+
sys.exit(1)
320+
month_range = (start_dt, end_dt)
321+
except ValueError:
322+
print("Invalid month-range format. Use YYYY-MM YYYY-MM.")
323+
sys.exit(1)
324+
elif args.month:
325+
try:
326+
datetime.datetime.strptime(args.month, "%Y-%m")
327+
month_filter = args.month
328+
except ValueError:
329+
print("Invalid month format. Use YYYY-MM.")
330+
sys.exit(1)
331+
else:
332+
month_filter = datetime.date.today().strftime("%Y-%m")
333+
334+
if month_range:
335+
output_filename = f"LP-report-{month_range[0].strftime('%Y-%m')}_to_{month_range[1].strftime('%Y-%m')}.md"
336+
else:
337+
report_month = datetime.datetime.strptime(month_filter, "%Y-%m")
338+
output_filename = f"LP-report-{report_month.strftime('%Y-%m')}.md"
339+
340+
with open(output_filename, "w") as f:
341+
with contextlib.ExitStack() as stack:
342+
stack.enter_context(contextlib.redirect_stdout(f))
343+
# Also print to original stdout
344+
class Tee:
345+
def __init__(self, *files):
346+
self.files = files
347+
def write(self, obj):
348+
for file in self.files:
349+
file.write(obj)
350+
def flush(self):
351+
for file in self.files:
352+
file.flush()
353+
tee = Tee(f, sys.__stdout__)
354+
with contextlib.redirect_stdout(tee):
355+
generate_report(month_filter=month_filter, month_range=month_range)
356+
print(f"Report written to {output_filename}")

‎tools/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,5 @@ pyspellchecker
55
better-profanity
66
setuptools
77
alive-progress
8+
requests
9+
BeautifulSoup4

0 commit comments

Comments
 (0)