Update extract_submissions.py

Author: reginankenchor

.github/scripts/extract_submissions.py

@@ -1,91 +1,68 @@
 import os
+import re
 import csv
 import random
-from datetime import datetime
 from collections import defaultdict
 from github import Github
 from openpyxl import Workbook
 from openpyxl.styles import Alignment, Font
 from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
 
-# Load GitHub issues
 print("📥 Fetching GitHub issues...")
-GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
-GITHUB_REPO = os.environ["GITHUB_REPOSITORY"]
-REPO = Github(GITHUB_TOKEN).get_repo(GITHUB_REPO)
 
-issues = REPO.get_issues(state="all", labels=["closed"])
+# Environment setup
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY")
+g = Github(GITHUB_TOKEN)
+repo = g.get_repo(GITHUB_REPOSITORY)
+
+issues = list(repo.get_issues(state='all', labels=['ambassador']))
+print(f"🔍 Found {len(issues)} total issues")
+
+# Helper to extract field from issue body
+def extract_field(label, body):
+    match = re.search(rf"{label}\s*\n\s*(.+)", body)
+    return match.group(1).strip() if match else ""
+
+# Step 1: Parse issues
 submissions_raw = []
 for issue in issues:
-    if not issue.body or "[Nomination]" not in issue.title:
-        continue
+    body = issue.body
     submission = {
         "Issue #": issue.number,
-        "Nominee Name": "",
-        "Nominee GitHub": "",
-        "Nominee Email": "",
-        "Organization": "",
-        "Location": "",
-        "Nominator Name": "",
-        "Nominator Email": "",
-        "Nominee Contributions": "",
-        "Ambassador Pitch": "",
-        "Additional Info": "",
-        "Created At": issue.created_at.strftime("%Y-%m-%d %H:%M:%S")
+        "Nominee Name": extract_field("Nominee Name", body),
+        "Nominee Email": extract_field("Nominee Email", body),
+        "GitHub Handle": extract_field("GitHub or GitLab Handle", body),
+        "Organization": extract_field("Organization / Affiliation", body),
+        "Location": extract_field("City, State/Province, Country", body),
+        "Contributions": extract_field("How has the nominee contributed to PyTorch?", body),
+        "Ambassador Pitch": extract_field("How Would the Nominee Contribute as an Ambassador?", body),
+        "Extra Notes": extract_field("Any additional details you'd like to share?", body),
     }
-
-    # Extract fields
-    lines = issue.body.splitlines()
-    current_key = ""
-    for line in lines:
-        if "**Nominee Name**" in line:
-            current_key = "Nominee Name"
-        elif "**Nominee Email**" in line:
-            current_key = "Nominee Email"
-        elif "**GitHub or GitLab Handle**" in line:
-            current_key = "Nominee GitHub"
-        elif "**Organization / Affiliation**" in line:
-            current_key = "Organization"
-        elif "**City, State/Province, Country**" in line:
-            current_key = "Location"
-        elif "**Your Name**" in line:
-            current_key = "Nominator Name"
-        elif "**Your Email (Optional)**" in line:
-            current_key = "Nominator Email"
-        elif "**How has the nominee contributed**" in line:
-            current_key = "Nominee Contributions"
-        elif "**How Would the Nominee Contribute as an Ambassador?**" in line:
-            current_key = "Ambassador Pitch"
-        elif "**Any additional details you'd like to share?**" in line:
-            current_key = "Additional Info"
-        elif line.strip() and current_key:
-            submission[current_key] += line.strip() + "\n"
-
     submissions_raw.append(submission)
 
-# Deduplicate by GitHub handle (latest entry kept)
-print("🧹 Deduplicating...")
-seen = {}
+# Step 2: Deduplicate by GitHub handle (keep latest by Issue #)
+submissions_by_handle = {}
 duplicates = []
-for s in sorted(submissions_raw, key=lambda x: x["Created At"]):
-    key = s["Nominee GitHub"].strip().lower()
-    if key in seen:
-        duplicates.append(s)
+for s in sorted(submissions_raw, key=lambda x: x["Issue #"], reverse=True):
+    handle = s.get("GitHub Handle", "").lower()
+    if handle and handle not in submissions_by_handle:
+        submissions_by_handle[handle] = s
     else:
-        seen[key] = s
+        duplicates.append(s)
 
-submissions = list(seen.values())
+submissions = list(submissions_by_handle.values())
+print(f"🧹 Deduplicated to {len(submissions)} unique submissions")
 
-# Save deduplicated CSV
+# Step 3: Write deduplicated CSV
 os.makedirs("ambassador", exist_ok=True)
-csv_path = "ambassador/ambassador_submissions_deduped.csv"
-with open(csv_path, "w", newline="", encoding="utf-8") as f:
-    writer = csv.DictWriter(f, fieldnames=list(submissions[0].keys()))
+with open("ambassador/ambassador_submissions_deduped.csv", "w", newline='', encoding='utf-8') as f:
+    writer = csv.DictWriter(f, fieldnames=submissions[0].keys())
     writer.writeheader()
     writer.writerows(submissions)
 
-# Save duplicates separately
+# Step 4: Save duplicates separately
 if duplicates:
     dup_wb = Workbook()
     ws = dup_wb.active
@@ -94,8 +71,13 @@
     for d in duplicates:
         ws.append([d.get(k, "") for k in ws[1]])
     dup_wb.save("ambassador/duplicates_removed.xlsx")
+    print("⚠️ Duplicates saved to ambassador/duplicates_removed.xlsx")
+
+# Step 5: Generate Reviewer Sheets
+print("📊 Generating reviewer sheets...")
+
+reviewers = [f"Reviewer {i}" for i in range(1, 8)]
 
-# Rubric
 rubric = [
     ("Technical Expertise", "Proficiency with the PyTorch Ecosystem", "Demonstrated knowledge and practical experience with PyTorch, including model building, traininga and deployment?"),
     ("Technical Expertise", "Proficiency with the PyTorch Ecosystem", "Familiarity with foundation-hosted projects, vLLM, DeepSpeed?"),
@@ -121,12 +103,7 @@
     ("Motivation and Vision", "Vision", "Proposed goals or initiatives that align with the mission of the PyTorch Foundation?")
 ]
 
-summary_categories = list(dict.fromkeys(cat for cat, _, _ in rubric))
-reviewers = [f"Reviewer {i}" for i in range(1, 8)]
-output_folder = "ambassador/reviewer_sheets_excel"
-os.makedirs(output_folder, exist_ok=True)
-
-# Assign reviewers evenly
+summary_categories = list({cat for cat, _, _ in rubric})
 assignments = []
 reviewer_counts = defaultdict(int)
 for submission in submissions:
@@ -135,17 +112,18 @@
         reviewer_counts[reviewer] += 1
         assignments.append((submission, reviewer))
 
-# Generate reviewer workbooks
+output_folder = "ambassador/reviewer_sheets_excel"
+os.makedirs(output_folder, exist_ok=True)
+
 for reviewer in reviewers:
     wb = Workbook()
     ws = wb.active
     ws.title = "Review Sheet"
     summary_ws = wb.create_sheet("Score Summary")
 
-    headers = [
-        "Submission ID", "First Name", "Last Name", "Submission Summary",
-        "Reviewer's Comment", "Category", "Subcategory", "Question", "Score"
-    ]
+    # Headers
+    headers = ["Submission ID", "First Name", "Last Name", "Submission Summary",
+               "Reviewer's Comment", "Category", "Subcategory", "Question", "Score"]
     ws.append(headers)
     for col in range(1, len(headers)+1):
         ws.cell(row=1, column=col).font = Font(bold=True)
@@ -164,18 +142,11 @@
         name = submission["Nominee Name"].split()
         fname = name[0]
         lname = name[-1] if len(name) > 1 else ""
+        summary = f"""Contributions:\n{submission.get("Contributions", "")}
 
-        # Submission Summary includes all fields except first 3
-        summary = f"""GitHub: {submission.get("Nominee GitHub", "")}
-Email: {submission.get("Nominee Email", "")}
-Organization: {submission.get("Organization", "")}
-Location: {submission.get("Location", "")}
-Nominator: {submission.get("Nominator Name", "")}
-Nominator Email: {submission.get("Nominator Email", "")}
-
-Contributions:\n{submission.get("Nominee Contributions", "")}
 Ambassador Pitch:\n{submission.get("Ambassador Pitch", "")}
-Additional Info:\n{submission.get("Additional Info", "")}"""
+
+Additional Notes:\n{submission.get("Extra Notes", "")}"""
 
         start = row_idx
         for cat, subcat, question in rubric:
@@ -184,41 +155,35 @@
         end = row_idx - 1
         candidate_ranges.append((sid, fname, lname, start, end))
 
-        for col in [1, 2, 3, 4, 5]:  # Merge ID, First, Last, Summary, Reviewer Comment
+        for col in [1, 2, 3, 4]:
             ws.merge_cells(start_row=start, end_row=end, start_column=col, end_column=col)
-            cell = ws.cell(row=start, column=col)
-            cell.alignment = Alignment(vertical="top", wrap_text=True)
+            ws.cell(row=start, column=col).alignment = Alignment(vertical="top", wrap_text=True)
 
         for r in range(start, end + 1):
             dv.add(ws[f"I{r}"])
 
-    for col in ws.columns:
-        max_len = max((len(str(cell.value)) if cell.value else 0) for cell in col)
-        ws.column_dimensions[get_column_letter(col[0].column)].width = min(max_len + 5, 50)
-
+    # Score summary tab
     summary_ws.append(["Submission ID", "First Name", "Last Name"] + summary_categories + ["Final Score"])
     for col in range(1, summary_ws.max_column + 1):
         summary_ws.cell(row=1, column=col).font = Font(bold=True)
 
     for sid, fname, lname, start, end in candidate_ranges:
-        category_rows = defaultdict(list)
+        cat_rows = defaultdict(list)
         for r in range(start, end + 1):
             cat = ws.cell(row=r, column=6).value
-            category_rows[cat].append(r)
+            cat_rows[cat].append(r)
 
+        row_num = summary_ws.max_row + 1
         formulas = []
         for cat in summary_categories:
-            if cat in category_rows:
-                rows = category_rows[cat]
+            if cat in cat_rows:
+                rows = cat_rows[cat]
                 formulas.append(f'=SUMPRODUCT(--(\'Review Sheet\'!I{rows[0]}:I{rows[-1]}="Yes"))')
             else:
                 formulas.append("0")
-
-        row_number = summary_ws.max_row + 1
-        total_formula = f"=SUM({','.join([f'{get_column_letter(i+4)}{row_number}' for i in range(len(formulas))])})"
+        total_formula = f"=SUM({','.join([f'{get_column_letter(i+4)}{row_num}' for i in range(len(formulas))])})"
         summary_ws.append([sid, fname, lname] + formulas + [total_formula])
 
-    filename = os.path.join(output_folder, f"{reviewer.replace(' ', '_').lower()}_sheet.xlsx")
-    wb.save(filename)
+    wb.save(os.path.join(output_folder, f"{reviewer.replace(' ', '_').lower()}_sheet.xlsx"))
 
-print("✅ All reviewer sheets generated successfully.")
+print("✅ All reviewer sheets and summaries generated.")