Update extract_submissions.py

Author: reginankenchor

.github/scripts/extract_submissions.py

@@ -1,18 +1,76 @@
 import os
-import csv
+import re
 import random
+import requests
 from collections import defaultdict
+from datetime import datetime
 from openpyxl import Workbook
 from openpyxl.styles import Alignment, Font
 from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
 
-# Load deduplicated submissions
-with open("ambassador/ambassador_submissions_deduped.csv", newline='', encoding='utf-8') as f:
-    reader = csv.DictReader(f)
-    submissions = list(reader)
-
-# Define reviewers
+# Set your GitHub repo details
+REPO = "pytorch-fdn/ambassador-program"
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+HEADERS = {"Authorization": f"Bearer {GITHUB_TOKEN}"}
+API_URL = f"https://api.github.com/repos/{REPO}/issues?state=all&labels=closed&per_page=100"
+
+# Output directories
+os.makedirs("ambassador/reviewer_sheets_excel", exist_ok=True)
+
+# Helper to extract structured data from the issue body
+def extract_submission(issue):
+    body = issue["body"]
+    def extract(label):  # Flexible line extractor
+        pattern = rf"\*\*{re.escape(label)}\*\*\s*\n([\s\S]*?)(?:\n\*\*|$)"
+        match = re.search(pattern, body, re.IGNORECASE)
+        return match.group(1).strip() if match else ""
+
+    return {
+        "Issue #": str(issue["number"]),
+        "Nominee Name": extract("Nominee Name"),
+        "Nominee Email": extract("Nominee Email"),
+        "GitHub Handle": extract("Nominee's GitHub or GitLab Handle"),
+        "Organization": extract("Organization / Affiliation"),
+        "Location": extract("City, State/Province, Country"),
+        "Nominator Name": extract("Your Name"),
+        "Nominator Email": extract("Your Email"),
+        "Contributions": extract("How has the nominee contributed to PyTorch?"),
+        "Ambassador Pitch": extract("How Would the Nominee Contribute as an Ambassador?"),
+        "Extra Notes": extract("Any additional details you'd like to share?"),
+        "Created At": issue["created_at"]
+    }
+
+# Step 1: Fetch and parse issues
+print("📥 Fetching GitHub issues...")
+all_issues = []
+page = 1
+while True:
+    response = requests.get(f"{API_URL}&page={page}", headers=HEADERS)
+    data = response.json()
+    if not data or "message" in data:
+        break
+    all_issues.extend(data)
+    page += 1
+
+submissions_raw = [extract_submission(issue) for issue in all_issues if "Nominee Name" in issue["body"]]
+
+# Step 2: Deduplicate by nominee name, keeping latest
+print("🧹 Deduplicating...")
+deduped, duplicates = {}, []
+for sub in submissions_raw:
+    key = sub["Nominee Name"].strip().lower()
+    dt = datetime.strptime(sub["Created At"], "%Y-%m-%dT%H:%M:%SZ")
+    if key not in deduped or dt > datetime.strptime(deduped[key]["Created At"], "%Y-%m-%dT%H:%M:%SZ"):
+        if key in deduped:
+            duplicates.append(deduped[key])
+        deduped[key] = sub
+    else:
+        duplicates.append(sub)
+
+submissions = list(deduped.values())
+
+# Step 3: Reviewer logic
 reviewers = [f"Reviewer {i}" for i in range(1, 8)]
 
 # Updated rubric including all categories from the latest file
@@ -46,109 +104,111 @@
     ("Credibility", "Community References", "References from other known community members?")
 ]
 
-# Dynamically detect unique rubric categories in order
 summary_categories = []
 for cat, _, _ in rubric:
     if cat not in summary_categories:
         summary_categories.append(cat)
 
-# Output directory
-output_folder = "ambassador/reviewer_sheets_excel"
-os.makedirs(output_folder, exist_ok=True)
-
-# Assign reviewers evenly
 assignments = []
 reviewer_counts = defaultdict(int)
-for submission in submissions:
+for sub in submissions:
     assigned = random.sample(sorted(reviewers, key=lambda r: reviewer_counts[r])[:4], 2)
-    for reviewer in assigned:
-        reviewer_counts[reviewer] += 1
-        assignments.append((submission, reviewer))
+    for r in assigned:
+        reviewer_counts[r] += 1
+        assignments.append((sub, r))
 
-# Generate Excel files per reviewer
+# Step 4: Generate reviewer sheets
 for reviewer in reviewers:
     wb = Workbook()
     ws = wb.active
     ws.title = "Review Sheet"
     summary_ws = wb.create_sheet("Score Summary")
 
-    # Review Sheet headers
     headers = [
         "Submission ID", "First Name", "Last Name", "Submission Summary",
         "Reviewer's Comment", "Category", "Subcategory", "Question", "Score"
     ]
     ws.append(headers)
-    for col in range(1, len(headers)+1):
-        ws.cell(row=1, column=col).font = Font(bold=True)
+    for c in range(1, len(headers)+1):
+        ws.cell(row=1, column=c).font = Font(bold=True)
 
-    # Add dropdown
     dv = DataValidation(type="list", formula1='"Yes,No,N/A"', allow_blank=True)
     ws.add_data_validation(dv)
 
     row_idx = 2
-    candidate_ranges = []
+    ranges = []
 
-    for submission, assigned_reviewer in assignments:
-        if assigned_reviewer != reviewer:
+    for sub, r in assignments:
+        if r != reviewer:
             continue
+        sid = sub["Issue #"]
+        name_parts = sub["Nominee Name"].split()
+        fname = name_parts[0]
+        lname = name_parts[-1] if len(name_parts) > 1 else ""
+        summary = f"""
+GitHub: {sub.get("GitHub Handle", "")}
+Org: {sub.get("Organization", "")}
+Location: {sub.get("Location", "")}
 
-        sid = submission["Issue #"]
-        name = submission["Nominee Name"].split()
-        fname = name[0]
-        lname = name[-1] if len(name) > 1 else ""
-        summary = f"""Contributions:\n{submission.get("Contributions", "")}
+Contributions:
+{sub.get("Contributions", "")}
 
-Ambassador Pitch:\n{submission.get("Ambassador Pitch", "")}
+Ambassador Pitch:
+{sub.get("Ambassador Pitch", "")}
 
-Additional Notes:\n{submission.get("Extra Notes", "")}"""
+Additional Info:
+{sub.get("Extra Notes", "")}
+""".strip()
 
         start = row_idx
         for cat, subcat, question in rubric:
             ws.append([sid, fname, lname, summary, "", cat, subcat, question, ""])
             row_idx += 1
         end = row_idx - 1
-        candidate_ranges.append((sid, fname, lname, start, end))
+        ranges.append((sid, fname, lname, start, end))
 
-        # Merge ID/name cells
-        for col in [1, 2, 3, 4]:
+        for col in [1, 2, 3, 4, 5]:  # Merge key fields
             ws.merge_cells(start_row=start, end_row=end, start_column=col, end_column=col)
-            cell = ws.cell(row=start, column=col)
-            cell.alignment = Alignment(vertical="top", wrap_text=True)
-
-        for r in range(start, end + 1):
+            ws.cell(row=start, column=col).alignment = Alignment(vertical="top", wrap_text=True)
+        for r in range(start, end+1):
             dv.add(ws[f"I{r}"])
 
     # Autofit columns
     for col in ws.columns:
-        max_len = max((len(str(cell.value)) if cell.value else 0) for cell in col)
-        ws.column_dimensions[get_column_letter(col[0].column)].width = min(max_len + 5, 50)
+        max_len = max((len(str(c.value)) if c.value else 0) for c in col)
+        ws.column_dimensions[get_column_letter(col[0].column)].width = min(max_len + 5, 60)
 
-    # Score Summary header
+    # Score Summary
     summary_ws.append(["Submission ID", "First Name", "Last Name"] + summary_categories + ["Final Score"])
     for col in range(1, summary_ws.max_column + 1):
         summary_ws.cell(row=1, column=col).font = Font(bold=True)
 
-    # Fill score summary
-    for sid, fname, lname, start, end in candidate_ranges:
-        category_rows = defaultdict(list)
+    for sid, fname, lname, start, end in ranges:
+        cat_rows = defaultdict(list)
         for r in range(start, end + 1):
             cat = ws.cell(row=r, column=6).value
-            category_rows[cat].append(r)
+            cat_rows[cat].append(r)
 
         formulas = []
         for cat in summary_categories:
-            if cat in category_rows:
-                rows = category_rows[cat]
+            if cat in cat_rows:
+                rows = cat_rows[cat]
                 formulas.append(f'=SUMPRODUCT(--(\'Review Sheet\'!I{rows[0]}:I{rows[-1]}="Yes"))')
             else:
                 formulas.append("0")
-
         row_number = summary_ws.max_row + 1
-        total_formula = f"=SUM({','.join([f'{get_column_letter(i+4)}{row_number}' for i in range(len(formulas))])})"
-        summary_ws.append([sid, fname, lname] + formulas + [total_formula])
+        final_formula = f"=SUM({','.join([f'{get_column_letter(i+4)}{row_number}' for i in range(len(formulas))])})"
+        summary_ws.append([sid, fname, lname] + formulas + [final_formula])
+
+    wb.save(f"ambassador/reviewer_sheets_excel/{reviewer.replace(' ', '_').lower()}_sheet.xlsx")
 
-    # Save
-    filename = os.path.join(output_folder, f"{reviewer.replace(' ', '_').lower()}_sheet.xlsx")
-    wb.save(filename)
+# Step 5: Save duplicates separately
+dup_wb = Workbook()
+ws = dup_wb.active
+ws.title = "Duplicates Removed"
+ws.append(list(duplicates[0].keys()))
+for d in duplicates:
+    ws.append([d.get(k, "") for k in ws[1]])
+dup_wb.save("ambassador/duplicates_removed.xlsx")
 
-print("✅ Reviewer sheets generated with updated rubric and corrected score summary.")
+print("✅ All reviewer sheets and duplicates file generated.")