improvements: cacheing and improved logging

Signed-off-by: Paul Jickling <paul.jickling@ethereum.org>

Author: pauljickling

tracker.py

@@ -2,7 +2,6 @@
 import logging
 import os
 import re
-import sys
 from datetime import datetime, timezone
 
 import requests
@@ -11,6 +10,8 @@
 logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 log = logging.getLogger(__name__)
 
+CACHE_VERSION = 1
+
 
 def parse_section(body: str, section: str) -> str:
     """Extract text under a ## section header, stripping HTML comments."""
@@ -25,7 +26,6 @@ def parse_section(body: str, section: str) -> str:
 def parse_duration(body: str, year_plus_months: int) -> tuple[str, relativedelta | None]:
     """Return (raw_value, expiry_offset). Offset is None for uncertain/indefinite."""
     raw = parse_section(body, "Resource Duration").lower()
-
     if "less than 6 months" in raw:
         return "Less than 6 months", relativedelta(months=6)
     if "6 months to a year" in raw:
@@ -34,12 +34,11 @@ def parse_duration(body: str, year_plus_months: int) -> tuple[str, relativedelta
         return "year+", relativedelta(months=year_plus_months)
     if "uncertain" in raw or "indefinite" in raw:
         return "uncertain/indefinite", None
-
     return f"unparseable: {raw[:80]}", None
 
 
-def fetch_github_issues(repo: str, token: str) -> list[dict]:
-    """Fetch all open non-PR issues from the repo, handling pagination."""
+def fetch_github_issues(repo: str, token: str, since: str | None = None) -> list[dict]:
+    """Fetch all non-PR issues (open and closed), optionally filtered by update time."""
     headers = {
         "Authorization": f"Bearer {token}",
         "Accept": "application/vnd.github+json",
@@ -48,10 +47,13 @@ def fetch_github_issues(repo: str, token: str) -> list[dict]:
     issues = []
     page = 1
     while True:
+        params = {"state": "all", "per_page": 100, "page": page}
+        if since:
+            params["since"] = since
         resp = requests.get(
             f"https://api.github.com/repos/{repo}/issues",
             headers=headers,
-            params={"state": "open", "per_page": 100, "page": page},
+            params=params,
             timeout=30,
         )
         resp.raise_for_status()
@@ -69,74 +71,148 @@ def check_netbox(project_name: str, netbox_url: str, token: str) -> tuple[bool,
         return False, None
     headers = {"Authorization": f"Token {token}"}
     for endpoint in ("virtualization/virtual-machines", "dcim/devices"):
-        resp = requests.get(
-            f"{netbox_url}/api/{endpoint}/",
-            headers=headers,
-            params={"name": project_name, "limit": 1},
-            timeout=30,
-        )
-        if resp.status_code == 200:
-            results = resp.json().get("results", [])
-            if results:
-                return True, results[0]["name"]
+        try:
+            resp = requests.get(
+                f"{netbox_url}/api/{endpoint}/",
+                headers=headers,
+                params={"name": project_name, "limit": 1},
+                timeout=30,
+            )
+            if resp.status_code == 200:
+                results = resp.json().get("results", [])
+                if results:
+                    return True, results[0]["name"]
+        except requests.exceptions.RequestException as e:
+            log.warning(f"Netbox request failed for '{project_name}' on {endpoint}: {e}")
     return False, None
 
 
+def load_cache(path: str) -> dict:
+    if os.path.exists(path):
+        try:
+            with open(path) as f:
+                return json.load(f)
+        except (json.JSONDecodeError, OSError) as e:
+            log.warning(f"Cache unreadable at {path}, starting fresh: {e}")
+    return {"version": CACHE_VERSION, "last_run": None, "issues": {}}
+
+
+def save_cache(cache: dict, path: str) -> None:
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(cache, f, indent=2)
+
+
+def compute_expiry(entry: dict, now: datetime, year_plus_months: int) -> None:
+    """Recompute lifecycle_status, expiry_date, and expired_days_ago in place."""
+    duration = entry.get("duration_stated", "").lower()
+    try:
+        created_at = datetime.fromisoformat(entry["created_at"]).replace(tzinfo=timezone.utc)
+    except (ValueError, KeyError) as e:
+        log.warning(f"Invalid created_at for issue #{entry.get('issue_number')}: {e}")
+        entry["lifecycle_status"] = "unknown"
+        return
+
+    if "less than 6 months" in duration:
+        offset = relativedelta(months=6)
+    elif "6 months to a year" in duration:
+        offset = relativedelta(months=12)
+    elif "year+" in duration:
+        offset = relativedelta(months=year_plus_months)
+    else:
+        entry["lifecycle_status"] = "no_expiry"
+        entry.pop("expiry_date", None)
+        entry.pop("expired_days_ago", None)
+        return
+
+    expiry = created_at + offset
+    days = (now - expiry).days
+    entry["expiry_date"] = expiry.date().isoformat()
+    entry["expired_days_ago"] = days
+    entry["lifecycle_status"] = "expired" if days >= 0 else "active"
+
+
+def process_issue(issue: dict, year_plus_months: int) -> dict | None:
+    """Parse a GitHub issue into a cache entry. Returns None if not a VM request."""
+    body = issue.get("body") or ""
+    if "## Resource Duration" not in body:
+        return None
+    try:
+        created_at = datetime.fromisoformat(issue["created_at"].replace("Z", "+00:00"))
+    except ValueError as e:
+        log.warning(f"Issue #{issue['number']} has unparseable created_at: {e}")
+        return None
+
+    duration_raw, _ = parse_duration(body, year_plus_months)
+    return {
+        "issue_number": issue["number"],
+        "issue_url": issue["html_url"],
+        "project_name": parse_section(body, "Project Name"),
+        "team": parse_section(body, "Team Owner"),
+        "contact": parse_section(body, "Team Contact"),
+        "created_at": created_at.date().isoformat(),
+        "duration_stated": duration_raw,
+    }
+
+
 def main() -> None:
     github_token = os.environ["GITHUB_TOKEN"]
     netbox_token = os.environ["NETBOX_TOKEN"]
     github_repo = os.environ["GITHUB_REPO"]
     netbox_url = os.environ.get("NETBOX_URL", "https://netbox.ethquokkaops.io").rstrip("/")
     year_plus_months = int(os.environ.get("THRESHOLD_YEAR_PLUS_MONTHS", "18"))
+    cache_file = os.environ.get("CACHE_FILE", "/data/cache.json")
 
     now = datetime.now(timezone.utc)
-
-    log.info(f"Fetching issues from {github_repo}")
-    issues = fetch_github_issues(github_repo, github_token)
-    log.info(f"Found {len(issues)} open issues")
-
-    expired = []
-    no_expiry = []
-
-    for issue in issues:
-        body = issue.get("body") or ""
-        if "## Resource Duration" not in body:
-            continue
-
-        created_at = datetime.fromisoformat(issue["created_at"].replace("Z", "+00:00"))
-        project_name = parse_section(body, "Project Name")
-        duration_raw, offset = parse_duration(body, year_plus_months)
-
-        netbox_match, netbox_vm = check_netbox(project_name, netbox_url, netbox_token)
-
-        entry = {
-            "issue_number": issue["number"],
-            "issue_url": issue["html_url"],
-            "project_name": project_name,
-            "team": parse_section(body, "Team Owner"),
-            "contact": parse_section(body, "Team Contact"),
-            "created_at": created_at.date().isoformat(),
-            "duration_stated": duration_raw,
-            "netbox_match": netbox_match,
-            "netbox_vm": netbox_vm,
-        }
-
-        if offset is None:
+    cache = load_cache(cache_file)
+    last_run = cache.get("last_run")
+
+    # Step 1: fetch new and updated issues from GitHub since last run
+    log.info(f"Fetching issues from {github_repo} (since {last_run or 'beginning'})")
+    try:
+        updated_issues = fetch_github_issues(github_repo, github_token, since=last_run)
+    except requests.exceptions.RequestException as e:
+        log.error(f"GitHub API request failed: {e}")
+        raise SystemExit(1)
+    log.info(f"Fetched {len(updated_issues)} new/updated issues")
+
+    for issue in updated_issues:
+        entry = process_issue(issue, year_plus_months)
+        if entry:
+            cache["issues"][str(issue["number"])] = entry
+
+    # Step 2: recheck Netbox and recompute expiry for all cached issues
+    log.info(f"Rechecking {len(cache['issues'])} cached issues")
+    expired, no_expiry, not_found = [], [], []
+
+    for entry in cache["issues"].values():
+        netbox_match, netbox_vm = check_netbox(entry["project_name"], netbox_url, netbox_token)
+        entry["netbox_match"] = netbox_match
+        entry["netbox_vm"] = netbox_vm
+        entry["last_checked"] = now.isoformat()
+
+        compute_expiry(entry, now, year_plus_months)
+
+        status = entry.get("lifecycle_status")
+        if not netbox_match:
+            not_found.append(entry)
+        elif status == "no_expiry":
             no_expiry.append(entry)
-            continue
-
-        expiry = created_at + offset
-        entry["expiry_date"] = expiry.date().isoformat()
-        entry["expired_days_ago"] = (now - expiry).days
-
-        if entry["expired_days_ago"] >= 0:
+        elif status == "expired":
             expired.append(entry)
 
-    expired.sort(key=lambda e: e["expired_days_ago"], reverse=True)
+    expired.sort(key=lambda e: e.get("expired_days_ago", 0), reverse=True)
 
-    log.info(f"Expired: {len(expired)}, no expiry set: {len(no_expiry)}")
+    cache["last_run"] = now.isoformat()
+    save_cache(cache, cache_file)
 
-    print(json.dumps({"generated_at": now.isoformat(), "expired": expired, "no_expiry": no_expiry}, indent=2))
+    log.info(f"Expired: {len(expired)}, no expiry: {len(no_expiry)}, not in Netbox: {len(not_found)}")
+    print(json.dumps({
+        "generated_at": now.isoformat(),
+        "expired": expired,
+        "no_expiry": no_expiry,
+        "not_found": not_found,
+    }, indent=2))
 
 
 if __name__ == "__main__":