Merge pull request #101 from Carreau/more-activity

Misc updates (manual activity dates for users)

Author: Carreau

last_user_activity.json

@@ -0,0 +1,6 @@
+# User : inviter : manual activity YYYY-MM : note
+JamiesHQ:fperez :2016-01:Hired admin at BIDS
+
+jhamrick:carreau:2016-01:Wrote nbgrader and was a regular member of the team at bids.
+mpacer  :carreau:2016-01:was a regular member of the team at bids on nbconvert
+

tools/all_repos.py

@@ -52,7 +52,7 @@ def get_packages(url):
 
 
 default_orgs = [
-    "binder-examples",
+    # "binder-examples",
     "binderhub-ci-repos",
     "ipython",
     "jupyter",
@@ -66,7 +66,7 @@ def get_packages(url):
     "jupyter-standards",
     "jupyter-widgets",
     "jupyter-xeus",
-    "jupytercon",
+    # "jupytercon",
     "jupyterhub",
     "jupyterlab",
     "voila-dashboards",

tools/last_user_activity.py

@@ -4,18 +4,20 @@
 It implements disk-based caching to minimize API requests and respect rate limits.
 """
 
+import argparse
+import asyncio
 import os
+import pathlib
 import sys
-import asyncio
-import aiohttp
-from rich import print
-from datetime import datetime, timezone, timedelta
-import humanize
+from datetime import datetime, timedelta, timezone
 from itertools import count
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import aiohttp
 import diskcache
-import pathlib
-from typing import Optional, List, Dict
-import argparse
+import humanize
+from rich import print
 
 default_orgs = [
     "binder-examples",
@@ -81,6 +83,7 @@ def get(self, key, default=None, retry=False):
 
 
 # Configure DiskCache in the current directory
+# todo: auto clear after ~24 hours
 CACHE_DIR = "github_cache"
 cache = DateTimeCache(CACHE_DIR)
 
@@ -247,12 +250,34 @@ async def check_user_admin(
         return (await response.json())["role"] == "admin"
 
 
-async def main(orgs, debug: bool, timelimit_days: int):
+async def main(orgs, debug: bool, timelimit_days: int, config_file: str):
     """Main execution function."""
     # Show cache status
     print(f"[blue]Cache directory: {CACHE_DIR} (size: {get_cache_size()})[/blue]")
     print(f"[blue]Cache contains {len(cache)} items[/blue]")
-
+    now = datetime.now(timezone.utc)
+    manual_users = {}
+    for line in Path(config_file).read_text().splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        user, inviter, date_last_activity, reason = [x.strip() for x in line.split(":")]
+        manual_users[user] = {
+            "manual_last_activity": datetime.strptime(
+                date_last_activity, "%Y-%m"
+            ).replace(tzinfo=timezone.utc),
+            "last_activity_reason": reason,
+            "inviter": inviter,
+        }
+        if not manual_users[user]["last_activity_reason"]:
+            print(
+                f"[yellow]Warning: No last_activity_reason for {user['username']}, skipping[/yellow]"
+            )
+            continue
+        if now < manual_users[user]["manual_last_activity"]:
+            print(
+                f"[red]Warning: manual_last_activity for {user['username']} is in the future, skipping[/red]"
+            )
+            continue
     # check who the current user is
     async with aiohttp.ClientSession() as session:
         async with session.get(
@@ -296,7 +321,7 @@ async def main(orgs, debug: bool, timelimit_days: int):
                 if member["login"] not in all_members:
                     all_members[member["login"]] = []
                 all_members[member["login"]].append(org)
-                if member["is_owner"]:
+                if member.get("is_owner", None):
                     org_owners.setdefault(org, []).append(member["login"])
 
         # Get activity for each user
@@ -312,16 +337,25 @@ async def main(orgs, debug: bool, timelimit_days: int):
         for (username, _), last_activity in zip(tasks, results):
             if last_activity is not None:
                 assert isinstance(last_activity, datetime), last_activity
+
+            if last_activity is None:
+                last_activity = manual_users.get(username, {}).get(
+                    "manual_last_activity", None
+                )
             user_activities.append(
                 (
                     username,
                     last_activity if last_activity is not None else None,
                     all_members[username],
                 )
             )
-        for org in orgs:
+
+        admin_check_tasks = [
+            check_user_admin(session, org, current_user) for org in orgs
+        ]
+        admin_check_results = await asyncio.gather(*admin_check_tasks)
+        for org, is_admin in zip(orgs, admin_check_results):
             print(f"[bold]{org}[/bold]")
-            is_admin = await check_user_admin(session, org, current_user)
             if is_admin:
                 if debug:
                     print(f"    [green]{current_user} is an admin in {org}[/green]")
@@ -347,6 +381,8 @@ async def main(orgs, debug: bool, timelimit_days: int):
                     - timedelta(days=timelimit_days)
                 ):
                     n_active += 1
+                    if debug:
+                        print(f"    [green]{username}[/green] is active in {org}")
                     continue
                 n_inactive += 1
                 last_activity_ago = (
@@ -356,9 +392,22 @@ async def main(orgs, debug: bool, timelimit_days: int):
                     if last_activity
                     else "[red]never[/red]"
                 )
-                orgs_str = ", ".join(user_orgs)
                 u_owner = " (owner)" if username in org_owners.get(org, []) else ""
-                print(f"    {username+u_owner:<20}: Last activity {last_activity_ago}")
+                inviter = manual_users.get(username, {}).get("inviter", None)
+                if inviter:
+                    inviter = f"[green]@{inviter}[/green]"
+                else:
+                    inviter = "[red]unknown[/red]"
+                reason = manual_users.get(username, {}).get(
+                    "last_activity_reason", None
+                )
+                if reason:
+                    reason = f"[green]{reason}[/green]"
+                else:
+                    reason = "[red]unknown[/red]"
+                print(
+                    f"    {username + u_owner:<20}: Last activity {last_activity_ago}: reason: {reason}, inviter: {inviter}"
+                )
             print(
                 f"    Found [red]{n_inactive} inactive[/red] and [green]{n_active} active[/green] users in {org} with last activity more recent than {timelimit_days} days."
             )
@@ -370,6 +419,12 @@ async def main(orgs, debug: bool, timelimit_days: int):
         "--clear-cache", action="store_true", help="Clear the cache before running"
     )
     parser.add_argument("--debug", action="store_true", help="Show debug information")
+    parser.add_argument(
+        "--config-file",
+        type=str,
+        default="last_user_activity.json",
+        help="Path to the config file (default: last_user_activity.json)",
+    )
 
     parser.add_argument(
         "--timelimit-days",
@@ -388,4 +443,4 @@ async def main(orgs, debug: bool, timelimit_days: int):
     if args.clear_cache:
         clear_cache()
 
-    asyncio.run(main(args.orgs, args.debug, args.timelimit_days))
+    asyncio.run(main(args.orgs, args.debug, args.timelimit_days, args.config_file))

tools/tide.py

@@ -25,6 +25,9 @@ def get_packages(url):
 
     if "A required part of this site couldn’t load" in response.text:
         print(f"Fastly is blocking us for {url}. Status code: 403")
+        print(
+            "You can try `Array.from(document.querySelectorAll('h3')).map(h3 => h3.innerText).join(' ');`, from js console when viewing a page from a browser and use the `--packages` option."
+        )
         exit(1)
 
     # Parse the HTML content