Merge branch 'master' into codex/create-crontab-script-for-daily-database-backup

Author: oesteban

README.md

@@ -31,6 +31,15 @@ python src/run.py --help
 
 for a description of all available options.
 
+Running:
+
+```bash
+python src/run.py plot
+```
+
+will generate the performance and version stream plots using the records stored
+in MongoDB.
+
 ## MongoDB backup script
 
 `scripts/backup_mongodb.sh` dumps a MongoDB database to a Dropbox-synced
@@ -42,7 +51,6 @@ Make it executable before scheduling it with `cron`:
 ```bash
 chmod +x scripts/backup_mongodb.sh
 ```
-
 Store `DBNAME` (and optional credentials) in environment variables rather than
 editing the script.  You may create a file named `~/.mongodb_backup_env` with
 content like:
@@ -54,4 +62,3 @@ export DBNAME="fmriprep_stats"
 ```
 
 The backup script will source this file if present.
-

scripts/backup_mongodb.sh

@@ -15,7 +15,7 @@ fi
 : "${DBNAME:?Set DBNAME, e.g., export DBNAME=your_db}"
 
 DATE=$(date +%Y-%m-%d)
-BACKUP_DIR="$HOME/Dropbox/backups"
+BACKUP_DIR="$HOME/Dropbox/fmriprep_stats"
 BACKUP_PATH="$BACKUP_DIR/db_backup_${DATE}"
 
 mkdir -p "$BACKUP_DIR"

src/db.py

@@ -0,0 +1,58 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+"""Database helpers used by the CLI and plotting utilities."""
+
+from __future__ import annotations
+
+import datetime
+from typing import Tuple
+
+import pandas as pd
+from pymongo import MongoClient
+
+
+def load_event(event_name: str, unique: bool = True) -> pd.DataFrame:
+    """Load one event collection from MongoDB."""
+    db = MongoClient().fmriprep_stats
+    data = pd.DataFrame(list(db[event_name].find()))
+    if len(data) == 0:
+        raise RuntimeError(f"No records of event '{event_name}'")
+
+    data["dateCreated"] = pd.to_datetime(data["dateCreated"])
+    data["date_minus_time"] = data["dateCreated"].apply(
+        lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day)
+    )
+    if unique:
+        data = data.drop_duplicates(subset=["run_uuid"])
+    return data
+
+
+def massage_versions(
+    started: pd.DataFrame, success: pd.DataFrame
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """Normalize version strings as done in the analysis notebook."""
+    started = started.copy()
+    success = success.copy()
+
+    started = started.fillna(value={"environment_version": "older"})
+    success = success.fillna(value={"environment_version": "older"})
+
+    started.loc[started.environment_version == "v0.0.1", "environment_version"] = "older"
+    success.loc[success.environment_version == "v0.0.1", "environment_version"] = "older"
+
+    started.loc[started.environment_version.str.startswith("20.0"), "environment_version"] = "older"
+    success.loc[success.environment_version.str.startswith("20.0"), "environment_version"] = "older"
+    started.loc[started.environment_version.str.startswith("20.1"), "environment_version"] = "older"
+    success.loc[success.environment_version.str.startswith("20.1"), "environment_version"] = "older"
+
+    versions = sorted(
+        {
+            ".".join(v.split(".")[:2])
+            for v in started.environment_version.unique()
+            if "." in str(v)
+        }
+    )
+    for ver in versions:
+        started.loc[started.environment_version.str.startswith(ver), "environment_version"] = ver
+        success.loc[success.environment_version.str.startswith(ver), "environment_version"] = ver
+
+    return started, success

src/run.py

@@ -28,6 +28,8 @@
 
 import click
 from api import parallel_fetch, ISSUES, DEFAULT_MAX_ERRORS
+from db import load_event, massage_versions
+from viz import plot_performance, plot_version_stream
 
 DEFAULT_DAYS_WINDOW = 90
 DEFAULT_CHUNK_DAYS = 1
@@ -149,6 +151,26 @@ def get(event, start_date, end_date, days, chunk_days, jobs, max_errors, cached_
     click.echo(f"{datetime.now(timezone.utc):%Y-%m-%d %H:%M:%S} [Finished]")
 
 
+@cli.command()
+@click.option("-o", "--output-dir", type=click.Path(file_okay=False, dir_okay=True, writable=True), default=".")
+@click.option("--drop-cutoff", default=None, help="Ignore versions older than this")
+def plot(output_dir, drop_cutoff):
+    """Generate plots using records stored in MongoDB."""
+    today = datetime.now().date().strftime("%Y%m%d")
+    out_perf = os.path.join(output_dir, f"{today}_weekly.png")
+    out_ver = os.path.join(output_dir, f"{today}_versionstream.png")
+
+    unique_started = load_event("started")
+    unique_success = load_event("success")
+
+    plot_performance(unique_started, unique_success, drop_cutoff=drop_cutoff, out_file=out_perf)
+    click.echo(f"Saved {out_perf}.")
+
+    started_v, success_v = massage_versions(unique_started, unique_success)
+    plot_version_stream(started_v, success_v, drop_cutoff=drop_cutoff, out_file=out_ver)
+    click.echo(f"Saved {out_ver}")
+
+
 if __name__ == "__main__":
     """ Install entry-point """
     cli()