record the start and end date of the query (#90)

* record the start and end date of the query

* add cli argument

Author: j-bowhay

devstats/__main__.py

@@ -1,4 +1,5 @@
 import collections
+import datetime
 import json
 import os
 import re
@@ -36,7 +37,14 @@ def cli():
     help="Output directory",
     show_default=True,
 )
-def query(repo_owner, repo_name, outdir):
+@click.option(
+    "-d",
+    "--start-date",
+    default="2020-01-01T00:00:00Z",
+    help="Start date for query",
+    show_default=True,
+)
+def query(repo_owner, repo_name, outdir, start_date):
     """Download and save issue and pr data for `repo_owner`/`repo_name`"""
     os.makedirs(outdir, exist_ok=True)
 
@@ -74,6 +82,7 @@ def query(repo_owner, repo_name, outdir):
             headers,
             repo_owner=repo_owner,
             repo_name=repo_name,
+            start_date=start_date,
         )
         data.get()
         ftype = {"issues": "issues", "pullRequests": "PRs"}
@@ -91,18 +100,22 @@ def query(repo_owner, repo_name, outdir):
 
     if response.status_code == 200:
         stargazers = response.json()
-        with open(f"{outdir}/{repo_name}_stars.json", "w") as outf:
-            simplified = [
-                {"starred_at": user["starred_at"], "login": user["user"]["login"]}
-                for user in stargazers
-            ]
-            json.dump(simplified, outf)
     else:
         print(
             "Request failed for collecting start with status code "
             f"{response.status_code}"
         )
 
+    with open(f"{outdir}/{repo_name}_misc.json", "w") as outf:
+        misc_data = {}
+        misc_data["query_start_date"] = start_date
+        misc_data["query_end_date"] = str(datetime.datetime.now())
+        misc_data["repo_stars"] = [
+            {"starred_at": user["starred_at"], "login": user["user"]["login"]}
+            for user in stargazers
+        ]
+        json.dump(misc_data, outf)
+
 
 cli.add_command(template)
 cli.add_command(publish)

devstats/queries/issue_activity_since_date.gql

@@ -1,6 +1,6 @@
 query {
   repository(owner: "_REPO_OWNER_", name: "_REPO_NAME_") {
-    issues(first: 100, filterBy: { since: "2020-01-01T00:00:00Z" }) {
+    issues(first: 100, filterBy: { since: "_START_DATE_" }) {
       totalCount
       edges {
         cursor

devstats/query.py

@@ -6,7 +6,7 @@
 endpoint = r"https://api.github.com/graphql"
 
 
-def load_query_from_file(fname, repo_owner="numpy", repo_name="numpy"):
+def load_query_from_file(fname, repo_owner, repo_name, start_date):
     """
     Load an 'issue' query from file and set the target repository, where
     the target repository has the format:
@@ -39,6 +39,7 @@ def load_query_from_file(fname, repo_owner="numpy", repo_name="numpy"):
         # Set target repo from template
         query = query.replace("_REPO_OWNER_", repo_owner)
         query = query.replace("_REPO_NAME_", repo_name)
+        query = query.replace("_START_DATE_", start_date)
     return query
 
 
@@ -178,7 +179,7 @@ class GithubGrabber:
     """
 
     def __init__(
-        self, query_fname, query_type, headers, repo_owner="numpy", repo_name="numpy"
+        self, query_fname, query_type, headers, repo_owner, repo_name, start_date
     ):
         """
         Create an object to send/recv queries related to the issue tracker
@@ -206,11 +207,15 @@ def __init__(
         self.repo_owner = repo_owner
         self.repo_name = repo_name
         self.raw_data = None
+        self.start_date = start_date
         self.load_query()
 
     def load_query(self):
         self.query = load_query_from_file(
-            self.query_fname, self.repo_owner, self.repo_name
+            self.query_fname,
+            self.repo_owner,
+            self.repo_name,
+            start_date=self.start_date,
         )
 
     def get(self):