Deal with unexpected problems in data when calculating statistics, and fix some bugs

Author: apdavison

api/dashboard/js/cumulative-user-count.js

@@ -51,7 +51,7 @@ d3.json("/statistics/cumulative-user-count", function (error, data) {
 });
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=BrainScaleS",
+  "/statistics/cumulative-user-count?hardware_platform=BrainScaleS",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -104,7 +104,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=BrainScaleS-2",
+  "/statistics/cumulative-user-count?hardware_platform=BrainScaleS-2",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -157,7 +157,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=SpiNNaker",
+  "/statistics/cumulative-user-count?hardware_platform=SpiNNaker",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -210,7 +210,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=Spikey",
+  "/statistics/cumulative-user-count?hardware_platform=Spikey",
   function (error, data) {
     if (error) return console.warn(error);
 

api/simqueue/db.py

@@ -291,7 +291,8 @@ async def update_log(job_id, log, append=False):
 
 
 def get_list_filter(attr, value):
-    if len(value) > 0:
+    assert isinstance(value, list)
+    if len(value) > 1:
         return attr.in_(value)
     else:
         return attr == value[0]

api/simqueue/resources/statistics.py

@@ -58,6 +58,7 @@ async def job_count(start: date = None, end: date = None, interval: int = 7):
             [
                 (timestamp["timestamp_completion"].date() - start).days
                 for timestamp in completion_timestamps
+                if timestamp["timestamp_completion"]
             ],
             dtype=int,
         )
@@ -178,6 +179,7 @@ async def job_duration(requested_max: int = None, n_bins: int = 50, scale: str =
                 status=[status],
                 hardware_platform=[platform],
                 size=100000,
+                fields=["timestamp_completion", "timestamp_submission"],
             )
 
             durations = np.array(
@@ -236,7 +238,7 @@ async def cumulative_project_count(
 
     submission_dates = [
         res["submission_date"]
-        for res in await db.query_projects(fields=["submission_date"], status=status)
+        for res in await db.query_projects(fields=["submission_date"], status=status, size=10000)
     ]
 
     submission_dates.append(date.today())
@@ -265,10 +267,16 @@ async def resource_usage(start: date = None, end: date = None, interval: int = 7
             fields=["timestamp_completion", "resource_usage"],
         )
         completed = np.array(
-            [(job["timestamp_completion"].date() - start).days for job in completed_jobs],
+            [
+                (job["timestamp_completion"].date() - start).days
+                for job in completed_jobs
+                if job["timestamp_completion"]
+            ],
             dtype=int,
         )
-        usage_per_job = np.array([job["resource_usage"] for job in completed_jobs])
+        usage_per_job = np.array(
+            [job["resource_usage"] for job in completed_jobs if job["timestamp_completion"]]
+        )
         index = completed // interval
         usage_per_interval[platform] = np.zeros((n_bins,))
         for i, usage in zip(index, usage_per_job):