Deal with unexpected problems in data when calculating statistics, and fix some bugs

Author: apdavison

api/dashboard/js/cumulative-user-count.js

@@ -51,7 +51,7 @@ d3.json("/statistics/cumulative-user-count", function (error, data) {
 });
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=BrainScaleS",
+  "/statistics/cumulative-user-count?hardware_platform=BrainScaleS",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -104,7 +104,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=BrainScaleS-2",
+  "/statistics/cumulative-user-count?hardware_platform=BrainScaleS-2",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -157,7 +157,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=SpiNNaker",
+  "/statistics/cumulative-user-count?hardware_platform=SpiNNaker",
   function (error, data) {
     if (error) return console.warn(error);
 
@@ -210,7 +210,7 @@ d3.json(
 );
 
 d3.json(
-  "/statistics/cumulative-user-count?platform=Spikey",
+  "/statistics/cumulative-user-count?hardware_platform=Spikey",
   function (error, data) {
     if (error) return console.warn(error);
 

api/simqueue/db.py

@@ -291,7 +291,8 @@ async def update_log(job_id, log, append=False):
 
 
 def get_list_filter(attr, value):
-    if len(value) > 0:
+    assert isinstance(value, list)
+    if len(value) > 1:
         return attr.in_(value)
     else:
         return attr == value[0]

api/simqueue/resources/statistics.py

@@ -58,6 +58,7 @@ async def job_count(start: date = None, end: date = None, interval: int = 7):
             [
                 (timestamp["timestamp_completion"].date() - start).days
                 for timestamp in completion_timestamps
+                if timestamp["timestamp_completion"]
             ],
             dtype=int,
         )
@@ -178,6 +179,7 @@ async def job_duration(requested_max: int = None, n_bins: int = 50, scale: str =
                 status=[status],
                 hardware_platform=[platform],
                 size=100000,
+                fields=["timestamp_completion", "timestamp_submission"],
             )
 
             durations = np.array(
@@ -236,7 +238,7 @@ async def cumulative_project_count(
 
     submission_dates = [
         res["submission_date"]
-        for res in await db.query_projects(fields=["submission_date"], status=status)
+        for res in await db.query_projects(fields=["submission_date"], status=status, size=10000)
     ]
 
     submission_dates.append(date.today())
@@ -265,10 +267,16 @@ async def resource_usage(start: date = None, end: date = None, interval: int = 7
             fields=["timestamp_completion", "resource_usage"],
         )
         completed = np.array(
-            [(job["timestamp_completion"].date() - start).days for job in completed_jobs],
+            [
+                (job["timestamp_completion"].date() - start).days
+                for job in completed_jobs
+                if job["timestamp_completion"]
+            ],
             dtype=int,
         )
-        usage_per_job = np.array([job["resource_usage"] for job in completed_jobs])
+        usage_per_job = np.array(
+            [job["resource_usage"] for job in completed_jobs if job["timestamp_completion"]]
+        )
         index = completed // interval
         usage_per_interval[platform] = np.zeros((n_bins,))
         for i, usage in zip(index, usage_per_job):
@@ -281,6 +289,7 @@ async def resource_usage(start: date = None, end: date = None, interval: int = 7
         interval_end = interval_start + timedelta(interval)
         for platform in STANDARD_QUEUES:
             usage_cumul[platform] += usage_per_interval[platform][i]
-        new_obj = {"start": interval_start, "end": interval_end, "value": usage_cumul}
+        new_obj = {"start": interval_start, "end": interval_end, "value": usage_cumul.copy()}
         results.append(new_obj)
+
     return results

api/simqueue/tests/test_statistics_router.py

@@ -12,24 +12,28 @@
         "timestamp_completion": datetime.fromisoformat("2022-10-03T02:44:28+00:00"),
         "user_id": "haroldlloyd",
         "status": "finished",
+        "resource_usage": 7.07,
     },
     {
         "timestamp_submission": datetime.fromisoformat("2022-10-11T02:44:23+00:00"),
         "timestamp_completion": datetime.fromisoformat("2022-10-11T02:44:38+00:00"),
         "user_id": "charliechaplin",
         "status": "finished",
+        "resource_usage": 13.13,
     },
     {
         "timestamp_submission": datetime.fromisoformat("2022-10-12T02:44:23+00:00"),
         "timestamp_completion": datetime.fromisoformat("2022-10-12T02:44:28+00:00"),
         "user_id": "haroldlloyd",
         "status": "finished",
+        "resource_usage": 19.19,
     },
     {
         "timestamp_submission": datetime.fromisoformat("2022-10-16T02:44:23+00:00"),
         "timestamp_completion": datetime.fromisoformat("2022-10-16T02:44:48+00:00"),
         "user_id": "haroldlloyd",
         "status": "error",
+        "resource_usage": 23.23,
     },
 ]
 
@@ -206,3 +210,55 @@ def test_job_duration(mocker):
             "max": 30,
         },
     ]
+
+
+def test_resource_usage(mocker):
+    mocker.patch("simqueue.db.query_jobs", mock_query_jobs)
+    response = client.get("/statistics/resource-usage?interval=7&start=2022-10-01&end=2022-10-28")
+    assert response.status_code == 200
+    assert response.json() == [
+        {
+            "start": "2022-10-01",
+            "end": "2022-10-08",
+            "value": {
+                "BrainScaleS": 0.0,
+                "BrainScaleS-ESS": 0.0,
+                "Spikey": 0.0,
+                "SpiNNaker": 7.07,
+                "BrainScaleS-2": 0.0,
+            },
+        },
+        {
+            "start": "2022-10-08",
+            "end": "2022-10-15",
+            "value": {
+                "BrainScaleS": 0.0,
+                "BrainScaleS-ESS": 0.0,
+                "Spikey": 0.0,
+                "SpiNNaker": 39.39,
+                "BrainScaleS-2": 0.0,
+            },
+        },
+        {
+            "start": "2022-10-15",
+            "end": "2022-10-22",
+            "value": {
+                "BrainScaleS": 0.0,
+                "BrainScaleS-ESS": 0.0,
+                "Spikey": 0.0,
+                "SpiNNaker": 62.620000000000005,
+                "BrainScaleS-2": 0.0,
+            },
+        },
+        {
+            "start": "2022-10-22",
+            "end": "2022-10-29",
+            "value": {
+                "BrainScaleS": 0.0,
+                "BrainScaleS-ESS": 0.0,
+                "Spikey": 0.0,
+                "SpiNNaker": 62.620000000000005,
+                "BrainScaleS-2": 0.0,
+            },
+        },
+    ]