contest: backend: add flakiness query

Signed-off-by: Jakub Kicinski <[email protected]>

Author: kuba-moo

contest/backend/query.py

@@ -18,6 +18,9 @@
 psql = psycopg2.connect(database=db_name)
 psql.autocommit = True
 
+# How many branches to query to get flakes for last month
+flake_cnt = 300
+
 
 @app.route('/')
 def hello():
@@ -190,3 +193,85 @@ def dev_info():
         data = [{columns[i]: value for i, value in enumerate(row)} for row in rows]
 
     return data
+
+
+@app.route('/flaky-tests')
+def flaky_tests():
+    """
+    Returns tests that are flaky (first try fails, retry passes, and no crash).
+    """
+    global flake_cnt
+    limit = request.args.get('limit')
+    try:
+        limit = int(limit)
+        month = False
+    except:
+        month = True # Default to querying last month
+        limit = flake_cnt  # Default limit
+
+    # Find branches with incomplete results, psql JSON helpers fail for them
+    t = datetime.datetime.now()
+    with psql.cursor() as cur:
+        query = f"""
+        SELECT branch
+        FROM results
+        WHERE json_normal NOT LIKE '%"results": [%'
+        GROUP BY branch;
+        """
+
+        cur.execute(query)
+        rows = cur.fetchall()
+        branches = ""
+        if rows:
+            branches = " AND branch != ".join([""] + [f"'{r[0]}'" for r in rows])
+    print(f"Query for in-prog execs took: {str(datetime.datetime.now() - t)}")
+
+    t = datetime.datetime.now()
+    with psql.cursor() as cur:
+        # Query for tests where first try failed, retry passed, and no crash
+        query = f"""
+        SELECT remote, executor, test, branch, branch_date
+            FROM results, jsonb_to_recordset(json_normal::jsonb->'results') as
+                x(test text, result text, retry text, crashes text)
+            WHERE x.result = 'fail'
+                AND x.retry = 'pass'
+                AND x.crashes IS NULL
+                {branches}
+            ORDER BY branch_date DESC LIMIT {limit};
+        """
+
+        cur.execute(query)
+        rows = cur.fetchall()
+
+    print(f"Query for flaky tests took: {str(datetime.datetime.now() - t)}")
+
+    target_date = datetime.datetime.now() - datetime.timedelta(days=14)
+    two_weeks = target_date.strftime("%Y-%m-%d--%H-%M")
+    target_date = datetime.datetime.now() - datetime.timedelta(days=28)
+    four_weeks = target_date.strftime("%Y-%m-%d--%H-%M")
+    cnt = 0
+    res = {}
+    for row in rows:
+        rem, exe, test, branch, br_date = row
+        key = (rem, exe, test)
+        if not month:
+            res[key] = res.get(key, 0) + 1
+        else:
+            if key not in res:
+                res[key] = [0, 0]
+            if br_date >= two_weeks:
+                res[key][0] += 1
+            elif br_date >= four_weeks:
+                res[key][1] += 1
+            else:
+                break
+        cnt += 1
+    # JSON needs a simple array, not a dict
+    data = []
+    for k, v in res.items():
+        data.append({"remote": k[0], "executor": k[1], "test": k[2], "count": v})
+
+    if month:
+        # Overcount by 30 to account for fluctuation in flakiness
+        flake_cnt = cnt + 30
+    return data

ui/status.html

@@ -52,6 +52,18 @@ <h3>Build processing</h3>
                 <th>Memory Use</th>
             </tr>
         </table>
+        <br />
+	<h3>Flakiest tests</h3>
+        <table id="flakes">
+            <tr>
+                <th>Remote</th>
+                <th>Executor</th>
+                <th>Test</th>
+                <th>Flakes (now - 2w)</th>
+                <th>Flakes (2w - 4w)</th>
+		<th>Ignored</th>
+            </tr>
+        </table>
     </div>
     <div class="column">
         <h3>Continuous testing results</h3>

ui/status.js

@@ -1005,6 +1005,37 @@ function branches_loaded(data_raw)
     loaded_one();
 }
 
+function flakes_doit(data_raw)
+{
+    let flakes = document.getElementById("flakes");
+
+    data_raw.sort(function(a, b){
+	if (a["count"][0] != b["count"][0])
+	    return b["count"][0] - a["count"][0];
+	if (a["count"][1] != b["count"][1])
+	    return b["count"][1] - a["count"][1];
+	return 0;
+    })
+
+    $.each(data_raw, function(i, v) {
+	let row = flakes.insertRow();
+	let reported = nipa_pw_reported(v, v);
+	let ignored = "";
+
+	if (v["count"][0] < 3 && reported)
+	    return 1;
+	if (!reported)
+	    ignored = "yes";
+
+	row.insertCell(0).innerText = v["remote"];
+	row.insertCell(1).innerText = v["executor"];
+	row.insertCell(2).innerText = v["test"];
+	row.insertCell(3).innerText = v["count"][0];
+	row.insertCell(4).innerText = v["count"][1];
+	row.insertCell(5).innerText = ignored;
+    });
+}
+
 function do_it()
 {
     /*
@@ -1028,4 +1059,7 @@ function do_it()
     $(document).ready(function() {
         $.get("static/nipa/branches-info.json", branches_loaded)
     });
+    $(document).ready(function() {
+        $.get("query/flaky-tests", flakes_doit)
+    });
 }