Skip to content

Commit 070046f

Browse files
moibra05moibra05
committed
feat(nimbus): add experiment error ui to new results page
1 parent 52b862c commit 070046f

File tree

4 files changed

+253
-9
lines changed

4 files changed

+253
-9
lines changed

‎experimenter/experimenter/experiments/models.py‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1345,6 +1345,19 @@ def timeline(self):
13451345

13461346
return timeline_entries
13471347

1348+
def metric_has_errors(self, metric_slug, analysis_basis, segment):
1349+
if self.results_data:
1350+
for error, details in (
1351+
self.results_data.get("v3", {}).get("errors", {}).items()
1352+
):
1353+
if (
1354+
error == metric_slug
1355+
and details[0].get("analysis_basis") == analysis_basis
1356+
and details[0].get("segment") == segment
1357+
):
1358+
return True
1359+
return False
1360+
13481361
def get_metric_areas(
13491362
self, analysis_basis, segment, reference_branch, window="overall"
13501363
):
@@ -1383,6 +1396,9 @@ def get_metric_areas(
13831396
"friendly_name", metric.slug
13841397
)
13851398
),
1399+
"has_errors": self.metric_has_errors(
1400+
metric.slug, analysis_basis, segment
1401+
),
13861402
}
13871403
if formatted_metric not in outcome_metrics:
13881404
outcome_metrics.append(formatted_metric)
@@ -1392,7 +1408,11 @@ def get_metric_areas(
13921408
metric_areas[outcome.friendly_name if outcome else slug] = outcome_metrics
13931409

13941410
metric_areas[NimbusUIConstants.OTHER_METRICS_AREA] = (
1395-
self.get_remaining_metrics_metadata(exclude_slugs=all_outcome_metric_slugs)
1411+
self.get_remaining_metrics_metadata(
1412+
exclude_slugs=all_outcome_metric_slugs,
1413+
analysis_basis=analysis_basis,
1414+
segment=segment,
1415+
)
13961416
)
13971417

13981418
window_results = self.get_window_results(analysis_basis, segment, window)
@@ -1423,7 +1443,9 @@ def is_metric_notable(slug, group):
14231443
)
14241444
return metric_areas
14251445

1426-
def get_remaining_metrics_metadata(self, exclude_slugs=None):
1446+
def get_remaining_metrics_metadata(
1447+
self, exclude_slugs=None, analysis_basis=None, segment=None
1448+
):
14271449
analysis_data = self.results_data.get("v3", {}) if self.results_data else {}
14281450
other_metrics = analysis_data.get("other_metrics", {})
14291451
metadata = analysis_data.get("metadata", {})
@@ -1442,6 +1464,9 @@ def get_remaining_metrics_metadata(self, exclude_slugs=None):
14421464
),
14431465
"group": group,
14441466
"friendly_name": metric_friendly_name,
1467+
"has_errors": self.metric_has_errors(
1468+
slug, analysis_basis, segment
1469+
),
14451470
}
14461471
)
14471472

@@ -1597,7 +1622,7 @@ def get_kpi_metrics(
15971622
)
15981623
for branch in diff_metrics:
15991624
if len(diff_metrics.get(branch, {}).get("all", [])) > 0:
1600-
kpi_metrics.append(NimbusConstants.DAU_METRIC)
1625+
kpi_metrics.append(NimbusConstants.DAU_METRIC.copy())
16011626
break
16021627
elif NimbusConstants.DAYS_OF_USE in other_metrics:
16031628
if (
@@ -1608,7 +1633,11 @@ def get_kpi_metrics(
16081633
)
16091634
> 0
16101635
):
1611-
kpi_metrics.append(NimbusConstants.DOU_METRIC)
1636+
kpi_metrics.append(NimbusConstants.DOU_METRIC.copy())
1637+
1638+
for kpi in kpi_metrics:
1639+
if self.metric_has_errors(kpi["slug"], analysis_basis, segment):
1640+
kpi["has_errors"] = True
16121641

16131642
return kpi_metrics
16141643

@@ -2253,6 +2282,14 @@ def audience_overlap_warnings(self):
22532282

22542283
return warnings
22552284

2285+
@property
2286+
def has_results_errors(self):
2287+
if self.results_data:
2288+
for error in self.results_data.get("v3", {}).get("errors", {}).values():
2289+
if error:
2290+
return True
2291+
return False
2292+
22562293
def get_invalid_fields_errors(self):
22572294
from experimenter.experiments.api.v5.serializers import NimbusReviewSerializer
22582295

‎experimenter/experimenter/experiments/tests/test_models.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5484,6 +5484,88 @@ def test_all_version_constants_parse_correctly(self):
54845484
f"('{version_string}'): {e}"
54855485
)
54865486

5487+
@parameterized.expand(
5488+
[
5489+
(
5490+
{
5491+
"v3": {
5492+
"errors": {
5493+
"ad_clicks": [
5494+
{"analysis_basis": "enrollments", "segment": "all"}
5495+
],
5496+
"experiment": [],
5497+
},
5498+
}
5499+
},
5500+
True,
5501+
),
5502+
(
5503+
{
5504+
"v3": {
5505+
"errors": {"experiment": []},
5506+
}
5507+
},
5508+
False,
5509+
),
5510+
]
5511+
)
5512+
def test_check_experiment_has_results_errors(self, results_data, expected_results):
5513+
experiment = NimbusExperimentFactory.create()
5514+
5515+
experiment.results_data = results_data
5516+
experiment.save()
5517+
5518+
self.assertEqual(experiment.has_results_errors, expected_results)
5519+
5520+
def test_experiment_kpi_metrics_have_errors(self):
5521+
experiment = NimbusExperimentFactory.create()
5522+
branch_a = NimbusBranchFactory.create(
5523+
experiment=experiment, name="Branch A", slug="branch-a"
5524+
)
5525+
5526+
experiment.results_data = {
5527+
"v3": {
5528+
"errors": {
5529+
"client_level_daily_active_users_v2": [
5530+
{"analysis_basis": "enrollments", "segment": "all"}
5531+
],
5532+
"experiment": [],
5533+
},
5534+
"overall": {
5535+
"enrollments": {
5536+
"all": {
5537+
"branch-a": {
5538+
"branch_data": {
5539+
"other_metrics": {
5540+
"client_level_daily_active_users_v2": {
5541+
"absolute": {"all": []},
5542+
"difference": {
5543+
"branch-a": {"all": [{}]},
5544+
},
5545+
}
5546+
}
5547+
}
5548+
},
5549+
}
5550+
}
5551+
},
5552+
}
5553+
}
5554+
5555+
experiment.save()
5556+
kpi_metrics = experiment.get_kpi_metrics("enrollments", "all", branch_a.slug)
5557+
5558+
self.assertIn(
5559+
{
5560+
"group": "other_metrics",
5561+
"friendly_name": "Daily Active Users",
5562+
"slug": "client_level_daily_active_users_v2",
5563+
"description": "Average number of client that sent a main ping per day.",
5564+
"has_errors": True,
5565+
},
5566+
kpi_metrics,
5567+
)
5568+
54875569

54885570
class TestNimbusBranch(TestCase):
54895571
def test_str(self):

‎experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-new-fragment.html‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,112 @@
11
<div class="d-flex flex-column gap-4" id="experiment-results-page">
2+
{% if experiment.has_results_errors %}
3+
<div class="alert p-4 py-3 mb-0 rounded-4 alert-warning" role="alert">
4+
<div class="row align-items-center">
5+
<div class="col-5 py-4 mt-0">
6+
<div class="d-flex align-items-center mb-2 gap-2">
7+
<i class="fa-solid fa-triangle-exclamation fs-5"></i>
8+
<h5 class="mb-0">Issues detected in a few metrics</h5>
9+
</div>
10+
<p class="text-muted">
11+
Your experiment is still running. Results for some metrics aren't available, but others are unaffected.
12+
</p>
13+
<button class="btn btn-secondary bg-secondary-subtle bg-opacity-25 border-0 text-body fw-semibold">
14+
Contact Experimenter Support
15+
</button>
16+
</div>
17+
<div class="col">
18+
<div class="card bg-warning bg-opacity-25 px-4 py-3 rounded-4 h-100 d-flex justify-content-center">
19+
<p class="text-muted mb-2">Possible Issues</p>
20+
<ul class="list-unstyled mb-0">
21+
{% for error_group, error_list in experiment.results_data.v3.errors.items %}
22+
{% for error in error_list %}
23+
{% if forloop.last or error_group == "experiment" %}
24+
<li class="d-flex align-items-center gap-1 mb-2 text-muted">
25+
<i class="fa-solid fa-triangle-exclamation"></i>
26+
<small class="fw-bold">
27+
{% if error.metric %}
28+
{{ error.metric }} metric encountered issues. —
29+
{% else %}
30+
{{ error.exception_type|default:"Unknown error" }} —
31+
{% endif %}
32+
</small>
33+
<button class="btn btn-link p-0"
34+
data-bs-toggle="modal"
35+
data-bs-target="#{{ experiment.slug }}-error-{{ forloop.counter }}">
36+
<small>View details</small>
37+
</button>
38+
</li>
39+
<div class="modal fade"
40+
id="{{ experiment.slug }}-error-{{ forloop.counter }}"
41+
tabindex="-1"
42+
aria-labelledby="exampleModalLabel"
43+
aria-hidden="true">
44+
<div class="modal-dialog modal-dialog-centered modal-lg">
45+
<div class="modal-content">
46+
<div class="modal-header">
47+
<button type="button"
48+
class="btn-close"
49+
data-bs-dismiss="modal"
50+
aria-label="Close"></button>
51+
</div>
52+
<div class="modal-body">
53+
<dl class="row">
54+
<small class="text-muted mb-3">[{{ error.timestamp }}]</small>
55+
<dt class="col-sm-4">Level</dt>
56+
<dd class="col-sm-8">
57+
{{ error.log_level }}
58+
</dd>
59+
<dt class="col-sm-4">Type</dt>
60+
<dd class="col-sm-8">
61+
{{ error.exception_type }}
62+
</dd>
63+
<dt class="col-sm-4">Message</dt>
64+
<dd class="col-sm-8 text-break">
65+
{{ error.message }}
66+
</dd>
67+
<dt class="col-sm-4">File</dt>
68+
<dd class="col-sm-8">
69+
{{ error.filename }}
70+
</dd>
71+
<dt class="col-sm-4">Function</dt>
72+
<dd class="col-sm-8">
73+
{{ error.func_name }}
74+
</dd>
75+
<dt class="col-sm-4">Metric</dt>
76+
<dd class="col-sm-8">
77+
{{ error.metric|default:"—" }}
78+
</dd>
79+
<dt class="col-sm-4">Segment</dt>
80+
<dd class="col-sm-8">
81+
{{ error.segment|default:"—" }}
82+
</dd>
83+
<dt class="col-sm-4">Analysis basis</dt>
84+
<dd class="col-sm-8">
85+
{{ error.analysis_basis|default:"—" }}
86+
</dd>
87+
{% if error.exception %}
88+
<details class="mt-3">
89+
<summary class="small">View full exception</summary>
90+
<code class="mt-2 mb-0 text-break">{{ error.exception }}</code>
91+
</details>
92+
{% endif %}
93+
</dl>
94+
</div>
95+
<div class="modal-footer">
96+
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
97+
</div>
98+
</div>
99+
</div>
100+
</div>
101+
{% endif %}
102+
{% endfor %}
103+
{% endfor %}
104+
</ul>
105+
</div>
106+
</div>
107+
</div>
108+
</div>
109+
{% endif %}
2110
<form hx-get="{{ request.path }}"
3111
hx-push-url="true"
4112
hx-trigger="change"

‎experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-new-inner.html‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -288,12 +288,29 @@ <h4 class="mb-0">{{ area }}</h4>
288288
{% for metric in metric_metadata %}
289289
{% for curr_metric_slug, metric_branch_data in metric_data.items %}
290290
{% if curr_metric_slug == metric.slug %}
291-
{% for curr_branch_slug, branch_metric_data in metric_branch_data.items %}
292-
{% if curr_branch_slug == branch.slug %}
293-
{% include "common/metric_card.html" with slug=branch.slug reference_branch=selected_reference_branch absolute_lower=branch_metric_data.absolute.0.lower absolute_upper=branch_metric_data.absolute.0.upper significance=branch_metric_data.relative.0.significance percentage=branch_metric_data.relative.0.avg_rel_change display_type=metric.display_type %}
294-
291+
{% if metric.has_errors %}
292+
{% if forloop.parentloop.parentloop.first %}
293+
<div class="d-flex align-items-center" style="height: 100px;">
294+
<div class="position-absolute alert alert-warning d-flex align-items-center justify-content-center gap-2 end-0 start-0 {% if branch_data|length > 4 %}mx-5{% else %}mx-3{% endif %} mb-0">
295+
<i class="fa-solid fa-triangle-exclamation fs-5"></i>
296+
<p class="mb-0 fw-semibold">Metric unavailable</p>
297+
<p class="mb-0">Other metrics are unaffected.</p>
298+
<button class="btn btn-secondary bg-secondary-subtle border-0 text-body fw-semibold">
299+
Contact Experimenter Support
300+
</button>
301+
</div>
302+
</div>
303+
{% else %}
304+
<div class="invisible" style="height: 100px;" aria-hidden="true"></div>
295305
{% endif %}
296-
{% endfor %}
306+
{% else %}
307+
{% for curr_branch_slug, branch_metric_data in metric_branch_data.items %}
308+
{% if curr_branch_slug == branch.slug %}
309+
{% include "common/metric_card.html" with slug=branch.slug reference_branch=selected_reference_branch absolute_lower=branch_metric_data.absolute.0.lower absolute_upper=branch_metric_data.absolute.0.upper significance=branch_metric_data.relative.0.significance percentage=branch_metric_data.relative.0.avg_rel_change display_type=metric.display_type %}
310+
311+
{% endif %}
312+
{% endfor %}
313+
{% endif %}
297314
{% endif %}
298315
{% endfor %}
299316
{% endfor %}

0 commit comments

Comments
 (0)