Fix #531: count partial executions of workflows (#541)

* Fix #531: count partial executions of workflows

* Rename partial to aborted

* Count incomplete workflows

* Use whiteboard tag in metrics names

Author: leplatrem

jbi/actions/default.py

@@ -10,6 +10,8 @@
 import logging
 from typing import Callable, Optional
 
+from statsd.defaults.env import statsd
+
 from jbi import ActionResult, Operation
 from jbi.actions import steps as steps_module
 from jbi.environment import get_settings
@@ -95,15 +97,30 @@ def __init__(self, steps: dict[Operation, list[Callable]], **parameters):
 
     def __call__(self, context: ActionContext) -> ActionResult:
         """Called from `runner` when the action is used."""
+        has_produced_request = False
+
         for step in self.steps[context.operation]:
             context = context.update(current_step=step.__name__)
             try:
                 context = step(context=context, **self.parameters)
             except IncompleteStepError as exc:
                 # Step did not execute all its operations.
                 context = exc.context
+                statsd.incr(
+                    f"jbi.action.{context.action.whiteboard_tag}.incomplete.count"
+                )
+            except Exception:
+                if has_produced_request:
+                    # Count the number of workflows that produced at least one request,
+                    # but could not complete entirely with success.
+                    statsd.incr(
+                        f"jbi.action.{context.action.whiteboard_tag}.aborted.count"
+                    )
+                raise
 
             step_responses = context.responses_by_step[step.__name__]
+            if step_responses:
+                has_produced_request = True
             for response in step_responses:
                 logger.debug(
                     "Received %s",

jbi/models.py

@@ -359,6 +359,7 @@ class RunnerContext(Context, extra=Extra.forbid):
 class ActionContext(Context, extra=Extra.forbid):
     """Logging context from actions"""
 
+    action: Action
     rid: str
     operation: Operation
     current_step: Optional[str]

jbi/runner.py

@@ -74,6 +74,7 @@ def execute_action(
         linked_issue_key: Optional[str] = bug.extract_from_see_also()
 
         action_context = ActionContext(
+            action=action,
             rid=request.rid,
             bug=bug,
             event=event,

tests/fixtures/factories.py

@@ -114,6 +114,7 @@ def comment_factory(**overrides):
 def action_context_factory(**overrides):
     return ActionContext.parse_obj(
         {
+            "action": action_factory(),
             "rid": token_hex(16),
             "operation": Operation.IGNORE,
             "bug": bug_factory(),

tests/unit/actions/test_default.py

@@ -2,11 +2,14 @@
 from unittest import mock
 
 import pytest
+import requests
 import responses
 
+from jbi import Operation
 from jbi.actions import default
 from jbi.environment import get_settings
 from jbi.models import ActionContext
+from tests.fixtures import factories
 
 
 def test_default_invalid_init():
@@ -95,3 +98,53 @@ def test_default_returns_callable_with_data(
     assert handled
     assert details["responses"][0] == {"key": "k"}
     assert details["responses"][1] == sentinel
+
+
+def test_counter_is_incremented_when_workflows_was_aborted(
+    mocked_bugzilla, mocked_jira
+):
+    context_create_example: ActionContext = factories.action_context_factory(
+        operation=Operation.CREATE,
+        action=factories.action_factory(whiteboard_tag="fnx"),
+    )
+    mocked_bugzilla.get_bug.return_value = context_create_example.bug
+    mocked_jira.create_or_update_issue_remote_links.side_effect = requests.HTTPError(
+        "Unauthorized"
+    )
+    callable_object = default.init(jira_project_key=context_create_example.jira.project)
+
+    with mock.patch("jbi.actions.default.statsd") as mocked:
+        with pytest.raises(requests.HTTPError):
+            callable_object(context=context_create_example)
+
+    mocked.incr.assert_called_with("jbi.action.fnx.aborted.count")
+
+
+def test_counter_is_incremented_when_workflows_was_incomplete(
+    mocked_bugzilla, mocked_jira
+):
+    context_create_example: ActionContext = factories.action_context_factory(
+        operation=Operation.CREATE,
+        action=factories.action_factory(whiteboard_tag="fnx"),
+        bug=factories.bug_factory(resolution="WONTFIX"),
+    )
+    mocked_bugzilla.get_bug.return_value = context_create_example.bug
+
+    callable_object = default.init(
+        jira_project_key=context_create_example.jira.project,
+        steps={
+            "new": [
+                "create_issue",
+                "maybe_update_issue_resolution",
+            ]
+        },
+        resolution_map={
+            # Not matching WONTFIX, `maybe_` step will not complete
+            "DUPLICATE": "Duplicate",
+        },
+    )
+
+    with mock.patch("jbi.actions.default.statsd") as mocked:
+        callable_object(context=context_create_example)
+
+    mocked.incr.assert_called_with("jbi.action.fnx.incomplete.count")