Fix #723: turn markdown into Jira syntax on comments and description (#839)

* Fix #723: turn markdown into Jira syntax on comments and description

* Add more tests about converted fields

* Add instructions in README

* Check pandoc install in healthcheck

* Check pandoc in built container

* Install pandoc for CI unit tests

* Truncate on last word, as suggested by @grahamalama

Author: leplatrem

.github/workflows/ci.yaml

@@ -28,6 +28,8 @@ jobs:
         id: setup-python
         with:
           python-version: "3.12"
+      - name: Install pandoc
+        run: sudo apt-get install -y pandoc
       - name: Install poetry
         run: pipx install poetry
       - uses: actions/cache@v3

Dockerfile

@@ -23,6 +23,11 @@ RUN $POETRY_HOME/bin/poetry install --without dev --no-root
 
 # `production` stage uses the dependencies downloaded in the `base` stage
 FROM python:3.12.2-slim as production
+
+# Install pandoc for markdown to Jira conversions.
+RUN apt-get -y update && \
+    apt-get -y install --no-install-recommends pandoc
+
 ENV PORT=8000 \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \

README.md

@@ -72,6 +72,8 @@ graph TD
 
 # Development
 
+We use [pandoc](https://pandoc.org) to convert markdown to the Jira syntax. Make sure the binary is found in path or [specify your custom location](https://github.com/JessicaTegner/pypandoc#specifying-the-location-of-pandoc-binaries).
+
 - `make start`: run the application locally (http://localhost:8000)
 - `make test`: run the unit tests suites
 - `make lint`: static analysis of the code base

bin/healthcheck.py

@@ -2,15 +2,29 @@
 
 import os
 
+import backoff
 import requests
-from requests.adapters import HTTPAdapter, Retry
 
-PORT = os.environ["PORT"]
+PORT = os.getenv("PORT", "8000")
 
-session = requests.Session()
 
-retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
-session.mount("http://", HTTPAdapter(max_retries=retries))
-if __name__ == "__main__":
-    response = session.get(f"http://0.0.0.0:{PORT}/")
+@backoff.on_exception(
+    backoff.expo,
+    requests.exceptions.RequestException,
+    max_tries=5,
+)
+def check_server():
+    url = f"http://0.0.0.0:{PORT}"
+    response = requests.get(f"{url}/")
     response.raise_for_status()
+
+    hb_response = requests.get(f"{url}/__heartbeat__")
+    hb_details = hb_response.json()
+    # Check that pandoc is installed, but ignore other checks
+    # like connection to Jira or Bugzilla.
+    assert hb_details["jira"]["pandoc_install"]
+    print("Ok")
+
+
+if __name__ == "__main__":
+    check_server()

jbi/jira/service.py

@@ -15,6 +15,7 @@
 
 from jbi import Operation, bugzilla, environment
 from jbi.common.instrument import ServiceHealth
+from jbi.jira.utils import markdown_to_jira
 from jbi.models import ActionContext
 
 from .client import JiraClient, JiraCreateError
@@ -66,6 +67,7 @@ def check_health(self, actions: Actions) -> ServiceHealth:
             and self._all_project_custom_components_exist(actions),
             "all_projects_issue_types_exist": is_up
             and self._all_project_issue_types_exist(actions),
+            "pandoc_install": markdown_to_jira("- Test") == "* Test",
         }
         return health
 
@@ -213,7 +215,9 @@ def create_jira_issue(
         fields: dict[str, Any] = {
             "summary": bug.summary,
             "issuetype": {"name": issue_type},
-            "description": description[:JIRA_DESCRIPTION_CHAR_LIMIT],
+            "description": markdown_to_jira(
+                description, max_length=JIRA_DESCRIPTION_CHAR_LIMIT
+            ),
             "project": {"key": context.jira.project},
         }
         logger.debug(
@@ -257,7 +261,7 @@ def add_jira_comment(self, context: ActionContext):
 
         issue_key = context.jira.issue
         formatted_comment = (
-            f"*({commenter})* commented: \n{{quote}}{comment.body}{{quote}}"
+            f"*{commenter}* commented: \n{markdown_to_jira(comment.body or "")}"
         )
         jira_response = self.client.issue_add_comment(
             issue_key=issue_key,
@@ -409,7 +413,9 @@ def update_issue_summary(self, context: ActionContext):
             bug.id,
             extra=context.model_dump(),
         )
-        truncated_summary = (bug.summary or "")[:JIRA_DESCRIPTION_CHAR_LIMIT]
+        truncated_summary = markdown_to_jira(
+            bug.summary or "", max_length=JIRA_DESCRIPTION_CHAR_LIMIT
+        )
         fields: dict[str, str] = {
             "summary": truncated_summary,
         }

jbi/jira/utils.py

@@ -0,0 +1,15 @@
+import logging
+
+import pypandoc  # type: ignore
+
+logging.getLogger("pypandoc").addHandler(logging.NullHandler())
+
+
+def markdown_to_jira(markdown: str, max_length: int = 0) -> str:
+    """
+    Convert markdown content into Jira specific markup language.
+    """
+    if max_length > 0 and len(markdown) > max_length:
+        # Truncate on last word.
+        markdown = markdown[:max_length].rsplit(maxsplit=1)[0]
+    return pypandoc.convert_text(markdown, "jira", format="md").strip()  # type: ignore

poetry.lock

@@ -20,6 +20,7 @@ statsd = "^4.0.1"
 requests = "^2.31.0"
 pydantic-settings = "^2.1.0"
 asgi-correlation-id = "^4.3.0"
+pypandoc = "^1.12"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.6.1"

pyproject.toml

@@ -131,7 +131,7 @@ def context_comment_example(action_context_factory) -> ActionContext:
         bug__see_also=["https://mozilla.atlassian.net/browse/JBI-234"],
         bug__with_comment=True,
         bug__comment__number=2,
-        bug__comment__body="hello",
+        bug__comment__body="> hello\n>\n\nworld",
         event__target="comment",
         event__user__login="[email protected]",
         jira__issue="JBI-234",

tests/conftest.py

@@ -0,0 +1,68 @@
+from textwrap import dedent
+
+import pytest
+
+from jbi.jira.utils import markdown_to_jira
+
+
+def test_markdown_to_jira():
+    markdown = dedent(
+        """
+    Mixed nested lists
+
+    * a
+    * bulleted
+      - with
+      - nested
+        1. nested-nested
+      - numbered
+    * list
+
+    this was `inline` value ``that`` is turned into ```monospace``` tag.
+
+    this sentence __has__ **bold** and _has_ *italic*.
+
+    this was ~~wrong~~.
+    """
+    ).lstrip()
+
+    jira = dedent(
+        """
+    Mixed nested lists
+
+    * a
+    * bulleted
+    ** with
+    ** nested
+    **# nested-nested
+    ** numbered
+    * list
+
+    this was {{inline}} value {{that}} is turned into {{monospace}} tag.
+
+    this sentence *has* *bold* and _has_ _italic_.
+
+    this was -wrong-.
+    """
+    ).strip()
+
+    assert markdown_to_jira(markdown) == jira
+
+
+def test_markdown_to_jira_with_malformed_input():
+    assert markdown_to_jira("[link|http://noend") == "\\[link|http://noend"
+
+
+@pytest.mark.parametrize(
+    "markdown, expected, max_length",
+    [
+        ("a" * 10, "aaaaa", 5),
+        ("aa aaa", "aa", 5),
+        ("aa\naaa", "aa", 5),
+        ("aa\taaa", "aa", 5),
+        ("aaaaaa", "aaaaa", 5),
+        ("aaaaa ", "aaaaa", 5),
+    ],
+)
+def test_markdown_to_jira_with_max_chars(markdown, expected, max_length):
+    assert markdown_to_jira(markdown, max_length=max_length) == expected