branch_worker: implement PR comment forwarding

Implement an optional generic PR comment forwarding feature.

Whenever a relevant patchwork subject is processed in
sync_relevant_subject(), inspect pull request comments and send
selected comment body via email according to KPD configuration.

This by design only affects pull request with corresponding up-to-date
patch series on patchwork.

To be forwarded the comment must pass the following filters:
- the comment hasn't been forwarded yet
  - KPD is stateless, and so we determine this by checking for other
    comments on the same PR that report previous notifications
- comment author's GitHub login is in configured allowlist
- comment body contains an "In-Reply-To-Subject" tag with the subject
  string of a patch message to reply to
  - rationale for such tag is the necessity for a mechanism to
    determine the exact message in a thread to reply to; commit hash
    or index can change between series version, and a subject string
    uniquely identifies a patch message within a series

When a comment is selected for forwarding KPD:
- posts a PR comment reporting the forward action
- sends an email to all original recipients of the patch message,
  filtered according to the configured allow/denylist

An example of PR comments forwarding configuration:

    "email": {
      ...
      "pr_comments_forwarding": {
        "enabled": true,
        "commenter_allowlist": ["kpd-bot[bot]"],
        "always_cc": ["[email protected]"],
        "recipient_denylist": [".*@foobar.org"]
      }
    }

Signed-off-by: Ihor Solodrai <[email protected]>

Author: theihor

kernel_patches_daemon/branch_worker.py

@@ -328,7 +328,7 @@ async def send_email(
     subject: str,
     body: str,
     in_reply_to: Optional[str] = None,
-):
+) -> str:
     """Send an email."""
     msg_id = generate_msg_id(config.smtp_host)
     curl_args, msg = build_email(
@@ -343,6 +343,8 @@ async def send_email(
         logger.error(f"failed to send email: {stdout.decode()} {stderr.decode()}")
         email_send_fail_counter.add(1)
 
+    return msg_id
+
 
 def ci_results_email_recipients(
     config: EmailConfig, series: Series
@@ -368,6 +370,61 @@ async def send_ci_results_email(
     await send_email(config, to_list, cc_list, subject, body, in_reply_to)
 
 
+def reply_email_recipients(
+    msg: EmailMessage,
+    allowlist: Sequence[re.Pattern] = None,
+    denylist: Sequence[re.Pattern] = None,
+) -> Tuple[List[str], List[str]]:
+    """
+    `msg` is an EmailMessage we are replying to
+    """
+    tos = msg.get_all("To", [])
+    ccs = msg.get_all("Cc", [])
+    cc_list = [a for (_, a) in email.utils.getaddresses(tos + ccs)]
+
+    (_, sender_address) = email.utils.parseaddr(msg.get("From"))
+    to_list = [sender_address]
+
+    if allowlist:
+        cc_list = [a for a in cc_list if email_matches_any(a, allowlist)]
+        if not email_matches_any(sender_address, allowlist):
+            to_list = []
+
+    if denylist:
+        cc_list = [a for a in cc_list if not email_matches_any(a, denylist)]
+        if email_matches_any(sender_address, denylist):
+            to_list = []
+
+    return (to_list, cc_list)
+
+
+async def send_pr_comment_email(
+    config: EmailConfig, msg: EmailMessage, body: str
+) -> Optional[str]:
+    """
+    `msg` is an EmailMessage we are replying to
+    """
+    if config is None or not config.is_pr_comment_forwarding_enabled():
+        return
+
+    cfg = config.pr_comments_forwarding
+    (to_list, cc_list) = reply_email_recipients(
+        msg, allowlist=cfg.recipient_allowlist, denylist=cfg.recipient_denylist
+    )
+    if not to_list:
+        to_list = cfg.always_cc
+    else:
+        cc_list += cfg.always_cc
+
+    if not to_list:
+        return
+
+    subject = "Re: " + msg.get("Subject")
+    in_reply_to = msg.get("Message-Id")
+
+    return await send_email(config, to_list, cc_list, subject, body, in_reply_to)
+
+
 def pr_has_label(pr: PullRequest, label: str) -> bool:
     for pr_label in pr.get_labels():
         if pr_label.name == label:
@@ -1307,3 +1364,77 @@ async def submit_pr_summary(
             description="PR summary",
         )
         pr_summary_report.add(1)
+
+    async def forward_pr_comments(self, pr: PullRequest, series: Series):
+        if (
+            self.email_config is None
+            or not self.email_config.is_pr_comment_forwarding_enabled()
+        ):
+            return
+        cfg = self.email_config.pr_comments_forwarding
+
+        comments = pr.get_issue_comments()
+
+        # Look for comments indicating what has already been forwarded
+        # to filter them out
+        forwarded_set = set()
+        for comment in comments:
+            match = re.search(
+                r"Forwarding comment \[([0-9]+)\].* via email",
+                comment.body,
+                re.MULTILINE,
+            )
+            if not match:
+                continue
+            forwarded_id = int(match.group(1))
+            forwarded_set.add(forwarded_id)
+
+        for comment in comments:
+            if (
+                comment.user.login not in cfg.commenter_allowlist
+                or comment.id in forwarded_set
+            ):
+                continue
+            # Determine the message to reply to
+            # Check for In-Reply-To-Subject tag in the comment body
+            match = re.search(
+                r"In-Reply-To-Subject: \`(.*)\`", comment.body, re.MULTILINE
+            )
+            if not match:
+                logger.info(
+                    f"Ignoring PR comment {comment.html_url} without In-Reply-To-Subject tag"
+                )
+                continue
+            subject = match.group(1)
+            patch = series.patch_by_subject(subject)
+            if not patch:
+                logger.warn(
+                    f"Ignoring PR comment {comment.html_url}, could not find relevant patch on patchwork"
+                )
+                continue
+
+            # first, post a comment that the message is being forwarded
+            msg_id = patch["msgid"]
+            patch_url = patch["web_url"]
+            message = f"Forwarding comment [{comment.id}]({comment.html_url}) via email"
+            message += f"\nIn-Reply-To: {msg_id}"
+            message += f"\nPatch: {patch_url}"
+            self._add_pull_request_comment(pr, message)
+
+            # then, load the message we are replying to
+            mbox = await series.pw_client.get_blob(patch["mbox"])
+            parser = email.parser.BytesParser(policy=email.policy.default)
+            patch_msg = parser.parsebytes(mbox, headersonly=True)
+
+            # and forward the target comment via email
+            sent_msg_id = await send_pr_comment_email(
+                self.email_config, patch_msg, comment.body
+            )
+            if sent_msg_id is not None:
+                logger.info(
+                    f"Forwarded PR comment {comment.html_url} via email, Message-Id: {sent_msg_id}"
+                )
+            else:
+                logger.warn(
+                    f"Failed to forward PR comment in reply to {msg_id}, to_list is empty"
+                )

kernel_patches_daemon/config.py

@@ -104,6 +104,29 @@ def from_json(cls, json: Dict) -> "BranchConfig":
         )
 
 
+@dataclass
+class PRCommentsForwardingConfig:
+    enabled: bool
+    always_cc: List[str]
+    commenter_allowlist: List[str]
+    recipient_allowlist: List[re.Pattern]
+    recipient_denylist: List[re.Pattern]
+
+    @classmethod
+    def from_json(cls, json: Dict) -> "PRCommentsForwardingConfig":
+        return cls(
+            enabled=json.get("enabled", False),
+            always_cc=json.get("always_cc", []),
+            recipient_allowlist=[
+                re.compile(pattern) for pattern in json.get("recipient_allowlist", [])
+            ],
+            recipient_denylist=[
+                re.compile(pattern) for pattern in json.get("recipient_denylist", [])
+            ],
+            commenter_allowlist=json.get("commenter_allowlist", []),
+        )
+
+
 @dataclass
 class EmailConfig:
     smtp_host: str
@@ -123,6 +146,7 @@ class EmailConfig:
     # Ignore the `submitter_allowlist` entries and send emails to all patch
     # submitters, unconditionally.
     ignore_allowlist: bool
+    pr_comments_forwarding: Optional[PRCommentsForwardingConfig]
 
     @classmethod
     def from_json(cls, json: Dict) -> "EmailConfig":
@@ -139,8 +163,17 @@ def from_json(cls, json: Dict) -> "EmailConfig":
                 re.compile(pattern) for pattern in json.get("submitter_allowlist", [])
             ],
             ignore_allowlist=json.get("ignore_allowlist", False),
+            pr_comments_forwarding=PRCommentsForwardingConfig.from_json(
+                json.get("pr_comments_forwarding", {})
+            ),
         )
 
+    def is_pr_comment_forwarding_enabled(self) -> bool:
+        if self.pr_comments_forwarding is not None:
+            return self.pr_comments_forwarding.enabled
+        else:
+            return False
+
 
 @dataclass
 class PatchworksConfig:

kernel_patches_daemon/github_sync.py

@@ -269,6 +269,7 @@ async def sync_relevant_subject(self, subject: Subject) -> None:
                 f"Created/updated {pr} ({pr.head.ref}): {pr.url} for series {series.id}"
             )
             await worker.sync_checks(pr, series)
+            await worker.forward_pr_comments(pr, series)
             # Close out other PRs if exists
             self.close_existing_prs_for_series(list(self.workers.values()), pr)
 

kernel_patches_daemon/patchwork.py

@@ -497,6 +497,12 @@ def submitter_email(self):
         """Retrieve the email address of the patch series submitter."""
         return self._submitter_email
 
+    def patch_by_subject(self, subject_str: str) -> Optional[Dict]:
+        for patch in self.patches:
+            if subject_str in patch["name"]:
+                return patch
+        return None
+
 
 class Patchwork:
     def __init__(

tests/data/fixtures/kpd_config.json

@@ -22,7 +22,13 @@
       "pass": "super-secret-is-king",
       "http_proxy": "http://example.com:8080",
       "submitter_allowlist": ["[email protected]", "[email protected]"],
-      "ignore_allowlist": true
+      "ignore_allowlist": true,
+      "pr_comments_forwarding": {
+        "enabled": true,
+        "always_cc": ["[email protected]"],
+        "commenter_allowlist": ["kpd-bot[bot]"],
+        "recipient_denylist": [".*@vger.kernel.org"]
+      }
     },
     "tag_to_branch_mapping": {
       "tag": [