Release report changes (boostorg#1862)

Author: daveoconnor

core/constants.py

@@ -4,3 +4,6 @@
 class SourceDocType(Enum):
     ASCIIDOC = "asciidoc"
     ANTORA = "antora"
+
+
+SLACK_URL = "https://cpplang.slack.com"

libraries/forms.py

@@ -26,7 +26,7 @@
 )
 from libraries.constants import SUB_LIBRARIES
 from mailing_list.models import EmailData
-from .utils import batched
+from .utils import batched, conditional_batched
 
 
 class LibraryForm(ModelForm):
@@ -668,6 +668,31 @@ def _get_dependency_data(self, library_order, version):
             diffs.append(diffs_by_id.get(lib_id, {}))
         return diffs
 
+    def get_library_data(self, libraries, library_order, prior_version, version):
+        library_data = [
+            {
+                "library": item[0],
+                "full_count": item[1],
+                "version_count": item[2],
+                "top_contributors_release": item[3],
+                "new_contributors_count": item[4],
+                "issues": item[5],
+                "library_version": item[6],
+                "deps": item[7],
+            }
+            for item in zip(
+                libraries,
+                self._get_library_full_counts(libraries, library_order),
+                self._get_library_version_counts(library_order, version),
+                self._get_top_contributors_for_library_version(library_order, version),
+                self._count_new_contributors(libraries, library_order, version),
+                self._count_issues(libraries, library_order, version, prior_version),
+                self._get_library_versions(library_order, version),
+                self._get_dependency_data(library_order, version),
+            )
+        ]
+        return [x for x in library_data if x["version_count"]["commit_count"] > 0]
+
     def get_stats(self):
         report_configuration = self.cleaned_data["report_configuration"]
         version = Version.objects.filter(name=report_configuration.version).first()
@@ -713,31 +738,16 @@ def get_stats(self):
             )
         )
 
-        library_data = [
-            {
-                "library": item[0],
-                "full_count": item[1],
-                "version_count": item[2],
-                "top_contributors_release": item[3],
-                "new_contributors_count": item[4],
-                "issues": item[5],
-                "library_version": item[6],
-                "deps": item[7],
-            }
-            for item in zip(
-                libraries,
-                self._get_library_full_counts(libraries, library_order),
-                self._get_library_version_counts(library_order, version),
-                self._get_top_contributors_for_library_version(library_order, version),
-                self._count_new_contributors(libraries, library_order, version),
-                self._count_issues(libraries, library_order, version, prior_version),
-                self._get_library_versions(library_order, version),
-                self._get_dependency_data(library_order, version),
-            )
-        ]
-        library_data = [
-            x for x in library_data if x["version_count"]["commit_count"] > 0
-        ]
+        library_data = self.get_library_data(
+            libraries, library_order, prior_version, version
+        )
+        AUTHORS_PER_PAGE_THRESHOLD = 6
+        batched_library_data = conditional_batched(
+            library_data,
+            2,
+            lambda x: x.get("top_contributors_release").count()
+            <= AUTHORS_PER_PAGE_THRESHOLD,
+        )
         top_contributors = self._get_top_contributors_for_version(version)
         # total messages sent during this release (version)
         total_mailinglist_count = EmailData.objects.filter(version=version).aggregate(
@@ -840,6 +850,7 @@ def get_stats(self):
             "version_commit_count": version_commit_count,
             "top_contributors_release_overall": top_contributors,
             "library_data": library_data,
+            "batched_library_data": batched_library_data,
             "top_libraries_for_version": top_libraries_for_version,
             "library_count": library_count,
             "library_count_prior": library_count_prior,

libraries/tests/test_utils.py

@@ -4,6 +4,7 @@
 from dateutil.relativedelta import relativedelta
 
 from libraries.utils import (
+    conditional_batched,
     decode_content,
     generate_fake_email,
     get_first_last_day_last_month,
@@ -232,3 +233,52 @@ def test_write_content_to_tempfile():
         file_content = f.read()
     assert file_content == content
     os.remove(temp_file.name)
+
+
+def test_conditional_batched():
+    # test basic functionality: batch consecutive items that pass condition
+    items = [1, 2, 4, 3, 6, 8, 5, 10, 12, 7]
+    # even numbers should be batched
+    result = list(conditional_batched(items, 2, lambda x: x % 2 == 0))
+
+    # consecutive even numbers get batched, odd numbers are individual, order preserved
+    assert result == [(1,), (2, 4), (3,), (6, 8), (5,), (10, 12), (7,)]
+
+
+def test_conditional_batched_all_pass():
+    # test when all items pass the condition
+    items = [2, 4, 6, 8, 10]
+    result = list(conditional_batched(items, 2, lambda x: x % 2 == 0))
+
+    assert result == [(2, 4), (6, 8), (10,)]
+
+
+def test_conditional_batched_all_fail():
+    # test when all items fail the condition
+    items = [1, 3, 5, 7, 9]
+    result = list(conditional_batched(items, 2, lambda x: x % 2 == 0))
+
+    assert result == [(1,), (3,), (5,), (7,), (9,)]
+
+
+def test_conditional_batched_strict_mode():
+    # test strict mode with incomplete batch
+    items = [2, 4, 6]
+    with pytest.raises(ValueError, match="conditional_batched\\(\\): incomplete batch"):
+        list(conditional_batched(items, 2, lambda x: x % 2 == 0, strict=True))
+
+
+def test_conditional_batched_strict_mode_complete():
+    # test strict mode with complete batches
+    items = [2, 4, 6, 8]
+    result = list(conditional_batched(items, 2, lambda x: x % 2 == 0, strict=True))
+
+    assert result == [(2, 4), (6, 8)]
+
+
+def test_conditional_batched_invalid_n():
+    # test invalid batch size
+    items = [1, 2, 3]
+
+    with pytest.raises(ValueError, match="n must be at least one"):
+        list(conditional_batched(items, 0, lambda x: True))

libraries/utils.py

@@ -218,6 +218,46 @@ def batched(iterable, n, *, strict=False):
         yield batch
 
 
+def conditional_batched(iterable, n: int, condition: callable, *, strict=False):
+    """
+    Batch items that pass a condition together, return items that fail individually.
+
+    Args:
+        iterable: Items to process
+        n: Batch size for items that pass the condition
+        condition: Function that returns True if item should be batched
+        strict: If True, raise error for incomplete final batch
+
+    Yields:
+        Tuples of batched items or single-item tuples for items that fail condition
+    """
+    if n < 1:
+        raise ValueError("n must be at least one")
+
+    batch = []
+
+    for item in iterable:
+        if condition(item):
+            # item passes condition - add to batch
+            batch.append(item)
+            if len(batch) == n:
+                # batch is full - yield it and start new batch
+                yield tuple(batch)
+                batch = []
+        else:
+            # item fails condition - yield any pending batch first, then item alone
+            if batch:
+                yield tuple(batch)
+                batch = []
+            yield (item,)
+
+    # handle any remaining items in batch
+    if strict and batch and len(batch) != n:
+        raise ValueError("conditional_batched(): incomplete batch")
+    if batch:
+        yield tuple(batch)
+
+
 def legacy_path_transform(content_path):
     if content_path and content_path.startswith(LEGACY_LATEST_RELEASE_URL_PATH_STR):
         content_path = re.sub(r"([a-zA-Z0-9\.]+)/(\S+)", r"latest/\2", content_path)

slack/management/commands/fetch_slack_activity.py

@@ -2,6 +2,7 @@
 import datetime
 import functools
 import time
+import re
 
 from slack_sdk import WebClient
 from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler
@@ -12,6 +13,7 @@
 from django.conf import settings
 from django.core.management import CommandError
 
+from core.constants import SLACK_URL
 from slack.models import (
     SlackUser,
     SlackActivityBucket,
@@ -289,14 +291,97 @@ def command(channels, debug):
         # materialize this generator so we can iterate multiple times
         selected_channels.extend(get_my_channels())
 
+    def interpolate_text_usernames(text):
+        user_mentions = re.findall(r"<@([A-Z0-9]+)>", text)
+        for user_id in user_mentions:
+            try:
+                slack_user = SlackUser.objects.get(id=user_id)
+                profile_url = f"{SLACK_URL}/team/{user_id}"
+                text = text.replace(
+                    f"<@{user_id}>", f'<a href="{profile_url}">@{slack_user.name}</a>'
+                )
+            except SlackUser.DoesNotExist:
+                logger.warning(f"SlackUser {user_id} not found in database")
+                continue
+
+        return text
+
+    def interpolate_text_slack_channels(text):
+        # match both <#CHANNELID> and <#CHANNELID|optional_text>
+        channel_mentions = re.findall(r"<#([A-Z0-9]+)(?:\|[^>]*)?>", text)
+        for channel_id in channel_mentions:
+            try:
+                channel = Channel.objects.get(id=channel_id)
+                channel_name = channel.name
+            except Channel.DoesNotExist:
+                try:
+                    # fetch channel info from Slack API
+                    channel_data = client.conversations_info(channel=channel_id)
+                    channel_name = channel_data.data["channel"]["name"]
+                    logger.info(
+                        f"Fetched channel name {channel_name} for {channel_id} from API"
+                    )
+                except Exception as e:
+                    logger.warning(f"Failed to get channel info for {channel_id}: {e}")
+                    continue
+
+            # replace the full match including any pipe content
+            pattern = f"<#{channel_id}(?:\\|[^>]*)?>"
+            text = re.sub(pattern, f"#{channel_name}", text)
+
+        return text
+
+    def interpolate_text_subteams(text):
+        # returns early because we don't have usergroups:read permission added and
+        # the only channel that really needs this data parsed at the moment is
+        # #general which doesn't appear in the release report. If we need it in
+        # the future, Sam says we'd need to create a new bot with that permission.
+        return text
+
+        # match <!subteam^SUBTEAMID> patterns
+        subteam_mentions = re.findall(r"<!subteam\^([A-Z0-9]+)>", text)
+        for subteam_id in subteam_mentions:
+            try:
+                usergroups_data = client.usergroups_list()
+                for usergroup in usergroups_data.data["usergroups"]:
+                    if usergroup["id"] == subteam_id:
+                        subteam_name = usergroup["handle"]
+                        text = text.replace(
+                            f"<!subteam^{subteam_id}>", f"@{subteam_name}"
+                        )
+                        break
+                else:
+                    logger.warning(f"Subteam {subteam_id} not found in usergroups list")
+            except Exception as e:
+                logger.warning(f"Failed to get subteam info for {subteam_id}: {e}")
+                continue
+
+        return text
+
+    def interpolate_text_urls_with_jinja_links(text):
+        return re.sub(r"<(https?://[^>]+)>", r'<a href="\1">\1</a>', text)
+
     for channel_data in selected_channels:
         with transaction.atomic():
+            topic = channel_data["topic"]["value"]
+            if topic:
+                topic = interpolate_text_usernames(topic)
+                topic = interpolate_text_slack_channels(topic)
+                topic = interpolate_text_subteams(topic)
+                topic = interpolate_text_urls_with_jinja_links(topic)
+            purpose = channel_data["purpose"]["value"]
+            if purpose:
+                purpose = interpolate_text_usernames(purpose)
+                purpose = interpolate_text_slack_channels(purpose)
+                purpose = interpolate_text_subteams(purpose)
+                purpose = interpolate_text_urls_with_jinja_links(purpose)
+
             channel, created = Channel.objects.update_or_create(
                 id=channel_data["id"],
                 defaults={
                     "name": channel_data["name"],
-                    "topic": channel_data["topic"]["value"],
-                    "purpose": channel_data["purpose"]["value"],
+                    "topic": topic,
+                    "purpose": purpose,
                 },
             )
             if created: