Added support for searching ecosystem and blog entries.

The blog results should have a property of whether it is included in
the search results. We should also limit the blogs that are searchable
for a version of Django based on the support end. This will allow us
to limit the inclusion of blog posts in the search based on the time
the entry was created, keeping the search results relevant to that
version of Django.

Author: tim-schilling

docs/migrations/0007_documentrelease_support_end.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2 on 2025-07-23 16:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('docs', '0006_alter_document_metadata_noop'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='documentrelease',
+            name='support_end',
+            field=models.DateField(blank=True, help_text='The end of support for this release of Django.', null=True),
+        ),
+    ]

docs/models.py

@@ -18,10 +18,13 @@
 from django.db import models, transaction
 from django.db.models import Q
 from django.db.models.fields.json import KeyTextTransform
+from django.test import RequestFactory
+from django.urls import resolve, reverse as reverse_path
 from django.utils.functional import cached_property
 from django.utils.html import strip_tags
 from django_hosts.resolvers import reverse
 
+from blog.models import Entry
 from releases.models import Release
 
 from . import utils
@@ -31,6 +34,7 @@
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
+    DocumentationCategory,
 )
 
 
@@ -95,6 +99,11 @@ class DocumentRelease(models.Model):
         on_delete=models.CASCADE,
     )
     is_default = models.BooleanField(default=False)
+    support_end = models.DateField(
+        null=True,
+        blank=True,
+        help_text="The end of support for this release of Django.",
+    )
 
     objects = DocumentReleaseQuerySet.as_manager()
 
@@ -212,6 +221,83 @@ def sync_to_db(self, decoded_documents):
             )
             document.save(update_fields=("metadata",))
 
+        self._sync_blog_to_db()
+        self._sync_views_to_db()
+
+    def _sync_blog_to_db(self):
+        """
+        Sync the blog entries into search based on the release documents
+        support end date.
+        """
+        if self.lang == "en" and self.support_end:
+            for entry in Entry.objects.published(self.support_end).searchable():
+                Document.objects.create(
+                    release=self,
+                    path=entry.get_absolute_url(),
+                    title=entry.headline,
+                    metadata={
+                        "body": entry.body_html,
+                        "breadcrumbs": [
+                            {
+                                "path": DocumentationCategory.WEBSITE.value,
+                                "title": "News",
+                            },
+                        ],
+                        "parents": DocumentationCategory.WEBSITE.value,
+                        "slug": entry.slug,
+                        "title": entry.headline,
+                        "toc": "",
+                    },
+                    config=TSEARCH_CONFIG_LANGUAGES.get(
+                        self.lang[:2], DEFAULT_TEXT_SEARCH_CONFIG
+                    ),
+                )
+
+    def _sync_views_to_db(self):
+        """
+        Sync the blog entries into search based on the release documents
+        support end date.
+        """
+        if self.lang == "en":
+            # The request needs to come through as a valid one, it's best if it
+            # matches the exact host we're looking for.
+            www_hosts = [
+                host for host in settings.ALLOWED_HOSTS if host.startswith("www.")
+            ]
+            if not www_hosts or not (www_host := www_hosts[0]):
+                return
+            synced_views = [
+                # Page title, url name, url kwargs
+                ("Django's Ecosystem", "community-ecosystem", {}),
+            ]
+            for title, url_name, kwargs in synced_views:
+                absolute_url = reverse(url_name, kwargs=kwargs, host="www")
+                path = reverse_path(url_name, kwargs=kwargs)
+                request = RequestFactory().get(path, HTTP_HOST=www_host)
+                body = resolve(path).func(request).render().text
+                # Need to parse the body element.
+                Document.objects.create(
+                    release=self,
+                    path=absolute_url,
+                    title=title,
+                    metadata={
+                        "body": body,
+                        "breadcrumbs": [
+                            {
+                                "path": DocumentationCategory.WEBSITE.value,
+                                "title": "Website",
+                            },
+                        ],
+                        "parents": DocumentationCategory.WEBSITE.value,
+                        "slug": url_name,
+                        "title": title,
+                        "toc": "",
+                    },
+                    config=TSEARCH_CONFIG_LANGUAGES.get(
+                        self.lang[:2], DEFAULT_TEXT_SEARCH_CONFIG
+                    ),
+                )
+
 
 def _clean_document_path(path):
     # We have to be a bit careful to reverse-engineer the correct
@@ -224,7 +310,9 @@ def _clean_document_path(path):
 
 
 def document_url(doc):
-    if doc.path:
+    if doc.metadata.get("parents") == DocumentationCategory.WEBSITE.value:
+        return doc.path
+    elif doc.path:
         kwargs = {
             "lang": doc.release.lang,
             "version": doc.release.version,

docs/search.py

@@ -65,6 +65,7 @@ class DocumentationCategory(TextChoices):
     TOPICS = "topics", _("Using Django")
     HOWTO = "howto", _("How-to guides")
     RELEASE_NOTES = "releases", _("Release notes")
+    WEBSITE = "weblog", _("Django Website")
 
     @classmethod
     def parse(cls, value, default=None):

docs/tests/test_models.py

@@ -4,10 +4,14 @@
 from django.conf import settings
 from django.db import connection
 from django.test import TestCase
+from django.utils import timezone
+from django_hosts import reverse
 
+from blog.models import Entry
 from releases.models import Release
 
 from ..models import DOCUMENT_SEARCH_VECTOR, Document, DocumentRelease
+from ..search import DocumentationCategory
 
 
 class ModelsTests(TestCase):
@@ -465,7 +469,21 @@ def test_search_title(self):
 class UpdateDocTests(TestCase):
     @classmethod
     def setUpTestData(cls):
-        cls.release = DocumentRelease.objects.create()
+        now = timezone.now()
+        cls.release = DocumentRelease.objects.create(
+            support_end=now + datetime.timedelta(days=1)
+        )
+        cls.entry = Entry.objects.create(
+            pub_date=now,
+            is_active=True,
+            is_searchable=True,
+            headline="Searchable post",
+            slug="a",
+            body_html="<h1>Searchable Blog Post</h1>",
+        )
+        cls.docs_documents = cls.release.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE.value
+        )
 
     def test_sync_to_db(self):
         self.release.sync_to_db(
@@ -477,8 +495,43 @@ def test_sync_to_db(self):
                 }
             ]
         )
-        document = self.release.documents.get()
-        self.assertEqual(document.path, "foo/bar")
+        document_paths = set(self.release.documents.values_list("path", flat=True))
+        self.assertEqual(
+            document_paths,
+            {
+                "foo/bar",
+                reverse("community-ecosystem", host="www"),
+                self.entry.get_absolute_url(),
+            },
+        )
+
+    def test_blog_to_db_skip_non_english(self):
+        """
+        Releases must be English to include the blog and website results in search.
+        """
+        non_english = DocumentRelease.objects.create(
+            lang="es",
+            release=Release.objects.create(version="88.0"),
+            support_end=self.release.support_end,
+        )
+        non_english.sync_to_db([])
+        self.assertFalse(non_english.documents.exists())
+
+    def test_blog_to_db_skip_no_end_support(self):
+        """
+        Releases must have an end support to include the blog.
+        """
+        no_end_support = DocumentRelease.objects.create(
+            lang="en",
+            release=Release.objects.create(version="99.0"),
+        )
+        no_end_support.sync_to_db([])
+        self.assertEqual(
+            set(no_end_support.documents.values_list("path", flat=True)),
+            {
+                reverse("community-ecosystem", host="www"),
+            },
+        )
 
     def test_clean_path(self):
         self.release.sync_to_db(
@@ -490,7 +543,7 @@ def test_clean_path(self):
                 }
             ]
         )
-        document = self.release.documents.get()
+        document = self.docs_documents.get()
         self.assertEqual(document.path, "foo/bar")
 
     def test_title_strip_tags(self):
@@ -504,7 +557,7 @@ def test_title_strip_tags(self):
             ]
         )
         self.assertQuerySetEqual(
-            self.release.documents.all(),
+            self.docs_documents.all(),
             ["This is the title"],
             transform=attrgetter("title"),
         )
@@ -520,7 +573,7 @@ def test_title_entities(self):
             ]
         )
         self.assertQuerySetEqual(
-            self.release.documents.all(),
+            self.docs_documents,
             ["Title & title"],
             transform=attrgetter("title"),
         )
@@ -533,7 +586,7 @@ def test_empty_documents(self):
                 {"current_page_name": "foo/3"},
             ]
         )
-        self.assertQuerySetEqual(self.release.documents.all(), [])
+        self.assertQuerySetEqual(self.docs_documents, [])
 
     def test_excluded_documents(self):
         """
@@ -562,3 +615,47 @@ def test_excluded_documents(self):
         )
         document = release.documents.get()
         self.assertEqual(document.path, "nonexcluded/bar")
+
+
+class DocumentUrlTests(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.release = DocumentRelease.objects.create(
+            release=Release.objects.create(version="1.2.3"),
+        )
+        documents = [
+            {
+                "metadata": {"parents": "topics http"},
+                "path": "topics/http/generic-views",
+                "release": cls.release,
+                "title": "Generic views",
+            },
+            # I'm not sure if this is valid or not.
+            {
+                "metadata": {},
+                "path": "",
+                "release": cls.release,
+                "title": "Index",
+            },
+        ]
+        # Include the static views in the document search
+        cls.release._sync_views_to_db()
+        Document.objects.bulk_create(Document(**doc) for doc in documents)
+        cls.document_index, cls.document_view, cls.document_detail = (
+            cls.release.documents.order_by("path")
+        )
+
+    def test_document_url(self):
+        self.assertEqual(
+            self.document_index.get_absolute_url(),
+            "http://docs.djangoproject.localhost:8000/en/1.2.3/",
+        )
+        self.assertEqual(
+            self.document_view.get_absolute_url(),
+            "http://www.djangoproject.localhost:8000/community/ecosystem/",
+        )
+        self.assertEqual(
+            self.document_detail.get_absolute_url(),
+            "http://docs.djangoproject.localhost:8000"
+            "/en/1.2.3/topics/http/generic-views/",
+        )

docs/tests/test_views.py

@@ -91,7 +91,7 @@ def test_search_type_filter_all(self):
         )
         self.assertEqual(response.status_code, 200)
         self.assertContains(
-            response, "4 results for <em>generic</em> in version 5.1", html=True
+            response, "5 results for <em>generic</em> in version 5.1", html=True
         )
         self.assertContains(response, self.active_filter, count=1)
         self.assertContains(response, f"{self.active_filter}All</a>", html=True)
@@ -122,7 +122,7 @@ def test_search_category_filter_invalid_doc_categories(self):
         )
         self.assertEqual(response.status_code, 200)
         self.assertContains(
-            response, "4 results for <em>generic</em> in version 5.1", html=True
+            response, "5 results for <em>generic</em> in version 5.1", html=True
         )
         self.assertContains(response, self.active_filter, count=1)
         self.assertContains(response, f"{self.active_filter}All</a>", html=True)