Made typos in docs searches return some results.

Co-authored-by: Paolo Melchiorre <[email protected]>

Author: sarahboyce

docs/models.py

@@ -250,8 +250,7 @@ def search(self, query_text, release):
                 query_text, config=models.F("config"), search_type="websearch"
             )
             search_rank = SearchRank(models.F("search"), search_query)
-            similarity = TrigramSimilarity("title", query_text)
-            return (
+            base_qs = (
                 self.prefetch_related(
                     Prefetch(
                         "release",
@@ -261,12 +260,8 @@ def search(self, query_text, release):
                         "release__release", queryset=Release.objects.only("version")
                     ),
                 )
-                .filter(
-                    release_id=release.id,
-                    search=search_query,
-                )
+                .filter(release_id=release.id)
                 .annotate(
-                    rank=search_rank + similarity,
                     headline=SearchHeadline(
                         "title",
                         search_query,
@@ -283,12 +278,28 @@ def search(self, query_text, release):
                     ),
                     breadcrumbs=models.F("metadata__breadcrumbs"),
                 )
-                .order_by("-rank")
                 .only(
                     "path",
                     "release",
                 )
             )
+            vector_qs = (
+                base_qs.alias(rank=search_rank)
+                .filter(search=search_query)
+                .order_by("-rank")
+            )
+            if not vector_qs:
+                return (
+                    base_qs.alias(
+                        similarity=TrigramSimilarity(
+                            "title", utils.sanitize_for_trigram(query_text)
+                        )
+                    )
+                    .filter(similarity__gt=0.3)
+                    .order_by("-similarity")
+                )
+            else:
+                return vector_qs
         else:
             return self.none()
 

docs/tests.py

@@ -20,7 +20,7 @@
 from .models import DOCUMENT_SEARCH_VECTOR, Document, DocumentRelease
 from .sitemaps import DocsSitemap
 from .templatetags.docs import get_all_doc_versions
-from .utils import get_doc_path
+from .utils import get_doc_path, sanitize_for_trigram
 
 
 class ModelsTests(TestCase):
@@ -262,6 +262,30 @@ def test_get_doc_path(self):
         path, filename = __file__.rsplit(os.path.sep, 1)
         self.assertEqual(get_doc_path(Path(path), filename), None)
 
+    def test_sanitize_for_trigram(self):
+        for query, sanitized_query in [
+            ("simple search", "simple search"),
+            ("Python Django -Flask", "Python Django"),
+            ('Python "Django Framework" -Flask', "Python Django Framework"),
+            ("Développement -'Framework Django' web", "Developpement web"),
+            (
+                "Γλώσσα προγραμματισμού Python -'Flask και Django'",
+                "Γλωσσα προγραμματισμου Python",
+            ),
+            (
+                "Pemrograman Python -'Flask dan Django' backend",
+                "Pemrograman Python backend",
+            ),
+            (
+                "Programmazione 'Python e Django' -Flask",
+                "Programmazione Python e Django",
+            ),
+            ("Linguagem Python -'Django e Flask' web", "Linguagem Python web"),
+            ("Desarrollo Python -'Django y Flask' rápido", "Desarrollo Python rapido"),
+        ]:
+            with self.subTest(query=query):
+                self.assertEqual(sanitize_for_trigram(query), sanitized_query)
+
 
 class UpdateDocTests(TestCase):
     @classmethod
@@ -548,18 +572,16 @@ def setUp(self):
     def test_search(self):
         expected_list = [
             (
-                0.96982837,
                 "releases/1.2.1",
-                "<mark>Django</mark> 1.2.1 release notes",
+                "<mark>Django</mark> 1.2.1 release notes",  # Ranked: 0.96982837.
                 (
                     "<mark>Django</mark> 1.2.1 release notes ¶  \n "
                     "<mark>Django</mark> 1.2.1 was released almost immediately after 1.2.0 to correct two small"
                 ),
             ),
             (
-                0.9490876,
                 "releases/1.9.4",
-                "<mark>Django</mark> 1.9.4 release notes",
+                "<mark>Django</mark> 1.9.4 release notes",  # Ranked: 0.9490876.
                 (
                     "<mark>Django</mark> 1.9.4 release notes ¶  \n  "
                     "March 5, 2016  \n "
@@ -570,24 +592,24 @@ def test_search(self):
         self.assertQuerySetEqual(
             Document.objects.search("django", self.release),
             expected_list,
-            transform=attrgetter("rank", "path", "headline", "highlight"),
+            transform=attrgetter("path", "headline", "highlight"),
         )
 
     def test_websearch(self):
         self.assertQuerySetEqual(
             Document.objects.search('django "release notes" -packaging', self.release),
-            [("Django 1.9.4 release notes", 1.5675676)],
-            transform=attrgetter("title", "rank"),
+            ["Django 1.9.4 release notes"],
+            transform=attrgetter("title"),
         )
 
     def test_multilingual_search(self):
         self.assertQuerySetEqual(
             Document.objects.search("publication", self.release_fr),
             [
-                ("Notes de publication de Django 1.2.1", 1.0693262),
-                ("Notes de publication de Django 1.9.4", 1.0458658),
+                "Notes de publication de Django 1.2.1",  # Ranked: 1.0693262.
+                "Notes de publication de Django 1.9.4",  # Ranked: 1.0458658.
             ],
-            transform=attrgetter("title", "rank"),
+            transform=attrgetter("title"),
         )
 
     def test_empty_search(self):
@@ -642,6 +664,15 @@ def test_search_highlight_stemmed(self):
             transform=attrgetter("headline", "highlight"),
         )
 
+    def test_search_title(self):
+        misspelled_query = Document.objects.search("viewss", self.release)
+        with self.assertNumQueries(2):
+            self.assertQuerySetEqual(
+                misspelled_query,
+                ["Generic views"],
+                transform=attrgetter("headline"),
+            )
+
 
 class TemplateTestCase(TestCase):
     def _assertOGTitleEqual(self, doc, expected):

docs/utils.py

@@ -1,3 +1,6 @@
+import re
+import unicodedata
+
 from django.conf import settings
 from django.http import Http404
 
@@ -39,3 +42,18 @@ def get_doc_path_or_404(docroot, subpath):
     if doc is None:
         raise Http404(doc)
     return doc
+
+
+def sanitize_for_trigram(text):
+    """
+    Sanitize search query for PostgreSQL Trigram search.
+
+    - Removes parts starting with '-'
+    - Normalizes Unicode characters (NFKD)
+    - Keeps only letters, numbers and spaces
+    - Removes multiple spaces and trims
+    """
+    text = re.sub(r'(\s|^)-[^\s"\']+|(\s|^)-["\'][^"\']+["\']', "", text)
+    text = unicodedata.normalize("NFKD", text)
+    text = re.sub(r"[^\w\s]", "", text, flags=re.UNICODE)
+    return " ".join(text.split())