feat: Add advanced search

Author: mrmn2

pdfding/e2e/test_pdf_overview_e2e.py

@@ -5,7 +5,7 @@
 from django.test import override_settings
 from django.urls import reverse
 from helpers import PdfDingE2ETestCase
-from pdf.models.pdf_models import Pdf
+from pdf.models.pdf_models import Metadata, Pdf
 from pdf.models.tag_models import Tag
 from playwright.sync_api import expect, sync_playwright
 from users.models import Profile
@@ -502,6 +502,23 @@ def test_search_filters(self):
             expect(self.page.locator("#search_filter")).not_to_be_visible()
             expect(self.page.locator("#tag_tag_filter")).not_to_be_visible()
 
+    def test_advanced_search(self):
+        pdf = Pdf.objects.get(name='pdf_1_1')
+        Metadata.objects.create(pdf=pdf, keywords='some_keyword')
+
+        with sync_playwright() as p:
+            self.open(reverse('advanced_search'), p)
+
+            self.page.locator("#id_name").click()
+            self.page.locator("#id_name").fill("pdf")
+            self.page.locator("#id_keywords").click()
+            self.page.locator("#id_keywords").fill("some_keyword")
+            self.page.get_by_role("button", name="Search").click()
+
+            # assert there is only one pdf matching the search
+            expect(self.page.locator("#pdf-link-1")).to_contain_text("pdf_1_1")
+            expect(self.page.locator("#pdf-link-2")).not_to_be_visible()
+
     def test_sort(self):
         self.user.profile.pdf_sorting = Profile.PdfSortingChoice.MOST_VIEWED
         self.user.profile.save()

pdfding/pdf/forms.py

@@ -623,6 +623,38 @@ class Meta:
         fields = ['description']
 
 
+class AdvancedSearchForm(forms.Form):
+    """Form for the advanced search."""
+
+    # normal fields
+    name = forms.CharField(required=False)
+    name.group = 'PDF Details'
+    tags = forms.CharField(required=False)
+    tags.group = 'PDF Details'
+    description = forms.CharField(required=False)
+    description.group = 'PDF Details'
+    notes = forms.CharField(required=False)
+    notes.group = 'PDF Details'
+
+    # metadata fields
+    title = forms.CharField(required=False)
+    title.group = 'PDF Metadata'
+    author = forms.CharField(required=False)
+    author.group = 'PDF Metadata'
+    abstract = forms.CharField(required=False)
+    abstract.group = 'PDF Metadata'
+    keywords = forms.CharField(required=False)
+    keywords.group = 'PDF Metadata'
+    doi = forms.CharField(required=False, label='DOI')
+    doi.group = 'PDF Metadata'
+    journal = forms.CharField(required=False)
+    journal.group = 'PDF Metadata'
+    publisher = forms.CharField(required=False)
+    publisher.group = 'PDF Metadata'
+    year = forms.CharField(required=False)
+    year.group = 'PDF Metadata'
+
+
 class CleanHelpers:
     @staticmethod
     def clean_file(file: File) -> File:

pdfding/pdf/templates/advanced_search.html

@@ -0,0 +1,31 @@
+{% extends 'layouts/add_box.html' %}
+{% load i18n %}
+
+{% block content %}
+
+<h1>{% trans 'Advanced Search' %}</h1>
+
+<form action="{% url 'advanced_search_query' %}" method="GET" id="advanced_search_form">
+  <div class="flex flex-col pb-4 gap-y-2">
+    {% regroup form by field.group as field_groups %}
+    {% for field_group in field_groups %}
+    <div class="text-2xl font-bold {% if field_group.grouper == 'PDF Metadata' %}pt-4{% endif %}">
+    {{ field_group.grouper }}
+    </div>
+    <div class="gap-y-2 md:grid md:grid-cols-2 md:gap-x-10">
+    {% for field in field_group.list %}
+      <div>
+        {{ field.errors }}
+        <div class="[&>input]:rounded-md! [&>input]:py-2!">
+          <span class="font-semibold">{{ field.label }}</span>
+          {{ field }}
+        </div>
+      </div>
+    {% endfor %}
+    </div>
+    {% endfor %}
+  </div>
+  <button type="submit" >{% trans 'Search' %}</button>
+</form>
+
+{% endblock %}

pdfding/pdf/templates/includes/pdf_overview/header_extra.html

@@ -1,11 +1,12 @@
 {% load i18n %}
 <div x-data="{ sortOpen: false, layoutOpen: false }" class="relative flex flex-row grow justify-end items-center md:gap-x-1">
-    <form class="flex grow md:max-w-120 lg:max-w-160 px-3 md:px-5" action="{% url 'pdf_overview_query' %}" method="GET" id="search_form"> <input class="border h-10 rounded-sm! align-text-center focus:border-primary!
-                      bg-slate-100 dark:bg-slate-800! border-slate-400 dark:border-slate-600"
-              type="text" placeholder="{% translate 'Search for PDFs or #tags' %}"
-                id="search_input"
-                name="search"
-                @keyup.enter="document.getElementById('search_form').submit()"
+    <form class="flex grow md:max-w-120 lg:max-w-160 px-3 md:px-5" action="{% url 'pdf_overview_query' %}" method="GET" id="search_form">
+      <input class="border h-10 rounded-sm! align-text-center focus:border-primary!
+                    bg-slate-100 dark:bg-slate-800! border-slate-400 dark:border-slate-600"
+             type="text" placeholder="{% translate 'Search for PDFs or #tags' %}"
+             id="search_input"
+             name="search"
+             @keyup.enter="document.getElementById('search_form').submit()"
         />
     </form>
     <a id="bulk_edit"

pdfding/pdf/templates/pdf_overview.html

@@ -23,12 +23,14 @@
                         md:ml-10 min-[850px]:ml-10! lg:mx-10!
                         {% endif %}
                         w-full pb-4">
-                {% if search_query or tag_query %}
+                {% if advanced_search %}
+                {% querystring %}
+                {% elif search_query or tag_query %}
                 {% include 'includes/pdf_overview/search_filters.html' %}
                 {% endif %}
                 {% if not page_obj %}
                 <div class="pt-4 *:flex *:justify-center">
-                    {% if search_query or tag_query or special_pdf_selection %}
+                    {% if search_query or tag_query or special_pdf_selection or advanced_search %}
                     <span class="text-2xl">{% trans "There aren't any PDFs matching the current filters" %}</span>
                     <span class="pt-4 text-lg">{% trans 'Please try different filters' %}.</span>
                     {% else %}

pdfding/pdf/tests/test_views/test_pdf_views.py

@@ -381,7 +381,8 @@ def setUp(self):
         self.user = None
         set_up(self)
 
-    def test_filter_objects(self):
+    @patch('pdf.views.pdf_views.OverviewMixin.advanced_search_filtering')
+    def test_filter_objects(self, mock_advanced_filtering):
         # create some pdfs
         for i in range(1, 15):
             pdf = Pdf.objects.create(collection=self.user.profile.current_collection, name=f'pdf_{i % 5}_{i}')
@@ -404,11 +405,14 @@ def test_filter_objects(self):
         pdf_2.tags.set(tags[2:3])
         pdf_3.tags.set(tags)
 
-        response = self.client.get(f'{reverse('pdf_overview')}?search=pdf_&tags=programming/python')
+        response = self.client.get(
+            f'{reverse('pdf_overview')}?search=pdf_&tags=programming/python&advanced_search=false'
+        )
 
         filtered_pdfs = pdf_views.OverviewMixin.filter_objects(response.wsgi_request)
 
         self.assertEqual(sorted(list(filtered_pdfs), key=lambda a: a.name), [pdf_1, pdf_2])
+        mock_advanced_filtering.assert_not_called()
 
     def test_filter_objects_starred(self):
         pdf_1 = Pdf.objects.create(
@@ -463,6 +467,18 @@ def test_filter_objects_archived(self):
 
         self.assertEqual(list(filtered_pdfs), [pdf_1])
 
+    @patch('pdf.views.pdf_views.OverviewMixin.advanced_search_filtering')
+    def test_filter_objects_advanced(self, mock_advanced_filtering):
+        pdf_1 = Pdf.objects.create(collection=self.user.profile.current_collection, name='pdf_to_be_found')
+        Pdf.objects.create(collection=self.user.profile.current_collection, name='pdf_not_to_be_found')
+
+        mock_advanced_filtering.return_value = self.user.profile.current_pdfs.filter(name='pdf_to_be_found')
+        response = self.client.get(f'{reverse('pdf_overview')}?advanced_search=true')
+
+        filtered_pdfs = pdf_views.OverviewMixin.filter_objects(response.wsgi_request)
+        assert filtered_pdfs.count() == 1
+        assert filtered_pdfs.first() == pdf_1
+
     def test_fuzzy_filter_pdfs(self):
         Pdf.objects.create(collection=self.user.profile.current_collection, name='pdf_not_to_be_found')
         pdf_self_hosted = Pdf.objects.create(
@@ -476,12 +492,32 @@ def test_fuzzy_filter_pdfs(self):
         filtered_pdfs = pdf_views.OverviewMixin.fuzzy_filter_pdfs(Pdf.objects.all(), 'self hosted')
         self.assertEqual(sorted(list(filtered_pdfs), key=lambda a: a.name), [pdf_self_hosting, pdf_self_hosted])
 
+    def test_advanced_search_filtering(self):
+        collection = self.user.profile.current_collection
+        pdf_1 = Pdf.objects.create(collection=collection, name='eggplant')
+        pdf_2 = Pdf.objects.create(collection=collection, name='eggplant_2', description='Vegetables')
+        pdf_3 = Pdf.objects.create(collection=collection, name='eggplant_3', description='Vegetables')
+        pdf_4 = Pdf.objects.create(collection=collection, name='tomato')
+        Pdf.objects.create(collection=collection, name='tomato_2', description='Vegetables')
+        Metadata.objects.create(pdf=pdf_1, title='Eggplant', year=2016)
+        Metadata.objects.create(pdf=pdf_2, title='All about vegetables: Eggplant', year=2016)
+        Metadata.objects.create(pdf=pdf_3, title='All about vegetables: Tomato', year=2017)
+        Metadata.objects.create(pdf=pdf_4, title='Tomato', year=2017)
+
+        filtered_pdfs = pdf_views.OverviewMixin.advanced_search_filtering(
+            Pdf.objects.all(), {'description': 'vegetable', 'title': 'About', 'year': 2016}
+        )
+
+        assert filtered_pdfs.count() == 1
+        assert filtered_pdfs.first() == pdf_2
+
     @patch('pdf.services.tag_services.TagServices.get_tag_info_dict', return_value='tag_info_dict')
     def test_get_extra_context(self, mock_get_tag_info_dict):
-        response = self.client.get(f'{reverse('pdf_overview')}?search=searching&tags=tagging')
+        response = self.client.get(f'{reverse('pdf_overview')}?search=searching&tags=tagging&advanced_search=false')
 
         generated_extra_context = pdf_views.OverviewMixin.get_extra_context(response.wsgi_request)
         expected_extra_context = {
+            'advanced_search': False,
             'search_query': 'searching',
             'tag_query': ['tagging'],
             'tag_info_dict': 'tag_info_dict',
@@ -501,6 +537,7 @@ def test_get_extra_context_selection(self, mock_get_tag_info_dict):
 
         generated_extra_context = pdf_views.OverviewMixin.get_extra_context(response.wsgi_request)
         expected_extra_context = {
+            'advanced_search': False,
             'search_query': '',
             'tag_query': [],
             'tag_info_dict': 'tag_info_dict',
@@ -514,12 +551,20 @@ def test_get_extra_context_selection(self, mock_get_tag_info_dict):
 
         self.assertEqual(generated_extra_context, expected_extra_context)
 
+    @patch('pdf.services.tag_services.TagServices.get_tag_info_dict', return_value='tag_info_dict')
+    def test_get_extra_context_advanced_search_true(self, mock_get_tag_info_dict):
+        response = self.client.get(f'{reverse('pdf_overview')}?advanced_search=true')
+
+        generated_extra_context = pdf_views.OverviewMixin.get_extra_context(response.wsgi_request)
+        assert generated_extra_context['advanced_search']
+
     @patch('pdf.services.tag_services.TagServices.get_tag_info_dict', return_value='tag_info_dict')
     def test_get_extra_context_selection_invalid(self, mock_get_tag_info_dict):
         response = self.client.get(f'{reverse('pdf_overview')}?selection=invalid')
 
         generated_extra_context = pdf_views.OverviewMixin.get_extra_context(response.wsgi_request)
         expected_extra_context = {
+            'advanced_search': False,
             'search_query': '',
             'tag_query': [],
             'tag_info_dict': 'tag_info_dict',
@@ -539,6 +584,7 @@ def test_get_extra_context_empty_queries(self, mock_get_tag_info_dict):
 
         generated_extra_context = pdf_views.OverviewMixin.get_extra_context(response.wsgi_request)
         expected_extra_context = {
+            'advanced_search': False,
             'search_query': '',
             'tag_query': [],
             'tag_info_dict': 'tag_info_dict',
@@ -1035,6 +1081,17 @@ def test_export_annotations_without_identifier(self, mock_export_annotations):
 
         mock_export_annotations.assert_called_once_with(self.user.profile, 'highlights')
 
+    def test_advanced_search_query(self):
+        queries = (
+            '?name=some_name&tags=&description=some_description&title=The+Title&doi=&journal=&publisher=&year=2017'
+        )
+        response = self.client.get(f'{reverse("advanced_search_query")}{queries}')
+
+        expected_queries = (
+            '?description=some_description&title=The+Title&year=2017&search=some_name&advanced_search=true'
+        )
+        self.assertRedirects(response, f'{reverse('pdf_overview')}{expected_queries}', status_code=302)
+
 
 class TestAnnotationMixin(TestCase):
     username = 'user'

pdfding/pdf/urls.py

@@ -9,6 +9,8 @@
     # pdf related views
     path('', pdf_views.Overview.as_view(), name='pdf_overview'),
     path('query/', pdf_views.OverviewQuery.as_view(), name='pdf_overview_query'),
+    path('advanced_search', pdf_views.AdvancedSearch.as_view(), name='advanced_search'),
+    path('advanced_search_query', pdf_views.AdvancedSearchQuery.as_view(), name='advanced_search_query'),
     path('get_next_overview_page/<int:page>/', pdf_views.Overview.as_view(), name='get_next_pdf_overview_page'),
     path('add', pdf_views.Add.as_view(), name='add_pdf'),
     path('bulk_add', pdf_views.BulkAdd.as_view(), name='bulk_add_pdfs'),

pdfding/pdf/views/pdf_views.py

@@ -9,6 +9,7 @@
 from django.forms import Textarea, ValidationError
 from django.http import FileResponse, Http404, HttpRequest, HttpResponse
 from django.shortcuts import redirect, render
+from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django.views import View
 from django_htmx.http import HttpResponseClientRedirect, HttpResponseClientRefresh
@@ -187,6 +188,9 @@ def filter_objects(cls, request: HttpRequest) -> QuerySet:
         if search:
             pdfs = cls.fuzzy_filter_pdfs(pdfs, search)
 
+        if request.GET.get('advanced_search', '') == 'true':
+            pdfs = cls.advanced_search_filtering(pdfs, request.GET)
+
         return pdfs
 
     @staticmethod
@@ -205,6 +209,22 @@ def fuzzy_filter_pdfs(pdfs: QuerySet, search: str) -> QuerySet:
 
         return pdfs
 
+    @staticmethod
+    def advanced_search_filtering(pdfs: QuerySet, query_dict: dict) -> QuerySet:
+        details_filter_dict = {
+            f'{key}__icontains': value for key, value in query_dict.items() if key in ['description', 'notes']
+        }
+        metadata_filter_dict = {
+            f'metadata__{key}__icontains': value
+            for key, value in query_dict.items()
+            if key in ['author', 'abstract', 'doi', 'keywords', 'journal', 'publisher', 'title', 'year']
+        }
+
+        pdfs = pdfs.filter(**details_filter_dict)
+        pdfs = pdfs.filter(**metadata_filter_dict)
+
+        return pdfs
+
     @staticmethod
     def get_extra_context(request: HttpRequest) -> dict:
         """get further information that needs to be passed to the template."""
@@ -221,12 +241,18 @@ def get_extra_context(request: HttpRequest) -> dict:
             special_pdf_selection = ''
             page = 'pdf_overview'
 
+        if request.GET.get('advanced_search', '') == 'true':
+            advanced_search = True
+        else:
+            advanced_search = False
+
         extra_context = {
             'layout': request.user.profile.layout,
             'page': page,
-            'search_query': request.GET.get('search', ''),
             'special_pdf_selection': special_pdf_selection,
             'tag_info_dict': TagServices.get_tag_info_dict(request.user.profile),
+            'advanced_search': advanced_search,
+            'search_query': request.GET.get('search', ''),
             'tag_query': tag_query,
             'current_collection_id': request.user.profile.current_collection_id,
             'current_collection_name': request.user.profile.current_collection_name,
@@ -1015,3 +1041,35 @@ def get(self, request: HttpRequest, kind: str, identifier: str = ''):
             response = FileResponse(annotations_buffer, as_attachment=True, filename='export.json')
 
             return response
+
+
+class AdvancedSearch(View):
+    """View for the advanced PDF search."""
+
+    def get(self, request: HttpRequest):  # pragma: no cover
+        """Display the form for adding an object."""
+
+        context = {'form': forms.AdvancedSearchForm()}
+
+        return render(request, 'advanced_search.html', context)
+
+
+class AdvancedSearchQuery(View):
+    """View for the advanced PDF search query."""
+
+    def get(self, request: HttpRequest):
+        query_dict = dict(request.GET)
+        query_dict['search'] = query_dict.pop('name', '')
+        query_dict['advanced_search'] = ['true']
+
+        # If sep is not specified or is None, a different splitting algorithm is applied:
+        # runs of consecutive whitespace are regarded as a single separator
+        # a query item looks like this: key: ['value']
+        query_string = '&'.join(
+            f'{key}={"+".join(query[0].strip().split())}'
+            for key, query in query_dict.items()
+            if query not in [[], ['']]
+        )
+        redirect_url = f'{reverse('pdf_overview')}?{query_string}'
+
+        return redirect(redirect_url)