Skip to content

Fix for issues with result sets that are too large for elasticsearch #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lbrant1 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Fix for issues with result sets that are too large for elasticsearch #45

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2d1c399
Fix for issues with result sets that are too large for elasticsearch
Dec 8, 2017
3458899
remove print statement
lbrant1 Dec 13, 2017
b316c6b
Set bootstrap warning on last page if paging is limited
Jul 5, 2018
8a02753
Fix warning so it shows when appropriate
Jul 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions seeker/templates/seeker/footer.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
{% load seeker %}
{% load i18n %}

<div class="pull-right">
{% seeker_pager results.hits.total page_size=page_size page=page querystring=querystring spread=page_spread %}
{% seeker_pager results.hits.total page_size=page_size page=page querystring=querystring spread=page_spread max_results=max_results %}
</div>
<p>
<em>Found {{ results.hits.total|intcomma }} result{{ results.hits.total|pluralize }} in {{ results.took }} millisecond{{ results.took|pluralize }}</em>
Expand Down
16 changes: 15 additions & 1 deletion seeker/templates/seeker/pager.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
{% load i18n %}

<ul class="pagination">
{% if 1 != page.number %}
<li><a href="?{{ querystring }}{% if querystring %}&amp;{% endif %}{{ param }}={{ page.number|add:"-1" }}" class="page-link" data-page="{{ page.number|add:"-1" }}">&larr;</a></li>
{% endif %}
{% if 1 not in page_range %}
<li><a href="?{{ querystring }}{% if querystring %}&amp;{% endif %}{{ param }}=1" class="page-link" data-page="1">1 &hellip;</a></li>
{% endif %}
{% for p in page_range %}
<li{% if p == page.number %} class="active"{% endif %}><a href="?{{ querystring }}{% if querystring %}&amp;{% endif %}{{ param }}={{ p }}" class="page-link" data-page="{{ p }}">{{ p }}</a></li>
{% endfor %}
{% if paginator.num_pages not in page_range %}
{% if paginator.num_pages not in page_range and show_last_page %}
<li><a href="?{{ querystring }}{% if querystring %}&amp;{% endif %}{{ param }}={{ paginator.num_pages }}" class="page-link" data-page="{{ paginator.num_pages }}">&hellip; {{ paginator.num_pages }}</a></li>
{% endif %}
{% if paginator.num_pages != page.number %}
<li><a href="?{{ querystring }}{% if querystring %}&amp;{% endif %}{{ param }}={{ page.number|add:"1" }}" class="page-link" data-page="{{ page.number|add:"1" }}">&rarr;</a></li>
{% endif %}
</ul>

{% if not show_last_page and page.number == paginator.num_pages %}
Copy link
Member

@dcwatson dcwatson Dec 18, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feels kind of gross to put an alert() on page load in this case. Why not stick a message under the pagination links?

<script>
document.addEventListener("DOMContentLoaded", function(event) {
alert('You have reached the end of the available results.\nTo view them all export the results.')
});
</script>
{% endif %}
12 changes: 10 additions & 2 deletions seeker/templatetags/seeker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@ def seeker_score(result, max_score=None, template='seeker/score.html'):


@register.simple_tag
def seeker_pager(total, page_size=10, page=1, param='p', querystring='', spread=7, template='seeker/pager.html'):
paginator = Paginator(range(total), page_size)
def seeker_pager(total, page_size=10, page=1, param='p', querystring='', spread=7, template='seeker/pager.html', max_results=None):
if max_results:
paginator = Paginator(range(min(total,(max_results//page_size * page_size))), page_size)
else:
paginator = Paginator(range(total), page_size)
if paginator.num_pages < 2:
return ''
page = paginator.page(page)
Expand All @@ -77,12 +80,17 @@ def seeker_pager(total, page_size=10, page=1, param='p', querystring='', spread=
page_range = range(start, end)
else:
page_range = paginator.page_range
if max_results and max_results < page_size * total:
show_last_page = False
else:
show_last_page = True
return loader.render_to_string(template, {
'page_range': page_range,
'paginator': paginator,
'page': page,
'param': param,
'querystring': querystring,
'show_last_page' : show_last_page
})

_phrase_re = re.compile(r'"([^"]*)"')
Expand Down
13 changes: 13 additions & 0 deletions seeker/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,12 @@ def required_display_fields(self):
The number of pages (not including first and last) to show in the paginator widget.
"""

max_results = 10000
"""
The number of results seeker can show. Most elasticsearch instances default to 10000.
Set it to avoid TransportError pages for large result sets
"""

can_save = True
"""
Whether searches for this view can be saved.
Expand Down Expand Up @@ -564,6 +570,12 @@ def render(self):
page = int(page) if page.isdigit() else 1
offset = (page - 1) * self.page_size
results_count = search[0:0].execute().hits.total

#check to see if the page is greater than the results in set or greater than the max_results seeker returns for a search
#if it is change page to first page.
if self.max_results and offset + self.page_size > self.max_results:
page = 1
offset = 0
if results_count < offset:
page = 1
offset = 0
Expand All @@ -586,6 +598,7 @@ def render(self):
'page': page,
'page_size': self.page_size,
'page_spread': self.page_spread,
'max_results': self.max_results,
'sort': sort,
'querystring': context_querystring,
'reset_querystring': self.normalized_querystring(ignore=['p', 's', 'saved_search']),
Expand Down