chg: [mail search] improve search perf, cache search, paginate mail domain search

Author: Terrtia

bin/lib/ail_core.py

@@ -187,6 +187,26 @@ def rreplace(s, old, new, occurrence):
     li = s.rsplit(old, occurrence)
     return new.join(li)
 
+def get_template_pagination(elems, total, page=1, nb=50):
+    if len(elems) > nb:
+        return paginate_iterator(elems, nb_obj=nb, page=page)
+    else:
+        dict_page = {'nb_all_elem': total}
+        nb_pages = dict_page['nb_all_elem'] / nb
+        if not nb_pages.is_integer():
+            nb_pages = int(nb_pages) + 1
+        else:
+            nb_pages = int(nb_pages)
+        dict_page['list_elem'] = elems
+        dict_page['page'] = page
+        dict_page['nb_pages'] = nb_pages
+        dict_page['nb_first_elem'] = (nb * (page - 1)) + 1
+        if page == nb_pages:
+            dict_page['nb_last_elem'] = total
+        else:
+            dict_page['nb_last_elem'] = (nb * page)
+        return dict_page
+
 def paginate_iterator(iter_elems, nb_obj=50, page=1):
     dict_page = {'nb_all_elem': len(iter_elems)}
     nb_pages = dict_page['nb_all_elem'] / nb_obj

bin/lib/objects/Mails.py

@@ -205,83 +205,194 @@ def is_indexed_username(username):
     return r_search.exists(f'm:u:{username}')
 
 
-def search_domain(s_domain=None, r_pos=False): # TODO paginate
+def search_domain(s_domain=None, r_pos=False, page=1, nb=500):
     if not s_domain:
-        return r_search.smembers('m:domains')  # TODO paginate
+        domains = []
+        total = r_search.scard('m:domains')
+        start = nb * (page - 1)
+        stop = start + nb - 1
+        cursor = 0
+        for domain in r_search.smembers('m:domains'):
+            if start <= cursor <= stop:
+                domains.append(domain)
+            elif cursor > stop:
+                break
+            cursor += 1
+        return total, domains
     else:
         domains = {}
         re_search = re.compile(s_domain)
-        for domain in sscan_iterator(r_search, 'm:domains'):
-            if s_domain in domain:
-                res = re.search(re_search, domain)
-                if res:
-                    domains[domain] = {}
-                    if r_pos:
+        total, results = get_cache_search_mail(s_domain=s_domain, page=page, nb=nb)
+        if results is None:
+            results = []
+            start = nb * (page - 1)
+            stop = start + nb - 1
+            cursor = 0
+            for domain in sscan_iterator(r_search, 'm:domains'):
+                if s_domain in domain:
+                    results.append(domain)
+                    if start <= cursor <= stop:
+                        domains[domain] = {}
+                        if r_pos:
+                            res = re.search(re_search, domain)
+                            if res:
+                                domains[domain]['hl-start'] = res.start()
+                                domains[domain]['hl-end'] = res.end()
+                                domains[domain]['content'] = domain
+                    cursor += 1
+            total = len(results)
+            if results:
+                cache_search_mail(results, s_domain=s_domain)
+        else:
+            for domain in results:
+                domains[domain] = {}
+                if r_pos:
+                    res = re.search(re_search, domain)
+                    if res:
                         domains[domain]['hl-start'] = res.start()
                         domains[domain]['hl-end'] = res.end()
                         domains[domain]['content'] = domain
-        return domains
+        return total, domains
 
-def search_domain_username(domain, s_username=None, r_pos=False): # TODO paginate
+def search_domain_username(domain, s_username=None, r_pos=False, page=1, nb=500):
     objs = {}
     if not s_username:
-        for username in r_search.smembers(f'm:d:{domain}'):  # TODO paginate + SSCAN ????
-            content = f'{username}@{domain}'
-            obj_id = get_mail_id(content)
-            objs[obj_id] = {}
-            objs[obj_id]['content'] = content
+        total = r_search.scard(f'm:d:{domain}')
+        start = nb * (page - 1)
+        stop = start + nb - 1
+        cursor = 0
+        for username in sscan_iterator(r_search, f'm:d:{domain}'):
+            if start <= cursor <= stop:
+                content = f'{username}@{domain}'
+                obj_id = get_mail_id(content)
+                objs[obj_id] = {}
+                objs[obj_id]['content'] = content
+            elif cursor > stop:
+                break
+            cursor += 1
     else:
         re_search = re.compile(s_username)
-        for username in sscan_iterator(r_search, f'm:d:{domain}'):
-            if s_username in username:
-                res = re.search(re_search, username)
-                if res:
-                    content = f'{username}@{domain}'
-                    obj_id = get_mail_id(content)
-                    objs[obj_id] = {}
-                    if r_pos:
+        total, results = get_cache_search_mail(domain=domain, s_username=s_username, page=page, nb=nb)
+        if results is None:
+            results = []
+            start = nb * (page - 1)
+            stop = start + nb - 1
+            cursor = 0
+            for username in sscan_iterator(r_search, f'm:d:{domain}'):
+                if s_username in username:
+                    results.append(username)
+                    if start <= cursor <= stop:
+                        content = f'{username}@{domain}'
+                        obj_id = get_mail_id(content)
+                        objs[obj_id] = {}
+                        if r_pos:
+                            res = re.search(re_search, username)
+                            if res:
+                                objs[obj_id]['hl-start'] = res.start()
+                                objs[obj_id]['hl-end'] = res.end()
+                                objs[obj_id]['content'] = content
+                    cursor += 1
+            total = len(results)
+            if results:
+                cache_search_mail(results, domain=domain, s_username=s_username)
+        else:
+            for user in results:
+                content = f'{user}@{domain}'
+                obj_id = get_mail_id(content)
+                objs[obj_id] = {}
+                if r_pos:
+                    res = re.search(re_search, user)
+                    if res:
                         objs[obj_id]['hl-start'] = res.start()
                         objs[obj_id]['hl-end'] = res.end()
                         objs[obj_id]['content'] = content
-    return objs
+    return total, objs
+
 
-def search_username_domain(username, s_domain=None, r_pos=False):
+def search_username_domain(username, s_domain=None, r_pos=False, page=1, nb=500):
     objs = {}
     if not s_domain:
-        for domain in r_search.smembers(f'm:u:{username}'):  # TODO paginate
-            obj_id = f'{username}@{domain}'
-            objs[obj_id] = {}
+        total = r_search.scard(f'm:u:{username}')
+        start = nb * (page - 1)
+        stop = start + nb - 1
+        cursor = 0
+        for domain in sscan_iterator(r_search, f'm:u:{username}'):
+            if start <= cursor <= stop:
+                content = f'{username}@{domain}'
+                obj_id = get_mail_id(content)
+                objs[obj_id] = {}
+                objs[obj_id]['content'] = content
+            if cursor > stop:
+                break
+            cursor += 1
     else:
+        total, results = get_cache_search_mail(username=username, s_domain=s_domain, page=page, nb=page)
         re_search = re.compile(s_domain)
-        for domain in sscan_iterator(r_search, f'm:u:{username}'):
-            if s_domain in domain:
-                res = re.search(re_search, domain)
-                if res:
-                    content = f'{username}@{domain}'
-                    obj_id = get_mail_id(content)
-                    objs[obj_id] = {}
-                    if r_pos:
+        if results is None:  # TODO no results
+            results = []
+            start = nb * (page - 1)
+            stop = start + nb
+            cursor = 0
+            for domain in sscan_iterator(r_search, f'm:u:{username}'):
+                if s_domain in domain:
+                    results.append(domain)
+                    if start <= cursor <= stop:
+                        content = f'{username}@{domain}'
+                        obj_id = get_mail_id(content)
+                        objs[obj_id] = {}
+                        if r_pos:
+                            res = re.search(re_search, domain)
+                            if res:
+                                objs[obj_id]['hl-start'] = len(username) + 1 + res.start()
+                                objs[obj_id]['hl-end'] = len(username) + 1 + res.end()
+                                objs[obj_id]['content'] = content
+                    cursor += 1
+            total = len(results)
+            if results:
+                cache_search_mail(results, username=username, s_domain=s_domain)
+        else:
+            for dom in results:
+                content = f'{username}@{dom}'
+                obj_id = get_mail_id(content)
+                objs[obj_id] = {}
+                if r_pos:
+                    res = re.search(re_search, dom)
+                    if res:
                         objs[obj_id]['hl-start'] = len(username) + 1 + res.start()
                         objs[obj_id]['hl-end'] = len(username) + 1 + res.end()
                         objs[obj_id]['content'] = content
-    return objs
+    return total, objs
 
 
-def search_mail(mail=None, username=None, domain=None, s_username=None, s_domain=None, r_pos=False):
-    if mail:  # TODO
+def search_mail(mail=None, username=None, domain=None, s_username=None, s_domain=None, r_pos=False, page=1, nb=500):
+    if mail:
         m = get_mail(mail)
         if m.exists():
-            return m.get_id()
+            return 1, m.get_id()
 
     if domain:
         if is_indexed_domain(domain):
-            return search_domain_username(domain, s_username=s_username, r_pos=r_pos)
+            return search_domain_username(domain, s_username=s_username, r_pos=r_pos, page=page, nb=nb)
     elif username:
         if is_indexed_username(username):
-            return search_username_domain(username, s_domain=s_domain, r_pos=r_pos)
+            return search_username_domain(username, s_domain=s_domain, r_pos=r_pos, page=page, nb=nb)
     elif s_domain:
-        return search_domain(s_domain=s_domain, r_pos=r_pos)
-    return None
+        return search_domain(s_domain=s_domain, r_pos=r_pos, page=page, nb=nb)
+    return None, None
+
+def cache_search_mail(to_cache, username='', domain='', s_username='', s_domain=''):
+    for result in to_cache:
+        r_cache.rpush(f'm:{username}:{domain}:{s_username}:{s_domain}', result)
+    r_cache.expire(f'm:{username}:{domain}:{s_username}:{s_domain}', 600)
+
+def get_cache_search_mail(username='', domain='', s_username='', s_domain='', page=1, nb=500):
+    total = r_cache.llen(f'm:{username}:{domain}:{s_username}:{s_domain}')
+    if not total:
+        return None, None
+    else:
+        start = nb * (page - 1)
+        stop = start + nb - 1
+        return total, r_cache.lrange(f'm:{username}:{domain}:{s_username}:{s_domain}', start, stop)
 
 
 class Mails(AbstractDaterangeObjects):

var/www/blueprints/objects_mail.py

@@ -19,7 +19,7 @@
 ##################################
 # Import Project packages
 ##################################
-from lib.ail_core import paginate_iterator
+from lib.ail_core import paginate_iterator, get_template_pagination
 from lib.objects import Mails
 from packages import Date
 from lib import search_engine
@@ -121,9 +121,9 @@ def objects_mail_search():
         username = in_username.lower()
         if in_domain:
             s_domain = in_domain.lower()
-            search_engine.log(user_id, 'mail', f'{mode}, {s_username} @ {s_domain}')
+            search_engine.log(user_id, 'mail', f'{mode}, {username} @ {s_domain}')
         else:
-            search_engine.log(user_id, 'mail', f'{mode}, {s_username} @')
+            search_engine.log(user_id, 'mail', f'{mode}, {username} @')
     elif mode == 'exact':
         mail = f'{in_username.lower()}@{in_domain.lower()}'
         search_engine.log(user_id, 'mail', f'{mode}, {mail}')
@@ -136,23 +136,15 @@ def objects_mail_search():
     else:
         return create_json_response({'error': 'Invalid search mode domain_search, domain to search not provided'}, 400)
 
-    if username and domain:
-        search_engine.log(user_id, 'mail', f'{mode}, {username} @ {domain}')
-    elif username:
-        search_engine.log(user_id, 'mail', f'{mode}, {username} @ {s_domain}')
-    elif domain:
-        search_engine.log(user_id, 'mail', f'{mode} {s_username} @ {domain}')
-    else:
-        search_engine.log(user_id, 'mail', f'{mode}, {username} @ {domain}')
-    search_result = Mails.search_mail(mail=mail, username=username, domain=domain, s_username=s_username, s_domain=s_domain, r_pos=True)
+    total, search_result = Mails.search_mail(mail=mail, username=username, domain=domain, s_username=s_username, s_domain=s_domain, r_pos=True, page=page, nb=500)
 
     if isinstance(search_result, str):
         return redirect(url_for('correlation.show_correlation', type='mail', id=search_result))
     else:
         if search_result:
             mails = Mails.Mails()
-            ids = sorted(search_result.keys())
-            dict_page = paginate_iterator(ids, nb_obj=500, page=page)
+            # ids = sorted(search_result.keys())
+            dict_page = get_template_pagination(search_result, total, nb=500, page=page)
             if mode == 'domain_search':
                 dict_objects = mails.get_domain_meta(dict_page['list_elem'])
             else:
@@ -167,5 +159,6 @@ def objects_mail_search():
             domain = s_domain
 
         return render_template("search_mail_result.html", dict_objects=dict_objects, search_result=search_result,
-                               dict_page=dict_page, mode=mode, username=username, domain=domain)
+                               dict_page=dict_page, mode=mode, username=username, domain=domain,
+                               object_name='mails')
 

var/www/templates/objects/mail/search_mail_result.html

@@ -70,7 +70,7 @@
                         <table id="table_objects" class="table table-striped table-bordered">
                             <thead class="bg-dark text-white">
                             <tr>
-                                <th></th>
+                                <th>Email</th>
                                 <th>First Seen</th>
                                 <th>Last Seen</th>
                                 <th>Total</th>
@@ -88,8 +88,8 @@
                                             {% endif %}
                                         </a>
                                     </td>
-                                    <td>{{ dict_objects[obj_id]['first_seen'] }}</td>
-                                    <td>{{ dict_objects[obj_id]['last_seen'] }}</td>
+                                    <td>{% if dict_objects[obj_id]['first_seen'] %}{{ dict_objects[obj_id]['first_seen'][0:4] }}-{{ dict_objects[obj_id]['first_seen'][4:6] }}-{{ dict_objects[obj_id]['first_seen'][6:8] }}{% endif %}</td>
+                                    <td>{% if dict_objects[obj_id]['last_seen'] %}{{ dict_objects[obj_id]['last_seen'][0:4] }}-{{ dict_objects[obj_id]['last_seen'][4:6] }}-{{ dict_objects[obj_id]['last_seen'][6:8] }}{% endif %}</td>
                                     <td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
                                 </tr>
                             {% endfor %}
@@ -130,7 +130,7 @@ <h3 class="text-center">No Results Found</h3>
         $('#table_objects').DataTable({
             "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
             "iDisplayLength": 10,
-            "order": [[ 3, "desc" ]]
+            "order": [[ 0, "asc" ]]
         });
   });