v1.7 update

Author: drlight17

CHANGES.md

@@ -1,17 +1,57 @@
 Unreleased VER.
-- add multiple simultanious text filters
-- minify
 - replace momentjs wih day.js or luxone(?)
 - make mailcow PR to integrate into
 - remove python privex helpers code (?)
 - save settings into cookies (for firefox private window mainly) + accept cookies modal (?)
 - fix dropdown update if use fomantic ui div based dropdown
-- fix sort icon to the same shevrons as main menu gui show/hide buttons  
 - upgrade procedure from GUI (?) if decided to do so use [this manual](https://stackoverflow.com/questions/32163955/how-to-run-shell-script-on-host-from-docker-container/63719458#63719458)
-- optimize and fix mail_to and log_lines queries with not(!)
-- BUG: fix process of '...and more' in stats
+- minify
+- add multiple simultanious text filters
+- MS Exchange multiple mail_to processing
 - [fix of non-dockerizeed deploy issue](https://github.com/drlight17/mta-log-parser/issues/10)
 
+VER. 1.7
+
+- ~~click arrows for modal view to get first and last message correspondly~~
+- ~~BUG: loading on top sometimes~~
+- ~~home and end key bindings for modal view to get first and last message respectively~~
+- ~~sort icon to the same shevrons as main menu gui show/hide buttons~~
+- ~~BUG: vertical scroll of modals by arrow keys~~
+- ~~TLS + status in the email details~~
+- ~~bigger bars and fonts for top charts~~
+- ~~main menu buttons border~~
+- ~~turn off blur by default for mobile devices (use is_mobile var)~~
+- ~~move settings and tips to modal menus~~
+- ~~change "Hide" to cross (times icon), cross (times icon) to eye icon, add Hide title, change clear exclude slash eye icon to trash alternate icon (or group two icons like [this](https://fomantic-ui.com/elements/icon.html#icons))~~
+- ~~multiple mail_to save as array~~
+- ~~remake multiple filtered stats~~
+- ~~remake multiple filtered emails and details view (TEST in different MTA!!!)~~
+- ~~remake addFilterLink for multiple (TEST in different MTA!!!)~~
+- ~~BUG: unknown with all recipients in exchange top 10 recipients stats~~
+- ~~BUG: trim comma in exchange recipients~~
+- ~~BUG: swipes pass 6 email instead of 1~~
+- ~~BUG: context menu is on the left (Macbook)~~
+- ~~BUG: multiple with no aliases link on the parent TD~~
+- ~~optimize and fix mail_to querie with and without not(!)~~
+- ~~show stats context menu with no data~~
+- ~~BUG: use stats context menu in mobile view~~
+- ~~make stats bars more thick in mobile view~~
+- ~~add tips about stats context menu (long press and right click)~~
+- ~~BUG: vertical scroll mobile view jump to top (modal details)~~
+- ~~BUG: change multiple li to span to fix overflow and hover effects~~
+- ~~add postfix orig_to (alias) to multiple mail_to li list (in brackets after to)~~
+- ~~BUG: email list and detail alias appearance (details look awful)~~
+- ~~BUG: fix stats top mail_to_alias appearance~~
+- ~~stats context menu max height with scroll for many hiddens + place context menu to the left from cursor~~
+- ~~mobile multiple view aliases in one row (hover effect must be fixed)~~
+- ~~BUG: emails list multiple overflow view~~
+- ~~BUG: prevent vertical scrolling when swiping left and right (modal details)~~
+- ~~BUG: prevent swipe when swiping log_lines in (modal details)~~
+- ~~BUG: fix postfix mail_to multiple regex (with orig_to)~~
+- ~~hidden addresses stats indicator~~
+- ~~show slow log_lines queries attention after choosing this filter (notie)~~
+- ~~BUG: emails list width is wider then other gui blocks with editable column width turned on~~  
+- ~~BUG: fix stat cached over hover details~~
 
 VER. 1.6
 - ~~use more then one CPU core when parsing (moment timestamp convert of related logs is variable)~~

mlp/core.py

@@ -85,11 +85,12 @@ async def get_rethink() -> Tuple[DB, DefaultConnection, RethinkDB]:
         idxs = await db.table(t).index_list().run(conn, array_limit=settings.rethink_arr_limit)
         for index in indexes:
             if index not in idxs:
-
                 log.debug('Index %s on table %s did not exist. Creating.', index, t)
                 if index == 'status_code':
                     # TODO need to test index creation
                     await db.table(t).index_create(index,r.row['status']['code'], multi=True).run(conn, array_limit=settings.rethink_arr_limit)
+                #if index == 'lines':
+                #    await db.table(t).index_create(index, lambda row: row['lines'].map(lambda line: [line['message']]).reduce(lambda left, right: left.set_union(right)), multi=True).run(conn, array_limit=settings.rethink_arr_limit)
                 else:
                     await db.table(t).index_create(index).run(conn, array_limit=settings.rethink_arr_limit)
 

mlp/main.py

@@ -28,17 +28,21 @@
 import email
 import base64
 import quopri
+import json
+
 from datetime import datetime, timedelta, timezone
 from quart import jsonify
 
+from collections import defaultdict
+
 import moment
 import datefinder
 
 log = logging.getLogger(__name__)
 
 # !!! change version upon update !!!
 global VERSION
-VERSION ="1.6"
+VERSION ="1.7"
 
 # postf_match += r'([A-F0-9]{11})\:[ \t]+?(.*)'
 #postf_match = r'([A-Za-z]+[ \t]+[0-9]+[ \t]+[0-9]+\:[0-9]+:[0-9]+).*'
@@ -156,7 +160,9 @@ async def housekeeping(housekeeping_days):
 async def import_log(logfile: str) -> Dict[str, PostfixMessage]:
     log.info('Opening log file %s', logfile)
     messages = {}
-    multiple_recipients = []
+    multiple_recipients_qids = []
+    multiple_recipients = defaultdict(list)
+    
     counter = 0
     same_qid = ''
     # avoid utf-8 codec error
@@ -217,8 +223,36 @@ async def import_log(logfile: str) -> Dict[str, PostfixMessage]:
             messages[qid].merge(await parse_line(msg))
             #print(messages[qid])
             #print(msg)
+
+            checking_mailto_alias = {}
+
+            if settings.mta == 'postfix':
+                if messages[qid].get('mail_to_alias') is not None:  
+                    if messages[qid].get('mail_to_alias') != {}:
+                        #print(messages[qid].get('mail_to'))
+                        #print(messages[qid].get('mail_to_alias'))
+                        #checking_mailto_alias_dict = messages[qid]['mail_to_alias']
+                        try:
+                            subdict = {}
+                            subdict['mail_to'] = messages[qid]['mail_to']
+                            subdict['mail_to_alias'] = messages[qid]['mail_to_alias'][messages[qid].get('mail_to')]
+
+                            checking_mailto_alias[qid] = subdict
+                            #print(checking_mailto_alias)
+                        except KeyError:
+                            continue
+
+            '''if checking_mailto_alias is not None and checking_mailto_alias != {}:
+                checking_mailto = checking_mailto_alias[qid]['mail_to']
+            else:
+                checking_mailto = messages[qid]['mail_to']'''
+
             checking_mailto = messages[qid]['mail_to']
-            if qid not in set(multiple_recipients):
+            # remove commas for ms exchange log
+            if settings.mta == 'exchange':
+                messages[qid]['mail_to'] = messages[qid]['mail_to'].replace(",", "")
+            
+            if qid not in set(multiple_recipients_qids):
                 if qid == same_qid or same_qid == '':
                     if messages[qid]['status'].get('code') is not None:
                         # check if there are already recipients in message and there are recipients parsed
@@ -227,22 +261,37 @@ async def import_log(logfile: str) -> Dict[str, PostfixMessage]:
                             if checking_mailto in msg:
                                 same_qid = qid
                                 counter += 1
+                                # don't add email duplicates
+                                if checking_mailto not in multiple_recipients[same_qid]:
+                                    # add alias dict if any
+                                    if checking_mailto_alias is not None and checking_mailto_alias != {}:
+                                        checking_mailto = checking_mailto_alias[qid]
+                                    multiple_recipients[same_qid].append(checking_mailto)
                 else:
                     #print("New message ID: ", qid)
                     #print("There are", counter,"recipients in message id",same_qid)
                     if counter > 1:
-                        multiple_recipients.append(same_qid)
+                        multiple_recipients_qids.append(same_qid)
                     counter = 0
                     same_qid = ''
 
             #print(await parse_line(msg[1]))
             messages[qid].lines.append(PostfixLog(timestamp=dtime, queue_id=qid, message=msg))
+        # fix for the last log line if multiple
+        if counter > 1:
+            multiple_recipients_qids.append(same_qid)
 
+    # clear multiple_recipients from the qids with < 2 recipients
+    for k in list(multiple_recipients):
+        if k not in multiple_recipients_qids:    
+            del multiple_recipients[k]
+
     #print(multiple_recipients)
     #print(messages)
     log.info('Finished parsing log file %s', logfile)
     output = dict();
     output['messages'] = messages
+    output['multiple_recipients_qids'] = multiple_recipients_qids
     output['multiple_recipients'] = multiple_recipients
     return output
 
@@ -306,6 +355,7 @@ async def main():
     log.info('Importing %s log file', settings.mta)
     import_output = await import_log(settings.mail_log)
     log.info('Converting %s log data into list',settings.mta)
+    multiple_recipients_qids = import_output['multiple_recipients_qids']
     multiple_recipients = import_output['multiple_recipients']
     msgs = import_output['messages']
     msg_list = [{"id": qid, **msg.clean_dict(convert_time=r_q.expr)} for qid, msg in msgs.items()]
@@ -355,8 +405,11 @@ async def main():
                 continue
             # check if there are many recipients
             if m.get('id') in set(multiple_recipients):
-                #print("There multiple recipients in ",m.get('id'))
-                m['mail_to'] += " and more (check log lines)"
+                #print("There are",len(multiple_recipients[m.get('id')]),"recipients in ",m.get('id'))
+                #print(multiple_recipients[m.get('id')])
+                #m['mail_to'] += " and more (check log lines)"
+                m['mail_to'] = json.dumps(multiple_recipients[m.get('id')])
+                #m['mail_to'] = multiple_recipients[m.get('id')]
                 m['status']['code'] = 'multiple'
                 m['status']['message'] = 'multiple, see log lines below'
 

mlp/objects.py

@@ -50,13 +50,14 @@ def clean_dict(self, convert_time=str) -> dict:
 
         return data
 
-
 @dataclass
 class PostfixMessage(Dictable):
     timestamp: datetime
     queue_id: str
     lines: List[PostfixLog] = field(default_factory=list)
     mail_to: str = ""
+    mail_to_alias: dict = field(default_factory=dict)
+    #mail_to_alias: str = ""
     #mail_to: dict = field(default_factory=dict)
     mail_from: str = ""
     subject: str = ""

mlp/parser.py

@@ -21,7 +21,10 @@
 log = logging.getLogger(__name__)
 
 # postfix regexp
-postf_to = re.compile(r'.*to=<([a-zA-Z0-9-+_.=]+@[a-zA-Z0-9-+_.]+)>')
+#OLD!!! postf_to = re.compile(r'.*to=<([a-zA-Z0-9-+_.=]+@[a-zA-Z0-9-+_.]+)>')
+#postf_to = re.compile(r'^.*to=<([^>]*)>,.*orig_to=<[^>]*>') <---- this is only for orig_to
+#postf_to = re.compile(r'^.*to=<([^>]*)>,.*orig_to=<[^>]*>|^.*to=<([^>]*)>')
+postf_to = re.compile(r'^.*to=<([^>]*)>,.*orig_to=<([^>]*)>|^.*to=<([^>]*)>')
 postf_from = re.compile(r'.*from=<([a-zA-Z0-9-+_.=]+@[a-zA-Z0-9-+_.]+)>')
 postf_subject = re.compile(r'.*header\sSubject:\s(.*)\sfrom\s')
 postf_size = re.compile(r'.*size=([0-9]{1,}),.*')
@@ -111,6 +114,8 @@
 
 async def parse_line(mline) -> dict:
     lm = {}
+    #aliases = []
+    aliases = {}
 
     _to = find_to.match(mline)
     #print(mline)
@@ -121,9 +126,22 @@ async def parse_line(mline) -> dict:
     _relay = find_relay.match(mline)
     _status = find_status.match(mline)
 
+    
     if _to is not None:
         if settings.mta == 'exchange':
             lm['mail_to'] = _to.group(1)[:-1]
+        if settings.mta == 'postfix':
+            if _to.group(2) is not None:
+                aliases = {_to.group(1):_to.group(2)}
+                #aliases.append(_to.group(2))
+                lm['mail_to_alias'] = aliases
+
+            if _to.group(1) is not None:
+                lm['mail_to'] = _to.group(1)
+                #if _to.group(2) is not None:
+                #    lm['mail_to'] +=' ('+_to.group(2)+')'
+            else:
+                lm['mail_to'] = _to.group(3)
         else:
             lm['mail_to'] = _to.group(1)
     if _subject is not None:
@@ -189,17 +207,17 @@ async def parse_line(mline) -> dict:
         if _client is not None: lm['client'] = dict(host=_client.group(1)[:-1], ip=_client.group(2)[:-1])
         if _status is not None:
             if _status.group(1) is not None:
-            	if _status.group(1)[:-1].lower() == 'deliver' or  _status.group(1)[:-1].lower()  == 'duplicatedeliver' or  _status.group(1)[:-1].lower()  == 'process' or  _status.group(1)[:-1].lower()  == 'receive' or  _status.group(1)[:-1].lower()  == 'redirect' or  _status.group(1)[:-1].lower()  == 'send' or  _status.group(1)[:-1].lower()  == 'submit':
-            		lm['status'] = lm['status'] = dict(code='sent', message="")
-            	elif _status.group(1)[:-1].lower() == 'fail' or  _status.group(1)[:-1].lower()  == 'hadiscard' or  _status.group(1)[:-1].lower()  == 'moderatorreject' or  _status.group(1)[:-1].lower()  == 'resubmitfail'  or  _status.group(1)[:-1].lower()  == 'submitfail'  or  _status.group(1)[:-1].lower()  == 'suppressed' or _status.group(1)[:-1].lower() == 'drop' or _status.group(1)[:-1].lower() == 'deliverfail':
-            		lm['status'] = lm['status'] = dict(code='reject', message="")
-            	elif _status.group(1)[:-1].lower() == 'dsn' or  _status.group(1)[:-1].lower()  == 'haredirect':
-            		lm['status'] = lm['status'] = dict(code='bounced', message="")
-            	elif _status.group(1)[:-1].lower() == 'defer' or _status.group(1)[:-1].lower() == 'resubmitdefer' or _status.group(1)[:-1].lower() == 'submitdefer':
-            		lm['status'] = lm['status'] = dict(code='deferred', message="")
-
-            if _status.group(2) is not None: 
-            	lm['status']['message'] = _status.group(2)[:-1]
+                if 'deliver' in _status.group(1)[:-1].lower() or  _status.group(1)[:-1].lower()  == 'process' or  _status.group(1)[:-1].lower()  == 'receive' or  _status.group(1)[:-1].lower()  == 'redirect' or  _status.group(1)[:-1].lower()  == 'send' or  _status.group(1)[:-1].lower()  == 'submit':
+                    lm['status'] = lm['status'] = dict(code='sent', message="")
+                elif 'fail' in _status.group(1)[:-1].lower() or 'discard' in _status.group(1)[:-1].lower() or 'reject' in _status.group(1)[:-1].lower()  or  _status.group(1)[:-1].lower()  == 'suppressed' or _status.group(1)[:-1].lower() == 'drop':
+                    lm['status'] = lm['status'] = dict(code='reject', message="")
+                elif 'redirect' in _status.group(1)[:-1].lower() or _status.group(1)[:-1].lower() == 'dsn':
+                    lm['status'] = lm['status'] = dict(code='bounced', message="")
+                elif 'defer' in _status.group(1)[:-1].lower():
+                    lm['status'] = lm['status'] = dict(code='deferred', message="")
+            if _status.group(2) is not None:
+                if _status.group(2)[:-1] != "":
+                    lm['status']['message'] = _status.group(2)[:-1]
 
     elif settings.mta == 'postfix':
         if _relay is not None: lm['relay'] = dict(host=_relay.group(1), ip=_relay.group(2), port=_relay.group(3))

mlp/settings.py

@@ -41,7 +41,7 @@
 rethink_arr_limit = int(env('RETHINK_ARR_LIMIT', 100000))
 
 rethink_tables = [
-    ('sent_mail', 'auth', ['status_code', 'mail_from', 'mail_to', 'timestamp', 'first_attempt', 'last_attempt']),
+    ('sent_mail', 'auth', ['status_code', 'mail_from', 'mail_to', 'mail_to_alias', 'timestamp', 'first_attempt', 'last_attempt']),
 ]
 
 # add ldap vars from env