v.1.6 update

some new gui features and bugfixes

Author: drlight17

CHANGES.md

@@ -6,9 +6,30 @@ Unreleased VER.
 - remove python privex helpers code (?)
 - save settings into cookies (for firefox private window mainly) + accept cookies modal (?)
 - fix dropdown update if use fomantic ui div based dropdown
-- add not (!) to text filters
 - fix sort icon to the same shevrons as main menu gui show/hide buttons  
 - upgrade procedure from GUI (?) if decided to do so use [this manual](https://stackoverflow.com/questions/32163955/how-to-run-shell-script-on-host-from-docker-container/63719458#63719458)
+- optimize and fix mail_to and log_lines queries with not(!)
+- BUG: fix process of '...and more' in stats
+- [fix of non-dockerizeed deploy issue](https://github.com/drlight17/mta-log-parser/issues/10)
+
+
+VER. 1.6
+- ~~use more then one CPU core when parsing (moment timestamp convert of related logs is variable)~~
+- ~~add ability to exclude some addresses from the TOP charts (frontend gui cookies + backend permanent using settings)~~
+- ~~add stats cached text background for better readability~~
+- ~~show all currently excluded in gui (backend in tips, frontend in gui)~~
+- ~~BUG: currently excluded unknown list fix~~
+- ~~BUG: fix currently excluded context menu (right click) doesn't appear in mobile view~~
+- ~~clear all currently excluded from cookies button~~
+- ~~BUG: exclude unknown~~
+- ~~add not (!) to text filters~~
+- ~~BUG: fix not(!) appearance in mobile mode~~
+- ~~BUGs with all stats after not(!) were added~~
+- ~~process sharelink, save and restore not (!) parameter value and from link~~
+- ~~BUG: stats_app is not defined~~
+- ~~fix "invalid data" in cached stats after cookie clear (replace with error text or copy last date to browser localstorage to use it in this case)~~
+- ~~[security fix](https://github.com/drlight17/mta-log-parser/security/dependabot/1)~~
+- ~~closed graphs and settings/tips by default~~
 
 VER. 1.5.5
 - ~~BUG: auto turn on "save filters" on every link come even without paramaters~~

Dockerfile

@@ -2,10 +2,12 @@
 FROM python:3.8-alpine
 RUN apk update
 RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
 RUN pip3 install -U pipenv
 WORKDIR /app
 COPY Pipfile /app/
 # uncomment to append ypur ca cert for TLS connections i.e. LDAPS
 #COPY ca.crt /usr/local/share/ca-certificates/ca.crt
 #RUN cat /usr/local/share/ca-certificates/ca.crt >> /etc/ssl/certs/ca-certificates.crt
-RUN pipenv install
+RUN pipenv install

Pipfile

@@ -12,8 +12,8 @@ python-dateutil = "*"
 pytz = "*"
 rethinkdb = ">=2.4.7"
 python-dotenv = "*"
-quart = "=0.18.4"
-werkzeug = "=2.3.7"
+quart = "=0.19.4"
+werkzeug = "=3.0.1"
 moment = "*"
 datefinder = "*"
 python-ldap = "*"

build/Dockerfile

@@ -2,6 +2,8 @@
 FROM python:3.8-alpine
 RUN apk update
 RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
 RUN pip3 install -U pipenv
 WORKDIR /app
 COPY . /app/

build/docker-compose.yaml

@@ -37,4 +37,4 @@ networks:
     driver: bridge
     driver_opts:
       com.docker.network.enable_ipv6: "false"
-      com.docker.network.bridge.name: mlp_iface
+      com.docker.network.bridge.name: mlp_iface

deploy/Dockerfile

@@ -0,0 +1,13 @@
+# Dockerfile for deploy
+FROM python:3.8-alpine
+RUN apk update
+RUN apk add bash ncurses musl-dev gcc flock openldap-dev
+# upgrade pip
+RUN pip install --upgrade pip
+RUN pip3 install -U pipenv
+WORKDIR /app
+COPY Pipfile /app/
+# uncomment to append ypur ca cert for TLS connections i.e. LDAPS
+#COPY ca.crt /usr/local/share/ca-certificates/ca.crt
+#RUN cat /usr/local/share/ca-certificates/ca.crt >> /etc/ssl/certs/ca-certificates.crt
+RUN pipenv install

deploy/docker-compose.yaml

@@ -0,0 +1,37 @@
+# compose file for deploy
+version: '3.9'
+
+services:
+  mlp:
+    image: drlight17/mta-log-parser:latest
+    container_name: mta-log-parser
+    command: /app/run.sh ${COMMAND}
+    ports:
+      - ${PORT}:${PORT}
+    restart: always
+    volumes:
+      - ./.env:/app/.env
+      - /tmp:/tmp
+      - ${MAIL_LOG_PATH}:${MAIL_LOG_PATH}
+    depends_on:
+      - rethinkdb
+    networks:
+      - internal
+
+  rethinkdb:
+    image: rethinkdb:2.4
+    container_name: rethinkdb
+    ports:
+      - ${RETHINK_EXP_GUI_PORT}:8080
+    restart: always
+    volumes:
+      - ./:/data
+    networks:
+      - internal
+
+networks:
+  internal:
+    driver: bridge
+    driver_opts:
+      com.docker.network.enable_ipv6: "false"
+      com.docker.network.bridge.name: mlp_iface

example.env

@@ -41,9 +41,18 @@ PORT=8487
 # make date time format (default is "HH:mm:ss, DD.MM.YYYY") to use in GUI for emails list timestamps and date picker according to moment.js templates (https://momentjs.com/)
 #DATETIME_FORMAT='HH:mm:ss, DD.MM.YYYY'
 
+# convert related logs timestamp according to the DATETIME_FORMAT. High CPU demanding! false by default
+#MAIL_LOG_TIMESTAMP_CONVERT=false
+
 # set MTA which logs you waunt to parse; available values are exim, postfix (default), sendmail and exchange
 #MTA=postfix
 
+# set email addresses to exclude from top senders stats permanently (comma separated)
+#EXCLUDE_FROM_TOP_SENDERS='user1@example.com,user2@example.org'
+
+# set email addresses to exclude from top recipients stats permanently (comma separated)
+#EXCLUDE_FROM_TOP_RECIPIENTS='user1@example.com,user2@example.org'
+
 # set startup command
 # dev to run in python developer mode
 # prod to run in production mode (default)

mlp/main.py

@@ -38,7 +38,7 @@
 
 # !!! change version upon update !!!
 global VERSION
-VERSION ="1.5.5"
+VERSION ="1.6"
 
 # postf_match += r'([A-F0-9]{11})\:[ \t]+?(.*)'
 #postf_match = r'([A-Za-z]+[ \t]+[0-9]+[ \t]+[0-9]+\:[0-9]+:[0-9]+).*'

mlp/objects.py

@@ -10,6 +10,9 @@
 from mlp.settings import log_timezone
 from mlp import settings, api
 
+import logging
+log = logging.getLogger(__name__)
+
 @dataclass
 class PostfixLog(Dictable):
     timestamp: datetime
@@ -32,14 +35,19 @@ def __str__(self):
 
     def clean_dict(self, convert_time=str) -> dict:
         data = dict(self)
-        # log lines timestamp convert before append
-        data['timestamp'] = moment.date(str(self.timestamp)).format(settings.datetime_format)
+        # log lines timestamp convert before append (for message related log view)
+        # this data conversion uses only one thread and takes too long so do it according to mail_log_timestamp_convert variable value
+        # log.info('Converting date and time %s using moment library according to settings',  data['timestamp'])
+        if settings.mail_log_timestamp_convert:
+            data['timestamp'] = moment.date(str(self.timestamp)).format(settings.datetime_format)
+        else:
         #print(data)
 
-        '''if is_false(convert_time):
-            data['timestamp'] = self.timestamp
-        else:
-            data['timestamp'] = convert_time(self.timestamp)'''
+            if is_false(convert_time):
+                data['timestamp'] = self.timestamp
+            else:
+                data['timestamp'] = convert_time(self.timestamp)
+
         return data