guard expire/fsreport file iteration against vanishing, improve reporting

also activates actual deletion (after quite some dry test runs on nine)

Author: hpk42

chatmaild/src/chatmaild/expire.py

@@ -22,11 +22,30 @@ def iter_mailboxes(basedir, maxnum):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
-    for name in os.listdir(basedir)[:maxnum]:
+    for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
             yield MailboxStat(basedir + "/" + name)
 
 
+def get_file_entry(path):
+    """return a FileEntry or None if the path does not exist or is not a regular file."""
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return None
+    if not S_ISREG(st.st_mode):
+        return None
+    return FileEntry(path, st.st_mtime, st.st_size)
+
+
+def os_listdir_if_exists(path):
+    """return a list of names obtained from os.listdir or an empty list if the path does not exist."""
+    try:
+        return os.listdir(path)
+    except FileNotFoundError:
+        return []
+
+
 class MailboxStat:
     last_login = None
 
@@ -40,19 +59,23 @@ def __init__(self, basedir):
 
         # scan all relevant files (without recursion)
         old_cwd = os.getcwd()
-        os.chdir(self.basedir)
-        for name in os.listdir("."):
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os.listdir(name):
-                    relpath = name + "/" + msg_name
-                    st = os.stat(relpath)
-                    self.messages.append(FileEntry(relpath, st.st_mtime, st.st_size))
+                for msg_name in os_listdir_if_exists(name):
+                    entry = get_file_entry(f"{name}/{msg_name}")
+                    if entry is not None:
+                        self.messages.append(entry)
+
             else:
-                st = os.stat(name)
-                if S_ISREG(st.st_mode):
-                    self.extrafiles.append(FileEntry(name, st.st_mtime, st.st_size))
+                entry = get_file_entry(name)
+                if entry is not None:
+                    self.extrafiles.append(entry)
                     if name == "password":
-                        self.last_login = st.st_mtime
+                        self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
         os.chdir(old_cwd)
 
@@ -80,9 +103,13 @@ def remove_mailbox(self, mboxdir):
             shutil.rmtree(mboxdir)
         self.del_mboxes += 1
 
-    def remove_file(self, path):
+    def remove_file(self, path, mtime=None):
         if self.verbose:
-            print_info(f"removing {path}")
+            if mtime is not None:
+                date = datetime.fromtimestamp(mtime).strftime("%b %d")
+                print_info(f"removing {date} {path}")
+            else:
+                print_info(f"removing {path}")
         if not self.dry:
             try:
                 os.unlink(path)
@@ -104,18 +131,27 @@ def process_mailbox_stat(self, mbox):
             return
 
         # all to-be-removed files are relative to the mailbox basedir
-        os.chdir(mbox.basedir)
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
-            print_info(f"checking for mailbox messages in: {mboxname}")
+            date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
+            if date:
+                print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
+            else:
+                print_info(f"checking mailbox (no last_login) {mboxname}")
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath)
+                self.remove_file(message.relpath, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
                 if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath)
+                    self.remove_file(message.relpath, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/tests/test_expire.py

@@ -6,7 +6,13 @@
 
 import pytest
 
-from chatmaild.expire import FileEntry, MailboxStat, iter_mailboxes
+from chatmaild.expire import (
+    FileEntry,
+    MailboxStat,
+    get_file_entry,
+    iter_mailboxes,
+    os_listdir_if_exists,
+)
 from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
@@ -127,3 +133,18 @@ def test_expiry_cli_old_files(capsys, example_config, mbox1):
                 pytest.fail(f"failed to remove {path}\n{err}")
 
     assert "shouldstay" not in err
+
+
+def test_get_file_entry(tmp_path):
+    assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
+    p = tmp_path.joinpath("x")
+    p.write_text("hello")
+    entry = get_file_entry(str(p))
+    assert entry.size == 5
+    assert entry.mtime
+
+
+def test_os_listdir_if_exists(tmp_path):
+    tmp_path.joinpath("x").write_text("hello")
+    assert len(os_listdir_if_exists(str(tmp_path))) == 1
+    assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0

cmdeploy/src/cmdeploy/service/chatmail-expire.service.f

@@ -5,5 +5,5 @@
 [Service]
 Type=oneshot
 User=vmail
-ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-expire /usr/local/lib/chatmaild/chatmail.ini -v
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-expire /usr/local/lib/chatmaild/chatmail.ini -v --remove