Adding some unit tests for listmonk sync

.github/workflows/unit.yml

@@ -0,0 +1,95 @@
+name: unit tests
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      listmonk: ${{ steps.filter.outputs.listmonk }}
+      guidebook: ${{ steps.filter.outputs.guidebook }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            listmonk:
+              - 'listmonk/**'
+            guidebook:
+              - 'guidebook/**'
+
+  test-listmonk:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.listmonk == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-listmonk-${{ hashFiles('listmonk/requirements.txt', 'listmonk/requirements-test.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-listmonk-
+
+      - name: Install dependencies
+        run: |
+          cd listmonk
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+
+      - name: Run tests
+        run: |
+          cd listmonk
+          pytest -v --tb=short
+
+  test-guidebook:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.guidebook == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-guidebook-${{ hashFiles('guidebook/requirements.txt', 'guidebook/requirements-test.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-guidebook-
+
+      - name: Install dependencies
+        run: |
+          cd guidebook
+          pip install -r requirements.txt
+          # Install test dependencies if they exist
+          if [ -f requirements-test.txt ]; then
+            pip install -r requirements-test.txt
+          fi
+
+      - name: Run tests
+        run: |
+          cd guidebook
+          # Run tests if they exist
+          if [ -d tests ] || ls test_*.py 2>/dev/null; then
+            pytest -v --tb=short || echo "No tests found or pytest not configured"
+          else
+            echo "No tests directory found yet - skipping tests"
+          fi

.gitignore

@@ -0,0 +1,2 @@
+.vscode
+__pycache__

.sugarjar.yaml

@@ -0,0 +1,3 @@
+on_push: [lint]
+lint:
+  - black -l 80 .

listmonk/pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = 
+    -v
+    --tb=short
+    --strict-markers
+markers =
+    slow: marks tests as slow (deselect with '-m "not slow"')
+    integration: marks tests as integration tests

listmonk/requirements-test.txt

@@ -0,0 +1,3 @@
+pytest>=7.4.0
+pytest-cov>=4.1.0
+pytest-mock>=3.11.1

listmonk/requirements.txt

@@ -0,0 +1,5 @@
+click>=8.1.0
+mysqlclient>=2.2.0
+requests>=2.31.0
+pyyaml>=6.0
+datadog-api-client>=2.0.0

listmonk/scale_email_sync.py

@@ -44,9 +44,10 @@ def get_csv_data(self, csv_url):
         reader = csv.DictReader(data.splitlines())
         subscribers = {}
         for row in reader:
-            subscribers[row["email"]] = {
+            email_lc = row["email"].lower()
+            subscribers[email_lc] = {
                 "id": row["id"],
-                "email": row["email"].lower(),
+                "email": email_lc,
                 "can_email": int(row["can_email"]),
             }
         return subscribers
@@ -71,7 +72,7 @@ def get_db_data(self):
         """
         )
         subscribers = {
-            row[0]: {
+            row[0].lower(): {
                 "email": row[0].lower(),
                 "name": row[1],
                 "can_email": row[2],
@@ -185,7 +186,9 @@ def get_all_subscribers(self):
         return subscribers
 
     def list_ids_to_names(self, list_ids):
-        return ", ".join([self.list_ids.get(lid, str(lid)) for lid in list_ids])
+        return ", ".join(
+            [self.list_names.get(lid, str(lid)) for lid in list_ids]
+        )
 
     def add_subscriber(self, email, lists):
         logging.debug(

listmonk/tests/README.md

@@ -0,0 +1,65 @@
+# Tests for scale_email_sync
+
+This directory contains tests for the `scale_email_sync.py` script.
+
+## Running Tests
+
+### Install test dependencies
+
+```bash
+pip install -r requirements-test.txt
+```
+
+### Run all tests
+
+```bash
+pytest
+```
+
+### Run with coverage
+
+```bash
+pytest --cov=scale_email_sync --cov-report=html
+```
+
+### Run specific test file
+
+```bash
+pytest tests/test_scale_email_sync.py
+```
+
+### Run specific test class or function
+
+```bash
+pytest tests/test_scale_email_sync.py::TestListMonk
+pytest tests/test_scale_email_sync.py::TestListMonk::test_add_subscriber_new
+```
+
+### Run with verbose output
+
+```bash
+pytest -v
+```
+
+## Test Structure
+
+- `test_scale_email_sync.py` - Main test file containing:
+   - `TestLoadConfig` - Tests for configuration loading
+   - `TestRegData` - Tests for RegData class (CSV and database operations)
+   - `TestListMonk` - Tests for ListMonk class (API interactions,
+     synchronization)
+   - `TestIntegration` - Integration tests for the full workflow
+
+## Coverage
+
+The tests aim to cover:
+
+- Configuration file loading
+- CSV data parsing
+- Database data fetching
+- Listmonk API interactions (GET, POST, PUT)
+- Subscriber synchronization logic
+- List management (expected, missing, extra lists)
+- Dry-run mode
+- Stats tracking and Datadog reporting
+- Error handling

listmonk/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for scale_email_sync"""