Merge pull request #126 from vnepomuceno/fix/refactor-restaurant-filters

Refactor restaurant filters

Author: vnepomuceno

.gitignore

@@ -4,4 +4,4 @@ auth
 **/output
 **/properties.yml
 **/__pycache__
-.env
+.env

gmail_fisher/__init__.py

@@ -1,10 +1,14 @@
 import logging
+import os
 
 import coloredlogs
 
 from gmail_fisher.utils.config import LOG_LEVEL, LOG_FORMAT
 
 
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
 def get_logger(name: str) -> logging.Logger:
     coloredlogs.install()
     custom_logger = logging.getLogger(name)

gmail_fisher/parsers/food.py

@@ -1,13 +1,14 @@
 import json
+import os
 import re
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Iterable, Optional, Final, Dict
+from typing import Iterable, Optional, Final
 
 import html2text as html2text
 from alive_progress import alive_bar
 
-from gmail_fisher import get_logger
+from gmail_fisher import get_logger, ROOT_DIR
 from gmail_fisher.api.gateway import GmailGateway
 from gmail_fisher.data.models import (
     GmailMessage,
@@ -16,10 +17,18 @@
     FoodExpense,
 )
 from gmail_fisher.parsers import print_header
+from gmail_fisher.utils.json_utils import JsonUtils
 
 logger = get_logger(__name__)
 
 
+def apply_restaurant_filter(func):
+    def wrapper(*args, **kwargs):
+        return FoodExpenseParser.apply_restaurant_filters(func(*args, **kwargs))
+
+    return wrapper
+
+
 class FoodExpenseParser(ABC):
     @classmethod
     @abstractmethod
@@ -34,51 +43,37 @@ def serialize_expenses_to_json_file(
         output_path = Path(output_path)
         if not output_path.parent.exists():
             output_path.parent.mkdir(exist_ok=True)
-        file = open(output_path, "w")
+
         sorted_expenses = sorted(expenses, key=lambda exp: exp.date, reverse=True)
         json_expenses = json.dumps(
             [expense.__dict__ for expense in sorted_expenses],
             ensure_ascii=False,
             indent=4,
         )
-        file.write(json_expenses)
-        file.close()
-        logger.success(f"Successfully written results to {output_path=}")
+
+        JsonUtils.write_to_json_file(json_expenses, output_path)
         return json_expenses
 
     @classmethod
-    def find_and_replace_string_value(
-        cls, string_value: str, filters: dict[str, str]
-    ) -> str:
+    def apply_restaurant_filters(cls, string_value: str) -> str:
         """
         Apply filters by finding the filters dict key and replacing it by the dict value
         """
-        for exclude_str, replace_str in filters.items():
-            string_value = string_value.replace(exclude_str, replace_str)
+        filters = JsonUtils.load_dict_from_json(
+            os.path.join(ROOT_DIR, "parsers/restaurant-filters.json")
+        )
+
+        for filter_key, filter_value in filters["replace"].items():
+            string_value = string_value.replace(filter_key, filter_value)
+        for trim_str in filters["trim"]:
+            string_value = string_value.replace(trim_str, "")
+
         return string_value
 
 
 class BoltFoodParser(FoodExpenseParser):
     sender_email: Final[str] = "portugal-food@bolt.eu"
     keywords: Final[str] = "Delivery from Bolt Food"
-    # TODO: Extract this list to an external file ignored by git and load it when script runs
-    # TODO: Make this filtering generic across parsers
-    restaurant_filters: Final[Dict[str, str]] = {
-        " - Saldanha Avenida Casal Ribeiro, 50 B , 1000-093 To Praça Aniceto do Rosário, Lisbon 1 Hamburguer X": "",
-        " - Saldanha Av. Miguel Bombarda, 23B": "",
-        " Rua do saco 50, 1150-284 Lisboa To Praça Aniceto do Rosário, Lisbon 1 🎁 2x1": "",
-        " - Av. Roma Avenida de Roma 74 B": "",
-        "'": "'",
-        " Rua Marquês de Fronteira 117F": "",
-        ", 1070-292 Lisboa To Praça Aniceto do Rosário, Lisbon 1 × 🎁 2x1": "",
-        " Av. Da República, 97 B": "",
-        ", 1070": "",
-        " Praça do Chile 8 Lisboa 1000": "",
-        " Rua da Penha de França": "",
-        " Centro de Lazer do Campo Pequeno loja 412": "",
-        " Av. Duque de Ávila 46B": "",
-        " Rua do saco 50, 1150": "",
-    }
 
     @classmethod
     def fetch_expenses(cls) -> Iterable[FoodExpense]:
@@ -120,6 +115,7 @@ def parse_expenses_from_messages(
         return expenses
 
     @classmethod
+    @apply_restaurant_filter
     def get_restaurant(cls, message: GmailMessage) -> Optional[str]:
         if re.search(r"From .* -", message.subject) is not None:
             restaurant = re.search(r"From .* -", message.subject).group()[5:-2]
@@ -136,9 +132,7 @@ def get_restaurant(cls, message: GmailMessage) -> Optional[str]:
         if restaurant.__contains__("-"):
             restaurant = restaurant.split("-")[0].strip()
 
-        return FoodExpenseParser.find_and_replace_string_value(
-            restaurant, cls.restaurant_filters
-        )
+        return restaurant
 
     @staticmethod
     def get_date(message: GmailMessage) -> str:
@@ -168,26 +162,6 @@ def get_total_payed(message: GmailMessage) -> Optional[float]:
 class UberEatsParser(FoodExpenseParser):
     sender_email: Final[str] = "uber.portugal@uber.com"
     keywords: Final[str] = "Total"
-    restaurant_filters: Final[Dict[str, str]] = {
-        "'": "'",
-        "&": "&",
-        "\u00ae": "",
-        " 🐠": "",
-        " (Marquês de Pombal)": "",
-        "® (Saldanha)": "",
-        " (Saldanha)": "",
-        " (General Roçadas)": "",
-        " (Fontes Pereira de Melo)": "",
-        " (São Sebastião)": "",
-        " (Graça)": "",
-        " (Monumental)": "",
-        " (Saldanha Residence)": "",
-        " (República)": "",
-        " (Sta": "",
-        " (Barata Salgueiro)": "",
-        " (Rossio)": "",
-        " - by Street Chow": "",
-    }
 
     @classmethod
     def fetch_expenses(cls) -> Iterable[FoodExpense]:
@@ -212,7 +186,7 @@ def parse_expenses_from_messages(
                 try:
                     expense = UberEatsExpense(
                         id=message.id,
-                        restaurant=cls.get_restaurant(message, cls.restaurant_filters),
+                        restaurant=cls.get_restaurant(message),
                         total=cls.get_total_payed(message),
                         date=cls.get_date(message),
                     )
@@ -227,9 +201,10 @@ def parse_expenses_from_messages(
         return expenses
 
     @staticmethod
-    def get_restaurant(message: GmailMessage, filters: dict[str, str]) -> Optional[str]:
+    @apply_restaurant_filter
+    def get_restaurant(message: GmailMessage) -> Optional[str]:
         restaurant = message.subject.split("receipt for ")[1].split(".")[0]
-        return FoodExpenseParser.find_and_replace_string_value(restaurant, filters)
+        return restaurant
 
     @staticmethod
     def get_total_payed(message: GmailMessage) -> float:

gmail_fisher/parsers/restaurant-filters.json

@@ -0,0 +1,37 @@
+{
+  "trim": [
+    "\u00ae",
+    " 🐠",
+    " (Marquês de Pombal)",
+    "® (Saldanha)",
+    " (Saldanha)",
+    " (General Roçadas)",
+    " (Fontes Pereira de Melo)",
+    " (São Sebastião)",
+    " (Graça)",
+    " (Monumental)",
+    " (Saldanha Residence)",
+    " (República)",
+    " (Sta",
+    " (Barata Salgueiro)",
+    " (Rossio)",
+    " - by Street Chow",
+    " - Saldanha Avenida Casal Ribeiro, 50 B , 1000-093 To Praça Aniceto do Rosário, Lisbon 1 Hamburguer X",
+    " - Saldanha Av. Miguel Bombarda, 23B",
+    " Rua do saco 50, 1150-284 Lisboa To Praça Aniceto do Rosário, Lisbon 1 🎁 2x1",
+    " - Av. Roma Avenida de Roma 74 B",
+    " Rua Marquês de Fronteira 117F",
+    ", 1070-292 Lisboa To Praça Aniceto do Rosário, Lisbon 1 × 🎁 2x1",
+    " Av. Da República, 97 B",
+    ", 1070",
+    " Praça do Chile 8 Lisboa 1000",
+    " Rua da Penha de França",
+    " Centro de Lazer do Campo Pequeno loja 412",
+    " Av. Duque de Ávila 46B",
+    " Rua do saco 50, 1150"
+  ],
+  "replace": {
+    "'": "'",
+    "&": "&"
+  }
+}

gmail_fisher/utils/json_utils.py

@@ -0,0 +1,21 @@
+import json
+
+from gmail_fisher import logger
+
+
+class JsonUtils:
+    @classmethod
+    def load_dict_from_json(cls, filters_path):
+        with open(filters_path) as filters:
+            json_content = filters.read()
+
+        filters_dict = json.loads(json_content)
+
+        return filters_dict
+
+    @classmethod
+    def write_to_json_file(cls, json_content, json_path):
+        file = open(json_path, "w")
+        file.write(json_content)
+        file.close()
+        logger.success(f"Successfully written results to {json_path=}")

tests/test_parsers.py tests/test_parsers_food.pytests/test_parsers.py renamed to tests/test_parsers_food.py

@@ -1,5 +1,29 @@
-from gmail_fisher.data.models import BoltFoodExpense, UberEatsExpense, GmailMessage
-from gmail_fisher.parsers.food import BoltFoodParser, UberEatsParser
+import pytest
+
+from gmail_fisher.data.models import UberEatsExpense, BoltFoodExpense
+from gmail_fisher.parsers.food import FoodExpenseParser, BoltFoodParser, UberEatsParser
+
+
+@pytest.mark.parametrize(
+    "input_value, expected_output",
+    [
+        ("Sushi Place (General Roçadas)", "Sushi Place"),
+        ("Honorato - Saldanha Av. Miguel Bombarda, 23B", "Honorato"),
+        ("Pizza Lizzy", "Pizza Lizzy"),
+        (
+            "Sabores do Campo Centro de Lazer do Campo Pequeno loja 412",
+            "Sabores do Campo",
+        ),
+        ("Chimarrão Praça do Chile 8 Lisboa 1000", "Chimarrão"),
+        ("Chickinho Rua Marquês de Fronteira 117F", "Chickinho"),
+        ("Hello Beijing Av. Da República, 97 B", "Hello Beijing"),
+        ("Udon Av. Duque de Ávila 46B", "Udon"),
+    ],
+)
+def test_apply_restaurant_filters(input_value, expected_output):
+    result = FoodExpenseParser.apply_restaurant_filters(input_value)
+
+    assert result == expected_output
 
 
 def test_parse_bolt_expenses_from_messages(bolt_food_messages):
@@ -36,9 +60,3 @@ def test_parse_uber_eats_expenses_from_messages(uber_eats_messages):
             date="2020-09-19",
         ),
     ]
-
-
-def test_total_payed(bolt_email_html_body):
-    BoltFoodParser.get_total_payed(
-        message=GmailMessage(id="id", subject="", date="", body=bolt_email_html_body)
-    )