Myeclpay logs (#703)

### Description

Please explain the changes you made here.

### Checklist

- [ ] Created tests which fail without the change (if possible)
- [ ] All tests passing
- [ ] Extended the documentation, if necessary

---------

Co-authored-by: Armand Didierjean <[email protected]>

Author: Rotheem

.env.template

@@ -66,6 +66,11 @@ CDR_PAYMENT_REDIRECTION_URL: str | None
 MYECLPAY_MAXIMUM_WALLET_BALANCE: int | None
 #MYECLPAY_DATA_VERIFIER_ACCESS_TOKEN = ""
 
+# S3 configuration for file storage
+#S3_MYECLPAY_LOGS_BUCKET_NAME = "hyperion"
+#S3_ACCESS_KEY_ID = "realaccesskey"
+#S3_SECRET_ACCESS_KEY = "realsecretkey"
+
 # Redis configuration #
 REDIS_HOST = "hyperion-redis" #May be left at "" during dev if you don't have a redis server running, in production it should be set to the name of the redis container
 REDIS_PORT = 6379

app/core/myeclpay/cruds_myeclpay.py

@@ -10,7 +10,6 @@
 from app.core.myeclpay import models_myeclpay, schemas_myeclpay
 from app.core.myeclpay.types_myeclpay import (
     TransactionStatus,
-    TransactionType,
     WalletDeviceStatus,
     WalletType,
 )
@@ -555,31 +554,24 @@ async def get_user_payment(
 
 
 async def create_transaction(
-    transaction_id: UUID,
-    debited_wallet_id: UUID,
+    transaction: schemas_myeclpay.Transaction,
     debited_wallet_device_id: UUID,
-    credited_wallet_id: UUID,
-    transaction_type: TransactionType,
-    seller_user_id: str | None,
-    total: int,
-    creation: datetime,
-    status: TransactionStatus,
     store_note: str | None,
     db: AsyncSession,
 ) -> None:
-    transaction = models_myeclpay.Transaction(
-        id=transaction_id,
-        debited_wallet_id=debited_wallet_id,
+    transaction_db = models_myeclpay.Transaction(
+        id=transaction.id,
+        debited_wallet_id=transaction.debited_wallet_id,
         debited_wallet_device_id=debited_wallet_device_id,
-        credited_wallet_id=credited_wallet_id,
-        transaction_type=transaction_type,
-        seller_user_id=seller_user_id,
-        total=total,
-        creation=creation,
-        status=status,
+        credited_wallet_id=transaction.credited_wallet_id,
+        transaction_type=transaction.transaction_type,
+        seller_user_id=transaction.seller_user_id,
+        total=transaction.total,
+        creation=transaction.creation,
+        status=transaction.status,
         store_note=store_note,
     )
-    db.add(transaction)
+    db.add(transaction_db)
 
 
 async def update_transaction_status(

app/core/myeclpay/endpoints_myeclpay.py

@@ -17,6 +17,12 @@
     get_user_active_membership_to_association_membership,
 )
 from app.core.myeclpay import cruds_myeclpay, schemas_myeclpay
+from app.core.myeclpay.integrity_myeclpay import (
+    format_cancel_log,
+    format_refund_log,
+    format_transaction_log,
+    format_transfer_log,
+)
 from app.core.myeclpay.models_myeclpay import Store, WalletDevice
 from app.core.myeclpay.types_myeclpay import (
     HistoryType,
@@ -73,6 +79,7 @@
 
 hyperion_error_logger = logging.getLogger("hyperion.error")
 hyperion_security_logger = logging.getLogger("hyperion.security")
+hyperion_myeclpay_logger = logging.getLogger("hyperion.myeclpay")
 
 
 @router.get(
@@ -1674,20 +1681,18 @@ async def add_transfer_by_admin(
             status_code=403,
             detail="Wallet balance would exceed the maximum allowed balance",
         )
-
-    await cruds_myeclpay.create_transfer(
-        db=db,
-        transfer=schemas_myeclpay.Transfer(
-            id=uuid.uuid4(),
-            type=transfer_info.transfer_type,
-            approver_user_id=user.id,
-            total=transfer_info.amount,
-            transfer_identifier="",  # TODO: require to provide an identifier
-            wallet_id=user_payment.wallet_id,
-            creation=datetime.now(UTC),
-            confirmed=True,
-        ),
+    creation_date = datetime.now(UTC)
+    transfer = schemas_myeclpay.Transfer(
+        id=uuid.uuid4(),
+        type=transfer_info.transfer_type,
+        approver_user_id=user.id,
+        total=transfer_info.amount,
+        transfer_identifier="",  # TODO: require to provide an identifier
+        wallet_id=user_payment.wallet_id,
+        creation=creation_date,
+        confirmed=True,
     )
+    await cruds_myeclpay.create_transfer(transfer, db)
     await cruds_myeclpay.increment_wallet_balance(
         wallet_id=user_payment.wallet_id,
         amount=transfer_info.amount,
@@ -1698,6 +1703,7 @@ async def add_transfer_by_admin(
     except Exception:
         await db.rollback()
         raise
+    hyperion_myeclpay_logger.info(format_transfer_log(transfer))
 
     message = Message(
         title="💳 Paiement - transfert",
@@ -2158,24 +2164,28 @@ async def store_scan_qrcode(
         db=db,
     )
     transaction_id = uuid.uuid4()
-    # We create a transaction
-    await cruds_myeclpay.create_transaction(
-        transaction_id=transaction_id,
+    creation_date = datetime.now(UTC)
+    transaction = schemas_myeclpay.Transaction(
+        id=transaction_id,
         debited_wallet_id=debited_wallet_device.wallet_id,
-        debited_wallet_device_id=debited_wallet_device.id,
         credited_wallet_id=store.wallet_id,
         transaction_type=TransactionType.DIRECT,
         seller_user_id=user.id,
         total=scan_info.tot,
-        creation=datetime.now(UTC),
+        creation=creation_date,
         status=TransactionStatus.CONFIRMED,
+    )
+    # We create a transaction
+    await cruds_myeclpay.create_transaction(
+        transaction=transaction,
+        debited_wallet_device_id=debited_wallet_device.id,
         store_note=None,
         db=db,
     )
 
     await db.commit()
 
-    # TODO: log the transaction
+    hyperion_myeclpay_logger.info(format_transaction_log(transaction))
 
     message = Message(
         title=f"💳 Paiement - {store.name}",
@@ -2187,16 +2197,7 @@ async def store_scan_qrcode(
         message=message,
     )
 
-    return schemas_myeclpay.Transaction(
-        id=transaction_id,
-        debited_wallet_id=debited_wallet_device.wallet_id,
-        credited_wallet_id=store.wallet_id,
-        transaction_type=TransactionType.DIRECT,
-        seller_user_id=user.id,
-        total=scan_info.tot,
-        creation=datetime.now(UTC),
-        status=TransactionStatus.CONFIRMED,
-    )
+    return transaction
 
 
 @router.post(
@@ -2325,18 +2326,21 @@ async def refund_transaction(
         db=db,
     )
 
+    creation_date = datetime.now(UTC)
+    refund = schemas_myeclpay.RefundBase(
+        id=uuid.uuid4(),
+        transaction_id=transaction_id,
+        total=refund_amount,
+        seller_user_id=user.id
+        if wallet_previously_credited.type == WalletType.STORE
+        else None,
+        credited_wallet_id=wallet_previously_debited.id,
+        debited_wallet_id=wallet_previously_credited.id,
+        creation=creation_date,
+    )
+
     await cruds_myeclpay.create_refund(
-        refund=schemas_myeclpay.RefundBase(
-            id=uuid.uuid4(),
-            transaction_id=transaction_id,
-            total=refund_amount,
-            seller_user_id=user.id
-            if wallet_previously_credited.type == WalletType.STORE
-            else None,
-            credited_wallet_id=wallet_previously_debited.id,
-            debited_wallet_id=wallet_previously_credited.id,
-            creation=datetime.now(UTC),
-        ),
+        refund=refund,
         db=db,
     )
 
@@ -2355,6 +2359,8 @@ async def refund_transaction(
 
     await db.commit()
 
+    hyperion_myeclpay_logger.info(format_refund_log(refund))
+
     if wallet_previously_debited.user is not None:
         message = Message(
             title="💳 Remboursement",
@@ -2499,6 +2505,8 @@ async def cancel_transaction(
 
     await db.commit()
 
+    hyperion_myeclpay_logger.info(format_cancel_log(transaction_id))
+
     if debited_wallet.user is not None:
         message = Message(
             title="💳 Paiement annulé",

app/core/myeclpay/integrity_myeclpay.py

@@ -0,0 +1,45 @@
+#########################################################################################
+#########################################################################################
+#### /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ ####
+####                                                                                 ####
+####     Following functions are used to format MyECLPay actions for S3 storage.     ####
+####                   Modifying them will break the verification                    ####
+####                   of MyECLPay's integrity via S3 validation.                    ####
+####                                                                                 ####
+####       Please do not modify them without understanding the consequences.         ####
+####                                                                                 ####
+#### /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ /!\ Warning /!\ ####
+#########################################################################################
+#########################################################################################
+
+
+from uuid import UUID
+
+from app.core.myeclpay import models_myeclpay, schemas_myeclpay
+from app.core.myeclpay.types_myeclpay import ActionType
+
+
+def format_transfer_log(
+    transfer: schemas_myeclpay.Transfer | models_myeclpay.Transfer,
+):
+    return f"{ActionType.TRANSFER.name} {transfer.id} {transfer.type.name} {transfer.total} {transfer.wallet_id}"
+
+
+def format_transaction_log(
+    transaction: schemas_myeclpay.Transaction,
+):
+    return f"{ActionType.TRANSACTION.name} {transaction.id} {transaction.debited_wallet_id} {transaction.credited_wallet_id} {transaction.total}"
+
+
+def format_refund_log(
+    refund: schemas_myeclpay.RefundBase,
+):
+    return (
+        f"{ActionType.REFUND.name} {refund.id} {refund.transaction_id} {refund.total}"
+    )
+
+
+def format_cancel_log(
+    transaction_id: UUID,
+):
+    return f"{ActionType.CANCEL.name} {transaction_id}"

app/core/myeclpay/types_myeclpay.py

@@ -54,6 +54,13 @@ class TransferType(str, Enum):
     BANK_TRANSFER = "bank_transfer"
 
 
+class ActionType(str, Enum):
+    TRANSFER = "transfer"
+    REFUND = "refund"
+    CANCEL = "cancel"
+    TRANSACTION = "transaction"
+
+
 class UnexpectedError(Exception):
     pass
 

app/core/myeclpay/utils_myeclpay.py

@@ -7,6 +7,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.myeclpay import cruds_myeclpay
+from app.core.myeclpay.integrity_myeclpay import format_transfer_log
 from app.core.myeclpay.models_myeclpay import UserPayment
 from app.core.myeclpay.schemas_myeclpay import (
     QRCodeContentData,
@@ -19,7 +20,7 @@
 from app.core.payment import schemas_payment
 
 hyperion_security_logger = logging.getLogger("hyperion.security")
-
+hyperion_myeclpay_logger = logging.getLogger("hyperion.myeclpay")
 hyperion_error_logger = logging.getLogger("hyperion.error")
 
 LATEST_TOS = 1
@@ -112,3 +113,4 @@ async def validate_transfer_callback(
     except Exception:
         await db.rollback()
         raise
+    hyperion_myeclpay_logger.info(format_transfer_log(transfer))

app/core/utils/config.py

@@ -86,6 +86,15 @@ class Settings(BaseSettings):
     REDIS_LIMIT: int
     REDIS_WINDOW: int
 
+    ####################
+    # S3 configuration #
+    ####################
+    # S3 configuration is needed to use the S3 storage for MyECLPay logs
+
+    S3_MYECLPAY_LOGS_BUCKET_NAME: str | None = None
+    S3_ACCESS_KEY_ID: str | None = None
+    S3_SECRET_ACCESS_KEY: str | None = None
+
     ##############
     # Google API #
     ##############

app/core/utils/log.py

@@ -74,6 +74,7 @@ class console_color:
 
     LOG_FORMAT: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     MATRIX_LOG_FORMAT: str = "%(asctime)s - %(name)s - <code>%(levelname)s</code> - <font color ='green'>%(message)s</font>"
+    MYECLPAY_LOG_FORMAT: str = "%(message)s"  # Do not change at any cost
 
     # Logging config
     # See https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
@@ -98,6 +99,9 @@ def get_config_dict(self, settings: Settings):
                     "format": self.MATRIX_LOG_FORMAT,
                     "datefmt": "%d-%b-%y %H:%M:%S",
                 },
+                "myeclpay": {
+                    "format": self.MYECLPAY_LOG_FORMAT,
+                },
             },
             "handlers": {
                 # Console handler is always active, even in production.
@@ -132,6 +136,16 @@ def get_config_dict(self, settings: Settings):
                     ),
                     "level": "INFO",
                 },
+                "myeclpay_s3": {
+                    "formatter": "myeclpay",
+                    "class": "app.utils.loggers_tools.s3_handler.S3LogHandler",
+                    "failure_logger": "hyperion.myeclpay.fallback",
+                    "s3_bucket_name": settings.S3_MYECLPAY_LOGS_BUCKET_NAME,
+                    "s3_access_key_id": settings.S3_ACCESS_KEY_ID,
+                    "s3_secret_access_key": settings.S3_SECRET_ACCESS_KEY,
+                    "folder": "myeclpay/",
+                    "retention": 365 * 10,
+                },
                 # There is a handler per log file #
                 # They are based on RotatingFileHandler to logs in multiple 1024 bytes files
                 # https://docs.python.org/3/library/logging.handlers.html#logging.handlers.RotatingFileHandler
@@ -163,6 +177,15 @@ def get_config_dict(self, settings: Settings):
                     "backupCount": 50,
                     "level": "INFO",
                 },
+                "file_myeclpay": {
+                    # file_myeclpay is there to log all operations related to MyECLPay that failed to be logged in the S3 bucket
+                    "formatter": "default",
+                    "class": "logging.handlers.RotatingFileHandler",
+                    "filename": "logs/myeclpay.log",
+                    "maxBytes": 1024 * 1024 * 40,  # ~ 40 MB
+                    "backupCount": 100,
+                    "level": "DEBUG",
+                },
                 "file_amap": {
                     # file_amap should receive informations about amap operation, every operation involving a cash modification.
                     "formatter": "default",
@@ -221,6 +244,21 @@ def get_config_dict(self, settings: Settings):
                     ],
                     "level": MINIMUM_LOG_LEVEL,
                 },
+                "hyperion.myeclpay.fallback": {
+                    "handlers": [
+                        "file_myeclpay",
+                        "matrix_errors",
+                        "console",
+                    ],
+                    "level": "DEBUG",
+                    "propagate": False,
+                },
+                "hyperion.myeclpay": {
+                    "handlers": [
+                        "myeclpay_s3",
+                    ],
+                    "level": "DEBUG",
+                },
                 "hyperion.amap": {
                     "handlers": [
                         "file_amap",