fix: remediate vulnerabilities and sync new shared config utlities

Author: mobious999

.github/workflows/Create Greetings on Pull Request.yml

@@ -15,7 +15,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Greet First-Time Contributors
-        uses: actions/first-interaction@v1
+        uses: actions/first-interaction@v2
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           issue-message: |

.github/workflows/Secret scanning.yml

@@ -1,14 +1,37 @@
 ---
 name: Secret Scanning
-on: [push, pull_request]
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    - cron: 0 3 * * 0  # weekly full scan on Sunday at 3 AM UTC
 jobs:
-  secret-scan:
+  # 🧪 PR Diff Scan
+  diff-scan:
+    name: TruffleHog PR Diff Scan
     runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v4
-      - name: Scan for Secrets
+      - name: Run TruffleHog on PR diff
         uses: trufflesecurity/trufflehog@main
         with:
-          path: .  # Scan the entire repo
+          base: ${{ github.event.pull_request.base.sha }}
+          head: ${{ github.event.pull_request.head.sha }}
+
+  # 🔍 Full Repo Scan
+  full-scan:
+    name: TruffleHog Full Repo Scan
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event_name == 'schedule'
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run TruffleHog on entire repo
+        uses: trufflesecurity/trufflehog@main
+        with:
+          path: .

src/app/config_shared.py

@@ -5,7 +5,6 @@
 """
 
 from functools import lru_cache
-from typing import List, Tuple
 
 from app.utils.config_utils import get_config_bool
 from app.utils.types import OutputMode
@@ -272,6 +271,16 @@ def get_log_level() -> str:
     return get_config_value_cached("LOG_LEVEL", "INFO")
 
 
+def get_log_format() -> str:
+    """Returns the configured log format.
+
+    Returns:
+        str: 'json' or 'text' (default is 'text').
+
+    """
+    return get_config_value_cached("LOG_FORMAT", "text").lower()
+
+
 @lru_cache
 def get_poller_type() -> str:
     """Retrieve the type/category of this poller.
@@ -1161,7 +1170,7 @@ def get_healthcheck_host() -> str:
     Defaults to '0.0.0.0' if not set.
 
     """
-    return get_config_value_cached("HEALTHCHECK_HOST", "0.0.0.0")
+    return get_config_value_cached("HEALTHCHECK_HOST", "127.0.0.1")
 
 
 @lru_cache
@@ -1201,7 +1210,7 @@ def get_metrics_bind_address() -> str:
     Defaults to '0.0.0.0' if not set.
 
     """
-    return get_config_value_cached("METRICS_BIND_ADDRESS", "0.0.0.0")
+    return get_config_value_cached("METRICS_BIND_ADDRESS", "127.0.0.1")
 
 
 @lru_cache

src/app/output_handler.py

@@ -19,6 +19,7 @@
     record_paper_trade_metrics,
     record_sink_metrics,
 )
+from app.utils.redactor import redact_dict
 from app.utils.setup_logger import setup_logger
 from app.utils.types import OutputMode, validate_list_of_dicts
 
@@ -115,7 +116,7 @@ def _output_to_log(self, data: list[dict[str, Any]]) -> None:
 
         """
         for item in data:
-            logger.info("📝 Processed message:\n%s", json.dumps(item, indent=4))
+            logger.info("📝 Processed message:\n%s", json.dumps(redact_dict(item), indent=4))
 
     def _output_to_stdout(self, data: list[dict[str, Any]]) -> None:
         """Print each item in the data list to standard output.
@@ -224,7 +225,7 @@ def _output_paper_trade_to_queue(self, data: dict[str, Any]) -> None:
         queue_name = config_shared.get_paper_trading_queue_name()
         exchange = config_shared.get_paper_trading_exchange()
         publish_to_queue([data], queue=queue_name, exchange=exchange)
-        logger.info("🪙 Paper trade sent to queue:\n%s", json.dumps(data, indent=4))
+        logger.info("🪙 Paper trade sent to queue:\n%s", json.dumps(redact_dict(data), indent=4))
         record_paper_trade_metrics("queue", success=True, duration_sec=0)
 
     def _output_paper_trade_to_database(self, data: dict[str, Any]) -> None:

src/app/utils/redactor.py

@@ -0,0 +1,40 @@
+# src/app/utils/redactor.py
+
+"""Utility for redacting sensitive fields from dictionaries before logging.
+
+This module is used to ensure that no passwords, tokens, or other secrets
+are ever written to logs in clear-text.
+"""
+
+from typing import Any
+
+# Fields that should never appear in logs
+SENSITIVE_KEYS = {
+    "password",
+    "secret",
+    "token",
+    "api_key",
+    "authorization",
+    "access_token",
+}
+
+
+def redact_dict(obj: Any) -> Any:
+    """Recursively redacts sensitive keys in a dictionary.
+
+    Args:
+        obj (Any): The input data (typically a dict or list of dicts).
+
+    Returns:
+        Any: The redacted version of the input, preserving structure.
+
+    """
+    if isinstance(obj, dict):
+        return {
+            k: "***REDACTED***" if k.lower() in SENSITIVE_KEYS else redact_dict(v)
+            for k, v in obj.items()
+        }
+    elif isinstance(obj, list):
+        return [redact_dict(item) for item in obj]
+    else:
+        return obj

src/app/utils/safe_logger.py

@@ -0,0 +1,82 @@
+"""Wrapper around the standard logger that applies redaction and structured logging.
+
+This ensures that sensitive fields are redacted and logs follow consistent formatting.
+"""
+
+import logging
+import os
+from typing import Any, Optional
+
+from app.utils.redactor import redact_dict
+from app.utils.setup_logger import setup_logger
+
+# Controls whether to log full payloads (only in dev/test)
+SAFE_LOG_FULL: bool = os.getenv("SAFE_LOG_FULL", "false").lower() == "true"
+SAFE_LOG_STRUCTURED: bool = os.getenv("SAFE_LOG_STRUCTURED", "false").lower() == "true"
+
+# Base logger instance
+logger: logging.Logger = setup_logger(__name__, structured=SAFE_LOG_STRUCTURED)
+
+
+def safe_info(message: str, data: Optional[dict[str, Any]] = None) -> None:
+    """
+    Logs an info-level message with optional sanitized payload metadata.
+
+    Args:
+        message (str): Human-readable log message.
+        data (Optional[dict]): Dictionary payload to log. Only logs redacted metadata unless SAFE_LOG_FULL is enabled.
+    """
+    if data is None:
+        logger.info(message)
+        return
+
+    payload_size = len(data) if SAFE_LOG_FULL else len(redact_dict(data))
+    logger.info("%s | payload_size=%d", message, payload_size)
+
+
+def safe_warning(message: str, data: Optional[dict[str, Any]] = None) -> None:
+    """
+    Logs a warning-level message with optional sanitized payload metadata.
+
+    Args:
+        message (str): Human-readable log message.
+        data (Optional[dict]): Dictionary payload to log. Only logs redacted metadata unless SAFE_LOG_FULL is enabled.
+    """
+    if data is None:
+        logger.warning(message)
+        return
+
+    payload_size = len(data) if SAFE_LOG_FULL else len(redact_dict(data))
+    logger.warning("%s | payload_size=%d", message, payload_size)
+
+
+def safe_error(message: str, data: Optional[dict[str, Any]] = None) -> None:
+    """
+    Logs an error-level message with optional sanitized payload metadata.
+
+    Args:
+        message (str): Human-readable log message.
+        data (Optional[dict]): Dictionary payload to log. Only logs redacted metadata unless SAFE_LOG_FULL is enabled.
+    """
+    if data is None:
+        logger.error(message)
+        return
+
+    payload_size = len(data) if SAFE_LOG_FULL else len(redact_dict(data))
+    logger.error("%s | payload_size=%d", message, payload_size)
+
+
+def safe_debug(message: str, data: Optional[dict[str, Any]] = None) -> None:
+    """
+    Logs a debug-level message with optional sanitized payload metadata.
+
+    Args:
+        message (str): Human-readable log message.
+        data (Optional[dict]): Dictionary payload to log. Only logs redacted metadata unless SAFE_LOG_FULL is enabled.
+    """
+    if data is None:
+        logger.debug(message)
+        return
+
+    payload_size = len(data) if SAFE_LOG_FULL else len(redact_dict(data))
+    logger.debug("%s | payload_size=%d", message, payload_size)

src/app/utils/setup_logger.py

@@ -1,52 +1,81 @@
+"""Configures and returns a logger with console, optional file, and optional JSON output.
+Supports redaction toggle from config_shared and multi-handler output.
+"""
+
 import logging
 import sys
 from logging import Logger
+from logging.handlers import RotatingFileHandler
+
+try:
+    from pythonjsonlogger.json import JsonFormatter
+except ImportError:
+    JsonFormatter = None  # JSON logging fallback
+
+from app import config_shared
 
 
 def setup_logger(
     name: str | None = None,
-    level: int = logging.INFO,
-    structured: bool = False,
+    level: int | None = None,
+    structured: bool | None = None,
+    log_file: str | None = None,
 ) -> Logger:
-    """Configure and return a logger with optional redaction and structured logging.
+    """Configure and return a logger with optional structured and file output.
 
     Args:
         name (Optional[str]): Logger name.
-        level (int): Log level (e.g., logging.INFO).
-        structured (bool): Enable structured (JSON) logging. (Not yet implemented)
+        level (Optional[int]): Logging level (overrides LOG_LEVEL config).
+        structured (Optional[bool]): Use structured (JSON) logging (overrides LOG_FORMAT config).
+        log_file (Optional[str]): Path to a log file (enables rotation if set).
 
     Returns:
         Logger: Configured logger instance.
 
     """
-    logger = logging.getLogger(name)
-    if logger.handlers:
+    logger = logging.getLogger(name or "app")
+
+    if logger.hasHandlers():
         return logger
 
-    # Delay import to avoid circular dependency
-    try:
-        from app import config_shared
+    # Resolve redaction
+    redact_enabled = config_shared.get_redact_sensitive_logs()
+
+    # Resolve level
+    level_name = config_shared.get_log_level()
+    resolved_level: int = level if level is not None else getattr(logging, level_name, logging.INFO)
+
+    # Resolve structured format
+    structured = structured if structured is not None else config_shared.get_log_format() == "json"
+
+    # Choose formatter
+    if structured and JsonFormatter:
+        formatter = JsonFormatter("%(asctime)s %(levelname)s %(name)s %(message)s")
+    else:
+        formatter = logging.Formatter(
+            fmt="%(asctime)s [%(levelname)s] %(name)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
 
-        redact = config_shared.get_redact_sensitive_logs()
-    except Exception:
-        redact = False
+    # Console handler
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
 
-    formatter = logging.Formatter(
-        fmt="%(asctime)s [%(levelname)s] %(name)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
-    )
+    # Optional rotating file handler
+    if log_file:
+        file_handler = RotatingFileHandler(log_file, maxBytes=5 * 1024 * 1024, backupCount=3)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
 
-    handler = logging.StreamHandler(sys.stdout)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(level)
+    logger.setLevel(resolved_level)
+    logger.propagate = False
 
-    if redact:
+    if redact_enabled:
         logger.info("🔒 Redaction of sensitive data is ENABLED")
     else:
         logger.info("🔓 Redaction of sensitive data is DISABLED")
 
-    # NOTE: 'structured' is accepted but not yet implemented
-    if structured:
-        logger.warning("⚠️ Structured logging requested but not yet supported.")
+    if structured and not JsonFormatter:
+        logger.warning("⚠️ Structured logging requested but 'python-json-logger' is not installed.")
 
     return logger