refactor: standardize config, vault client, and test structure across all repos

- Updated all `config.py` files to include repo-specific getters for poller name, queue, and DLQ
- Unified usage of `config_shared.py` and removed legacy config references
- Replaced old `get_secret_from_vault` with `get_secret_or_env` in `vault_client.py`
- Cleaned up and standardized `test_vault_client.py` across all repos
- Synced `main.py`, `db_writer.py`, and related config imports to match shared structure
- Removed redundant or outdated test and config files

Author: mobious999

.github/workflows/SBOM and License Audit.yml

@@ -8,6 +8,8 @@ jobs:
   sbom:
     name: Generate SBOM & License Report
     runs-on: ubuntu-latest
+    permissions:
+      contents: read  
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

.hooks/check-compiled-requirements.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """Pre-commit hook to check and auto-fix requirements.txt and requirements-dev.txt
 if they are out of sync with requirements.in and requirements-dev.in.
 """
@@ -7,18 +6,23 @@
 import sys
 from pathlib import Path
 from shutil import which
-
-PIP_COMPILE = which("pip-compile")
-if not PIP_COMPILE:
-    print("[Error] pip-compile not found in PATH.")
-    sys.exit(1)
+from typing import Optional
 
 
 def recompile(in_file: str, out_file: str) -> bool:
+    """Recompile the output requirements file from its corresponding .in file.
+
+    Args:
+        in_file: Path to the .in file.
+        out_file: Path to the output .txt file.
+
+    Returns:
+        True if successful, False otherwise.
+    """
     print(f"[Fix] Recompiling {in_file} -> {out_file}")
     try:
         subprocess.run(  # nosec B603
-            [PIP_COMPILE, in_file, "--resolver=backtracking", "--output-file", out_file],
+            ["pip-compile", in_file, "--resolver=backtracking", "--output-file", out_file],
             check=True,
         )
         return True
@@ -28,13 +32,25 @@ def recompile(in_file: str, out_file: str) -> bool:
 
 
 def check_file(in_file: str, out_file: str, autofix: bool = True) -> bool:
+    """Check if output requirements file is in sync with its .in file and optionally fix it.
+
+    Args:
+        in_file: Path to the .in file.
+        out_file: Path to the output .txt file.
+        autofix: Whether to fix mismatches automatically.
+
+    Returns:
+        True if up to date or successfully fixed, False otherwise.
+    """
     if not Path(in_file).exists():
-        return True  # skip if .in file doesn't exist
+        return True  # Skip if input file does not exist
 
     print(f"[Check] {in_file} -> {out_file}")
+    tmp_out = Path(out_file + ".tmp")
+
     try:
         subprocess.run(  # nosec B603
-            [PIP_COMPILE, in_file, "--resolver=backtracking", "--output-file", out_file + ".tmp"],
+            ["pip-compile", in_file, "--resolver=backtracking", "--output-file", str(tmp_out)],
             check=True,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
@@ -43,27 +59,30 @@ def check_file(in_file: str, out_file: str, autofix: bool = True) -> bool:
         print(f"[Error] pip-compile failed during check for {in_file}: {e}")
         return False
 
-    expected = Path(out_file + ".tmp").read_text(encoding="utf-8")
+    expected = tmp_out.read_text(encoding="utf-8")
     actual = Path(out_file).read_text(encoding="utf-8") if Path(out_file).exists() else ""
 
-    Path(out_file + ".tmp").unlink()
+    tmp_out.unlink()
 
     if expected != actual:
         print(f"[Mismatch] {out_file} is out of sync with {in_file}")
-        if autofix:
-            return recompile(in_file, out_file)
-        return False
+        return recompile(in_file, out_file) if autofix else False
 
     return True
 
 
-def main():
+def main() -> int:
+    """Main entry point for the hook."""
+    if not which("pip-compile"):
+        print("[Error] pip-compile not found in PATH.")
+        return 1
+
     ok1 = check_file("requirements.in", "requirements.txt", autofix=True)
     ok2 = check_file("requirements-dev.in", "requirements-dev.txt", autofix=True)
 
     if not (ok1 and ok2):
         print("[Error] Could not fix all mismatches. Check pip-compile output.")
-        sys.exit(1)
+        return 1
 
     print("[OK] All requirements files are now up to date.")
     return 0

Makefile

@@ -1,3 +1,4 @@
+
 .PHONY: help install test lint audit format clean clean-all compile preflight precommit security bump docker-build sbom sign-image watch
 
 help:
@@ -72,35 +73,26 @@ sync:
 
 sync-apply:
 	@echo "🚀 Applying sync to all repositories..."
-	python sync_if_needed.py --apply && echo "✅ Sync complete. Changes logged in sync.log"	
+	python sync_if_needed.py --apply && echo "✅ Sync complete. Changes logged in sync.log"
 
 # -----------------------------------------------------------------------------
 # Attestation and SBOM Targets
 # -----------------------------------------------------------------------------
 
-# ✅ Generate CycloneDX SBOM for Python dependencies
-# Requires: pip install cyclonedx-bom
 sbom-py:
 	cyclonedx-py -o bom.json
 
-# ✅ Generate full SBOM from Docker image context using Syft
-# Requires: https://github.com/anchore/syft
 sbom-image:
 	syft . -o spdx-json > sbom.spdx.json
 
-# ✅ Run both SBOM generators + audit (pip check + pip-audit + deptry)
 attest: sbom-py sbom-image audit
 
 # -----------------------------------------------------------------------------
 # Optional Utilities (enable as needed)
 # -----------------------------------------------------------------------------
 
-# 🟡 OPTIONAL: Sign Docker image using cosign
-# Requires: https://github.com/sigstore/cosign
 # sign-image:
 # 	cosign sign $(shell basename $(PWD)):latest
 
-# 🟡 OPTIONAL: Pytest watch mode for test-driven development
-# Requires: pip install pytest-watch
 # watch:
 # 	ptw --onfail "notify-send 'Test failed!'"

src/app/utils/rate_limit.py

@@ -1,5 +1,6 @@
 """Thread-safe rate limiter using the token bucket algorithm."""
 
+import hashlib
 import re
 import threading
 import time
@@ -24,14 +25,44 @@
 
 
 def _sanitize_context(context: str) -> str:
-    """Sanitize context string for safe logging."""
+    """Sanitize context string for safe metric labeling.
+
+    Replaces unsafe characters and truncates the string for Prometheus label compatibility.
+
+    Args:
+        context (str): Original context string.
+
+    Returns:
+        str: Sanitized and truncated context label.
+    """
     return re.sub(r"[^\w\-:.]", "_", context)[:64]
 
 
+def _hash_context(context: str) -> str:
+    """Hash the context string to produce a short, opaque identifier for logs.
+
+    Args:
+        context (str): The original context string.
+
+    Returns:
+        str: SHA-256-based short hash of the context.
+    """
+    return hashlib.sha256(context.encode()).hexdigest()[:8]
+
+
 class RateLimiter:
     """Token bucket rate limiter with Prometheus support."""
 
     def __init__(self, max_requests: int, time_window: float) -> None:
+        """Initialize a new RateLimiter instance.
+
+        Args:
+            max_requests (int): Maximum number of allowed requests in the time window.
+            time_window (float): Time window in seconds.
+
+        Raises:
+            ValueError: If max_requests or time_window is non-positive.
+        """
         if max_requests <= 0:
             raise ValueError("max_requests must be greater than 0")
         if time_window <= 0:
@@ -44,7 +75,19 @@ def __init__(self, max_requests: int, time_window: float) -> None:
         self._last_check = time.time()
 
     def acquire(self, context: str = "RateLimiter") -> None:
+        """Acquire a token, blocking if needed to respect rate limits.
+
+        Replenishes tokens based on elapsed time and sleeps if tokens are unavailable.
+        Also updates Prometheus metrics and logs token state.
+
+        Args:
+            context (str): An identifier for the caller (e.g., service name).
+
+        Returns:
+            None
+        """
         context = _sanitize_context(context)
+        context_id = _hash_context(context)
 
         with self._lock:
             current_time: float = time.time()
@@ -57,21 +100,21 @@ def acquire(self, context: str = "RateLimiter") -> None:
             rate_limiter_tokens_remaining.labels(context=context).set(self._tokens)
 
             logger.debug(
-                f"[{context}] Replenished {tokens_to_add:.2f} tokens. "
-                f"Available tokens: {self._tokens:.2f}"
+                f"[ctx:{context_id}] Replenished {tokens_to_add:.2f} tokens. "
+                f"Available: {self._tokens:.2f}"
             )
 
             if self._tokens < 1:
                 sleep_time: float = (1 - self._tokens) * (self._time_window / self._max_requests)
                 sleep_time = min(sleep_time, self._time_window)
 
                 logger.info(
-                    f"[{context}] Rate limit reached. Sleeping for {sleep_time:.2f} seconds."
+                    f"[ctx:{context_id}] Rate limit hit. Sleeping for {sleep_time:.2f} seconds."
                 )
                 rate_limiter_blocked_total.labels(context=context).inc()
                 time.sleep(sleep_time)
                 self._tokens = 1
 
             self._tokens -= 1
             rate_limiter_tokens_remaining.labels(context=context).set(self._tokens)
-            logger.debug(f"[{context}] Consumed a token. Remaining: {self._tokens:.2f}")
+            logger.debug(f"[ctx:{context_id}] Token consumed. Remaining: {self._tokens:.2f}")