Merge pull request #1173 from i-dot-ai/feature/move-logging-to-core-logger

Change logging to use i_dot_ai_utilities logger and formatted all files

Author: 252afh

backend/consultations/api/views/candidate_theme.py

@@ -19,16 +19,16 @@ class CandidateThemeViewSet(ModelViewSet):
     def get_queryset(self):
         consultation_uuid = self.kwargs["consultation_pk"]
         question_uuid = self.kwargs["question_pk"]
-        
+
         queryset = models.CandidateTheme.objects.filter(
             question__consultation_id=consultation_uuid,
             question_id=question_uuid,
         )
-        
+
         # Staff users can see all candidate themes, non-staff users only see themes for assigned consultations
         if not self.request.user.is_staff:
             queryset = queryset.filter(question__consultation__users=self.request.user)
-            
+
         return queryset.order_by("-approximate_frequency")
 
     def list(self, request, consultation_pk=None, question_pk=None):

backend/consultations/api/views/consultation.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Any
 from uuid import UUID
 
@@ -58,11 +57,11 @@ def get_permissions(self):
         Override permissions for specific actions
         """
         permission_classes = self.permission_classes
-        
-        if self.action == 'destroy':
+
+        if self.action == "destroy":
             # Only admin users can delete consultations
             permission_classes = [IsAuthenticated, IsAdminUser]
-            
+
         return [permission() for permission in permission_classes]
 
     def perform_destroy(self, instance):
@@ -302,7 +301,7 @@ def submit_consultation_export(self, request) -> Response:
 
             for id in validated["question_ids"]:
                 try:
-                    logging.info("Exporting theme audit data - sending to queue")
+                    logger.info("Exporting theme audit data - sending to queue")
                     export_user_theme_job.delay(question_id=UUID(id), s3_key=validated["s3_key"])
                 except Exception:
                     return Response(

backend/consultations/api/views/question.py

@@ -21,13 +21,13 @@ class QuestionViewSet(ModelViewSet):
 
     def get_queryset(self):
         consultation_uuid = self.kwargs["consultation_pk"]
-        
+
         queryset = models.Question.objects.filter(consultation_id=consultation_uuid)
-        
+
         # Staff users can see all questions, non-staff users only see questions for assigned consultations
         if not self.request.user.is_staff:
             queryset = queryset.filter(consultation__users=self.request.user)
-            
+
         return queryset.annotate(response_count=Count("response")).order_by("number")
 
     @action(

backend/consultations/api/views/respondent.py

@@ -15,11 +15,11 @@ class RespondentViewSet(ModelViewSet):
 
     def get_queryset(self):
         consultation_uuid = self.kwargs["consultation_pk"]
-        
+
         queryset = models.Respondent.objects.filter(consultation_id=consultation_uuid)
-        
+
         # Staff users can see all respondents, non-staff users only see respondents for assigned consultations
         if not self.request.user.is_staff:
             queryset = queryset.filter(consultation__users=self.request.user)
-            
+
         return queryset.order_by("-created_at")

backend/consultations/api/views/selected_theme.py

@@ -20,16 +20,16 @@ class SelectedThemeViewSet(ModelViewSet):
     def get_queryset(self):
         consultation_uuid = self.kwargs["consultation_pk"]
         question_uuid = self.kwargs["question_pk"]
-        
+
         queryset = models.SelectedTheme.objects.filter(
             question__consultation_id=consultation_uuid,
             question_id=question_uuid,
         )
-        
+
         # Staff users can see all themes, non-staff users only see themes for assigned consultations
         if not self.request.user.is_staff:
             queryset = queryset.filter(question__consultation__users=self.request.user)
-            
+
         return queryset
 
     def perform_create(self, serializer):

backend/consultations/api/views/theme.py

@@ -13,11 +13,11 @@ class ThemeViewSet(ReadOnlyModelViewSet):
 
     def get_queryset(self):
         consultation_uuid = self.kwargs["consultation_pk"]
-        
+
         queryset = models.CrossCuttingTheme.objects.filter(consultation_id=consultation_uuid)
-        
+
         # Staff users can see all themes, non-staff users only see themes for assigned consultations
         if not self.request.user.is_staff:
             queryset = queryset.filter(consultation__users=self.request.user)
-            
+
         return queryset.order_by("-created_at")

backend/consultations/export_user_theme.py

@@ -97,7 +97,9 @@ def export_user_theme(question_id: uuid.UUID, s3_key: str) -> None:
     filename = f"{timestamp}_question_{question_number}_theme_changes.csv"
 
     if not output:
-        logger.warning(f"No responses found for question {question_number}")
+        logger.warning(
+            "No responses found for question {question_number}", question_number=question_number
+        )
         return
 
     if settings.ENVIRONMENT == "local":
@@ -127,7 +129,9 @@ def export_user_theme(question_id: uuid.UUID, s3_key: str) -> None:
         )
         csv_buffer.close()
     logger.info(
-        f"Finishing export for question {question_number} for consultation {question.consultation.title}"
+        "Finishing export for question {question_number} for consultation {consultation_title}",
+        question_number=question_number,
+        consultation_title=question.consultation.title,
     )
 
 

backend/data_pipeline/batch.py

@@ -36,7 +36,11 @@ def submit_job(
 
     batch = boto3.client("batch")
 
-    logger.info(f"Submitting {job_type} job to AWS Batch for consultation: {consultation_name}")
+    logger.info(
+        "Submitting {job_type} job to AWS Batch for consultation: {consultation_name}",
+        job_type=job_type,
+        consultation_name=consultation_name,
+    )
 
     response = batch.submit_job(
         jobName=job_name,
@@ -47,6 +51,6 @@ def submit_job(
     )
 
     job_id = response["jobId"]
-    logger.info(f"Batch job submitted: {job_id}")
+    logger.info("Batch job submitted: {job_id}", job_id=job_id)
 
     return response

backend/data_pipeline/s3.py

@@ -34,7 +34,7 @@ def read_jsonl(
         response = s3_client.get_object(Bucket=bucket_name, Key=key)
     except ClientError as e:
         if not raise_if_missing and e.response["Error"]["Code"] == "NoSuchKey":
-            logger.info(f"File not found (skipping): {key}")
+            logger.info("File not found (skipping): {key}", key=key)
             return []
         raise
 
@@ -73,7 +73,7 @@ def read_json(
         return data
     except ClientError as e:
         if not raise_if_missing and e.response["Error"]["Code"] == "NoSuchKey":
-            logger.info(f"File not found (skipping): {key}")
+            logger.info("File not found (skipping): {key}", key=key)
             return None
         raise
 

backend/data_pipeline/sync/candidate_themes.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Dict, List, Optional
 
 from django.conf import settings
@@ -9,7 +8,7 @@
 from backend.consultations.models import CandidateTheme, Consultation, Question
 from backend.data_pipeline.models import CandidateThemeBatch, ThemeNodeList
 
-logger = logging.getLogger(__name__)
+logger = settings.LOGGER
 
 
 # ============================================================================
@@ -46,21 +45,23 @@ def load_candidate_themes_from_s3(
     # Build S3 key for clustered_themes.json
     key = f"app_data/consultations/{consultation_code}/outputs/sign_off/{timestamp}/question_part_{question_number}/clustered_themes.json"
 
-    logger.info(f"Loading candidate themes from {key}")
+    logger.info("Loading candidate themes from {key}", key=key)
 
     # Read and parse JSON file
     theme_data = s3.read_json(
         bucket_name=bucket_name_str, key=key, s3_client=s3_client, raise_if_missing=False
     )
 
     if theme_data is None:
-        logger.info(f"No candidate themes file found at {key}")
+        logger.info("No candidate themes file found at {key}", key=key)
         return []
 
     validated_themes = ThemeNodeList.model_validate(theme_data).theme_nodes
 
     logger.info(
-        f"Loaded and validated {len(validated_themes)} candidate themes for question {question_number}"
+        "Loaded and validated {theme_count} candidate themes for question {question_number}",
+        theme_count=len(validated_themes),
+        question_number=question_number,
     )
 
     return validated_themes
@@ -106,8 +107,10 @@ def load_candidate_themes_batch(
             )
 
     logger.info(
-        f"Loading candidate themes for consultation '{consultation_code}' "
-        f"(timestamp: {timestamp}) across {len(question_numbers)} questions"
+        "Loading candidate themes for consultation '{consultation_code}' (timestamp: {timestamp}) across {question_count} questions",
+        consultation_code=consultation_code,
+        timestamp=timestamp,
+        question_count=len(question_numbers),
     )
 
     # Load themes for each question
@@ -133,7 +136,9 @@ def load_candidate_themes_batch(
 
     total_themes = sum(len(themes) for themes in themes_by_question.values())
     logger.info(
-        f"Loaded {total_themes} total candidate themes across {len(themes_by_question)} questions"
+        "Loaded {total_themes} total candidate themes across {question_count} questions",
+        total_themes=total_themes,
+        question_count=len(themes_by_question),
     )
 
     return batch
@@ -159,18 +164,27 @@ def _import_candidate_themes_for_question(question: Question, themes: List[Theme
         themes: List of validated ThemeNode objects
     """
     if not themes:
-        logger.info(f"No candidate themes to import for question {question.number}")
+        logger.info(
+            "No candidate themes to import for question {question_number}",
+            question_number=question.number,
+        )
         return
 
     # Delete existing candidate themes for this question (idempotent)
     existing_count = CandidateTheme.objects.filter(question=question).count()
     if existing_count > 0:
         logger.info(
-            f"Deleting {existing_count} existing candidate themes for question {question.number}"
+            "Deleting {existing_count} existing candidate themes for question {question_number}",
+            existing_count=existing_count,
+            question_number=question.number,
         )
         CandidateTheme.objects.filter(question=question).delete()
 
-    logger.info(f"Importing {len(themes)} candidate themes for question {question.number}")
+    logger.info(
+        "Importing {theme_count} candidate themes for question {question_number}",
+        theme_count=len(themes),
+        question_number=question.number,
+    )
 
     # First pass: create all themes without parent relationships
     themes_to_create = [
@@ -209,15 +223,24 @@ def _import_candidate_themes_for_question(question: Question, themes: List[Theme
             themes_to_update.append(candidate_theme)
         else:
             logger.warning(
-                f"Parent theme with topic_id '{parent_id}' not found for theme '{candidate_theme.name}'"
+                "Parent theme with topic_id '{parent_id}' not found for theme '{theme_name}'",
+                parent_id=parent_id,
+                theme_name=candidate_theme.name,
             )
 
     # Bulk update parent relationships
     if themes_to_update:
         CandidateTheme.objects.bulk_update(themes_to_update, ["parent"])
-        logger.info(f"Set {len(themes_to_update)} parent relationships")
+        logger.info(
+            "Set {relationship_count} parent relationships",
+            relationship_count=len(themes_to_update),
+        )
 
-    logger.info(f"Created {len(created_themes)} candidate themes for question {question.number}")
+    logger.info(
+        "Created {theme_count} candidate themes for question {question_number}",
+        theme_count=len(created_themes),
+        question_number=question.number,
+    )
 
 
 @transaction.atomic
@@ -247,12 +270,17 @@ def import_candidate_themes(batch: CandidateThemeBatch) -> None:
             "Base consultation data must be imported before candidate themes."
         )
 
-    logger.info(f"Starting candidate themes import for consultation '{consultation.title}'")
+    logger.info(
+        "Starting candidate themes import for consultation '{consultation_title}'",
+        consultation_title=consultation.title,
+    )
 
     # Update consultation timestamp
     if consultation.timestamp != batch.timestamp:
         logger.info(
-            f"Updating consultation timestamp from '{consultation.timestamp}' to '{batch.timestamp}'"
+            "Updating consultation timestamp from '{old_timestamp}' to '{new_timestamp}'",
+            old_timestamp=consultation.timestamp,
+            new_timestamp=batch.timestamp,
         )
         consultation.timestamp = batch.timestamp
         consultation.save(update_fields=["timestamp"])
@@ -278,8 +306,9 @@ def import_candidate_themes(batch: CandidateThemeBatch) -> None:
         total_themes_created += len(themes)
 
     logger.info(
-        f"Candidate themes import complete: "
-        f"{total_themes_created} themes across {questions_imported} questions"
+        "Candidate themes import complete: {total_themes_created} themes across {questions_imported} questions",
+        total_themes_created=total_themes_created,
+        questions_imported=questions_imported,
     )
 
 
@@ -311,8 +340,9 @@ def import_candidate_themes_from_s3(
         ValueError: If consultation or questions don't exist
     """
     logger.info(
-        f"Starting candidate themes import for consultation '{consultation_code}' "
-        f"(timestamp: {timestamp})"
+        "Starting candidate themes import for consultation '{consultation_code}' (timestamp: {timestamp})",
+        consultation_code=consultation_code,
+        timestamp=timestamp,
     )
 
     # Load from S3
@@ -325,4 +355,7 @@ def import_candidate_themes_from_s3(
     # Import into database
     import_candidate_themes(batch)
 
-    logger.info(f"Candidate themes import complete for consultation '{consultation_code}'")
+    logger.info(
+        "Candidate themes import complete for consultation '{consultation_code}'",
+        consultation_code=consultation_code,
+    )