Refactoring FOBT + Surveillance regression tests

Refactoring Investigation dataset ui app to accomodate new changes
Refactoring Subject assertion to givebetter error messages
Fixing incossistend KIT logging

Author: adrianoaru-nhs

classes/repositories/kit_service_management_repository.py

@@ -1,8 +1,10 @@
 import logging
-from typing import Optional
+from typing import Optional, Any
 from utils.oracle.oracle import OracleDB
 from classes.kits.kit_service_management_record import KitServiceManagementRecord
 from classes.entities.kit_service_management_entity import KitServiceManagementEntity
+import numpy as np
+from decimal import Decimal
 
 
 class KitServiceManagementRepository:
@@ -284,6 +286,22 @@ def update_kit_service_management_entity(
             if entity.put_attempts is not None:
                 params["put_attempts"] = entity.put_attempts
 
+            params = {k: _sanitize_param(v) for k, v in params.items()}
             self.oracle_db.update_or_insert_data_to_table(sql_query, params)
         except Exception as ex:
             raise RuntimeError(f"Error updating KIT_QUEUE record: {ex}")
+
+
+def _sanitize_param(val: Any) -> Any:
+    """
+    Sanitizes a parameter value for database operations.
+    Args:
+        val: The parameter value to sanitize.
+    Returns:
+        The sanitized parameter value.
+    """
+    if isinstance(val, np.generic):
+        return val.item()
+    if isinstance(val, Decimal):
+        return float(val)
+    return val

docs/InvestigationDatasetBuilderApplication.md

@@ -329,7 +329,7 @@ Examples include `"yes_no"` and `"therapeutic_diagnostic"`, which are handled as
 
 1. Choose a unique string for `"type"` (e.g., `"yes_no"`, `"therapeutic_diagnostic"`).
 2. In your JSON field definition, set `"type"` to this string.
-3. Ensure your `render_field` function in `investigation_dataset_ui.py` has a case for your custom type, rendering the appropriate widget (usually a dropdown/selectbox).
+3. Ensure your `render_field` function in `investigation_dataset_ui.py` has a case for your custom type, rendering the appropriate widget (dropdown, radio, etc.).
    - For `"yes_no"`, the UI will show a dropdown with "yes" and "no".
    - For `"therapeutic_diagnostic"`, the UI will show a dropdown with "therapeutic" and "diagnostic".
 4. You can add more custom types by extending the `render_field` function with new cases in the `match-case` or `if` dispatch.

investigation_dataset_ui.py

@@ -546,7 +546,23 @@ def show_section_with_imports(section_name: str) -> None:
         )
     else:
         import_block = ""
-    st.code(f"{import_block}{section_name} = {pretty_dict(result)}", language="python")
+    # Map section to correct fill method
+    fill_methods = {
+        "general_information": "fill_out_general_information",
+        "drug_information": "fill_out_drug_information",
+        "endoscopy_information": "fill_endoscopy_information",
+        "completion_information": "fill_out_completion_information",
+        "failure_information": "fill_out_failure_information",
+        "radiology_information": "fill_out_radiology_information",
+        "suspected_findings": "fill_out_suspected_findings",
+    }
+    method = fill_methods.get(section_name, f"fill_{section_name}")
+    st.code(
+        f"InvestigationDatasetCompletion(page).{method}({pretty_dict(result)})",
+        language="python",
+    )
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def show_drug_group_section_with_imports(section_name: str) -> None:
@@ -569,14 +585,28 @@ def show_drug_group_section_with_imports(section_name: str) -> None:
             all_fields.extend(group["fields"])
             _render_drug_group(section_name, group, result)
 
+    # No special merging; each section is handled independently
+
     enums = get_enums_used(all_fields)
     if enums:
         import_block = (
             enum_import_string + new_indented_line_string.join(sorted(enums)) + "\n)\n"
         )
     else:
         import_block = ""
-    st.code(f"{import_block}{section_name} = {pretty_dict(result)}", language="python")
+    # Map section to correct fill method
+    fill_methods = {
+        "drug_information": "fill_out_drug_information",
+        "contrast_tagging_and_drug": "fill_out_contrast_tagging_and_drug_information",
+        "tagging_agent_given_drug_information": "fill_out_tagging_agent_given_drug_information",
+    }
+    method = fill_methods.get(section_name, f"fill_{section_name}")
+    st.code(
+        f"InvestigationDatasetCompletion(page).{method}({pretty_dict(result)})",
+        language="python",
+    )
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def _render_single_entry_fields(fields: list, result: dict) -> None:
@@ -662,29 +692,63 @@ def show_polyp_information_and_intervention_and_histology() -> None:
     num_polyps = st.number_input(
         "Number of polyps", min_value=0, max_value=20, value=1, step=1
     )
-    polyp_information = []
-    polyp_intervention = []
-    polyp_histology = []
+    polyp_info_dicts = {}
+    polyp_histology_dicts = {}
+    polyp_interventions_dicts = {}
 
     for pi in range(1, num_polyps + 1):
         st.markdown(f"### Polyp {pi}")
-        polyp_information.append(_render_polyp_info(polyp_info_fields, pi))
-        polyp_intervention.append(_render_interventions(polyp_intervention_fields, pi))
-        polyp_histology.append(_render_histology(polyp_histology_fields, pi))
+        polyp_info = _render_polyp_info(polyp_info_fields, pi)
+        polyp_info_dicts[pi] = polyp_info
+        interventions = _render_interventions(polyp_intervention_fields, pi)
+        # If interventions is a list of length 1, store as dict, else as list
+        if isinstance(interventions, list) and len(interventions) == 1:
+            polyp_interventions_dicts[pi] = interventions[0]
+        else:
+            polyp_interventions_dicts[pi] = interventions
+        polyp_histology = _render_histology(polyp_histology_fields, pi)
+        polyp_histology_dicts[pi] = polyp_histology
 
     st.markdown("#### Output")
-    st.code(
-        f"{import_block}polyp_information = {pretty_list(polyp_information)}",
-        language="python",
-    )
-    st.code(
-        f"polyp_intervention = {pretty_list(polyp_intervention)}",
-        language="python",
-    )
-    st.code(
-        f"polyp_histology = {pretty_list(polyp_histology)}",
-        language="python",
-    )
+    output_blocks = []
+    for pi in range(1, num_polyps + 1):
+        info_dict = polyp_info_dicts[pi]
+        hist_dict = polyp_histology_dicts[pi]
+        intervention = polyp_interventions_dicts[pi]
+        code_lines = []
+        # Information
+        code_lines.append(
+            f"InvestigationDatasetCompletion(page).fill_polyp_x_information({pretty_dict(info_dict)}, {pi})"
+        )
+        # Intervention
+        if isinstance(intervention, list):
+            if len(intervention) > 1:
+                code_lines.append(
+                    f"InvestigationDatasetCompletion(page).fill_polyp_x_multiple_interventions({pretty_list(intervention)}, {pi})"
+                )
+            elif len(intervention) == 1 and isinstance(intervention[0], dict):
+                code_lines.append(
+                    f"InvestigationDatasetCompletion(page).fill_polyp_x_intervention({pretty_dict(intervention[0])}, {pi})"
+                )
+            else:
+                code_lines.append(f"# No intervention for polyp {pi}")
+        elif isinstance(intervention, dict):
+            code_lines.append(
+                f"InvestigationDatasetCompletion(page).fill_polyp_x_intervention({pretty_dict(intervention)}, {pi})"
+            )
+        else:
+            code_lines.append(f"# No intervention for polyp {pi}")
+        # Histology
+        if not hist_dict:
+            code_lines.append(f"# No histology for polyp {pi}")
+        else:
+            code_lines.append(
+                f"InvestigationDatasetCompletion(page).fill_polyp_x_histology({pretty_dict(hist_dict)}, {pi})"
+            )
+        output_blocks.append("\n".join(code_lines))
+        st.code("\n".join(code_lines), language="python")
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def _render_polyp_info(fields: list, pi: int) -> dict:
@@ -718,7 +782,7 @@ def _render_interventions(fields: list, pi: int) -> list:
         f"Add interventions for polyp {pi}?", key=f"add_interventions_{pi}"
     )
     if not add_interventions:
-        return interventions
+        return []
     num_int = st.number_input(
         f"Number of interventions for polyp {pi}",
         min_value=0,
@@ -735,6 +799,8 @@ def _render_interventions(fields: list, pi: int) -> list:
             if val is not None:
                 int_dict[field["key"]] = val
         interventions.append(int_dict)
+    if len(interventions) == 1:
+        return interventions[0]
     return interventions
 
 
@@ -799,7 +865,7 @@ def _render_histology(fields: list, pi: int) -> dict:
     "endoscopy_information": show_section_with_imports,
     "completion_information": show_section_with_imports,
     "failure_information": show_section_with_imports,
-    "polyp_information_and_intervention_and_histology": lambda _: show_polyp_information_and_intervention_and_histology(),  # If you want imports here, update similarly
+    "polyp_information_and_intervention_and_histology": lambda _: show_polyp_information_and_intervention_and_histology(),
     "contrast_tagging_and_drug": show_drug_group_section_with_imports,
     "tagging_agent_given_drug_information": show_drug_group_section_with_imports,
     "radiology_information": show_section_with_imports,

pages/contacts_list/maintain_contacts_page.py

@@ -65,3 +65,14 @@ def click_person_link_from_forename(self, forename: str) -> None:
             forename (str): The forename of the subject
         """
         self.click(self.page.get_by_role("link", name=forename).last)
+
+    def select_person_by_id(self, person_id: int) -> None:
+        """
+        Selects a person by their unique person ID by clicking the corresponding link.
+        Requires a search to have been conducted and returned at least one result.
+        Args:
+            person_id (int): The unique ID of the person to select
+        """
+        href_selector = f"a[href*='{person_id}']"
+        person_link = self.page.locator(href_selector).first
+        self.click(person_link)

pages/datasets/investigation_dataset_page.py

@@ -419,6 +419,7 @@ def click_save_dataset_button(self) -> None:
         It clicks on the save dataset button.
         """
         self.safe_accept_dialog(self.save_dataset_button)
+        self.page.wait_for_timeout(3000)  # 3 second timeout to allow page to update
 
     def click_save_dataset_button_assert_dialog(self, expected_text: str) -> None:
         """
@@ -427,7 +428,7 @@ def click_save_dataset_button_assert_dialog(self, expected_text: str) -> None:
         Args:
             expected_text (str): The expected text in the resultant dialog
         """
-        self.assert_dialog_text("High-risk findings")
+        self.assert_dialog_text(expected_text)
         self.click(self.save_dataset_button)
 
     def expect_text_to_be_visible(self, text: str) -> None: