Feature/bcss 22223 refactor investigation dataset util (#164)

<!-- markdownlint-disable-next-line first-line-heading -->
## Description

<!-- Describe your changes in detail. -->
Changing the logic around populating the investigation dataset form to
be more inline with selenium and be easier to manage.
Refactoring FOBT + Surveillance regression tests
Refactoring Investigation dataset UI app to accommodate new changes
Refactoring Subject assertion to give better error messages
Fixing inconsistent KIT logging

## Context

<!-- Why is this change required? What problem does it solve? -->
Changing the logic around populating the investigation dataset form to
be more inline with selenium and be easier to manage.
Refactoring FOBT + Surveillance regression tests
Refactoring Investigation dataset UI app to accommodate new changes
Refactoring Subject assertion to give better error messages
Fixing inconsistent KIT logging

## Type of changes

<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply. -->

- [x] Refactoring (non-breaking change)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would change existing
functionality)
- [ ] Bug fix (non-breaking change which fixes an issue)

## Checklist

<!-- Go over all the following points, and put an `x` in all the boxes
that apply. -->

- [x] I am familiar with the [contributing
guidelines](https://github.com/nhs-england-tools/playwright-python-blueprint/blob/main/CONTRIBUTING.md)
- [x] I have followed the code style of the project
- [x] I have added tests to cover my changes (where appropriate)
- [x] I have updated the documentation accordingly
- [ ] This PR is a result of pair or mob programming

---

## Sensitive Information Declaration

To ensure the utmost confidentiality and protect your and others
privacy, we kindly ask you to NOT including [PII (Personal Identifiable
Information) / PID (Personal Identifiable
Data)](https://digital.nhs.uk/data-and-information/keeping-data-safe-and-benefitting-the-public)
or any other sensitive data in this PR (Pull Request) and the codebase
changes. We will remove any PR that do contain any sensitive
information. We really appreciate your cooperation in this matter.

- [x] I confirm that neither PII/PID nor sensitive data are included in
this PR and the codebase changes.

Author: adrianoaru-nhs

classes/repositories/kit_service_management_repository.py

@@ -1,8 +1,10 @@
 import logging
-from typing import Optional
+from typing import Optional, Any
 from utils.oracle.oracle import OracleDB
 from classes.kits.kit_service_management_record import KitServiceManagementRecord
 from classes.entities.kit_service_management_entity import KitServiceManagementEntity
+import numpy as np
+from decimal import Decimal
 
 
 class KitServiceManagementRepository:
@@ -284,6 +286,22 @@ def update_kit_service_management_entity(
             if entity.put_attempts is not None:
                 params["put_attempts"] = entity.put_attempts
 
+            params = {k: _sanitize_param(v) for k, v in params.items()}
             self.oracle_db.update_or_insert_data_to_table(sql_query, params)
         except Exception as ex:
             raise RuntimeError(f"Error updating KIT_QUEUE record: {ex}")
+
+
+def _sanitize_param(val: Any) -> Any:
+    """
+    Sanitizes a parameter value for database operations.
+    Args:
+        val: The parameter value to sanitize.
+    Returns:
+        The sanitized parameter value.
+    """
+    if isinstance(val, np.generic):
+        return val.item()
+    if isinstance(val, Decimal):
+        return float(val)
+    return val

docs/InvestigationDatasetBuilderApplication.md

@@ -329,7 +329,7 @@ Examples include `"yes_no"` and `"therapeutic_diagnostic"`, which are handled as
 
 1. Choose a unique string for `"type"` (e.g., `"yes_no"`, `"therapeutic_diagnostic"`).
 2. In your JSON field definition, set `"type"` to this string.
-3. Ensure your `render_field` function in `investigation_dataset_ui.py` has a case for your custom type, rendering the appropriate widget (usually a dropdown/selectbox).
+3. Ensure your `render_field` function in `investigation_dataset_ui.py` has a case for your custom type, rendering the appropriate widget (dropdown, radio, etc.).
    - For `"yes_no"`, the UI will show a dropdown with "yes" and "no".
    - For `"therapeutic_diagnostic"`, the UI will show a dropdown with "therapeutic" and "diagnostic".
 4. You can add more custom types by extending the `render_field` function with new cases in the `match-case` or `if` dispatch.

investigation_dataset_ui.py

@@ -546,7 +546,23 @@ def show_section_with_imports(section_name: str) -> None:
         )
     else:
         import_block = ""
-    st.code(f"{import_block}{section_name} = {pretty_dict(result)}", language="python")
+    # Map section to correct fill method
+    fill_methods = {
+        "general_information": "fill_out_general_information",
+        "drug_information": "fill_out_drug_information",
+        "endoscopy_information": "fill_endoscopy_information",
+        "completion_information": "fill_out_completion_information",
+        "failure_information": "fill_out_failure_information",
+        "radiology_information": "fill_out_radiology_information",
+        "suspected_findings": "fill_out_suspected_findings",
+    }
+    method = fill_methods.get(section_name, f"fill_{section_name}")
+    st.code(
+        f"InvestigationDatasetCompletion(page).{method}({pretty_dict(result)})",
+        language="python",
+    )
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def show_drug_group_section_with_imports(section_name: str) -> None:
@@ -569,14 +585,28 @@ def show_drug_group_section_with_imports(section_name: str) -> None:
             all_fields.extend(group["fields"])
             _render_drug_group(section_name, group, result)
 
+    # No special merging; each section is handled independently
+
     enums = get_enums_used(all_fields)
     if enums:
         import_block = (
             enum_import_string + new_indented_line_string.join(sorted(enums)) + "\n)\n"
         )
     else:
         import_block = ""
-    st.code(f"{import_block}{section_name} = {pretty_dict(result)}", language="python")
+    # Map section to correct fill method
+    fill_methods = {
+        "drug_information": "fill_out_drug_information",
+        "contrast_tagging_and_drug": "fill_out_contrast_tagging_and_drug_information",
+        "tagging_agent_given_drug_information": "fill_out_tagging_agent_given_drug_information",
+    }
+    method = fill_methods.get(section_name, f"fill_{section_name}")
+    st.code(
+        f"InvestigationDatasetCompletion(page).{method}({pretty_dict(result)})",
+        language="python",
+    )
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def _render_single_entry_fields(fields: list, result: dict) -> None:
@@ -638,6 +668,51 @@ def _render_drug_entry(fields: list, index: int, result: dict) -> None:
         result[dose_field["key"]] = ddose
 
 
+def _polyp_output_block(
+    pi: int, info_dict: dict, intervention: object, hist_dict: dict
+) -> str:
+    """
+    Generate the code block for a polyp's information, intervention(s), and histology.
+    Args:
+        pi (int): The polyp index (1-based).
+        info_dict (dict): The polyp information dictionary.
+        intervention (object): The polyp intervention(s), can be dict or list of dicts.
+        hist_dict (dict): The polyp histology dictionary.
+    Returns:
+        str: The generated code block.
+    """
+    code_lines = []
+    code_lines.append(
+        f"InvestigationDatasetCompletion(page).fill_polyp_x_information({pretty_dict(info_dict)}, {pi})"
+    )
+    # Intervention
+    if isinstance(intervention, list):
+        if len(intervention) > 1:
+            code_lines.append(
+                f"InvestigationDatasetCompletion(page).fill_polyp_x_multiple_interventions({pretty_list(intervention)}, {pi})"
+            )
+        elif len(intervention) == 1 and isinstance(intervention[0], dict):
+            code_lines.append(
+                f"InvestigationDatasetCompletion(page).fill_polyp_x_intervention({pretty_dict(intervention[0])}, {pi})"
+            )
+        else:
+            code_lines.append(f"# No intervention for polyp {pi}")
+    elif isinstance(intervention, dict):
+        code_lines.append(
+            f"InvestigationDatasetCompletion(page).fill_polyp_x_intervention({pretty_dict(intervention)}, {pi})"
+        )
+    else:
+        code_lines.append(f"# No intervention for polyp {pi}")
+    # Histology
+    if not hist_dict:
+        code_lines.append(f"# No histology for polyp {pi}")
+    else:
+        code_lines.append(
+            f"InvestigationDatasetCompletion(page).fill_polyp_x_histology({pretty_dict(hist_dict)}, {pi})"
+        )
+    return "\n".join(code_lines)
+
+
 def show_polyp_information_and_intervention_and_histology() -> None:
     """
     Show the Polyp Information, Intervention & Histology section, allowing multiple polyps and interventions.
@@ -652,39 +727,39 @@ def show_polyp_information_and_intervention_and_histology() -> None:
     # Collect all fields for import analysis
     all_fields = polyp_info_fields + polyp_intervention_fields + polyp_histology_fields
     enums = get_enums_used(all_fields)
-    if enums:
-        import_block = (
-            enum_import_string + new_indented_line_string.join(sorted(enums)) + "\n)\n"
-        )
-    else:
-        import_block = ""
+    import_block = (
+        enum_import_string + new_indented_line_string.join(sorted(enums)) + "\n)\n"
+        if enums
+        else ""
+    )
 
     num_polyps = st.number_input(
         "Number of polyps", min_value=0, max_value=20, value=1, step=1
     )
-    polyp_information = []
-    polyp_intervention = []
-    polyp_histology = []
+    polyp_info_dicts = {}
+    polyp_histology_dicts = {}
+    polyp_interventions_dicts = {}
 
     for pi in range(1, num_polyps + 1):
         st.markdown(f"### Polyp {pi}")
-        polyp_information.append(_render_polyp_info(polyp_info_fields, pi))
-        polyp_intervention.append(_render_interventions(polyp_intervention_fields, pi))
-        polyp_histology.append(_render_histology(polyp_histology_fields, pi))
+        polyp_info_dicts[pi] = _render_polyp_info(polyp_info_fields, pi)
+        interventions = _render_interventions(polyp_intervention_fields, pi)
+        polyp_interventions_dicts[pi] = (
+            interventions[0]
+            if isinstance(interventions, list) and len(interventions) == 1
+            else interventions
+        )
+        polyp_histology_dicts[pi] = _render_histology(polyp_histology_fields, pi)
 
     st.markdown("#### Output")
-    st.code(
-        f"{import_block}polyp_information = {pretty_list(polyp_information)}",
-        language="python",
-    )
-    st.code(
-        f"polyp_intervention = {pretty_list(polyp_intervention)}",
-        language="python",
-    )
-    st.code(
-        f"polyp_histology = {pretty_list(polyp_histology)}",
-        language="python",
-    )
+    for pi in range(1, num_polyps + 1):
+        info_dict = polyp_info_dicts[pi]
+        hist_dict = polyp_histology_dicts[pi]
+        intervention = polyp_interventions_dicts[pi]
+        code_block = _polyp_output_block(pi, info_dict, intervention, hist_dict)
+        st.code(code_block, language="python")
+    if import_block:
+        st.code(import_block, language="python")
 
 
 def _render_polyp_info(fields: list, pi: int) -> dict:
@@ -718,7 +793,7 @@ def _render_interventions(fields: list, pi: int) -> list:
         f"Add interventions for polyp {pi}?", key=f"add_interventions_{pi}"
     )
     if not add_interventions:
-        return interventions
+        return []
     num_int = st.number_input(
         f"Number of interventions for polyp {pi}",
         min_value=0,
@@ -735,6 +810,8 @@ def _render_interventions(fields: list, pi: int) -> list:
             if val is not None:
                 int_dict[field["key"]] = val
         interventions.append(int_dict)
+    if len(interventions) == 1:
+        return interventions[0]
     return interventions
 
 
@@ -799,7 +876,7 @@ def _render_histology(fields: list, pi: int) -> dict:
     "endoscopy_information": show_section_with_imports,
     "completion_information": show_section_with_imports,
     "failure_information": show_section_with_imports,
-    "polyp_information_and_intervention_and_histology": lambda _: show_polyp_information_and_intervention_and_histology(),  # If you want imports here, update similarly
+    "polyp_information_and_intervention_and_histology": lambda _: show_polyp_information_and_intervention_and_histology(),
     "contrast_tagging_and_drug": show_drug_group_section_with_imports,
     "tagging_agent_given_drug_information": show_drug_group_section_with_imports,
     "radiology_information": show_section_with_imports,

pages/contacts_list/maintain_contacts_page.py

@@ -65,3 +65,14 @@ def click_person_link_from_forename(self, forename: str) -> None:
             forename (str): The forename of the subject
         """
         self.click(self.page.get_by_role("link", name=forename).last)
+
+    def select_person_by_id(self, person_id: int) -> None:
+        """
+        Selects a person by their unique person ID by clicking the corresponding link.
+        Requires a search to have been conducted and returned at least one result.
+        Args:
+            person_id (int): The unique ID of the person to select
+        """
+        href_selector = f"a[href*='{person_id}']"
+        person_link = self.page.locator(href_selector).first
+        self.click(person_link)

pages/datasets/investigation_dataset_page.py

@@ -96,6 +96,7 @@ def __init__(self, page: Page):
         self.diagnostic_test_result = self.page.locator(
             "#datasetContent > div:nth-child(1) > div:nth-child(7) > span.userInput"
         )
+        self.show_details_links = self.page.locator('a:has-text("Show details")')
 
         # Repeat strings:
         self.bowel_preparation_administered_string = "Bowel Preparation Administered"
@@ -361,10 +362,13 @@ def select_diagnostic_procedure_type(self) -> None:
 
     def click_show_completion_proof_information(self) -> None:
         """
-        This method is designed to click on the show completion proof information link.
-        It clicks on the show completion proof information link.
+        Clicks on the show completion proof information link if it contains the text "show" (case-insensitive).
         """
-        self.click(self.show_completion_proof_information_details)
+        if (
+            "show"
+            in self.show_completion_proof_information_details.inner_text().lower()
+        ):
+            self.click(self.show_completion_proof_information_details)
 
     def click_show_failure_information(self) -> None:
         """
@@ -415,6 +419,17 @@ def click_save_dataset_button(self) -> None:
         It clicks on the save dataset button.
         """
         self.safe_accept_dialog(self.save_dataset_button)
+        self.page.wait_for_timeout(3000)  # 3 second timeout to allow page to update
+
+    def click_save_dataset_button_assert_dialog(self, expected_text: str) -> None:
+        """
+        Clicks on the save dataset button and performs an assertion of the resulting dialog text.
+        Once done it dismisses the dialog.
+        Args:
+            expected_text (str): The expected text in the resultant dialog
+        """
+        self.assert_dialog_text(expected_text)
+        self.click(self.save_dataset_button)
 
     def expect_text_to_be_visible(self, text: str) -> None:
         """
@@ -1193,6 +1208,24 @@ def assert_test_result(self, expected_text: str) -> None:
             actual_text.lower() == expected_text.lower()
         ), f"Expected '{expected_text}', but found '{actual_text}'"
 
+    def open_all_details_tabs(self) -> None:
+        """
+        Clicks all visible "Show details" links on the page to open all details tabs.
+        """
+        count = self.show_details_links.count()
+        for i in range(count):
+            link = self.show_details_links.nth(i)
+            if link.is_visible():
+                self.click(link)
+
+    def open_all_minimized_sections(self) -> None:
+        """
+        Opens all the minimized sections in the investigation dataset form.
+        """
+        self.open_all_details_tabs()
+        # Then do it again to get the internal sections
+        self.open_all_details_tabs()
+
 
 def normalize_label(text: str) -> str:
     """