Merge remote branch into local branch

Author: Akol125

README.md

@@ -147,6 +147,10 @@ Steps:
     AWS_PROFILE={your_profile}
     IMMUNIZATION_ENV=local
     ```
+    For unit tests to run successfully, you may also need to add an environment variable for PYTHONPATH. This should be:
+    ```
+    PYTHONPATH=src:tests
+    ```
 
 4. Configure `direnv` by creating a `.envrc` file in the backend folder. This points direnv to the `.venv` created by poetry and loads env variables specified in the `.env` file
     ```
@@ -163,6 +167,10 @@ Steps:
     ```
     Test if environment variables have been loaded into shell: `echo $IMMUNIZATION_ENV`.
 
+#### Running Unit Tests from the Command Line
+
+It is not necessary to activate the virtual environment (using `source .venv/bin/activate`) before running a unit test suite from the command line; `direnv` will pick up the correct configurations for us. Run `pip list` to verify that the expected packages are installed. You should for example see that `recordprocessor` is specifically running `moto` v4, regardless of which if any `.venv` is active.
+
 ### Setting up the root level environment
 The root-level virtual environment is primarily used for linting, as we create separate virtual environments for each folder that contains Lambda functions.
 Steps: 
@@ -178,20 +186,38 @@ The current team uses VS Code mainly. So this setup is targeted towards VS code.
 
 ### VS Code
 
-The project must be opened as a multi-root workspace for VS Code to know that `backend` has its own environment.
+The project must be opened as a multi-root workspace for VS Code to know that specific lambdas (e.g. `backend`) have their own environment.
+This example is for `backend`; substitute another lambda name in where applicable.
 
 - Open the workspace `immunisation-fhir-api.code-workspace`.
 - Copy `backend/.vscode/settings.json.default` to `backend/.vscode/settings.json`, or merge the contents with
   your existing file.
+- Similarly, copy or merge `backend/.vscode/launch.json.default` to `backend/.vscode/launch.json`.
 
 VS Code will automatically use the `backend` environment when you're editing a file under `backend`.
 
 Depending on your existing setup VS Code might automatically choose the wrong virtualenvs. Change it
 with `Python: Select Interpreter`.
 
-The root (`immunisation-fhir-api`) should point to `/mnt/d/Source/immunisation-fhir-api/.venv/bin/python`.
+The root (`immunisation-fhir-api`) should point to the root `.venv`, e.g. `/mnt/d/Source/immunisation-fhir-api/.venv/bin/python`.
+
+Meanwhile, `backend` should be pointing at (e.g.) `/mnt/d/Source/immunisation-fhir-api/backend/.venv/bin/python`
+
+#### Running Unit Tests
+
+Note that unit tests can be run from the command line without VSCode configuration.
+
+In order that VSCode can resolve modules in unit tests, it needs the PYTHONPATH. This should be setup in `backend/.vscode/launch.json` (see above).
 
-`backend` should be pointing at `/mnt/d/Source/immunisation-fhir-api/backend/.venv/bin/python`
+**NOTE:** In order to run unit test suites, you may need to manually switch to the correct virtual environment each time you wish to
+run a different set of tests. To do this: 
+- Show and Run Commands (Ctrl-Shift-P on Windows)
+    - Python: Create Environment
+    - Venv
+    - Select the `.venv` named for the test suite you wish to run, e.g. `backend`
+    - Use Existing
+- VSCode should now display a toast saying that the following environment is selected: 
+    - (e.g.) `/mnt/d/Source/immunisation-fhir-api/backend/.venv/bin/python`
 
 ### IntelliJ
 

ack_backend/src/constants.py

@@ -2,7 +2,7 @@
 
 import os
 
-SOURCE_BUCKET_NAME = f"immunisation-batch-{os.getenv('ENVIRONMENT')}-data-sources"
+SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
 ACK_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_FILENAME_GSI = "filename_index"

ack_backend/tests/utils/mock_environment_variables.py

@@ -21,5 +21,5 @@ class Firehose:
     "ACK_BUCKET_NAME": BucketNames.DESTINATION,
     "FIREHOSE_STREAM_NAME": Firehose.STREAM_NAME,
     "AUDIT_TABLE_NAME": "immunisation-batch-internal-dev-audit-table",
-    "ENVIRONMENT": "internal-dev",
+    "SOURCE_BUCKET_NAME": BucketNames.SOURCE
 }

backend/.vscode/launch.json.default

@@ -0,0 +1,33 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "File",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "unittest",
+      "args": [
+        "${fileBasenameNoExtension}" // Run tests in the current file
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}:${workspaceFolder}/src:${workspaceFolder}/tests" // Add root, src, and tests
+      }
+    },
+    {
+      "name": "Here",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "unittest",
+      "args": [
+        "-k",
+        "${selectedText}" // Run the test method or class under the cursor
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}:${workspaceFolder}/src:${workspaceFolder}/tests" // Add root, src, and tests
+      }
+    }
+  ]
+}

backend/src/models/fhir_immunization_pre_validators.py

@@ -626,16 +626,16 @@ def pre_validate_protocol_applied(self, values: dict) -> dict:
             PreValidation.for_list(field_value, "protocolApplied", defined_length=1)
         except KeyError:
             pass
-
+    DOSE_NUMBER_MAX_VALUE = 9
     def pre_validate_dose_number_positive_int(self, values: dict) -> dict:
         """
         Pre-validate that, if protocolApplied[0].doseNumberPositiveInt (legacy CSV field : dose_sequence)
-        exists, then it is an integer from 1 to 9
+        exists, then it is an integer from 1 to 9 (DOSE_NUMBER_MAX_VALUE)
         """
         field_location = "protocolApplied[0].doseNumberPositiveInt"
         try:
             field_value = values["protocolApplied"][0]["doseNumberPositiveInt"]
-            PreValidation.for_positive_integer(field_value, field_location)
+            PreValidation.for_positive_integer(field_value, field_location, self.DOSE_NUMBER_MAX_VALUE)
         except (KeyError, IndexError):
             pass
 

backend/tests/test_immunization_pre_validator.py

@@ -25,6 +25,7 @@
 from utils.pre_validation_test_utils import ValidatorModelTests
 from utils.values_for_tests import ValidValues, InvalidValues
 from models.constants import Constants
+from models.fhir_immunization_pre_validators import PreValidators
 
 class TestImmunizationModelPreValidationRules(unittest.TestCase):
     """Test immunization pre validation rules on the FHIR model using the covid sample data"""
@@ -819,6 +820,33 @@ def test_pre_validate_protocol_applied(self):
         )
 
     def test_pre_validate_protocol_applied_dose_number_positive_int(self):
+        """
+        Test pre_validate_protocol_applied_dose_number_positive_int accepts valid values and
+        rejects invalid values
+        """
+        for value in range(1, PreValidators.DOSE_NUMBER_MAX_VALUE + 1):
+            data = {
+                "protocolApplied": [
+                    {"doseNumberPositiveInt": value}
+                ]
+            }
+            validator = PreValidators(data)
+            # Should not raise
+            validator.pre_validate_dose_number_positive_int(data)
+
+    def test_out_of_range_dose_number(self):
+        # Invalid: doseNumberPositiveInt < 1 or > 9
+        for value in [0, PreValidators.DOSE_NUMBER_MAX_VALUE + 1, -1]:
+            data = {
+                "protocolApplied": [
+                    {"doseNumberPositiveInt": value}
+                ]
+            }
+            validator = PreValidators(data)
+            with self.assertRaises(ValueError):
+                validator.pre_validate_dose_number_positive_int(data)
+
+    def test_test_positive_integer_value(self):
         """
         Test pre_validate_protocol_applied_dose_number_positive_int accepts valid values and
         rejects invalid values

config/dev/permissions_config.json

@@ -1,12 +1,28 @@
 [
   {
     "supplier": "DPSFULL",
-    "permissions": ["COVID19.CRUDS", "FLU.CRUDS", "MMR.CRUDS", "RSV.CRUDS"],
+    "permissions": [
+      "COVID19.CRUDS",
+      "FLU.CRUDS",
+      "MMR.CRUDS",
+      "RSV.CRUDS",
+      "HPV.CRUDS",
+      "MENACWY.CRUDS",
+      "3IN1.CRUDS"
+    ],
     "ods_codes": ["DPSFULL"]
   },
   {
     "supplier": "DPSREDUCED",
-    "permissions": ["COVID19.CRUD", "FLU.CRUD", "MMR.CRUD", "RSV.CRUD"],
+    "permissions": [
+      "COVID19.CRUDS",
+      "FLU.CRUDS",
+      "MMR.CRUDS",
+      "RSV.CRUDS",
+      "HPV.CRUDS",
+      "MENACWY.CRUDS",
+      "3IN1.CRUDS"
+    ],
     "ods_codes": ["DPSREDUCED"]
   },
   {
@@ -33,11 +49,7 @@
   {
     "supplier": "RAVS",
     "permissions": ["RSV.CRUDS", "MMR.CRUDS"],
-    "ods_codes": ["X26"]
-  },
-  {
-    "supplier": "AGEM-RAVS-Integration",
-    "permissions": ["RSV.CRUDS", "MMR.CRUDS"]
+    "ods_codes": ["X26", "X8E5B"]
   },
   {
     "supplier": "EMIS",
@@ -125,7 +137,6 @@
       "RSV.CRUDS",
       "HPV.CRUDS",
       "MENACWY.CRUDS",
-      "PERTUSSIS.CRUDS",
       "3IN1.CRUDS"
     ]
   },
@@ -138,16 +149,7 @@
       "RSV.CRUDS",
       "HPV.CRUDS",
       "MENACWY.CRUDS",
-      "PERTUSSIS.CRUDS",
       "3IN1.CRUDS"
     ]
-  },
-  {
-    "supplier": "EVA",
-    "ods_codes": ["8HA94"]
-  },
-  {
-    "supplier": "SONAR",
-    "ods_codes": ["8HK48"]
   }
 ]

config/preprod/permissions_config.json

@@ -1,12 +1,28 @@
 [
   {
     "supplier": "DPSFULL",
-    "permissions": ["COVID19.CRUDS", "FLU.CRUDS", "MMR.CRUDS", "RSV.CRUDS"],
+    "permissions": [
+      "COVID19.CRUDS",
+      "FLU.CRUDS",
+      "MMR.CRUDS",
+      "RSV.CRUDS",
+      "HPV.CRUDS",
+      "MENACWY.CRUDS",
+      "3IN1.CRUDS"
+    ],
     "ods_codes": ["DPSFULL"]
   },
   {
     "supplier": "DPSREDUCED",
-    "permissions": ["COVID19.CRUDS", "FLU.CRUDS", "MMR.CRUDS", "RSV.CRUDS"],
+    "permissions": [
+      "COVID19.CRUDS",
+      "FLU.CRUDS",
+      "MMR.CRUDS",
+      "RSV.CRUDS",
+      "HPV.CRUDS",
+      "MENACWY.CRUDS",
+      "3IN1.CRUDS"
+    ],
     "ods_codes": ["DPSREDUCED"]
   },
   {
@@ -23,11 +39,7 @@
   {
     "supplier": "RAVS",
     "permissions": ["RSV.CRUDS", "MMR.CRUDS"],
-    "ods_codes": ["X26"]
-  },
-  {
-    "supplier": "AGEM-RAVS-Integration",
-    "permissions": ["RSV.CRUDS", "MMR.CRUDS"]
+    "ods_codes": ["X26", "X8E5B"]
   },
   {
     "supplier": "EMIS",
@@ -71,7 +83,6 @@
       "RSV.CRUDS",
       "HPV.CRUDS",
       "MENACWY.CRUDS",
-      "PERTUSSIS.CRUDS",
       "3IN1.CRUDS"
     ]
   },
@@ -84,7 +95,6 @@
       "RSV.CRUDS",
       "HPV.CRUDS",
       "MENACWY.CRUDS",
-      "PERTUSSIS.CRUDS",
       "3IN1.CRUDS"
     ]
   },

config/prod/permissions_config.json

@@ -16,7 +16,7 @@
   {
     "supplier": "RAVS",
     "permissions": ["RSV.RS"],
-    "ods_codes": ["X26"]
+    "ods_codes": ["X26", "X8E5B"]
   },
   {
     "supplier": "PINNACLE",

filenameprocessor/src/constants.py

@@ -1,6 +1,8 @@
 """Constants for the filenameprocessor lambda"""
 
 import os
+from enum import StrEnum
+
 from errors import (
     VaccineTypePermissionsError,
     InvalidFileKeyError,
@@ -15,6 +17,8 @@
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_QUEUE_NAME_GSI = "queue_name_index"
 AUDIT_TABLE_FILENAME_GSI = "filename_index"
+DATA_SOURCES_BUCKET_SUFFIX = "data-sources"
+VALID_VERSIONS = ["V5"]
 
 SUPPLIER_PERMISSIONS_HASH_KEY = "supplier_permissions"
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
@@ -31,7 +35,7 @@
 }
 
 
-class FileStatus:
+class FileStatus(StrEnum):
     """File status constants"""
 
     QUEUED = "Queued"
@@ -40,16 +44,11 @@ class FileStatus:
     DUPLICATE = "Not processed - duplicate"
 
 
-class AuditTableKeys:
+class AuditTableKeys(StrEnum):
     """Audit table keys"""
 
     FILENAME = "filename"
     MESSAGE_ID = "message_id"
     QUEUE_NAME = "queue_name"
     STATUS = "status"
     TIMESTAMP = "timestamp"
-
-
-class Constants:
-    """Constants for the filenameprocessor lambda"""
-    VALID_VERSIONS = ["V5"]