clean tests

yaml env
pythonpath

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -1,110 +1,110 @@
 name: SonarCloud
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled, opened, synchronize, reopened, unlabeled]
+    push:
+        branches:
+            - master
+    pull_request:
+        types: [labeled, opened, synchronize, reopened, unlabeled]
 
 jobs:
-  sonarcloud:
-    name: SonarCloud
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependency') }}
-    runs-on: ubuntu-latest
+    sonarcloud:
+        name: SonarCloud
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependency') }}
+        runs-on: ubuntu-latest
 
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  fetch-depth: 0
 
-      - name: Set up AWS credentials
-        env:
-          AWS_ACCESS_KEY_ID: "FOOBARKEY"
-          AWS_SECRET_ACCESS_KEY: "FOOBARSECRET"
-        run: |
-          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+            - name: Set up AWS credentials
+              env:
+                  AWS_ACCESS_KEY_ID: "FOOBARKEY"
+                  AWS_SECRET_ACCESS_KEY: "FOOBARSECRET"
+              run: |
+                  aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+                  aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Initialize Unit Test Failure Tracker
-        run: echo "false" > test_failed.txt
+            - name: Initialize Unit Test Failure Tracker
+              run: echo "false" > test_failed.txt
 
-      # - name: Run unittest with filenameprocessor-coverage
-      #   id: filenameprocessor
-      #   continue-on-error: true
-      #   run: |
-      #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson pandas freezegun responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20 fakeredis
-      #     poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
+            # - name: Run unittest with filenameprocessor-coverage
+            #   id: filenameprocessor
+            #   continue-on-error: true
+            #   run: |
+            #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson pandas freezegun responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20 fakeredis
+            #     poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
 
-      # - name: Run unittest with recordprocessor-coverage
-      #   id: recordprocessor
-      #   continue-on-error: true
-      #   run: |
-      #     poetry run coverage run --source=recordprocessor -m unittest discover -s recordprocessor || echo "recordprocessor tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-coverage-recordprocessor-coverage.xml
+            # - name: Run unittest with recordprocessor-coverage
+            #   id: recordprocessor
+            #   continue-on-error: true
+            #   run: |
+            #     poetry run coverage run --source=recordprocessor -m unittest discover -s recordprocessor || echo "recordprocessor tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-coverage-recordprocessor-coverage.xml
 
-      # - name: Run unittest with recordforwarder-coverage
-      #   id: recordforwarder
-      #   continue-on-error: true
-      #   run: |
-      #     PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
+            # - name: Run unittest with recordforwarder-coverage
+            #   id: recordforwarder
+            #   continue-on-error: true
+            #   run: |
+            #     PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
 
-      # - name: Run unittest with coverage-ack-lambda
-      #   id: acklambda
-      #   continue-on-error: true
-      #   run: |
-      #     pip install poetry==1.8.4 coverage moto==4.2.11 freezegun
-      #     poetry run coverage run --source=ack_backend -m unittest discover -s ack_backend || echo "ack-lambda tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-coverage-ack-lambda.xml
+            # - name: Run unittest with coverage-ack-lambda
+            #   id: acklambda
+            #   continue-on-error: true
+            #   run: |
+            #     pip install poetry==1.8.4 coverage moto==4.2.11 freezegun
+            #     poetry run coverage run --source=ack_backend -m unittest discover -s ack_backend || echo "ack-lambda tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-coverage-ack-lambda.xml
 
-      - name: Run unittest with coverage-delta
-        id: delta
-        env:
-           PYTHONPATH: delta_backend/src:delta_backend/tests:./src:./tests
-        continue-on-error: true
-        run: |
-          echo ">>>>  SAW: use delta.firehose_log:"
-          pip install poetry==1.8.4 moto==4.2.11 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
-          export PYTHONPATH=src:tests
-          PYTHONPATH=delta_backend/src:delta_backend/tests:./src:./tests poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend/tests || echo "delta tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-delta.xml
+            - name: Run unittest with coverage-delta
+              id: delta
+              env:
+                  PYTHONPATH: delta_backend/src:delta_backend/tests
+              continue-on-error: true
+              run: |
+                  echo ">>>>  SAW: use delta.firehose_log:"
+                  pip install poetry==1.8.4 moto==4.2.11 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
+                  poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend/tests || echo "delta tests failed" >> failed_tests.txt
+                  poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend/tests || echo "delta tests failed" >> failed_tests.txt
+                  poetry run coverage xml -o sonarcloud-coverage-delta.xml
 
-      # - name: Run unittest with coverage-fhir-api
-      #   id: fhirapi
-      #   continue-on-error: true
-      #   run: |
-      #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
-      #     poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-coverage.xml
+            # - name: Run unittest with coverage-fhir-api
+            #   id: fhirapi
+            #   continue-on-error: true
+            #   run: |
+            #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+            #     poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-coverage.xml
 
-      # - name: Run unittest with coverage-mesh-processor
-      #   id: meshprocessor
-      #   continue-on-error: true
-      #   run: |
-      #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
-      #     poetry run coverage run --source=mesh_processor -m unittest discover -s mesh_processor || echo "mesh_processor tests failed" >> failed_tests.txt
-      #     poetry run coverage xml -o sonarcloud-mesh_processor-coverage.xml
+            # - name: Run unittest with coverage-mesh-processor
+            #   id: meshprocessor
+            #   continue-on-error: true
+            #   run: |
+            #     pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+            #     poetry run coverage run --source=mesh_processor -m unittest discover -s mesh_processor || echo "mesh_processor tests failed" >> failed_tests.txt
+            #     poetry run coverage xml -o sonarcloud-mesh_processor-coverage.xml
 
-      - name: Run Test Failure Summary
-        id: check_failure
-        run: |
-          if [ -s failed_tests.txt ]; then
-            echo "The following tests failed:"
-            cat failed_tests.txt
+            - name: Run Test Failure Summary
+              id: check_failure
+              run: |
+                  if [ -s failed_tests.txt ]; then
+                    echo "The following tests failed:"
+                    cat failed_tests.txt
 
-            while IFS= read -r line; do
-              echo "##[error]Test Failures: $line"
-            done < failed_tests.txt
-            
-            exit 1
-          else
-            echo "All tests passed."
-          fi
+                    while IFS= read -r line; do
+                      echo "##[error]Test Failures: $line"
+                    done < failed_tests.txt
+                    
+                    exit 1
+                  else
+                    echo "All tests passed."
+                  fi
 
-      - name: SonarCloud Scan
-        uses: SonarSource/sonarqube-scan-action@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+            - name: SonarCloud Scan
+              uses: SonarSource/sonarqube-scan-action@master
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
+                  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

delta_backend/failed_tests.txt

@@ -4,7 +4,11 @@ version = "0.1.0"
 description = ""
 authors = ["Your Name <you@example.com>"]
 readme = "README.md"
-packages    = [{include = "src"}]
+packages = [
+    {include = "src"},
+    {include = "tests"},
+    {include = "tests/sample_data"}
+]
 
 [tool.poetry.dependencies]
 python = "~3.10"

delta_backend/pyproject.toml

@@ -14,18 +14,17 @@
     get_valid_address,
 )
 
-# Converter variables
-FHIRData = ""
-SchemaFile = {}
-imms = []
-Converted = {}
-ErrorRecords = []
-
-
 # Converter
 class Converter:
 
     def __init__(self, fhir_data):
+        #Converter variables
+        self.FHIRData = ""
+        self.SchemaFile = {}
+        self.imms = []
+        self.Converted = {}
+        self.ErrorRecords = []
+
         self.FHIRData = fhir_data  # Store JSON data directly
         self.SchemaFile = ConversionLayout.ConvertLayout
 
@@ -56,7 +55,7 @@ def _convertData(self, ConversionValidate, expression, dataParser, json_data):
         except Exception as e:
             message = "Data get value Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
             p = {"code": ExceptionMessages.PARSING_ERROR, "message": message}
-            ErrorRecords.append(p)
+            self.ErrorRecords.append(p)
             return p
 
         for conversionValue in conversionValues:
@@ -66,7 +65,7 @@ def _convertData(self, ConversionValidate, expression, dataParser, json_data):
             if "address" in FHIRFieldName or "performer" in FHIRFieldName or "name" in FHIRFieldName:
                 convertedData = self.extract_patient_details(json_data, FlatFieldName)
             if convertedData is not None:
-                Converted[FlatFieldName] = convertedData
+                self.Converted[FlatFieldName] = convertedData
 
     # run the conversion against the data
     def runConversion(self, json_data, summarise=False, report_unexpected_exception=True):
@@ -76,7 +75,7 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "FHIR Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.ErrorRecords.append(p)
                 return p
 
         try:
@@ -85,7 +84,7 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "Schema Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.ErrorRecords.append(p)
                 return p
 
         try:
@@ -94,7 +93,7 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "Expression Checker Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.ErrorRecords.append(p)
                 return p
 
         # get list of expressions
@@ -104,17 +103,17 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "Expression Getter Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.ErrorRecords.append(p)
                 return p
 
         for conversion in conversions:
             rows = self._convertData(ConversionValidate, conversion, dataParser, json_data)
 
-        imms.append(Converted)
-        return imms
+        self.imms.append(self.Converted)
+        return self.imms
 
     def getErrorRecords(self):
-        return ErrorRecords
+        return self.ErrorRecords
 
     def extract_patient_details(self, json_data, FlatFieldName):
         if not hasattr(self, "_cached_values"):

delta_backend/src/Converter.py

@@ -2,3 +2,4 @@
 import sys
 
 sys.path.append(f"{os.path.dirname(os.path.abspath(__file__))}/../src")
+

delta_backend/tests/__init__.py

@@ -22,9 +22,9 @@
     "SOURCE": "test-source",
 }
 request_json_data = ValuesForTests.json_data
-with patch.dict("os.environ", MOCK_ENV_VARS):
+with patch.dict("os.environ", MOCK_ENV_VARS, clear=True):
     from delta import handler, Converter
-    from Converter import imms, ErrorRecords
+    # from Converter import imms, ErrorRecords
 
 
 @patch.dict("os.environ", MOCK_ENV_VARS, clear=True)
@@ -111,7 +111,7 @@ def assert_dynamodb_record(self, operation_flag, items, expected_values, expecte
 
     def test_fhir_converter_json_direct_data(self):
         """it should convert fhir json data to flat json"""
-        imms.clear()
+        # imms.clear()
         json_data = json.dumps(ValuesForTests.json_data)
 
         start = time.time()
@@ -135,12 +135,13 @@ def test_fhir_converter_json_direct_data(self):
         end = time.time()
         print(end - start)
 
+        # These tests fail
     def test_fhir_converter_json_error_scenario(self):
         """it should convert fhir json data to flat json - error scenarios"""
         error_test_cases = [ErrorValuesForTests.missing_json, ErrorValuesForTests.json_dob_error]
 
         for test_case in error_test_cases:
-            imms.clear()
+            # imms.clear()
             json_data = json.dumps(test_case)
 
             start = time.time()
@@ -177,7 +178,7 @@ def test_handler_imms_convert_to_flat_json(self):
 
         for test_case in expected_action_flags:
             with self.subTest(test_case["Operation"]):
-                imms.clear()
+                # imms.clear()
 
                 event = self.get_event(operation=test_case["Operation"])
 
@@ -257,6 +258,9 @@ def test_conversion_checker_exception(self, mock_conversion_checker):
         )
         self.assertEqual(converter.getErrorRecords()[0]["code"], 0)
 
+
+
+
     @patch("Converter.SchemaParser.getConversions")
     def test_get_conversions_exception(self, mock_get_conversions):
         # Mock getConversions to raise an exception
@@ -266,7 +270,7 @@ def test_get_conversions_exception(self, mock_get_conversions):
         response = converter.runConversion(ValuesForTests.json_data)
 
         # Check if the error message was added to ErrorRecords
-        self.assertEqual(len(converter.getErrorRecords()), 3)
+        self.assertEqual(len(converter.getErrorRecords()), 1)
         self.assertIn(
             "FHIR Parser Unexpected exception [JSONDecodeError]: Expecting value: line 1 column 1 (char 0)",
             converter.getErrorRecords()[0]["message"],
@@ -278,7 +282,7 @@ def test_get_conversions_exception(self, mock_get_conversions):
     def test_conversion_exceptions(self, mock_get_key_value, mock_get_conversions):
         mock_get_conversions.side_effect = Exception("Error while getting conversions")
         mock_get_key_value.side_effect = Exception("Key value retrieval failed")
-        ErrorRecords.clear()
+        # ErrorRecords.clear()
         converter = Converter(fhir_data="some_data")
 
         schema = {