handler decorator tests

Author: nhsdevws

redis_sync/src/record_processor.py

@@ -16,7 +16,6 @@ def process_record(record: S3EventRecord) -> dict:
         file_key = record.get_object_key()
 
         base_log_data = {
-            "bucket_name": bucket_name,
             "file_key": file_key
         }
 

redis_sync/src/redis_sync.py

@@ -26,16 +26,20 @@ def handler(event, _):
                 return {"status": "success", "message": "No records found in event"}
             else:
                 error_count = 0
+                file_keys = []
                 for record in s3_event.get_s3_records():
                     record_result = process_record(record)
+                    file_keys.append(record_result["file_key"])
                     if record_result["status"] == "error":
                         error_count += 1
                 if error_count > 0:
                     logger.error("Processed %d records with %d errors", record_count, error_count)
-                    return {"status": "error", "message": f"Processed {record_count} records with {error_count} errors"}
+                    return {"status": "error", "message": f"Processed {record_count} records with {error_count} errors",
+                            "file_keys": file_keys}
                 else:
                     logger.info("Successfully processed all %d records", record_count)
-                    return {"status": "success", "message": f"Successfully processed {record_count} records"}
+                    return {"status": "success", "message": f"Successfully processed {record_count} records",
+                            "file_keys": file_keys}
         else:
             logger.info("No records found in event")
             return {"status": "success", "message": "No records found in event"}

redis_sync/tests/test_handler.py

@@ -46,9 +46,12 @@ def test_handler_success(self):
             mock_event = {'Records': [self.s3_vaccine]}
             self.mock_get_s3_records.return_value = [self.s3_vaccine]
             with patch("redis_sync.process_record") as mock_record_processor:
-                mock_record_processor.return_value = {'status': 'success', 'message': 'Processed successfully'}
+                mock_record_processor.return_value = {'status': 'success', 'message': 'Processed successfully',
+                                                      'file_key': 'test-key'}
                 result = redis_sync.handler(mock_event, None)
-                self.assertEqual(result, {'status': 'success', 'message': 'Successfully processed 1 records'})
+                self.assertEqual(result["status"], "success")
+                self.assertEqual(result["message"], "Successfully processed 1 records")
+                self.assertEqual(result["file_keys"], ['test-key'])
 
     def test_handler_failure(self):
         with patch("log_decorator.logging_decorator", lambda prefix=None: (lambda f: f)):
@@ -57,7 +60,7 @@ def test_handler_failure(self):
             mock_event = {'Records': [self.s3_vaccine]}
             with patch("redis_sync.process_record") as mock_record_processor:
                 self.mock_get_s3_records.return_value = [self.s3_vaccine]
-                mock_record_processor.side_effect = Exception("Processing error")
+                mock_record_processor.side_effect = Exception("Processing error 1")
 
                 result = redis_sync.handler(mock_event, None)
 
@@ -77,7 +80,7 @@ def test_handler_exception(self):
             mock_event = {'Records': [self.s3_vaccine]}
             self.mock_get_s3_records.return_value = [self.s3_vaccine]
             with patch("redis_sync.process_record") as mock_record_processor:
-                mock_record_processor.side_effect = Exception("Processing error")
+                mock_record_processor.side_effect = Exception("Processing error 2")
                 result = redis_sync.handler(mock_event, None)
                 self.assertEqual(result, {'status': 'error', 'message': 'Error processing S3 event'})
 
@@ -99,9 +102,15 @@ def test_handler_multi_record(self):
             # ]
             self.mock_get_s3_records.return_value = [self.s3_vaccine, self.s3_supplier]
             with patch("redis_sync.process_record") as mock_record_processor:
-                mock_record_processor.return_value = {'status': 'success', 'message': 'Processed successfully'}
+                mock_record_processor.side_effect = [{'status': 'success', 'message': 'Processed successfully',
+                                                      'file_key': 'test-key1'},
+                                                     {'status': 'success', 'message': 'Processed successfully',
+                                                     'file_key': 'test-key2'}]
                 result = redis_sync.handler(mock_event, None)
-                self.assertEqual(result, {'status': 'success', 'message': 'Successfully processed 2 records'})
+                self.assertEqual(result['status'], 'success')
+                self.assertEqual(result['message'], 'Successfully processed 2 records')
+                self.assertEqual(result['file_keys'][0], 'test-key1')
+                self.assertEqual(result['file_keys'][1], 'test-key2')
 
     def test_handler_read_event(self):
         with patch("log_decorator.logging_decorator", lambda prefix=None: (lambda f: f)):

redis_sync/tests/test_handler_with_decorator.py

@@ -8,6 +8,11 @@
 
 
 class TestHandlerDecorator(unittest.TestCase):
+    """ Unit tests for the handler decorator in redis_sync.py
+        these will check what is sent to firehose and the logging
+        Note: test_handler.py will check the actual business logic of the handler
+        the decorator is used to log the function execution and send logs to firehose
+    """
     s3_vaccine = {
         's3': {
             'bucket': {'name': 'test-bucket1'},
@@ -39,87 +44,91 @@ def setUp(self):
     def tearDown(self):
         patch.stopall()
 
-    def test_handler_decorator_success(self):  # , mock_firehose_client):
+    def test_handler_decorator_success(self):
         mock_event = {'Records': [self.s3_vaccine]}
         self.mock_get_s3_records.return_value = [self.s3_vaccine]
+        bucket_name = self.s3_vaccine['s3']['bucket']['name']
+        file_key = self.s3_vaccine['s3']['object']['key']
+        self.mock_record_processor.return_value = {'status': 'success',
+                                                   'message': 'Successfully processed 1 records',
+                                                   'bucket_name': bucket_name,
+                                                   'file_key': file_key}
 
-        result = handler(mock_event, None)
+        handler(mock_event, None)
 
-        self.assertEqual(result, {'status': 'success', 'message': 'Successfully processed 1 records'})
-
-        self.mock_firehose_client.put_record.assert_called()
-
-        # Get the actual call arguments
+        # Get put_record arguments
         args, kwargs = self.mock_firehose_client.put_record.call_args
         record = kwargs.get("Record") or args[1]
         data_bytes = record["Data"]
         log_json = data_bytes.decode("utf-8")
         log_dict = json.loads(log_json)
 
-        # Now check for the expected content
+        # check expected content
         event = log_dict["event"]
         self.assertIn("function_name", event)
         self.assertEqual(event["function_name"], "redis_sync_handler")
         self.assertEqual(event["status"], "success")
         self.assertEqual(event["message"], "Successfully processed 1 records")
+        self.assertEqual(event["file_keys"], [file_key])
 
-    def test_handler_failure1(self):
+    def test_handler_decorator_failure(self):
         mock_event = {'Records': [self.s3_vaccine]}
 
         self.mock_get_s3_records.return_value = [self.s3_vaccine]
-        self.mock_record_processor.side_effect = Exception("Processing error")
-
-        result = handler(mock_event, None)
-
-        self.assertEqual(result, {'status': 'error', 'message': 'Error processing S3 event'})
-        self.mock_firehose_client.put_record.assert_called()
-
-        # Get the actual call arguments
-        args, kwargs = self.mock_firehose_client.put_record.call_args
-        record = kwargs.get("Record") or args[1]
-        data_bytes = record["Data"]
-        log_json = data_bytes.decode("utf-8")
-        log_dict = json.loads(log_json)
-        # Now check for the expected content
-        event = log_dict["event"]
-        self.assertIn("function_name", event)
-        self.assertEqual(event["function_name"], "redis_sync_handler")
-        self.assertEqual(event["status"], "error")
-        self.assertEqual(event["message"], "Error processing S3 event")
+        bucket_name = self.s3_vaccine['s3']['bucket']['name']
+        file_key = self.s3_vaccine['s3']['object']['key']
+        with patch("redis_sync.process_record") as mock_record_processor:
+            mock_record_processor.return_value = {'status': 'error',
+                                                  'message': 'my-error',
+                                                  'bucket_name': bucket_name,
+                                                  'file_key': file_key}
+
+            handler(mock_event, None)
+
+            # Get put_record arguments
+            args, kwargs = self.mock_firehose_client.put_record.call_args
+            record = kwargs.get("Record") or args[1]
+            data_bytes = record["Data"]
+            log_json = data_bytes.decode("utf-8")
+            log_dict = json.loads(log_json)
+            # check expected content
+            event = log_dict["event"]
+            self.assertIn("function_name", event)
+            self.assertEqual(event["function_name"], "redis_sync_handler")
+            self.assertEqual(event["status"], "error")
+            self.assertEqual(event["message"], 'Processed 1 records with 1 errors')
+            self.assertEqual(event["file_keys"], [file_key])
 
     def test_handler_decorator_no_records1(self):
         mock_event = {'Records': []}
 
         self.mock_get_s3_records.return_value = []
         self.mock_record_processor.return_value = {'status': 'success', 'message': 'No records found in event'}
 
-        result = handler(mock_event, None)
+        handler(mock_event, None)
 
-        self.assertEqual(result, {'status': 'success', 'message': 'No records found in event'})
-        self.mock_firehose_client.put_record.assert_called()
-
-        # Get the actual call arguments
+        # Get put_record arguments
         args, kwargs = self.mock_firehose_client.put_record.call_args
         record = kwargs.get("Record") or args[1]
         data_bytes = record["Data"]
         log_json = data_bytes.decode("utf-8")
         log_dict = json.loads(log_json)
-        # Now check for the expected content
+        # check expected content
         event = log_dict["event"]
         self.assertIn("function_name", event)
         self.assertEqual(event["function_name"], "redis_sync_handler")
         self.assertEqual(event["status"], "success")
+        # filename is empty since no records were processed
+        self.assertEqual(event["message"], "No records found in event")
+        self.assertNotIn("file_key", event)
 
-    def test_handler_exception1(self):
+    def test_handler_decorator_exception(self):
         mock_event = {'Records': [self.s3_vaccine]}
-        self.mock_get_s3_records.return_value = [self.s3_vaccine]
-        self.mock_record_processor.side_effect = Exception("Processing error")
+        self.mock_get_s3_records.side_effect = Exception("Test exception")
 
-        result = handler(mock_event, None)
+        handler(mock_event, None)
 
-        self.assertEqual(result, {'status': 'error', 'message': 'Error processing S3 event'})
-        self.mock_firehose_client.put_record.assert_called()
-        # Get the actual call arguments
+        # check put_record arguments
         args, kwargs = self.mock_firehose_client.put_record.call_args
         record = kwargs.get("Record") or args[1]
         data_bytes = record["Data"]
@@ -134,43 +143,71 @@ def test_handler_exception1(self):
     def test_handler_with_empty_event(self):
         self.mock_get_s3_records.return_value = []
 
-        result = handler({}, None)
-
-        self.assertEqual(result, {'status': 'success', 'message': 'No records found in event'})
-
-    def test_handler_multi_record(self):
-        mock_event = {'Records': [self.s3_vaccine, self.s3_supplier]}
-
-        self.mock_get_s3_records.return_value = [
-            S3EventRecord(self.s3_vaccine),
-            S3EventRecord(self.s3_supplier)
-        ]
-        self.mock_record_processor.return_value = {'status': 'success', 'message': 'Processed successfully'}
+        handler({}, None)
 
-        result = handler(mock_event, None)
-
-        self.assertEqual(result, {'status': 'success', 'message': 'Successfully processed 2 records'})
-        self.mock_firehose_client.put_record.assert_called()
-        # Get the actual call arguments
+        # get put_record arguments
         args, kwargs = self.mock_firehose_client.put_record.call_args
         record = kwargs.get("Record") or args[1]
         data_bytes = record["Data"]
         log_json = data_bytes.decode("utf-8")
         log_dict = json.loads(log_json)
-        # Now check for the expected content
+        # check expected content
         event = log_dict["event"]
         self.assertIn("function_name", event)
         self.assertEqual(event["function_name"], "redis_sync_handler")
         self.assertEqual(event["status"], "success")
+        self.assertEqual(event["message"], "No records found in event")
+
+    def test_handler_multi_record(self):
+        mock_event = {'Records': [self.s3_vaccine, self.s3_supplier]}
+
+        with patch.object(self.mock_get_s3_records,
+                          "return_value", 
+                          [S3EventRecord(self.s3_vaccine), S3EventRecord(self.s3_supplier)]):
+            # repatch mock_record_processor
+            with patch("redis_sync.process_record") as mock_record_processor:
+                # Mock the return value for each record
+                mock_record_processor.side_effect = [
+                    {'status': 'success', 'message': 'Processed successfully',
+                     'file_key': RedisCacheKey.DISEASE_MAPPING_FILE_KEY},
+                    {'status': 'success', 'message': 'Processed successfully',
+                     'file_key': RedisCacheKey.PERMISSIONS_CONFIG_FILE_KEY}
+                ]
+
+                handler(mock_event, None)
+
+                # Get put_record arguments
+                args, kwargs = self.mock_firehose_client.put_record.call_args
+                record = kwargs.get("Record") or args[1]
+                data_bytes = record["Data"]
+                log_json = data_bytes.decode("utf-8")
+                log_dict = json.loads(log_json)
+                # check expected content
+                event = log_dict["event"]
+                self.assertIn("function_name", event)
+                self.assertEqual(event["function_name"], "redis_sync_handler")
+                self.assertEqual(event["status"], "success")
 
     # test to check that event_read is called when "read" key is passed in the event
     def test_handler_read_event(self):
         mock_event = {'read': 'myhash'}
-        mock_read_event_response = {'field1': 'value1'}
+        return_key = 'field1'
+        return_value = 'value1'
+        mock_read_event_response = {return_key: return_value}
 
         with patch('redis_sync.read_event') as mock_read_event:
             mock_read_event.return_value = mock_read_event_response
-            result = handler(mock_event, None)
-
-            mock_read_event.assert_called_once()
-            self.assertEqual(result, mock_read_event_response)
+            handler(mock_event, None)
+
+            # get put_record arguments
+            args, kwargs = self.mock_firehose_client.put_record.call_args
+            record = kwargs.get("Record") or args[1]
+            data_bytes = record["Data"]
+            log_json = data_bytes.decode("utf-8")
+            log_dict = json.loads(log_json)
+            # check expected content
+            event = log_dict["event"]
+            self.assertIn("function_name", event)
+            self.assertEqual(event["function_name"], "redis_sync_handler")
+            actual_return_value = event.get(return_key)
+            self.assertEqual(actual_return_value, return_value)

redis_sync/tests/test_record_processor.py

@@ -40,7 +40,6 @@ def test_record_processor_success(self):
         result = process_record(S3EventRecord(self.s3_vaccine))
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(result["bucket_name"], "test-bucket1")
         self.assertEqual(result["file_key"], RedisCacheKey.DISEASE_MAPPING_FILE_KEY)
 
     def test_record_processor_failure(self):
@@ -50,7 +49,6 @@ def test_record_processor_failure(self):
         result = process_record(S3EventRecord(self.s3_vaccine))
 
         self.assertEqual(result["status"], "error")
-        self.assertEqual(result["bucket_name"], "test-bucket1")
         self.assertEqual(result["file_key"], RedisCacheKey.DISEASE_MAPPING_FILE_KEY)
 
     def test_record_processor_exception(self):