[PRMT-462] Schedule start and stop for bulk upload ingestion (#349)

* PRMT-462- added infrastructure to allow enabling/disabling of bulk upload ingestion

* PRMT-462- run pre commit

* PRMT-462- updated call to iam

* PRMT-462- run pre commit

* PRMT-462- added code for debugging

* PRMT-462- precommit

* PRMT-462- updated setup

* PRMT-462- fixed typo

* PRMT-462- cleaned code

* PRMT-462- renaming

* PRMT-463- removed TODO

* PRMT-462- merged and formated lines

Author: PedroSoaresNHS

infrastructure/iam.tf

@@ -176,6 +176,27 @@ resource "aws_iam_policy" "s3_document_data_policy_for_ods_report_lambda" {
   })
 }
 
+data "aws_iam_policy_document" "lambda_toggle_bulk_upload_document" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "lambda:UpdateEventSourceMapping",
+      "lambda:GetEventSourceMapping"
+    ]
+
+    resources = [
+      aws_lambda_event_source_mapping.bulk_upload_lambda.arn
+    ]
+  }
+}
+
+resource "aws_iam_policy" "lambda_toggle_bulk_upload_policy" {
+  name   = "${terraform.workspace}_lambda_toggle_bulk_upload_policy"
+  policy = data.aws_iam_policy_document.lambda_toggle_bulk_upload_document.json
+}
+
+
 data "aws_iam_policy_document" "assume_role_policy_for_ods_report_lambda" {
   statement {
     actions = ["sts:AssumeRole"]

infrastructure/lambda-bulk-upload.tf

@@ -55,11 +55,11 @@ module "bulk-upload-lambda" {
   ]
 }
 
-
 resource "aws_lambda_event_source_mapping" "bulk_upload_lambda" {
-  event_source_arn = module.sqs-lg-bulk-upload-metadata-queue.endpoint
+  event_source_arn = module.sqs-lg-bulk-upload-metadata-queue.sqs_arn
   function_name    = module.bulk-upload-lambda.lambda_arn
-
+  enabled          = false # Disabled by default; scheduler lambda will control
+  batch_size       = 10
   scaling_config {
     maximum_concurrency = local.bulk_upload_lambda_concurrent_limit
   }

infrastructure/lambda-toggle-bulk-upload.tf

@@ -0,0 +1,19 @@
+module "toggle-bulk-upload-lambda" {
+  source         = "./modules/lambda"
+  name           = "ToggleBulkUploadLambda"
+  handler        = "handlers.toggle_bulk_upload_handler.lambda_handler"
+  lambda_timeout = 60
+  memory_size    = 128
+
+  iam_role_policy_documents = [
+    data.aws_iam_policy_document.lambda_toggle_bulk_upload_document.json
+  ]
+
+  lambda_environment_variables = {
+    ESM_UUID = aws_lambda_event_source_mapping.bulk_upload_lambda.uuid
+  }
+
+  is_gateway_integration_needed = false
+  is_invoked_from_gateway       = false
+}
+

infrastructure/modules/lambda/variable.tf

@@ -22,6 +22,7 @@ variable "lambda_environment_variables" {
 variable "rest_api_id" {
   description = "ID of the associated API Gateway REST API."
   type        = string
+  default     = ""
 }
 
 variable "resource_id" {
@@ -51,6 +52,7 @@ variable "http_methods" {
 variable "api_execution_arn" {
   description = "Execution ARN of the API Gateway used for granting invoke permissions."
   type        = string
+  default     = ""
 }
 
 variable "iam_role_policy_documents" {

infrastructure/schedules.tf

@@ -184,3 +184,44 @@ resource "aws_lambda_permission" "nhs_oauth_token_generator_schedule" {
   principal     = "events.amazonaws.com"
   source_arn    = aws_cloudwatch_event_rule.nhs_oauth_token_generator_schedule.arn
 }
+resource "aws_cloudwatch_event_rule" "bulk_upload_enable_rule" {
+  name                = "${terraform.workspace}_bulk_upload_enable"
+  description         = "Enable Bulk Upload ingestion"
+  schedule_expression = "cron(0 19 ? * MON-FRI *)"
+}
+
+resource "aws_cloudwatch_event_rule" "bulk_upload_disable_rule" {
+  name                = "${terraform.workspace}_bulk_upload_disable"
+  description         = "Disable Bulk Upload ingestion"
+  schedule_expression = "cron(0 7 ? * TUE-SAT *)"
+}
+
+resource "aws_cloudwatch_event_target" "bulk_upload_enable_target" {
+  rule      = aws_cloudwatch_event_rule.bulk_upload_enable_rule.name
+  target_id = "toggle-bulk-upload-enable"
+  arn       = module.toggle-bulk-upload-lambda.lambda_arn
+  input     = jsonencode({ action = "enable" })
+}
+
+resource "aws_cloudwatch_event_target" "bulk_upload_disable_target" {
+  rule      = aws_cloudwatch_event_rule.bulk_upload_disable_rule.name
+  target_id = "toggle-bulk-upload-disable"
+  arn       = module.toggle-bulk-upload-lambda.lambda_arn
+  input     = jsonencode({ action = "disable" })
+}
+
+resource "aws_lambda_permission" "toggle_bulk_upload_enable_permission" {
+  statement_id  = "AllowExecutionFromCloudWatchEnable"
+  action        = "lambda:InvokeFunction"
+  function_name = module.toggle-bulk-upload-lambda.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.bulk_upload_enable_rule.arn
+}
+
+resource "aws_lambda_permission" "toggle_bulk_upload_disable_permission" {
+  statement_id  = "AllowExecutionFromCloudWatchDisable"
+  action        = "lambda:InvokeFunction"
+  function_name = module.toggle-bulk-upload-lambda.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.bulk_upload_disable_rule.arn
+}