Skip to content

Commit 69f53bd

Browse files
authored
Merge pull request #462 from NASA-IMPACT/iss1373_s3_direct_access
s3credentials enabled on UAT and PROD Testing with Smoke Test on UAT
2 parents 0b2de3d + d683191 commit 69f53bd

File tree

9 files changed

+235
-155
lines changed

9 files changed

+235
-155
lines changed

.github/workflows/terraspace.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ env:
4141
#
4242
jobs:
4343
terraspace:
44-
runs-on: ubuntu-20.04
44+
runs-on: ubuntu-22.04
4545
environment: ${{ inputs.TS_ENV }}
4646
env:
4747
TS_ENV: ${{ inputs.TS_ENV }}
@@ -102,6 +102,11 @@ jobs:
102102
run: |
103103
yarn --non-interactive install
104104
105+
- name: Set Github Secrets as Environment Variables
106+
run: |
107+
echo "TF_ENV_URS_EDL_TEA_CLIENT_ID=${{ secrets.URS_EDL_TEA_CLIENT_ID }}" >> $GITHUB_ENV
108+
echo "TF_ENV_URS_EDL_TEA_CLIENT_PASS=${{ secrets.URS_EDL_TEA_CLIENT_PASS }}" >> $GITHUB_ENV
109+
105110
- name: Run Post Deploy Mods setup
106111
run: |
107112
sh app/stacks/post-deploy-mods/resources/lambdas/pre-filter-DistributionApiEndpoints/zip_lambda.sh

app/stacks/cumulus/main.tf

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ data "aws_secretsmanager_secret_version" "launchpad_pfx" {
8080
secret_id = data.aws_secretsmanager_secret.launchpad_pfx.id
8181
}
8282

83+
# Helps enable s3credentials endpoint
84+
data "aws_lambda_function" "sts_credentials" { function_name = "gsfc-ngap-sh-s3-sts-get-keys" }
85+
data "aws_lambda_function" "sts_policy_helper" { function_name = "gsfc-ngap-sh-sts-policy-helper" }
86+
8387
#-------------------------------------------------------------------------------
8488
# RESOURCES
8589
#-------------------------------------------------------------------------------
@@ -418,6 +422,9 @@ module "cumulus_distribution" {
418422
deploy_to_ngap = true
419423
lambda_subnet_ids = module.vpc.subnets.ids
420424

425+
sts_credentials_lambda_function_arn = data.aws_lambda_function.sts_credentials.arn
426+
sts_policy_helper_lambda_function_arn = data.aws_lambda_function.sts_policy_helper.arn
427+
421428
oauth_client_id = data.aws_ssm_parameter.csdap_client_id.value
422429
oauth_client_password = data.aws_ssm_parameter.csdap_client_password.value
423430
oauth_host_url = var.csdap_host_url
@@ -428,6 +435,12 @@ module "cumulus_distribution" {
428435
system_bucket = var.system_bucket
429436
tags = local.tags
430437
vpc_id = module.vpc.vpc_id
438+
439+
# Additional available options not used
440+
#default_log_retention_days = var.default_log_retention_days
441+
#cloudwatch_log_retention_periods = var.cloudwatch_log_retention_periods
442+
#lambda_timeouts = var.lambda_timeouts
443+
#lambda_memory_sizes = var.lambda_memory_sizes
431444
}
432445

433446
module "discover_granules_workflow" {
@@ -503,10 +516,9 @@ module "cumulus" {
503516
rds_security_group = local.rds_security_group
504517
rds_user_access_secret_arn = local.rds_user_access_secret_arn
505518

506-
# These are no longer used, but are required by the module, so we simply set
507-
# them to empty strings.
508-
urs_client_id = ""
509-
urs_client_password = ""
519+
# URS connection info (added for s3 credentials endpoint)
520+
urs_client_id = var.urs_edl_tea_client_id
521+
urs_client_password = var.urs_edl_tea_client_pass
510522

511523
# <% if !in_sandbox? then %>
512524
metrics_es_host = var.metrics_es_host
@@ -563,9 +575,32 @@ module "cumulus" {
563575
# <% end %>
564576
additional_log_groups_to_elk = var.additional_log_groups_to_elk
565577

578+
# These lines must be commented when using Cumulus Distribution and NOT using TEA
579+
# Thin Egress App settings. Uncomment to use TEA.
580+
# must match stage_name variable for thin-egress-app module
581+
#tea_api_gateway_stage = var.api_gateway_stage
582+
#tea_external_api_endpoint = module.thin_egress_app.api_endpoint
583+
#tea_internal_api_endpoint = module.thin_egress_app.internal_api_endpoint
584+
#tea_rest_api_id = module.thin_egress_app.rest_api.id
585+
#tea_rest_api_root_resource_id = module.thin_egress_app.rest_api.root_resource_id
586+
587+
588+
# Cumulus Distribution settings. This line is Uncommented when using using the Cumulus Distribution API instead of TEA.
566589
tea_external_api_endpoint = var.cumulus_distribution_url
590+
567591
deploy_cumulus_distribution = true
568-
deploy_distribution_s3_credentials_endpoint = false
592+
deploy_distribution_s3_credentials_endpoint = true
593+
# The above var is the correct one for using the cumulus_distribution api path for s3credentials endpoint
594+
595+
# This is for when using the s3credentials with the TEA module -- The other time this appears in the code is for using the Cumulus_Distribution module
596+
#sts_credentials_lambda_function_arn = data.aws_lambda_function.sts_credentials.arn
597+
#sts_policy_helper_lambda_function_arn = data.aws_lambda_function.sts_policy_helper.arn
598+
#cmr_acl_based_credentials = true
599+
600+
# Ok so there are two places where 'deploy_s3_credentials_endpoint' exist.
601+
# # If wanting this feature enabled, and using the attribute 'deploy_s3_credentials_endpoint' in the ThinEgressApp Module,
602+
# # then the one here needs to be set ot false - so they don't conflict.
603+
#deploy_s3_credentials_endpoint = true
569604

570605
tags = local.tags
571606

app/stacks/cumulus/tfvars/prod.tfvars

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,5 @@ s3_replicator_target_bucket = "esdis-metrics-inbound-prod-csdap-distribution"
2626
# <% end %>
2727

2828
s3_replicator_target_prefix = "input/s3_access/csdapprod"
29+
30+
urs_url = "https://urs.earthdata.nasa.gov"

app/stacks/cumulus/tfvars/uat.tfvars

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,5 @@ metrics_es_host = "https://dmzza2al43z4f.cloudfront.net/"
1414

1515
s3_replicator_target_bucket = "cloud-metrics-inbound-uat-csdap-distribution"
1616
s3_replicator_target_prefix = "input/s3_access/csdapuat"
17+
18+
urs_url = "https://uat.urs.earthdata.nasa.gov"

app/stacks/cumulus/variables.tf

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,18 @@ variable "private_archive_api_gateway" {
165165
default = true
166166
}
167167

168+
variable "rsa_priv_key" {
169+
description = "The Private Key part of the JWT Token used by TEA Module as part of the TEA s3 credentials access"
170+
type = string
171+
default = ""
172+
}
173+
174+
variable "rsa_pub_key" {
175+
description = "The Public Key part of the JWT Token used by TEA Module as part of the TEA s3 credentials access"
176+
type = string
177+
default = ""
178+
}
179+
168180
variable "s3_replicator_target_bucket" {
169181
type = string
170182
default = null
@@ -200,3 +212,21 @@ variable "tags" {
200212
type = map(string)
201213
default = {}
202214
}
215+
216+
variable "urs_edl_tea_client_id" {
217+
description = "The Client ID of the Earthdata login (URS) application for TEA s3 credentials access"
218+
type = string
219+
default = ""
220+
}
221+
222+
variable "urs_edl_tea_client_pass" {
223+
description = "The Password of the Earthdata login (URS) application for TEA s3 credentials access"
224+
type = string
225+
default = ""
226+
}
227+
228+
variable "urs_url" {
229+
description = "The URL of the Earthdata login (URS) site"
230+
type = string
231+
default = "https://uat.urs.earthdata.nasa.gov"
232+
}
Lines changed: 1 addition & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -1,149 +1,3 @@
11
# main.tf for post-deploy-mods
22

3-
# Define the Lambda Function
4-
resource "aws_lambda_function" "pre_filter_DistApiEndpoints" {
5-
# function_name = "ks-test-pre-filter-DistributionApiEndpoints"
6-
function_name = "${var.prefix}-pre-filter-DistApiEndpoints"
7-
filename = "${path.module}/resources/lambdas/pre-filter-DistributionApiEndpoints/distro/lambda.zip"
8-
role = aws_iam_role.lambda_exec_pre_filter_DistApiEndpoints.arn
9-
handler = "lambda_function.lambda_handler" #"index.preFilterDistApiEndpoints"
10-
runtime = "python3.10" #local.lambda_runtime
11-
timeout = 300
12-
memory_size = 3008
13-
14-
source_code_hash = filebase64sha256("${path.module}/resources/lambdas/pre-filter-DistributionApiEndpoints/distro/lambda.zip")
15-
16-
lifecycle {
17-
create_before_destroy = true
18-
prevent_destroy = true
19-
}
20-
}
21-
22-
# Define the Execution Role and Policy
23-
resource "aws_iam_role" "lambda_exec_pre_filter_DistApiEndpoints" {
24-
#name = "lambda_exec_role_pre_filter_DistributionApiEndpoints"
25-
name = "${var.prefix}-lamb_exe_role_pf_DistApiEndpoints" # Must be 64 chars or less
26-
27-
assume_role_policy = jsonencode({
28-
Version = "2012-10-17"
29-
Statement = [
30-
{
31-
Action = "sts:AssumeRole"
32-
Effect = "Allow"
33-
Sid = ""
34-
Principal = {
35-
Service = "lambda.amazonaws.com"
36-
}
37-
},
38-
]
39-
})
40-
41-
# lifecycle {
42-
# prevent_destroy = true
43-
# }
44-
}
45-
46-
# Define an attachment to the aws_iam_role above
47-
resource "aws_iam_role_policy_attachment" "lambda_exec_policy" {
48-
role = aws_iam_role.lambda_exec_pre_filter_DistApiEndpoints.name
49-
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
50-
}
51-
52-
# Define another policy attachment to allow invoking of another lambda
53-
resource "aws_iam_policy" "lambda_invoke_policy" {
54-
#name = "lambda_invoke_policy"
55-
name = "${var.prefix}-lambda_pf_invoke_policy"
56-
description = "Policy to allow Lambda functions to invoke other Lambda functions"
57-
policy = jsonencode({
58-
Version = "2012-10-17"
59-
Statement = [
60-
{
61-
Effect = "Allow"
62-
Action = [
63-
"lambda:InvokeFunction"
64-
]
65-
Resource = "*"
66-
}
67-
]
68-
})
69-
}
70-
71-
# Attach the Policy, which allows a Lambda to be Invoked, to the Lambda Role
72-
resource "aws_iam_role_policy_attachment" "lambda_invoke_policy_attachment" {
73-
role = aws_iam_role.lambda_exec_pre_filter_DistApiEndpoints.name
74-
policy_arn = aws_iam_policy.lambda_invoke_policy.arn
75-
}
76-
77-
# Attach an AWS managed Policy for DynamoDB Read Only access
78-
resource "aws_iam_role_policy_attachment" "dynamodb_readonly_policy" {
79-
role = aws_iam_role.lambda_exec_pre_filter_DistApiEndpoints.name
80-
policy_arn = "arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess"
81-
}
82-
83-
# Fetch existing API Gateway
84-
data "aws_api_gateway_rest_api" "distribution_api" {
85-
name = "${var.prefix}-distribution" # Example "cumulus-uat-distribution"
86-
}
87-
88-
# Fetch the proxy resource (API Gateway "/{proxy+}" prop)
89-
data "aws_api_gateway_resource" "proxy_resource" {
90-
rest_api_id = data.aws_api_gateway_rest_api.distribution_api.id
91-
path = "/{proxy+}"
92-
}
93-
94-
# No need to update the root resource
95-
# The way this is all set up, we only want to override where the file is downloaded
96-
# That happens only when the proxy is invoked
97-
#
98-
# # If we need to update the root resource than, uncomment this code
99-
# Fetch the root resource (API Gateway "/" prop)
100-
#
101-
#data "aws_api_gateway_resource" "root_resource" {
102-
# rest_api_id = data.aws_api_gateway_rest_api.distribution_api.id
103-
# path = "/"
104-
#}
105-
#
106-
#
107-
## Update the integration for the root resource with GET method
108-
#resource "aws_api_gateway_integration" "root_lambda_integration" {
109-
# rest_api_id = data.aws_api_gateway_rest_api.distribution_api.id
110-
# resource_id = data.aws_api_gateway_resource.root_resource.id
111-
# http_method = "GET"
112-
# integration_http_method = "POST" #"GET"
113-
# type = "AWS_PROXY"
114-
# uri = aws_lambda_function.pre_filter_DistApiEndpoints.invoke_arn
115-
#}
116-
117-
# Update the integration for the root resource with GET method
118-
resource "aws_api_gateway_integration" "proxy_lambda_integration" {
119-
rest_api_id = data.aws_api_gateway_rest_api.distribution_api.id
120-
resource_id = data.aws_api_gateway_resource.proxy_resource.id
121-
http_method = "ANY"
122-
integration_http_method = "POST" #"GET"
123-
type = "AWS_PROXY"
124-
uri = aws_lambda_function.pre_filter_DistApiEndpoints.invoke_arn
125-
}
126-
127-
# Ensure the Lambda function as the necessary permissions to be invoked by API Gateway
128-
resource "aws_lambda_permission" "api_gateway" {
129-
statement_id = "AllowAPIGatewayInvoke"
130-
action = "lambda:InvokeFunction"
131-
function_name = aws_lambda_function.pre_filter_DistApiEndpoints.function_name
132-
principal = "apigateway.amazonaws.com"
133-
source_arn = "${data.aws_api_gateway_rest_api.distribution_api.execution_arn}/*/*"
134-
}
135-
136-
# Ensure the API Gateway redeploys after the update
137-
resource "aws_api_gateway_deployment" "api_deployment" {
138-
depends_on = [aws_api_gateway_integration.proxy_lambda_integration]
139-
140-
rest_api_id = data.aws_api_gateway_rest_api.distribution_api.id
141-
stage_name = "dev" # The existing cumulus deployment for this API Gateway Stage is always called dev (in all environments)
142-
143-
triggers = {
144-
redeployment = sha1(jsonencode({
145-
lambda_version = aws_lambda_function.pre_filter_DistApiEndpoints.source_code_hash
146-
integration_uri = aws_api_gateway_integration.proxy_lambda_integration.uri
147-
}))
148-
}
149-
}
3+
# Temp Disable this stack -- Copy contents of file: main__tf_Working.txt back in here to reinstate

0 commit comments

Comments
 (0)