Skip to content

Commit 649f796

Browse files
bo-lubo-lu
authored
Merge pull request #95 from Canadian-Geospatial-Platform/dev
App-geo-ca-v2 harvester
2 parents 8791e36 + 61d35cc commit 649f796

File tree

6 files changed

+117
-7
lines changed

6 files changed

+117
-7
lines changed

‎docs/cloudformation/geocore-eo-rcm-ard-harvester.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ Resources:
118118
GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}'
119119
GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json'
120120
ITEM_LINK_BUCKET_NAME: !Sub 'eo-sg-datacube-item-links-${Environment}'
121-
PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataS3Bucket
121+
PROCESSED_DATA_BUCKET_NAME: !Sub 'eo-sg-processed-data-rcm-ard-${Environment}'
122122
API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac'
123123
ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API'
124124
SOURCE: 'eodms'
@@ -175,7 +175,7 @@ Resources:
175175
Variables:
176176
GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}'
177177
GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json'
178-
PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataS3Bucket
178+
PROCESSED_DATA_BUCKET_NAME: !Sub 'eo-sg-processed-data-rcm-ard-${Environment}'
179179
API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac'
180180
ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API'
181181
SOURCE: 'eodms'
@@ -405,7 +405,7 @@ Resources:
405405
Timeout: 900
406406
Environment:
407407
Variables:
408-
GEOJSON_BUCKET_NAME: !Ref ProcessedDataS3Bucket
408+
GEOJSON_BUCKET_NAME: !Sub 'eo-sg-processed-data-rcm-ard-${Environment}'
409409
PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}'
410410
DYNAMODB_TABLE: 'analytics_popularity'
411411
PARQUET_FILENAME: 'rcm-ard.parquet'

‎docs/cloudformation/geocore-hnap-harvest-app-geo-ca-v2.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: 'AWS::Serverless-2016-10-31'
3+
Description: Deploys HNAP JSON harvester solution + GeoNetwork to geocore translation
4+
5+
Parameters:
6+
Environment:
7+
Type: AWS::SSM::Parameter::Value<String>
8+
Default: /webpresence/environment
9+
Description: SSM parameter name for environment
10+
DeploymentBucket:
11+
Type: AWS::SSM::Parameter::Value<String>
12+
Default: /webpresence/deployment-bucket
13+
Description: S3 bucket where all deployment files are stored
14+
SvelteBucketString:
15+
Type: String
16+
Description: Unique string generated by the svelte app.geo.ca v2 deployment
17+
18+
Conditions:
19+
IsProd: !Equals [prod, !Ref Environment]
20+
IsStage: !Equals [stage, !Ref Environment]
21+
IsDev: !Equals [dev, !Ref Environment]
22+
23+
Resources:
24+
HnapJsonHarvesterAppGeoCav2:
25+
Type: AWS::Serverless::Function
26+
Properties:
27+
Runtime: python3.9
28+
Role: !GetAtt LambdaExecutionRole.Arn
29+
CodeUri:
30+
Bucket: !Ref DeploymentBucket
31+
Key:
32+
Fn::If:
33+
- IsProd
34+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
35+
- Fn::If:
36+
- IsStage
37+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
38+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
39+
MemorySize: 3009
40+
Handler: app.lambda_handler
41+
Timeout: 900
42+
Environment:
43+
Variables:
44+
BUCKET_NAME: !Sub 'app-geo-ca-pv-datalake-${SvelteBucketString}/hnap'
45+
GEOJSON_BUCKET_NAME: !Sub 'app-geo-ca-pv-datalake-${SvelteBucketString}/geojson'
46+
BASE_URL: 'https://maps.canada.ca'
47+
GN_JSON_RECORD_URL_START: 'https://maps.canada.ca/geonetwork/srv/api/0.1/records/'
48+
RUN_INTERVAL_MINUTES: 11
49+
Layers:
50+
- arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8
51+
52+
HnapJsonHarvesterRule:
53+
Type: AWS::Events::Rule
54+
Properties:
55+
Name: !Sub 'harvester-10mins-app-geo-ca-v2-${Environment}'
56+
Description: Harvester on a 10 minute interval
57+
State: DISABLED
58+
ScheduleExpression: 'rate(10 minutes)'
59+
Targets:
60+
-
61+
Arn:
62+
Fn::GetAtt:
63+
- HnapJsonHarvesterAppGeoCav2
64+
- Arn
65+
Id: !Ref HnapJsonHarvesterAppGeoCav2
66+
67+
PermissionForEventsToInvokeHarvesterLambda:
68+
Type: AWS::Lambda::Permission
69+
Properties:
70+
FunctionName: !Ref HnapJsonHarvesterAppGeoCav2
71+
Action: 'lambda:InvokeFunction'
72+
Principal: 'events.amazonaws.com'
73+
SourceArn: !GetAtt
74+
- HnapJsonHarvesterRule
75+
- Arn
76+
77+
LambdaExecutionRole:
78+
Type: AWS::IAM::Role
79+
Properties:
80+
AssumeRolePolicyDocument:
81+
Version: '2012-10-17'
82+
Statement:
83+
- Effect: Allow
84+
Principal:
85+
Service:
86+
- lambda.amazonaws.com
87+
Action:
88+
- sts:AssumeRole
89+
ManagedPolicyArns:
90+
- arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
91+
Policies:
92+
- PolicyName: 'policy'
93+
PolicyDocument:
94+
Version: '2012-10-17'
95+
Statement:
96+
- Effect: 'Allow'
97+
Action:
98+
- 's3:*'
99+
Resource:
100+
- !Sub arn:aws:s3:::app-geo-ca-pv-datalake-${SvelteBucketString}
101+
- !Sub arn:aws:s3:::app-geo-ca-pv-datalake-${SvelteBucketString}/*
102+
103+
LogGroup:
104+
Type: Custom::LogGroup
105+
Properties:
106+
ServiceToken: !ImportValue LogGroupHelperLambdaArn
107+
LogGroupName: !Sub '/${Environment}/webpresence/hnap_json_harvest_app-geo-ca-v2'
108+
RetentionInDays: 3653

‎docs/cloudformation/geocore-hnap-harvest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,11 +84,11 @@ Resources:
8484
Key:
8585
Fn::If:
8686
- IsProd
87-
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip
87+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
8888
- Fn::If:
8989
- IsStage
90-
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip
91-
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip
90+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
91+
- cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip
9292
MemorySize: 3009
9393
Handler: app.lambda_handler
9494
Timeout: 900

‎docs/cloudformation/geocore-stack.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4139,7 +4139,9 @@ Resources:
41394139
,\r\n \"type\": \"$input.params('type')\",\r\n \"org\": \"$input.params('org')\"\
41404140
,\r\n \"min\": \"$input.params('min')\",\r\n \"max\": \"$input.params('max')\"\
41414141
,\r\n \"foundational\": \"$input.params('foundational')\"
4142-
,\r\n \"sort\": \"$input.params('sort')\"
4142+
,\r\n \"sort\": \"$input.params('sort')\"
4143+
,\r\n \"begin\": \"$input.params('begin')\"
4144+
,\r\n \"end\": \"$input.params('end')\"
41434145
,\r\n \"source_system\": \"$input.params('sourcesystemname')\"
41444146
,\r\n \"eo_collection\": \"$input.params('eocollection')\"
41454147
,\r\n \"polarization\": \"$input.params('polarization')\"

‎docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip‎

4.08 KB
Binary file not shown.

‎docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20241212-1100.zip‎

4.39 KB
Binary file not shown.

0 commit comments

Comments
 (0)