Skip to content

Commit cdf1a66

Browse files
hrodmnjjfrench
hrodmn
and
jjfrench
authored
fix: refactor to keep database pgstac and ingestor pypgstac in sync (#137)
* fix: refactor to keep database pgstac and ingestor pypgstac in sync * add default value for optional jwks_url settings parameter * add ingestor with no authentication to integration test deployment Co-authored-by: Jamison French <[email protected]> --------- Co-authored-by: Jamison French <[email protected]>
1 parent 151fe6f commit cdf1a66

File tree

7 files changed

+85
-40
lines changed

7 files changed

+85
-40
lines changed

‎integration_tests/cdk/app.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
1-
from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_rds
1+
from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_iam, aws_rds
22
from config import AppConfig, build_app_config
33
from constructs import Construct
44
from eoapi_cdk import (
55
PgStacApiLambda,
66
PgStacDatabase,
7+
StacIngestor,
78
TiPgApiLambda,
89
TitilerPgstacApiLambda,
910
)
1011

12+
PGSTAC_VERSION = "0.9.5"
13+
1114

1215
class VpcStack(Stack):
1316
def __init__(
@@ -81,7 +84,7 @@ def __init__(
8184
instance_type=aws_ec2.InstanceType(app_config.db_instance_type),
8285
add_pgbouncer=True,
8386
removal_policy=RemovalPolicy.DESTROY,
84-
pgstac_version="0.9.5",
87+
pgstac_version=PGSTAC_VERSION,
8588
)
8689

8790
assert pgstac_db.security_group
@@ -137,6 +140,31 @@ def __init__(
137140
},
138141
)
139142

143+
s3_read_only_role = aws_iam.Role(
144+
self,
145+
"S3ReadOnlyRole",
146+
assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"),
147+
description="Role with read-only access to S3 buckets",
148+
)
149+
150+
s3_read_only_role.add_managed_policy(
151+
aws_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3ReadOnlyAccess")
152+
)
153+
154+
StacIngestor(
155+
self,
156+
"ingestor",
157+
data_access_role=s3_read_only_role,
158+
stac_db_secret=pgstac_db.pgstac_secret,
159+
stac_db_security_group=pgstac_db.security_group,
160+
stac_url=stac_api.url,
161+
stage="test",
162+
pgstac_version=PGSTAC_VERSION,
163+
api_env={
164+
"JWKS_URL": "", # no authentication!
165+
},
166+
)
167+
140168

141169
app = App()
142170

‎lib/database/index.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@ import {
1010
aws_logs,
1111
} from "aws-cdk-lib";
1212
import { Construct } from "constructs";
13-
import { CustomLambdaFunctionProps } from "../utils";
13+
import { CustomLambdaFunctionProps, DEFAULT_PGSTAC_VERSION } from "../utils";
1414
import { PgBouncer } from "./PgBouncer";
1515

1616
const instanceSizes: Record<string, number> = require("./instance-memory.json");
17-
const DEFAULT_PGSTAC_VERSION = "0.9.5";
1817

1918
let defaultPgSTACCustomOptions: { [key: string]: any } = {
2019
context: "FALSE",

‎lib/ingestor-api/index.ts‎

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
Stack,
1414
} from "aws-cdk-lib";
1515
import { Construct } from "constructs";
16-
import { CustomLambdaFunctionProps } from "../utils";
16+
import { CustomLambdaFunctionProps, DEFAULT_PGSTAC_VERSION } from "../utils";
1717

1818
export class StacIngestor extends Construct {
1919
table: dynamodb.Table;
@@ -39,10 +39,10 @@ export class StacIngestor extends Construct {
3939
assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
4040
managedPolicies: [
4141
iam.ManagedPolicy.fromAwsManagedPolicyName(
42-
"service-role/AWSLambdaBasicExecutionRole",
42+
"service-role/AWSLambdaBasicExecutionRole"
4343
),
4444
iam.ManagedPolicy.fromAwsManagedPolicyName(
45-
"service-role/AWSLambdaVPCAccessExecutionRole",
45+
"service-role/AWSLambdaVPCAccessExecutionRole"
4646
),
4747
],
4848
});
@@ -57,6 +57,7 @@ export class StacIngestor extends Construct {
5757
dbSecurityGroup: props.stacDbSecurityGroup,
5858
subnetSelection: props.subnetSelection,
5959
lambdaFunctionOptions: props.apiLambdaFunctionOptions,
60+
pgstacVersion: props.pgstacVersion,
6061
});
6162

6263
this.buildApiEndpoint({
@@ -74,7 +75,8 @@ export class StacIngestor extends Construct {
7475
dbVpc: props.vpc,
7576
dbSecurityGroup: props.stacDbSecurityGroup,
7677
subnetSelection: props.subnetSelection,
77-
lambdaFunctionOptions: props.ingestorLambdaFunctionOptions
78+
lambdaFunctionOptions: props.ingestorLambdaFunctionOptions,
79+
pgstacVersion: props.pgstacVersion,
7880
});
7981

8082
this.registerSsmParameter({
@@ -110,20 +112,23 @@ export class StacIngestor extends Construct {
110112
dbSecret: secretsmanager.ISecret;
111113
dbVpc: undefined | ec2.IVpc;
112114
dbSecurityGroup: ec2.ISecurityGroup;
113-
subnetSelection: undefined | ec2.SubnetSelection
115+
subnetSelection: undefined | ec2.SubnetSelection;
114116
lambdaFunctionOptions?: CustomLambdaFunctionProps;
117+
pgstacVersion?: string;
115118
}): lambda.Function {
116-
117119
const handler = new lambda.Function(this, "api-handler", {
118120
// defaults
119121
runtime: lambda.Runtime.PYTHON_3_11,
120122
handler: "src.handler.handler",
121123
memorySize: 2048,
122124
logRetention: aws_logs.RetentionDays.ONE_WEEK,
123125
timeout: Duration.seconds(30),
124-
code:lambda.Code.fromDockerBuild(__dirname, {
126+
code: lambda.Code.fromDockerBuild(__dirname, {
125127
file: "runtime/Dockerfile",
126-
buildArgs: { PYTHON_VERSION: '3.11' },
128+
buildArgs: {
129+
PYTHON_VERSION: "3.11",
130+
PGSTAC_VERSION: props.pgstacVersion || DEFAULT_PGSTAC_VERSION,
131+
},
127132
}),
128133
allowPublicSubnet: true,
129134
vpc: props.dbVpc,
@@ -139,7 +144,7 @@ export class StacIngestor extends Construct {
139144

140145
// Allow handler to connect to DB
141146

142-
if (props.dbVpc){
147+
if (props.dbVpc) {
143148
props.dbSecurityGroup.addIngressRule(
144149
handler.connections.securityGroups[0],
145150
ec2.Port.tcp(5432),
@@ -160,10 +165,9 @@ export class StacIngestor extends Construct {
160165
dbSecurityGroup: ec2.ISecurityGroup;
161166
subnetSelection: undefined | ec2.SubnetSelection;
162167
lambdaFunctionOptions?: CustomLambdaFunctionProps;
168+
pgstacVersion?: string;
163169
}): lambda.Function {
164-
165-
166-
const handler = new lambda.Function(this, "stac-ingestor",{
170+
const handler = new lambda.Function(this, "stac-ingestor", {
167171
// defaults
168172
runtime: lambda.Runtime.PYTHON_3_11,
169173
handler: "src.ingestor.handler",
@@ -172,7 +176,10 @@ export class StacIngestor extends Construct {
172176
timeout: Duration.seconds(180),
173177
code: lambda.Code.fromDockerBuild(__dirname, {
174178
file: "runtime/Dockerfile",
175-
buildArgs: { PYTHON_VERSION: '3.11' },
179+
buildArgs: {
180+
PYTHON_VERSION: "3.11",
181+
PGSTAC_VERSION: props.pgstacVersion || DEFAULT_PGSTAC_VERSION,
182+
},
176183
}),
177184
vpc: props.dbVpc,
178185
vpcSubnets: props.subnetSelection,
@@ -187,15 +194,15 @@ export class StacIngestor extends Construct {
187194
props.dbSecret.grantRead(handler);
188195

189196
// Allow handler to connect to DB
190-
if (props.dbVpc){
197+
if (props.dbVpc) {
191198
props.dbSecurityGroup.addIngressRule(
192199
handler.connections.securityGroups[0],
193200
ec2.Port.tcp(5432),
194201
"Allow connections from STAC Ingestor"
195202
);
196203
}
197204

198-
// Allow handler to write results back to DBƒ
205+
// Allow handler to write results back to DB
199206
props.table.grantWriteData(handler);
200207

201208
// Trigger handler from writes to DynamoDB table
@@ -221,7 +228,6 @@ export class StacIngestor extends Construct {
221228
endpointConfiguration?: apigateway.EndpointConfiguration;
222229
ingestorDomainNameOptions?: apigateway.DomainNameOptions;
223230
}): apigateway.LambdaRestApi {
224-
225231
return new apigateway.LambdaRestApi(
226232
this,
227233
`${Stack.of(this).stackName}-ingestor-api`,
@@ -236,10 +242,12 @@ export class StacIngestor extends Construct {
236242
endpointConfiguration: props.endpointConfiguration,
237243
policy: props.policy,
238244

239-
domainName: props.ingestorDomainNameOptions ? {
240-
domainName: props.ingestorDomainNameOptions.domainName,
241-
certificate: props.ingestorDomainNameOptions.certificate,
242-
} : undefined,
245+
domainName: props.ingestorDomainNameOptions
246+
? {
247+
domainName: props.ingestorDomainNameOptions.domainName,
248+
certificate: props.ingestorDomainNameOptions.certificate,
249+
}
250+
: undefined,
243251
}
244252
);
245253
}
@@ -316,20 +324,26 @@ export interface StacIngestorProps {
316324
/**
317325
* Custom Domain Name Options for Ingestor API
318326
*/
319-
readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
327+
readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
320328

321329
/**
322-
* Can be used to override the default lambda function properties.
323-
*
324-
* @default - default settings are defined in the construct.
325-
*/
330+
* Can be used to override the default lambda function properties.
331+
*
332+
* @default - default settings are defined in the construct.
333+
*/
326334
readonly apiLambdaFunctionOptions?: CustomLambdaFunctionProps;
327335

328336
/**
329-
* Can be used to override the default lambda function properties.
330-
*
331-
* @default - default settings are defined in the construct.
332-
*/
333-
readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps;
337+
* Can be used to override the default lambda function properties.
338+
*
339+
* @default - default settings are defined in the construct.
340+
*/
341+
readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps;
334342

343+
/**
344+
* pgstac version - must match the version installed on the pgstac database
345+
*
346+
* @default - default settings are defined in the construct
347+
*/
348+
readonly pgstacVersion?: string;
335349
}

‎lib/ingestor-api/runtime/Dockerfile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
ARG PYTHON_VERSION
2+
23
FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
34

45
WORKDIR /tmp
@@ -8,7 +9,10 @@ RUN yum install -y git
89
RUN python -m pip install pip -U
910

1011
COPY runtime/requirements.txt requirements.txt
11-
RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
12+
13+
ARG PGSTAC_VERSION
14+
RUN echo "pypgstac==${PGSTAC_VERSION}" > constraints.txt
15+
RUN python -m pip install -r requirements.txt -c constraints.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
1216

1317
RUN mkdir -p /asset/src
1418
COPY runtime/src/*.py /asset/src/

‎lib/ingestor-api/runtime/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@ orjson>=3.6.8
55
psycopg[binary,pool]>=3.0.15
66
pydantic_ssm_settings>=1.0,<2.0
77
pydantic>=1.9.0
8-
pypgstac==0.8.5
8+
pypgstac
99
requests>=2.27.1
1010
stac-pydantic>=3.0,<4.0

‎lib/ingestor-api/runtime/src/config.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ class Settings(BaseSettings):
2222
root_path: Optional[str] = Field(description="Path from where to serve this URL.")
2323

2424
jwks_url: Optional[HttpUrlString] = Field(
25-
description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json" # noqa
25+
default=None,
26+
description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json", # noqa
2627
)
2728

2829
stac_url: HttpUrlString = Field(description="URL of STAC API")

‎lib/utils/index.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
import {
2-
aws_lambda as lambda,
3-
} from "aws-cdk-lib";
1+
import { aws_lambda as lambda } from "aws-cdk-lib";
42

53
export type CustomLambdaFunctionProps = lambda.FunctionProps | any;
4+
export const DEFAULT_PGSTAC_VERSION = "0.9.5";

0 commit comments

Comments
 (0)