Skip to content

Commit bac0487

Browse files
hobuhobu
committed
use Terraform to config and manage
1 parent 93920ad commit bac0487

27 files changed

+635
-264
lines changed

‎README.rst‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,44 @@ containers are often quite slow.
1717
AWS_SECRET_ACCESS_KEY=somethingelse
1818
AWS_DEFAULT_REGION=us-east-1
1919

20-
1. Build the containers. It should make both an arm64 and amd64 image
20+
1. (optional) Edit your variables
2121

2222
::
2323

24-
$ ./build.sh pdal-lambda
24+
cat terraform/pdal.tfvars
2525

26-
2. Create an ECR repository in your account for the ``pdal-lambda``
27-
image
26+
environment_name="pdal-lambda"
27+
arch="arm64"
28+
29+
2. Initialize your Terraform Environment
2830

2931
::
3032

31-
aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION
32-
aws ecr create-repository \
33-
--repository-name pdal-lambda \
34-
--region $AWS_DEFAULT_REGION
33+
cd terraform
34+
terraform init
35+
terraform validate
36+
terraform apply -var-file pdal.tfvars
37+
38+
The Terraform configuration will create some resources including an ECR
39+
repository to store the image, a role for execution of the lambda,
40+
and the lambda itself. Adjust your configuration as needed in `./terraform/resources`
41+
42+
3. Test locally
3543

36-
3. Build the containers. It should make both an arm64 and amd64 image
44+
Fire up the Lambda Docker container in one terminal:
3745

3846
::
3947

40-
$ ./build.sh pdal-lambda
48+
cd docker
49+
./run-local.sh /var/task/python-entry.sh pdal_lambda.ecr.info.handler
4150

42-
4. Push the containers
51+
In another terminal, issue the test. Note that it currently defaults to running
52+
on port 9000. Adjust the script as necessary.
4353

4454
::
4555

46-
$ ./push.sh pdal-lambda
56+
cd docker
57+
./test-local.sh info-event.json
58+
59+
60+

‎build.sh‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

Dockerfile renamed to docker/Dockerfile.runner

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ ARG TARGETARCH
4141
ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}
4242
ENV TARGETARCH=${TARGETARCH:-amd64}
4343

44+
45+
4446
ENV CONDAENV "/var/task"
4547
ENV CONDA_PREFIX "/var/task"
4648
ENV TARGETPLATFORM "${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ ENV PROJ_LIB ${CONDAENV}/share/proj
5254
ENV PROJ_NETWORK=TRUE
5355
ENV PATH $PATH:${CONDAENV}/bin
5456
ENV LD_LIBRARY_PATH=${CONDAENV}/lib
57+
ENV HOME=/var/task/
5558

5659
RUN /var/task/bin/python -m pip install awslambdaric==2.0.11
5760
ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-${RIE_ARCH} /usr/bin/aws-lambda-rie
@@ -61,7 +64,6 @@ RUN chmod +x /usr/bin/aws-lambda-rie
6164

6265
WORKDIR /var/task
6366
COPY python-entry.sh ./
64-
COPY pdal_handler.py ./
67+
COPY handlers/python/ /var/task/lib/python3.11/site-packages/pdal_lambda
6568
COPY root-bashrc /root/.bashrc
6669
ENTRYPOINT [ "/var/task/python-entry.sh" ]
67-
CMD ["pdal_handler.handler"]

build-environment.yml renamed to docker/build-environment.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,9 @@ channels:
33
- conda-forge
44
dependencies:
55
- conda-pack
6-
- mamba
6+
- compilers
7+
- ninja
8+
- cmake
9+
- pdal
10+
- eigen
11+
- unzip

‎docker/info-event.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"Records": [
3+
{
4+
"eventVersion": "2.1",
5+
"eventSource": "aws:s3",
6+
"awsRegion": "us-east-1",
7+
"eventTime": "2019-09-03T19:37:27.192Z",
8+
"eventName": "ObjectCreated:Put",
9+
"userIdentity": {
10+
"principalId": "AWS:AIDAINPONIXQXHT3IKHL2"
11+
},
12+
"requestParameters": {
13+
"sourceIPAddress": "205.255.255.255"
14+
},
15+
"responseElements": {
16+
"x-amz-request-id": "D82B88E5F771F645",
17+
"x-amz-id-2": "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo="
18+
},
19+
"s3": {
20+
"s3SchemaVersion": "1.0",
21+
"configurationId": "828aa6fc-f7b5-4305-8584-487c791949c1",
22+
"bucket": {
23+
"name": "pdal",
24+
"ownerIdentity": {
25+
"principalId": "A3I5XTEXAMAI3E"
26+
},
27+
"arn": "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df"
28+
},
29+
"object": {
30+
"key": "test.laz",
31+
"size": 1305107,
32+
"eTag": "b21b84d653bb07b05b1e6b33684dc11b",
33+
"sequencer": "0C0F6F405D6ED209E1"
34+
}
35+
}
36+
}
37+
]
38+
}
39+

python-entry.sh renamed to docker/python-entry.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/bin/sh
22

3+
env
34
if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
45
exec /usr/bin/aws-lambda-rie /var/task/bin/python -m awslambdaric $1
56
else

root-bashrc renamed to docker/root-bashrc

File renamed without changes.

‎docker/run-environment.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: pdal
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- python-pdal
6+
- python=3.11
7+
- aws-sdk-cpp
8+
- shapely
9+
- gdal
10+
- pyproj
11+
- pystac
12+
- stactools
13+
- nodejs
14+
- whitebox_tools
15+
- boto3
16+
- awscli
17+
- pip
18+
- eigen
19+
- matplotlib
20+
- unzip
21+
- requests
22+
- geocoder
23+
- untwine
24+
- pip:
25+
- trenchrun
26+
- cloudpathlib
27+
- ept-python
28+
29+
30+
31+
32+
33+

‎docker/run-local.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
3+
# ./run-local.sh /var/task/python-entry.sh pdal_lambda.ecr.info.handler
4+
5+
entrypoint="$1"
6+
command="$2"
7+
8+
9+
PREFIX=$(cat ../terraform/terraform.tfstate | jq '.outputs.environment.value // empty' -r)
10+
STAGE=$(cat ../terraform/terraform.tfstate | jq '.outputs.stage.value // empty' -r)
11+
ARCH=$(cat ../terraform/terraform.tfstate | jq '.outputs.arch.value // empty' -r)
12+
13+
if [ -z "$STAGE" ]
14+
then
15+
echo "STAGE must be set!"
16+
exit 1;
17+
fi
18+
19+
if [ -z "$PREFIX" ]
20+
then
21+
echo "PREFIX must be set!"
22+
exit 1;
23+
fi
24+
25+
if [ -z "$ARCH" ]
26+
then
27+
echo "ARCH must be set!"
28+
exit 1;
29+
fi
30+
31+
CONTAINER="$PREFIX-$STAGE-pdal_runner"
32+
33+
REGION=$AWS_DEFAULT_REGION
34+
if [ -z "$REGION" ]
35+
then
36+
echo "AWS_DEFAULT_REGION must be set!"
37+
exit 1;
38+
fi
39+
40+
LOCALPORT=9000
41+
REMOTEPORT=8080
42+
43+
identity=$(aws sts get-caller-identity --query 'Account' --output text)
44+
45+
KEY_ID=$(aws --profile $AWS_DEFAULT_PROFILE configure get aws_access_key_id)
46+
SECRET_ID=$(aws --profile $AWS_DEFAULT_PROFILE configure get aws_secret_access_key)
47+
48+
49+
echo "running $identity.dkr.ecr.$region.amazonaws.com/$container:$ARCH"
50+
51+
if [ -z "$entrypoint" ]
52+
then
53+
echo "executing default entrypoint using $command"
54+
docker run -p $LOCALPORT:$REMOTEPORT \
55+
-e AWS_DEFAULT_REGION=$REGION \
56+
-e AWS_ACCESS_KEY_ID=${KEY_ID} \
57+
-e AWS_SECRET_ACCESS_KEY=${SECRET_ID} \
58+
$identity.dkr.ecr.$REGION.amazonaws.com/$CONTAINER:$ARCH "$command"
59+
else
60+
echo "executing with $entrypoint and command '$command'"
61+
docker run -p $LOCALPORT:$REMOTEPORT \
62+
-e AWS_DEFAULT_REGION=$REGION \
63+
-e AWS_ACCESS_KEY_ID=$KEY_ID \
64+
-e AWS_SECRET_ACCESS_KEY=$SECRET_ID \
65+
-t -i \
66+
-v $(pwd):/data \
67+
--entrypoint=$entrypoint \
68+
$identity.dkr.ecr.$REGION.amazonaws.com/$CONTAINER:$ARCH \
69+
"$command"
70+
fi
71+

‎docker/test-event.json‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
{
2+
"pipeline": [
3+
{
4+
"bounds": "([-10425171.940, -10423171.940], [5164494.710, 5166494.710])",
5+
"filename": "https://s3-us-west-2.amazonaws.com/usgs-lidar-public/IA_FullState/ept.json",
6+
"resolution":5,
7+
"requests":1,
8+
"type": "readers.ept",
9+
"tag": "readdata"
10+
},
11+
{
12+
"expression": "Classification != 7",
13+
"type": "filters.expression",
14+
"tag": "nonoise"
15+
},
16+
{
17+
"assignment": "Classification[:]=0",
18+
"tag": "wipeclasses",
19+
"type": "filters.assign"
20+
},
21+
{
22+
"out_srs": "EPSG:26915",
23+
"tag": "reprojectUTM",
24+
"type": "filters.reprojection"
25+
},
26+
{
27+
"tag": "groundify",
28+
"type": "filters.smrf"
29+
},
30+
{
31+
"expression": "Classification == 2",
32+
"type": "filters.expression",
33+
"tag": "classify"
34+
},
35+
{
36+
"filename": "iowa.laz",
37+
"inputs": [ "classify" ],
38+
"tag": "writerslas",
39+
"type": "writers.las"
40+
},
41+
{
42+
"filename": "iowa.tif",
43+
"gdalopts": "tiled=yes, compress=deflate",
44+
"inputs": [ "writerslas" ],
45+
"nodata": -9999,
46+
"output_type": "idw",
47+
"resolution": 5,
48+
"type": "writers.gdal",
49+
"window_size": 3
50+
}
51+
],
52+
"Records": [
53+
{
54+
"eventVersion": "2.1",
55+
"eventSource": "aws:s3",
56+
"awsRegion": "us-east-1",
57+
"eventTime": "2019-09-03T19:37:27.192Z",
58+
"eventName": "ObjectCreated:Put",
59+
"userIdentity": {
60+
"principalId": "AWS:AIDAINPONIXQXHT3IKHL2"
61+
},
62+
"requestParameters": {
63+
"sourceIPAddress": "205.255.255.255"
64+
},
65+
"responseElements": {
66+
"x-amz-request-id": "D82B88E5F771F645",
67+
"x-amz-id-2": "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo="
68+
},
69+
"s3": {
70+
"s3SchemaVersion": "1.0",
71+
"configurationId": "828aa6fc-f7b5-4305-8584-487c791949c1",
72+
"bucket": {
73+
"name": "grid-dev-lidarscans",
74+
"ownerIdentity": {
75+
"principalId": "A3I5XTEXAMAI3E"
76+
},
77+
"arn": "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df"
78+
},
79+
"object": {
80+
"key": "RiPS/jpg/INGLEFIELD_RIPS_puck_20190708_210000.jpg",
81+
"size": 1305107,
82+
"eTag": "b21b84d653bb07b05b1e6b33684dc11b",
83+
"sequencer": "0C0F6F405D6ED209E1"
84+
}
85+
}
86+
}
87+
]
88+
89+
}

0 commit comments

Comments
 (0)