Skip to content

Commit ed6d98f

Browse files
Merge pull request #1 from program--/main
add hfsubset files
2 parents 57e0bbf + dbf5262 commit ed6d98f

File tree

8 files changed

+471
-0
lines changed

8 files changed

+471
-0
lines changed

.dockerignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
cli
2+
.git
3+
.github
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
on:
2+
push:
3+
paths:
4+
- runtime.R
5+
- Dockerfile
6+
7+
name: Build and Deploy Lambda
8+
9+
jobs:
10+
container:
11+
name: Deploy container to ECR
12+
runs-on: ubuntu-latest
13+
env:
14+
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
15+
ECR_REPOSITORY: hfsubset
16+
IMAGE_TAG: ${{ github.sha }}
17+
steps:
18+
- name: Checkout
19+
uses: actions/checkout@v3
20+
21+
- name: Configure AWS Credentials
22+
id: aws-credentials
23+
uses: aws-actions/configure-aws-credentials@v2
24+
with:
25+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
26+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
27+
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
28+
29+
- name: Login to AWS ECR
30+
id: login-ecr
31+
uses: aws-actions/amazon-ecr-login@v1
32+
33+
- name: Verify ECR Repository
34+
env:
35+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
36+
run: |
37+
(aws ecr describe-repositories --repository-names ${ECR_REPOSITORY} \
38+
&& docker pull ${ECR_REGISTRY}/${ECR_REPOSITORY}:latest) \
39+
|| aws ecr create-repository --repository-name ${ECR_REPOSITORY}
40+
41+
- name: Build and tag image
42+
env:
43+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
44+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
45+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
46+
run: |
47+
docker build \
48+
--cache-from "${ECR_REGISTRY}/${ECR_REPOSITORY}:latest" \
49+
--tag="${ECR_REGISTRY}/${ECR_REPOSITORY}:${IMAGE_TAG}" \
50+
--tag="${ECR_REGISTRY}/${ECR_REPOSITORY}:latest" .
51+
52+
- name: Push image to ECR
53+
env:
54+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
55+
run: docker push ${ECR_REGISTRY}/${ECR_REPOSITORY} --all-tags
56+
57+
- name: Check Lambda function
58+
id: lambda-function
59+
continue-on-error: true
60+
env:
61+
LAMBDA_FUNCTION: hfsubset
62+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
63+
run: aws lambda wait function-exists --function-name ${LAMBDA_FUNCTION} > /dev/null 2>&1
64+
65+
- name: Create Lambda function
66+
if: steps.lambda-function.outcome == 'failure'
67+
env:
68+
LAMBDA_FUNCTION: hfsubset
69+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
70+
AWS_ACCOUNT_ID: ${{ steps.aws-credentials.outputs.aws-account-id }}
71+
run: |
72+
aws iam create-role \
73+
--role-name ${LAMBDA_FUNCTION}-role \
74+
--assume-role-policy-document '{"Version": "2012-10-17","Statement": [{ "Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"}]}'
75+
76+
aws iam attach-role-policy \
77+
--role-name ${LAMBDA_FUNCTION}-role \
78+
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
79+
80+
aws lambda create-function \
81+
--function-name ${LAMBDA_FUNCTION} \
82+
--role "arn:aws:iam:${AWS_ACCOUNT_ID}:role/${LAMBDA_FUNCTION}-role" \
83+
--memory-size 3008 \
84+
--code "ImageUri=${ECR_REGISTRY}/${ECR_REPOSTIORY}:${IMAGE_TAG}" \
85+
--ephemeral-storage 2048
86+
87+
# TODO: put this behind an API Gateway?
88+
output=$(aws lambda create-function-url-config --function-name ${LAMBDA_FUNCTION} --auth-type NONE)
89+
90+
echo ":rocket: lambda function running at $(echo $output | jq .FunctionUrl)" >> $GITHUB_STEP_SUMMARY
91+
92+
- name: Update Lambda function
93+
if: steps.lambda-function.outcome == 'success'
94+
env:
95+
LAMBDA_FUNCTION: hfsubset
96+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
97+
run: |
98+
aws lambda update-function-code \
99+
--function-name ${LAMBDA_FUNCTION} \
100+
--image-uri "${ECR_REGISTRY}/${ECR_REPOSITORY}:${IMAGE_TAG}"
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
on:
2+
push:
3+
paths:
4+
- cli/**.go
5+
6+
name: Build and Upload hfsubset Executables
7+
8+
jobs:
9+
generate:
10+
name: Generate cross-platform builds
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: write
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v3
17+
18+
- name: Setup Golang
19+
uses: actions/setup-go@v4
20+
with:
21+
go-version-file: 'cli/go.mod'
22+
23+
- name: Package MacOS executable
24+
working-directory: ./cli
25+
run: |
26+
GOOS=darwin GOARCH=amd64 go build
27+
tar -czvf hfsubset-macos_amd64.tar.gz hfsubset
28+
rm hfsubset
29+
30+
- name: Package Linux executable
31+
working-directory: ./cli
32+
run: |
33+
GOOS=linux GOARCH=amd64 go build
34+
tar -czvf hfsubset-linux_amd64.tar.gz hfsubset
35+
rm hfsubset
36+
37+
- name: Package Windows executable
38+
working-directory: ./cli
39+
run: |
40+
GOOS=windows GOARCH=amd64 go build
41+
zip hfsubset-windows_amd64.zip hfsubset.exe
42+
rm hfsubset.exe
43+
44+
- name: Upload packages
45+
uses: ncipollo/release-action@v1
46+
with:
47+
tag: hfsubset-release-${{ github.run_number }}
48+
commit: ${{ github.sha }}
49+
artifacts: "cli/hfsubset-*"

Dockerfile

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
FROM rocker/geospatial:4.3.0
2+
3+
COPY --from=public.ecr.aws/lambda/provided:al2.2023.05.13.00 /lambda-entrypoint.sh /lambda-entrypoint.sh
4+
COPY --from=public.ecr.aws/lambda/provided:al2.2023.05.13.00 /usr/local/bin/aws-lambda-rie /usr/local/bin/aws-lambda-rie
5+
ENV LAMBDA_TASK_ROOT=/var/task
6+
ENV LAMBDA_RUNTIME_DIR=/var/runtime
7+
RUN mkdir /var/runtime
8+
RUN mkdir /var/task
9+
WORKDIR /var/task
10+
ENTRYPOINT ["/lambda-entrypoint.sh"]
11+
12+
# Tnstall Apache Arrow/build tools
13+
ENV HF_BUILD_PKGS="build-essential git"
14+
ENV HF_ARROW_PKGS="libarrow-dev libarrow-glib-dev libarrow-dataset-dev libarrow-dataset-glib-dev libarrow-flight-dev libarrow-flight-glib-dev libparquet-dev libparquet-glib-dev"
15+
RUN apt update \
16+
&& apt install -y -V ca-certificates lsb-release wget \
17+
&& wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
18+
&& apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
19+
&& apt update \
20+
&& apt install -y -V ${HF_BUILD_PKGS} ${HF_ARROW_PKGS}
21+
22+
# Setup hydrofabric location
23+
RUN mkdir -p /hydrofabric
24+
25+
# Install CRAN Packages
26+
ENV HF_CRAN_PKGS="cli arrow crayon dplyr DBI RSQLite sf terra lambdr glue rstudioapi purrr magrittr"
27+
RUN cd /hydrofabric \
28+
&& . /etc/lsb-release \
29+
&& echo "options(Ncpus = $(nproc --all), repos=c(CRAN = 'https://packagemanager.rstudio.com/cran/__linux__/${DISTRIB_CODENAME}/latest'))" >> .Rprofile \
30+
&& install2.r devtools \
31+
&& install2.r -n 6 -s ${HF_CRAN_PKGS}
32+
33+
# Install GH Packages
34+
ENV HF_GH_PKGS="mikejohnson51/hydrofab mikejohnson51/ngen.hydrofab mikejohnson51/zonal mikejohnson51/climateR DOI-USGS/nhdplusTools"
35+
RUN cd /hydrofabric \
36+
&& installGithub.r ${HF_GH_PKGS}
37+
38+
RUN git clone https://github.com/NOAA-OWP/hydrofabric.git /hydrofabric \
39+
&& mkdir -p /hydrofabric/subset
40+
41+
COPY . /hydrofabric/subset
42+
43+
# Setup Lambdr
44+
RUN cd /hydrofabric \
45+
&& Rscript -e "devtools::install_local()" \
46+
&& chmod 755 subset/runtime.R \
47+
&& printf "#!/bin/sh\ncd /hydrofabric/subset\nRscript runtime.R" > /var/runtime/bootstrap \
48+
&& chmod +x /var/runtime/bootstrap
49+
50+
CMD ["subset"]

cli/go.mod

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
module hfsubset
2+
3+
go 1.20
4+
5+
require github.com/schollz/progressbar/v3 v3.13.1
6+
7+
require (
8+
github.com/mattn/go-runewidth v0.0.14 // indirect
9+
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
10+
github.com/rivo/uniseg v0.4.4 // indirect
11+
golang.org/x/sys v0.8.0 // indirect
12+
golang.org/x/term v0.8.0 // indirect
13+
)

cli/go.sum

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
3+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4+
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
5+
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
6+
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
7+
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
8+
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
9+
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
10+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
11+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
12+
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
13+
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
14+
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
15+
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
16+
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
17+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
18+
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
19+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
20+
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
21+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
22+
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
23+
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
24+
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
25+
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
26+
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=

0 commit comments

Comments
 (0)