diff --git a/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/.lambdaignore b/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/.lambdaignore new file mode 100644 index 0000000..63f1219 --- /dev/null +++ b/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/.lambdaignore @@ -0,0 +1,40 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/bash-completion/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/Dockerfile b/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/Dockerfile new file mode 100644 index 0000000..4f99ba3 --- /dev/null +++ b/terraform/layers/python3.10/rasterio_no_numpy_1.3.10/Dockerfile @@ -0,0 +1,23 @@ +FROM --platform=linux/amd64 ghcr.io/lambgeo/lambda-gdal:3.8-python3.10 + +ENV NUMPY_VERSION=1.26.4 +ENV RASTERIO_VERSION=1.3.10 + +WORKDIR /opt +RUN mkdir -p python + +# Install rasterio from source because the binary pulls in its own GDAL +# subset, and we already have it. Otherwise install binary packages, +# because they tend to be smaller than those you build yourself. +RUN pip install NumPy==${NUMPY_VERSION} rasterio==${RASTERIO_VERSION} --no-binary rasterio -t python + +# Reduce size of the C libs +# TODO: Try stripping bin as well? +RUN find lib -name \*.so\* -exec strip {} \; + +# Compress all files except those listed in .lambdaignore +RUN yum install -y zip +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -r9q layer.zip python -x && \ + cat .lambdaignore | xargs zip -r9q --symlinks layer.zip lib/*.so* share -x && \ + zip -r9q --symlinks layer.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack \ No newline at end of file diff --git a/terraform/layers/python3.13/geopandas_0.14.4/.lambdaignore b/terraform/layers/python3.13/geopandas_0.14.4/.lambdaignore new file mode 100644 index 0000000..6886be2 --- /dev/null +++ b/terraform/layers/python3.13/geopandas_0.14.4/.lambdaignore @@ -0,0 +1,37 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/pandas/* +python/pandas.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/pytz/* +python/regex/test/* +python/s3transfer/* +python/setuptools/* +python/shapely/* +python/Shapely.libs/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/bash-completion/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/geopandas_0.14.4/Dockerfile b/terraform/layers/python3.13/geopandas_0.14.4/Dockerfile new file mode 100644 index 0000000..25aaf20 --- /dev/null +++ b/terraform/layers/python3.13/geopandas_0.14.4/Dockerfile @@ -0,0 +1,31 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV NUMPY_VERSION="2.3.3" +ENV SHAPELY_VERSION="2.1.2" +ENV PANDAS_VERSION="2.3.3" +ENV GEOPANDAS_VERSION="0.14.4" + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install \ + "numpy==${NUMPY_VERSION}" \ + "shapely==${SHAPELY_VERSION}" \ + "pandas==${PANDAS_VERSION}" \ + "geopandas==${GEOPANDAS_VERSION}" \ + -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/geopandas_1.1.2/.lambdaignore b/terraform/layers/python3.13/geopandas_1.1.2/.lambdaignore new file mode 100644 index 0000000..6886be2 --- /dev/null +++ b/terraform/layers/python3.13/geopandas_1.1.2/.lambdaignore @@ -0,0 +1,37 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/pandas/* +python/pandas.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/pytz/* +python/regex/test/* +python/s3transfer/* +python/setuptools/* +python/shapely/* +python/Shapely.libs/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/bash-completion/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/geopandas_1.1.2/Dockerfile b/terraform/layers/python3.13/geopandas_1.1.2/Dockerfile new file mode 100644 index 0000000..03f603a --- /dev/null +++ b/terraform/layers/python3.13/geopandas_1.1.2/Dockerfile @@ -0,0 +1,31 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV NUMPY_VERSION="2.3.5" +ENV SHAPELY_VERSION="2.1.2" +ENV PANDAS_VERSION="2.3.3" +ENV GEOPANDAS_VERSION="1.1.2" + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install \ + "numpy==${NUMPY_VERSION}" \ + "shapely==${SHAPELY_VERSION}" \ + "pandas==${PANDAS_VERSION}" \ + "geopandas==${GEOPANDAS_VERSION}" \ + -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/mercantile_1.2.1/.lambdaignore b/terraform/layers/python3.13/mercantile_1.2.1/.lambdaignore new file mode 100644 index 0000000..2f6f212 --- /dev/null +++ b/terraform/layers/python3.13/mercantile_1.2.1/.lambdaignore @@ -0,0 +1,37 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/mercantile_1.2.1/Dockerfile b/terraform/layers/python3.13/mercantile_1.2.1/Dockerfile new file mode 100644 index 0000000..890b8ce --- /dev/null +++ b/terraform/layers/python3.13/mercantile_1.2.1/Dockerfile @@ -0,0 +1,23 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV MERCANTILE_VERSION=1.2.1 + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install mercantile==${MERCANTILE_VERSION} -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/numpy_2.3.5/.lambdaignore b/terraform/layers/python3.13/numpy_2.3.5/.lambdaignore new file mode 100644 index 0000000..4854865 --- /dev/null +++ b/terraform/layers/python3.13/numpy_2.3.5/.lambdaignore @@ -0,0 +1,29 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/numpy_2.3.5/Dockerfile b/terraform/layers/python3.13/numpy_2.3.5/Dockerfile new file mode 100644 index 0000000..9f6b1b1 --- /dev/null +++ b/terraform/layers/python3.13/numpy_2.3.5/Dockerfile @@ -0,0 +1,23 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV NUMPY_VERSION="2.3.5" + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install "numpy==${NUMPY_VERSION}" -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/pandas_2.3.3/.lambdaignore b/terraform/layers/python3.13/pandas_2.3.3/.lambdaignore new file mode 100644 index 0000000..44cd558 --- /dev/null +++ b/terraform/layers/python3.13/pandas_2.3.3/.lambdaignore @@ -0,0 +1,39 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/pandas_2.3.3/Dockerfile b/terraform/layers/python3.13/pandas_2.3.3/Dockerfile new file mode 100644 index 0000000..a01bb60 --- /dev/null +++ b/terraform/layers/python3.13/pandas_2.3.3/Dockerfile @@ -0,0 +1,24 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV NUMPY_VERSION="2.3.5" +ENV PANDAS_VERSION="2.3.3" + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install "numpy==${NUMPY_VERSION}" "pandas==${PANDAS_VERSION}" -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/pillow_12.0.0/.lambdaignore b/terraform/layers/python3.13/pillow_12.0.0/.lambdaignore new file mode 100644 index 0000000..2f6f212 --- /dev/null +++ b/terraform/layers/python3.13/pillow_12.0.0/.lambdaignore @@ -0,0 +1,37 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/pillow_12.0.0/Dockerfile b/terraform/layers/python3.13/pillow_12.0.0/Dockerfile new file mode 100644 index 0000000..ec53dff --- /dev/null +++ b/terraform/layers/python3.13/pillow_12.0.0/Dockerfile @@ -0,0 +1,23 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV PILLOW_VERSION=12.0.0 + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install pillow==${PILLOW_VERSION} -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/layers/python3.13/rasterio_1.5.0/.lambdaignore b/terraform/layers/python3.13/rasterio_1.5.0/.lambdaignore new file mode 100644 index 0000000..63f1219 --- /dev/null +++ b/terraform/layers/python3.13/rasterio_1.5.0/.lambdaignore @@ -0,0 +1,40 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/bash-completion/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/rasterio_1.5.0/Dockerfile b/terraform/layers/python3.13/rasterio_1.5.0/Dockerfile new file mode 100644 index 0000000..6b33eb4 --- /dev/null +++ b/terraform/layers/python3.13/rasterio_1.5.0/Dockerfile @@ -0,0 +1,53 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV GDAL_VERSION="3.12.1" +ENV NUMPY_VERSION="2.3.5" +ENV RASTERIO_VERSION="1.5.0" + +SHELL ["/bin/bash", "-c"] + +ENV PREFIX=/opt +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y gcc gcc-c++ wget cmake proj-devel sqlite-devel tar libcurl-devel libtiff-devel libpng-devel libjpeg-devel libxml2-devel zlib-devel pkgconfig findutils zip && \ + dnf clean all + +# Build gdal from source +RUN mkdir -p /tmp/gdal \ + && curl -sfL https://github.com/OSGeo/gdal/archive/refs/tags/v${GDAL_VERSION}.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ + && cd /tmp/gdal \ + && mkdir build \ + && cd build \ + && cmake3 .. \ + -DGDAL_USE_EXTERNAL_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DGDAL_SET_INSTALL_RELATIVE_RPATH=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ + -DGDAL_USE_LERC_INTERNAL=OFF \ + -DBUILD_PYTHON_BINDINGS=OFF \ + && make -j $(nproc) --silent \ + && make install \ + && rm -rf /tmp/gdal + +# Build rasterio from source because the binary pulls in its own GDAL +# subset, and we already have it. Otherwise install binary packages, +# because they tend to be smaller than those you build yourself. + RUN pip install NumPy==${NUMPY_VERSION} rasterio==${RASTERIO_VERSION} --no-binary rasterio -t /opt/python + +# Reduce size of the C libs +# TODO: Try stripping bin as well? +RUN cd /opt && \ + find lib -name \*.so\* -exec strip {} \; + +# Compress all files except those listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -r9q layer.zip python -x && \ + cat .lambdaignore | xargs zip -r9q --symlinks layer.zip lib/*.so* share -x && \ + zip -r9q --symlinks layer.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack diff --git a/terraform/layers/python3.13/shapely_2.1.2/.lambdaignore b/terraform/layers/python3.13/shapely_2.1.2/.lambdaignore new file mode 100644 index 0000000..44cd558 --- /dev/null +++ b/terraform/layers/python3.13/shapely_2.1.2/.lambdaignore @@ -0,0 +1,39 @@ +*-info/* +*/tests/* +python/awscli/examples/* +python/boto3/* +python/botocore/* +python/dateutil/* +python/docker/* +python/docutils/* +python/easy_install/* +python/future/backports/test/* +python/future/moves/test/* +python/future/tests/* +python/jmespath/* +python/jsonschema/tests/* +python/numpy/* +python/numpy.libs/* +python/past/tests/* +python/pip/* +python/pkg_resources/* +python/regex/test/* +python/s3transfer/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-dotnet/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-golang/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-gradle/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-java-maven/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs6/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/tests/* +python/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/* +python/setuptools/* +python/tzlocal/test_data/* +python/urllib3/* +python/websocket/tests/* +python/wheel/* +share/cryptopp/* +share/doc/* +share/gtk-doc/* +share/libxml2-2.9.10/* +share/man/* \ No newline at end of file diff --git a/terraform/layers/python3.13/shapely_2.1.2/Dockerfile b/terraform/layers/python3.13/shapely_2.1.2/Dockerfile new file mode 100644 index 0000000..7ef3fae --- /dev/null +++ b/terraform/layers/python3.13/shapely_2.1.2/Dockerfile @@ -0,0 +1,25 @@ +FROM public.ecr.aws/lambda/python:3.13 + +ENV NUMPY_VERSION="2.3.5" +ENV PYAML_VERSION="6.0.3" +ENV SHAPELY_VERSION="2.1.2" + +SHELL ["/bin/bash", "-c"] + +WORKDIR /opt +RUN mkdir -p python + +RUN dnf update -y && \ + dnf install -y zip findutils + +RUN pip install NumPy==${NUMPY_VERSION} PyYAML==${PYAML_VERSION} shapely==${SHAPELY_VERSION} -t python + +# Precompile all python packages and remove .py files +RUN python -m compileall . +RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-313//'); cp $f $n; done; +RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f + +# Compress all source codes except files listed in .lambdaignore +COPY .lambdaignore . +RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x diff --git a/terraform/locals.tf b/terraform/locals.tf index e8cf683..0dda311 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -1,6 +1,4 @@ locals { - bucket_suffix = var.environment == "production" ? "" : "-${var.environment}" - tf_state_bucket = "gfw-terraform${local.bucket_suffix}" project = "lambda-layers" name_suffix = terraform.workspace == "default" ? "" : "-${terraform.workspace}" tags = { diff --git a/terraform/main.tf b/terraform/main.tf index 8c0d624..21640c0 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -7,18 +7,17 @@ terraform { } } -data "terraform_remote_state" "core" { - backend = "s3" - config = { - bucket = local.tf_state_bucket - region = "us-east-1" - key = "core.tfstate" - } +data "aws_ssm_parameter" "core_contract" { + name = "/infra/${var.environment}/gfw-aws-core-infra/contract" +} + +locals { + core = jsondecode(data.aws_ssm_parameter.core_contract.value) } module "py310_geopandas_0144" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "geopandas" module_version = "0.14.4" runtime = "python3.10" @@ -27,7 +26,7 @@ module "py310_geopandas_0144" { module "py310_mercantile_121" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "mercantile" module_version = "1.2.1" runtime = "python3.10" @@ -36,7 +35,7 @@ module "py310_mercantile_121" { module "py310_numpy_1264" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "numpy" module_version = "1.26.4" runtime = "python3.10" @@ -45,7 +44,7 @@ module "py310_numpy_1264" { module "py310_pandas_153" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "pandas" module_version = "1.5.3" runtime = "python3.10" @@ -54,7 +53,7 @@ module "py310_pandas_153" { module "py310_pillow_950" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "pillow" module_version = "9.5.0" runtime = "python3.10" @@ -63,16 +62,25 @@ module "py310_pillow_950" { module "py310_rasterio_138" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "rasterio" module_version = "1.3.8" runtime = "python3.10" layer_path = "${path.root}/layers/python3.10/rasterio_1.3.8/" } +module "py310_rasterio_no_numpy_1310" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "rasterio_no_numpy_1310" + module_version = "1.3.10" + runtime = "python3.10" + layer_path = "${path.root}/layers/python3.10/rasterio_no_numpy_1.3.10/" +} + module "py310_shapely_185" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "shapely" module_version = "1.8.5" runtime = "python3.10" @@ -83,7 +91,7 @@ module "py310_shapely_185" { # These are temporary: module "py310_geopandas_no_numpy_no_pandas" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "geopandas_no_numpy_no_pandas" module_version = "0.14.4" runtime = "python3.10" @@ -92,7 +100,7 @@ module "py310_geopandas_no_numpy_no_pandas" { module "py310_numpy" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "numpy" module_version = "1.26.4" runtime = "python3.10" @@ -101,7 +109,7 @@ module "py310_numpy" { module "py310_pandas_no_numpy" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "pandas_no_numpy" module_version = "1.5.3" runtime = "python3.10" @@ -110,7 +118,7 @@ module "py310_pandas_no_numpy" { module "py310_rasterio_no_numpy" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "rasterio_no_numpy" module_version = "1.3.8" runtime = "python3.10" @@ -119,7 +127,7 @@ module "py310_rasterio_no_numpy" { module "py310_shapely_no_numpy" { source = "./modules/lambda_layer" - bucket = data.terraform_remote_state.core.outputs.pipelines_bucket + bucket = local.core.gfw_pipelines_bucket_name name = "shapely_no_numpy" module_version = "1.8.5" runtime = "python3.10" @@ -127,6 +135,79 @@ module "py310_shapely_no_numpy" { layer_path = "${path.root}/layers/python3.10/shapely_no_numpy/" } +# These are new: +module "py313_geopandas_112" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_geopandas_112" + module_version = "1.1.2" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/geopandas_1.1.2/" +} + +module "py313_geopandas_0144" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_geopandas_0144" + module_version = "0.14.4" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/geopandas_0.14.4/" +} + +module "py313_mercantile_121" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_mercantile_121" + module_version = "1.2.1" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/mercantile_1.2.1/" +} +module "py313_numpy_235" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_numpy_235" + module_version = "2.3.5" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/numpy_2.3.5/" +} + +module "py313_pandas_233" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_pandas_233" + module_version = "2.3.3" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/pandas_2.3.3/" +} + +module "py313_pillow_1200" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_pillow_1200" + module_version = "12.0.0" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/pillow_12.0.0/" +} + +module "py313_rasterio_150" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_rasterio_150" + module_version = "1.5.0" + runtime = "python3.13" + layer_path = "${path.root}/layers/python3.13/rasterio_1.5.0/" +} + +module "py313_shapely_212" { + source = "./modules/lambda_layer" + bucket = local.core.gfw_pipelines_bucket_name + name = "py313_shapely_212" + module_version = "2.1.2" + runtime = "python3.13" + name_suffix = local.name_suffix + layer_path = "${path.root}/layers/python3.13/shapely_2.1.2/" +} + module "ssm" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/ssm?ref=v0.4.2.8" environment = var.environment @@ -140,6 +221,14 @@ module "ssm" { py310_pillow_950_arn = module.py310_pillow_950.layer_arn py310_rasterio_no_numpy_arn = module.py310_rasterio_no_numpy.layer_arn py310_shapely_no_numpy_arn = module.py310_shapely_no_numpy.layer_arn + py313_geopandas_0144_arn = module.py313_geopandas_0144.layer_arn + py313_geopandas_112_arn = module.py313_geopandas_112.layer_arn + py313_mercantile_121_arn = module.py313_mercantile_121.layer_arn + py313_numpy_235_arn = module.py313_numpy_235.layer_arn + py313_pandas_233_arn = module.py313_pandas_233.layer_arn + py313_pillow_1200_arn = module.py313_pillow_1200.layer_arn + py313_rasterio_150_arn = module.py313_rasterio_150.layer_arn + py313_shapely_212_arn = module.py313_shapely_212.layer_arn } lists = {} strings = {} diff --git a/terraform/modules/lambda_layer/scripts/build.sh b/terraform/modules/lambda_layer/scripts/build.sh index e9448dc..bb34e5a 100755 --- a/terraform/modules/lambda_layer/scripts/build.sh +++ b/terraform/modules/lambda_layer/scripts/build.sh @@ -12,7 +12,7 @@ CONTAINER_NAME="container_$(sha1sum ${LAYER_PATH}/foo.txt |cut -c 1-8)" pushd "${LAYER_PATH}" echo "BUILD image ${IMAGE}" -docker build --no-cache -t "${IMAGE}" . +docker build --no-cache -t "${IMAGE}" . --platform=linux/amd64 echo "CREATE container ${CONTAINER_NAME}" docker run -itd --name "${CONTAINER_NAME}" "${IMAGE}" /bin/bash diff --git a/terraform/modules/lambda_layer/variables.tf b/terraform/modules/lambda_layer/variables.tf index 528089b..5798991 100644 --- a/terraform/modules/lambda_layer/variables.tf +++ b/terraform/modules/lambda_layer/variables.tf @@ -7,6 +7,7 @@ variable "name_suffix" { type = string default = "" } + variable "hash_script" { description = "Path to script to generate hash of source contents" type = string diff --git a/terraform/outputs.tf b/terraform/outputs.tf index cbbe577..be8caa1 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -22,6 +22,10 @@ output "py310_rasterio_138_arn" { value = module.py310_rasterio_138.layer_arn } +output "py310_rasterio_no_numpy_1310_arn" { + value = module.py310_rasterio_no_numpy_1310.layer_arn +} + output "py310_shapely_185_arn" { value = module.py310_shapely_185.layer_arn } @@ -46,3 +50,36 @@ output "py310_rasterio_no_numpy_arn" { output "py310_shapely_no_numpy_arn" { value = module.py310_shapely_no_numpy.layer_arn } + +# New: +output "py313_geopandas_0144_arn" { + value = module.py313_geopandas_0144.layer_arn +} + +output "py313_geopandas_112_arn" { + value = module.py313_geopandas_112.layer_arn +} + +output "py313_mercantile_121_arn" { + value = module.py313_mercantile_121.layer_arn +} + +output "py313_numpy_235_arn" { + value = module.py313_numpy_235.layer_arn +} + +output "py313_pandas_233_arn" { + value = module.py313_pandas_233.layer_arn +} + +output "py313_pillow_1200_arn" { + value = module.py313_pillow_1200.layer_arn +} + +output "py313_rasterio_150_arn" { + value = module.py313_rasterio_150.layer_arn +} + +output "py313_shapely_212_arn" { + value = module.py313_shapely_212.layer_arn +} diff --git a/terraform/versions.tf b/terraform/versions.tf index 86be3dd..570fe89 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,9 +2,9 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4, < 5" + version = ">= 5, < 6" region = "us-east-1" } } - required_version = ">= 0.13, < 0.14" + required_version = "= 0.13.3" } \ No newline at end of file