diff --git a/.gitignore b/.gitignore index 87ee7939..6f57f480 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +secrets/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index 4ba49214..cd17ef58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,5 @@ -# check=skip=JSONArgsRecommended FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim -ARG EARTHDATA_USERNAME -ARG EARTHDATA_PASSWORD - -# Check if EARTHDATA_USERNAME and EARTHDATA_PASSWORD are provided -RUN <<'EOF' -if [ -z "$EARTHDATA_USERNAME" ] || [ -z "$EARTHDATA_PASSWORD" ]; then - echo "Error: EARTHDATA_USERNAME and EARTHDATA_PASSWORD build args must be provided" - exit 1 -fi - -echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME} password ${EARTHDATA_PASSWORD}" > ~/.netrc -unset EARTHDATA_USERNAME EARTHDATA_PASSWORD -EOF - RUN <<'EOF' apt-get update apt-get -y --no-install-recommends install libexpat1 @@ -24,11 +9,16 @@ EOF WORKDIR /app COPY pyproject.toml uv.lock README.md LICENSE ./ +RUN uv sync --no-dev --frozen --extra uvicorn + COPY titiler ./titiler -RUN uv sync --no-dev --frozen --extra uvicorn +RUN \ + --mount=type=secret,id=earthdata-username,required,env=EARTHDATA_USERNAME \ + --mount=type=secret,id=earthdata-password,required,env=EARTHDATA_PASSWORD \ + echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME:?} password ${EARTHDATA_PASSWORD:?}" > ~/.netrc # http://www.uvicorn.org/settings/ ENV HOST=0.0.0.0 ENV PORT=80 -CMD uv run --no-dev uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload +CMD ["/bin/sh", "-c", "uv run --no-dev uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload"] diff --git a/README.md b/README.md index 4dc05795..7fa168f6 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,27 @@ echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME} password ${EART ## Docker deployment -You can run the application in a docker container using the docker-compose.yml file. -The docker container is configured to read the `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables so make sure set those before starting the docker network. +You can run the application in a docker container using the `docker-compose.yml` +file. The docker container is configured to use secrets for your Earthdata Login +credentials, so make sure the secrets exist by first running the following +script: + +```bash +uv run scripts/write-secrets.py +``` + +The script will look for your Earthdata Login credentials in the following +places, in descending order of precedence: + +- Exported `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables +- A `.env.secrets` file +- A `.env` file +- A netrc file (defaults to `~/.netrc`, but can be set via `NETRC` environment + variable) +- As a last resort, prompts for input + +If you ever change your credentials, rerun the script to repopulate the secrets. +Once the secrets are generated, you can start the application as follows: ```bash docker compose up --build diff --git a/docker-compose.yml b/docker-compose.yml index 86fbf9dd..07438e02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,26 @@ +secrets: + # To generate the secret files, run the following command: + # + # uv run scripts/write-secrets.py + # + # See comments in the script describing how it obtains the values to write. + # + # Unfortunately, when specifying secrets via docker compose, we are currently + # forced to use secret files rather than the more convenient environment + # variable option due to a bug in how docker compose passes env-var-based + # secret values to the docker build process. + # + # Although the following issues are closed, there still appears to be broken + # behavior, and there are other related issues mentioned in various comments: + # + # - https://github.com/docker/compose/issues/13255 + # - https://github.com/docker/compose/issues/13235 + # + earthdata-username: + file: ./secrets/earthdata-username.txt + earthdata-password: + file: ./secrets/earthdata-password.txt + services: tiler: container_name: tiler-cmr @@ -5,9 +28,9 @@ services: build: context: . dockerfile: Dockerfile - args: - EARTHDATA_USERNAME: ${EARTHDATA_USERNAME} - EARTHDATA_PASSWORD: ${EARTHDATA_PASSWORD} + secrets: + - earthdata-username + - earthdata-password ports: - "8081:8081" volumes: @@ -20,9 +43,6 @@ services: - TITILER_CMR_DEBUG=TRUE - TITILER_CMR_S3_AUTH_STRATEGY=netrc - TITILER_CMR_S3_AUTH_ACCESS=external - # earthdata authentication - - EARTHDATA_USERNAME=${EARTHDATA_USERNAME} - - EARTHDATA_PASSWORD=${EARTHDATA_PASSWORD} # GDAL Config # This option controls the default GDAL raster block cache size. # If its value is small (less than 100000), it is assumed to be measured in megabytes, otherwise in bytes. diff --git a/scripts/write-secrets.py b/scripts/write-secrets.py new file mode 100644 index 00000000..9f23fb98 --- /dev/null +++ b/scripts/write-secrets.py @@ -0,0 +1,122 @@ +"""Initialize Docker secrets with Earthdata credentials for local development. + +Run this once prior to running "docker compose up --build". There is no need +to run it again unless your Earthdata credentials change. + +Creates the following secret files within the ./secrets directory (git-ignored) +for use with Docker: + +- earthdata-username.txt +- earthdata-password.txt + +If necessary, creates the ./secrets directory with permissions 700, and creates +files within it with permissions 600. + +Attempts to populate each secret as follows, in descending order of precedence: + +1. Looks for corresponding environment variable (EARTHDATA_USERNAME and + EARTHDATA_PASSWORD). +2. Loads from .env.secrets file, if exists. +3. Loads from .env file, if exists. +4. Loads from netrc file, if exists (defaults to ~/.netrc, but can set NETRC env var) +5. Prompts user as last resort. + +This will ALWAYS overwrite existing secret files. +""" + +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "python-dotenv>=1.2.1", +# ] +# /// +import logging +import netrc +import os +from getpass import getpass +from pathlib import Path +from typing import Mapping + +from dotenv import dotenv_values + +logging.basicConfig(level=logging.INFO) + + +def read_env() -> Mapping[str, str]: + """Read environment variables, env files, and netrc as unified 'environment'. + + Returns a mapping of "environment variables", obtained from the following + sources in descending order of precedence: + + - Environment variables (highest precedence) + - Variables from .env.secrets, if it exists + - Variables from .env, if it exists + - netrc file, if it exists (default location can be overridden via NETRC + environment variable): if an entry is found for host urs.earthdata.nasa.gov, + sets default values on returned mapping for EARTHDATA_USERNAME and + EARTHDATA_PASSWORD (i.e., only visible if these are not already set by + a source listed above, with higher precedence) + """ + env = { + **dotenv_values(".env"), + **dotenv_values(".env.secrets"), + **os.environ, + } + + if auth := netrc.netrc(env.get("NETRC")).authenticators("urs.earthdata.nasa.gov"): + username, _, password = auth + env.setdefault("EARTHDATA_USERNAME", username) + env.setdefault("EARTHDATA_PASSWORD", password) + + return env + + +def ensure_secrets_dir() -> Path: + """Make sure the secrets directory exists with mode 0o700 (rwx for user only).""" + path = Path("./secrets") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + + return path + + +def to_kebab_case(name: str) -> str: + """Convert variable name to kebab case. + + Example: EARTHDATA_USERNAME becomes earthdata-username + """ + return name.lower().replace("_", "-") + + +def ask_user(prompt: str, sensitive: bool) -> str: + """Prompt user for a possibly sensitive value.""" + return getpass(prompt) if sensitive else input(prompt) + + +def write_secret(secrets_dir: Path, name: str, value: str) -> None: + """Write a secret (with only rw perms for user) to the secrets directory.""" + file = (secrets_dir / name).with_suffix(".txt") + file.write_text(value) + file.chmod(0o600) + + +def main(): + """Write Earthdata credentials as secrets for Docker. + + Looks for EDL credentials in the environment, and if not found, prompts + user to enter values, then writes them as secret files for Docker to use. + """ + env = read_env() + secrets_dir = ensure_secrets_dir() + + descriptors = ( + ("EARTHDATA_USERNAME", "Earthdata username: ", False), + ("EARTHDATA_PASSWORD", "Earthdata password: ", True), + ) + + for name, prompt, sensitive in descriptors: + value = env.get(name) or ask_user(prompt, sensitive) + write_secret(secrets_dir, to_kebab_case(name), value) + + +if __name__ == "__main__": + main()