Use docker secrets for EDL credentials

.gitignore

@@ -1,3 +1,5 @@
+secrets/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Dockerfile

@@ -1,20 +1,5 @@
-# check=skip=JSONArgsRecommended
 FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
 
-ARG EARTHDATA_USERNAME
-ARG EARTHDATA_PASSWORD
-
-# Check if EARTHDATA_USERNAME and EARTHDATA_PASSWORD are provided
-RUN <<'EOF'
-if [ -z "$EARTHDATA_USERNAME" ] || [ -z "$EARTHDATA_PASSWORD" ]; then
-  echo "Error: EARTHDATA_USERNAME and EARTHDATA_PASSWORD build args must be provided"
-  exit 1
-fi
-
-echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME} password ${EARTHDATA_PASSWORD}" > ~/.netrc
-unset EARTHDATA_USERNAME EARTHDATA_PASSWORD
-EOF
-
 RUN <<'EOF'
 apt-get update
 apt-get -y --no-install-recommends install libexpat1
@@ -24,11 +9,16 @@ EOF
 WORKDIR /app
 
 COPY pyproject.toml uv.lock README.md LICENSE ./
+RUN uv sync --no-dev --frozen --extra uvicorn
+
 COPY titiler ./titiler
 
-RUN uv sync --no-dev --frozen --extra uvicorn
+RUN \
+  --mount=type=secret,id=earthdata-username,required,env=EARTHDATA_USERNAME \
+  --mount=type=secret,id=earthdata-password,required,env=EARTHDATA_PASSWORD \
+  echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME:?} password ${EARTHDATA_PASSWORD:?}" > ~/.netrc
 
 # http://www.uvicorn.org/settings/
 ENV HOST=0.0.0.0
 ENV PORT=80
-CMD uv run --no-dev uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload
+CMD ["/bin/sh", "-c", "uv run --no-dev uvicorn titiler.cmr.main:app --host ${HOST} --port ${PORT} --log-level debug --reload"]

README.md

@@ -103,8 +103,27 @@ echo "machine urs.earthdata.nasa.gov login ${EARTHDATA_USERNAME} password ${EART
 
 ## Docker deployment
 
-You can run the application in a docker container using the docker-compose.yml file.
-The docker container is configured to read the `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables so make sure set those before starting the docker network.
+You can run the application in a docker container using the `docker-compose.yml`
+file. The docker container is configured to use secrets for your Earthdata Login
+credentials, so make sure the secrets exist by first running the following
+script:
+
+```bash
+uv run scripts/write-secrets.py
+```
+
+The script will look for your Earthdata Login credentials in the following
+places, in descending order of precedence:
+
+- Exported `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables
+- A `.env.secrets` file
+- A `.env` file
+- A netrc file (defaults to `~/.netrc`, but can be set via `NETRC` environment
+  variable)
+- As a last resort, prompts for input
+
+If you ever change your credentials, rerun the script to repopulate the secrets.
+Once the secrets are generated, you can start the application as follows:
 
 ```bash
 docker compose up --build

docker-compose.yml

@@ -1,13 +1,36 @@
+secrets:
+  # To generate the secret files, run the following command:
+  #
+  #    uv run scripts/write-secrets.py
+  #
+  # See comments in the script describing how it obtains the values to write.
+  #
+  # Unfortunately, when specifying secrets via docker compose, we are currently
+  # forced to use secret files rather than the more convenient environment
+  # variable option due to a bug in how docker compose passes env-var-based
+  # secret values to the docker build process.
+  #
+  # Although the following issues are closed, there still appears to be broken
+  # behavior, and there are other related issues mentioned in various comments:
+  #
+  # - https://github.com/docker/compose/issues/13255
+  # - https://github.com/docker/compose/issues/13235
+  #
+  earthdata-username:
+    file: ./secrets/earthdata-username.txt
+  earthdata-password:
+    file: ./secrets/earthdata-password.txt
+
 services:
   tiler:
     container_name: tiler-cmr
     platform: linux/amd64
     build:
       context: .
       dockerfile: Dockerfile
-      args:
-        EARTHDATA_USERNAME: ${EARTHDATA_USERNAME}
-        EARTHDATA_PASSWORD: ${EARTHDATA_PASSWORD}
+      secrets:
+        - earthdata-username
+        - earthdata-password
     ports:
       - "8081:8081"
     volumes:
@@ -20,9 +43,6 @@ services:
       - TITILER_CMR_DEBUG=TRUE
       - TITILER_CMR_S3_AUTH_STRATEGY=netrc
       - TITILER_CMR_S3_AUTH_ACCESS=external
-      # earthdata authentication
-      - EARTHDATA_USERNAME=${EARTHDATA_USERNAME}
-      - EARTHDATA_PASSWORD=${EARTHDATA_PASSWORD}
       # GDAL Config
       # This option controls the default GDAL raster block cache size.
       # If its value is small (less than 100000), it is assumed to be measured in megabytes, otherwise in bytes.

scripts/write-secrets.py

@@ -0,0 +1,122 @@
+"""Initialize Docker secrets with Earthdata credentials for local development.
+
+Run this once prior to running "docker compose up --build".  There is no need
+to run it again unless your Earthdata credentials change.
+
+Creates the following secret files within the ./secrets directory (git-ignored)
+for use with Docker:
+
+- earthdata-username.txt
+- earthdata-password.txt
+
+If necessary, creates the ./secrets directory with permissions 700, and creates
+files within it with permissions 600.
+
+Attempts to populate each secret as follows, in descending order of precedence:
+
+1. Looks for corresponding environment variable (EARTHDATA_USERNAME and
+   EARTHDATA_PASSWORD).
+2. Loads from .env.secrets file, if exists.
+3. Loads from .env file, if exists.
+4. Loads from netrc file, if exists (defaults to ~/.netrc, but can set NETRC env var)
+5. Prompts user as last resort.
+
+This will ALWAYS overwrite existing secret files.
+"""
+
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "python-dotenv>=1.2.1",
+# ]
+# ///
+import logging
+import netrc
+import os
+from getpass import getpass
+from pathlib import Path
+from typing import Mapping
+
+from dotenv import dotenv_values
+
+logging.basicConfig(level=logging.INFO)
+
+
+def read_env() -> Mapping[str, str]:
+    """Read environment variables, env files, and netrc as unified 'environment'.
+
+    Returns a mapping of "environment variables", obtained from the following
+    sources in descending order of precedence:
+
+    - Environment variables (highest precedence)
+    - Variables from .env.secrets, if it exists
+    - Variables from .env, if it exists
+    - netrc file, if it exists (default location can be overridden via NETRC
+      environment variable): if an entry is found for host urs.earthdata.nasa.gov,
+      sets default values on returned mapping for EARTHDATA_USERNAME and
+      EARTHDATA_PASSWORD (i.e., only visible if these are not already set by
+      a source listed above, with higher precedence)
+    """
+    env = {
+        **dotenv_values(".env"),
+        **dotenv_values(".env.secrets"),
+        **os.environ,
+    }
+
+    if auth := netrc.netrc(env.get("NETRC")).authenticators("urs.earthdata.nasa.gov"):
+        username, _, password = auth
+        env.setdefault("EARTHDATA_USERNAME", username)
+        env.setdefault("EARTHDATA_PASSWORD", password)
+
+    return env
+
+
+def ensure_secrets_dir() -> Path:
+    """Make sure the secrets directory exists with mode 0o700 (rwx for user only)."""
+    path = Path("./secrets")
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+
+    return path
+
+
+def to_kebab_case(name: str) -> str:
+    """Convert variable name to kebab case.
+
+    Example: EARTHDATA_USERNAME becomes earthdata-username
+    """
+    return name.lower().replace("_", "-")
+
+
+def ask_user(prompt: str, sensitive: bool) -> str:
+    """Prompt user for a possibly sensitive value."""
+    return getpass(prompt) if sensitive else input(prompt)
+
+
+def write_secret(secrets_dir: Path, name: str, value: str) -> None:
+    """Write a secret (with only rw perms for user) to the secrets directory."""
+    file = (secrets_dir / name).with_suffix(".txt")
+    file.write_text(value)
+    file.chmod(0o600)
+
+
+def main():
+    """Write Earthdata credentials as secrets for Docker.
+
+    Looks for EDL credentials in the environment, and if not found, prompts
+    user to enter values, then writes them as secret files for Docker to use.
+    """
+    env = read_env()
+    secrets_dir = ensure_secrets_dir()
+
+    descriptors = (
+        ("EARTHDATA_USERNAME", "Earthdata username: ", False),
+        ("EARTHDATA_PASSWORD", "Earthdata password: ", True),
+    )
+
+    for name, prompt, sensitive in descriptors:
+        value = env.get(name) or ask_user(prompt, sensitive)
+        write_secret(secrets_dir, to_kebab_case(name), value)
+
+
+if __name__ == "__main__":
+    main()