Merge pull request #4 from nsidc/add-automated-testing

Add automated testing

Author: trey-stafford

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/action@v3.0.1

.github/workflows/test.yml

@@ -0,0 +1,27 @@
+name: tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: "Run backend tests"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "Check out repository"
+        uses: "actions/checkout@v4"
+
+      - name: "Build container image"
+        run: |
+          docker build --tag "test" .
+
+      # mypy
+      - name: "Run mypy"
+        run: "docker run test mypy dat_backend/ test/"
+
+      # Unit tests
+      - name: "Run unit tests"
+        run: "docker run test pytest test/unit"

.gitignore

@@ -1,2 +1,2 @@
 **/__pycache__/
-nginx/logs/*
+nginx/logs/*

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+default_language_version:
+  python: "python3.12"
+
+repos:
+  - repo: "https://github.com/pre-commit/pre-commit-hooks"
+    rev: "v5.0.0"
+    hooks:
+      - id: "check-added-large-files"
+      - id: "check-vcs-permalinks"
+      - id: "end-of-file-fixer"
+
+  - repo: "https://github.com/charliermarsh/ruff-pre-commit"
+    rev: "v0.9.9"
+    hooks:
+      - id: "ruff"
+        # NOTE: "--exit-non-zero-on-fix" is important for CI to function
+        # correctly!
+        args:
+          ["--fix", "--exit-non-zero-on-fix", "--verbose", "--line-length=88"]
+
+  - repo: "https://github.com/psf/black"
+    rev: "25.1.0"
+    hooks:
+      - id: "black"
+
+  - repo: "https://github.com/jendrikseipp/vulture"
+    rev: "v2.14"
+    hooks:
+      - id: "vulture"
+        args:
+          # Ignore flask routes that we define with the `@api.route` decorator.
+          - "--ignore-decorators"
+          - "@api.route"
+          # Ignore names of attrs in a ctx object in python_script.py that
+          # vulture incorrectly believes are unused.
+          - "--ignore-names"
+          - "check_hostname,verify_mode"
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: "v2.4.1"
+    hooks:
+      - id: codespell
+
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: "v3.5.2"
+    hooks:
+      - id: prettier
+        types_or: [yaml, markdown, html, css, scss, javascript, json]
+        args: [--prose-wrap=always]

Dockerfile

@@ -5,12 +5,14 @@ COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/env.yaml
 RUN micromamba install -y -n base -f /tmp/env.yaml && \
     micromamba clean --all --yes
 
-COPY src/* .
+COPY src/dat_backend/ ./dat_backend/
+COPY test/ ./test/
+COPY pyproject.toml .
 
 EXPOSE 5000
 
 # TODO: handle ssl w/ gunicorn? Self-signed cert on this image easiest?
 # CMD /bin/bash -c "gunicorn --bind 0.0.0.0:5000 --workers 3 app:app"
 
 # TODO this might not work
-CMD /bin/bash -c "PYTHONPATH=./ python dat_backend/app.py"
+CMD /bin/bash -c "PYTHONPATH=./ python dat_backend/app.py"

README.md

@@ -1,37 +1,68 @@
 # Data Access Tool (DAT) Backend
 
-Backend services for the [Data Access Tool UI](https://github.com/nsidc/data-access-tool-ui).
+Backend services for the
+[Data Access Tool UI](https://github.com/nsidc/data-access-tool-ui).
 
 A Flask-based API provides:
 
-* The`python_script.py` download script for data granules matching a user's filters.
-* The
+- The`python_script.py` download script for data granules matching a user's
+  filters.
+- The
   [getLinks](https://github.com/nasa/earthdata-download/blob/main/docs/GET_LINKS.md)
-  service required for DAT integration with the [NASA Earthdata
-  Downloader](https://github.com/nasa/earthdata-download).
-
+  service required for DAT integration with the
+  [NASA Earthdata Downloader](https://github.com/nasa/earthdata-download).
 
 Note that this service was originally a part of the
 [hermes-api](https://bitbucket.org/nsidc/hermes-api/src). It was moved to a
-standalone service to support the decomissioning of ECS and the rest of the
+standalone service to support the decommissioning of ECS and the rest of the
 hermes stack planned for July 2026.
 
-
 ## Dev
 
-### Testing the download script
+### Pre-commit
+
+This project uses [pre-commit](https://pre-commit.com/) to run pre-commit hooks
+that check and format this project's code for stylistic consistency (e.g., using
+`ruff` and `black`) .
+
+The pre-commit configuration for this project can be found in
+`.pre-commit-config.yaml`. Configuration for specific tools (e.g., `vulture`) is
+given in the included `pyproject.toml`.
+
+For more information about using `pre-commit`, please sese the
+[Scientific Python Library Development Guide's section on pre-commit](https://learn.scientific-python.org/development/guides/gha-basic/#pre-commit).
+
+To install pre-commit to run checks for each commit you make:
+
+```
+$ pre-commit install
+```
+
+To manually run the pre-commit hooks without a commit:
+
+```
+$ pre-commit run --all-files
+```
+
+> [!NOTE] GitHub actions are configured to run pre-commit for all PRs and pushes
+> to the `main` branch. See
+> [.github/workflows/pre-commit.yml](.github/workflows/pre-commit.yml).
 
-The tests for the download script assumes a .netrc file is setup for the current
-user to login to earthdata. Setup a `.netrc` with credentials for earthdata
-login by e.g., [using the earthaccess
-library](https://earthaccess.readthedocs.io/en/latest/howto/authenticate/)
+### Running tests
 
-Once a `.netrc` file is setup:
+Before manually running tests, setup the `EARTHDATA_USERNAME` and
+`EARTHDATA_PASSWORD` envvars, which are necessary for integration tests.
+
+Next, to run all tests:
 
 ```
-PYTHONPATH=./src/ pytest test/
+scripts/run_tests.sh
 ```
 
+> [!NOTE] GitHub actions are configured to run unit tests that do not require
+> Earthdata login credentials for all PRs and pushes to the `main` branch. See
+> [.github/workflows/test.yml](.github/workflows/test.yml).
+
 ### Testing the EDD integration
 
 An example deep-link to initiate EDD downloads:
@@ -46,16 +77,14 @@ that looks like the above.
 The GET request to `earthdata-download://startDownload` should include the
 following query parameters:
 
-* `getLinks`: URI for `/api/get-links/`. This URI will specify the
+- `getLinks`: URI for `/api/get-links/`. This URI will specify the
   `cmr_request_params` query-parameter, which is a string representing the CMR
   query parameters mapping to a user's selections in the DAT.
-* `downloadId`: The dataset ID and version for the current order (e.g., ATL06 v6
+- `downloadId`: The dataset ID and version for the current order (e.g., ATL06 v6
   is `atl06_06`)
-* `authUrl`: URI for `/api/earthdata/auth/`. EDD will
-  use this to initiate a token exchange with URS to authenticate user
-  downloads. This URL must include
+- `authUrl`: URI for `/api/earthdata/auth/`. EDD will use this to initiate a
+  token exchange with URS to authenticate user downloads. This URL must include
   `eddRedirect=earthdata-download%3A%2F%2FauthCallback` as a query parameter.
 
-> [!WARNING]
-> As of this writing, the CMR query parameters are hard-coded to always return a
-> small subset of ATL06 v6 data.
+> [!WARNING] As of this writing, the CMR query parameters are hard-coded to
+> always return a small subset of ATL06 v6 data.

docker-compose-test.yml

@@ -0,0 +1,12 @@
+services:
+  # TODO: rename this.
+  downloader-script:
+    # TODO: dev stuff extracted to dev env yaml
+    build: .
+    volumes:
+      - "./src/dat_backend:/tmp/dat_backend/"
+      - "./test:/tmp/test/"
+    command: 'bash -c "mypy dat_backend/ test/ && pytest test/"'
+    environment:
+      - EARTHDATA_USERNAME=${EARTHDATA_USERNAME:?EARTHDATA_USERNAME must be set}
+      - EARTHDATA_PASSWORD=${EARTHDATA_PASSWORD:?EARTHDATA_PASSWORD must be set}

environment.yml

@@ -13,4 +13,6 @@ dependencies:
 
   # Dev deps
   - pytest ~=8.2
-  - pyopenssl  # required for test server w/ ssl
+  - pyopenssl # required for test server w/ ssl
+  - pre-commit ~=4.1
+  - mypy ~=1.15

nginx/Dockerfile

@@ -12,4 +12,4 @@ RUN useradd --uid 1000 --user-group vagrant && \
     touch /var/run/nginx.pid && \
     chown -R vagrant:vagrant /var/run/nginx.pid && \
     chown -R vagrant:vagrant /var/cache/nginx && \
-    chown -R vagrant:vagrant /etc/nginx
+    chown -R vagrant:vagrant /etc/nginx

nginx/dat.conf

@@ -46,4 +46,4 @@ server {
     proxy_set_header      X-Script-Name $http_x_script_name;
   }
 
-}
+}