diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..ce6b0b4 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,72 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package test + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '0 8 * * 1,4' + +jobs: + unit-tests: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-22.04", "ubuntu-latest"] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Lint + run: | + python -m pip install pre-commit + pre-commit run --show-diff-on-failure --all-files + + - name: Install dependencies + env: + CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt + run: | + sudo apt-add-repository -y ppa:ubuntugis/ubuntugis-unstable + sudo apt-get -y update + sudo apt-get install -y gdal-bin libgdal-dev libproj-dev libgeos-dev + python -m pip install --upgrade pip wheel + pip install -e .[complete,test] + pip freeze + + - name: Start containers + run: docker compose -f "docker-compose.yml" up -d + + # run tests + - name: run + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: pytest -v --cov mapchete-hub-cli --cov-report xml:coverage.xml --cov-report=term-missing:skip-covered --junitxml=pytest.xml + + # Upload the report to Codecov + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: mapchete/mapchete-hub-cli + + # this will let the workflow fail if coverage is below 100% + - name: Pytest coverage + run: coverage report --skip-covered --show-missing + # --fail-under 100 + + - name: Stop containers + if: always() + run: docker compose -f "docker-compose.yml" down diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..0499260 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Build and publish + env: + HATCH_INDEX_USER: "__token__" + HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} + run: | + hatch build + hatch publish diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9f10361..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,127 +0,0 @@ -stages: - - codecheck - - build - - test - - package - -# python packaging & EOX PyPy releases # -######################################## - -precommit: - stage: codecheck - tags: - - privileged - image: registry.gitlab.eox.at/maps/docker-base/python:3.11-slim-buster - before_script: - - apt update && apt install -y --no-install-recommends git - - pip install pre-commit - script: - - pre-commit run --all-files - -# build python packages -build_python: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - stage: build - artifacts: - paths: - - dist/ - before_script: - - pip install --upgrade pip hatch - script: - - hatch build - -# test python packages -test_wheel: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - needs: ["build_python"] - stage: test - before_script: - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - -r requirements_test.txt - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - --upgrade - `ls dist/mapchete_hub_cli*.tar.gz -1 | head -1`[test] - script: - - mhub --help - - pytest -v --cov=mapchete_hub_cli tests/ && coverage report - -test_tar: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - needs: ["build_python"] - stage: test - before_script: - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - -r requirements_test.txt - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - --upgrade - `ls dist/mapchete_hub_cli*.tar.gz -1 | head -1`[test] - script: - - mhub --help - - pytest -v --cov=mapchete_hub_cli tests/ && coverage report - - -# INTEGRATION TESTS CURRENTLY NOT WORKING ON OUR CI -# # full test suite # -# test_integration: -# image: docker/compose:1.26.0 -# needs: ["build_python"] -# stage: test -# timeout: 30m -# retry: 2 -# before_script: -# - docker info -# - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY -# - export MHUB_PORT=$(( 5000 + $RANDOM % 1000 )) -# script: -# # run tests -# - echo "run mhub on port ${MHUB_PORT}" -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# up \ -# --exit-code-from mhub_cli_tester \ -# --build -# after_script: -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# down \ -# -v \ -# --rmi all \ -# --remove-orphans \ -# || true -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# rm -fv \ -# || true - - -# release package -package: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - stage: package - before_script: - - pip install --upgrade pip hatch - script: - - hatch publish -r https://gitlab.eox.at/api/v4/projects/255/packages/pypi dist/mapchete_hub_cli* - only: - - tags diff --git a/LICENSE b/LICENSE index bb9a60b..64946c6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 - 2024 EOX IT Services +Copyright (c) 2018 - 2025 EOX IT Services Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index 3a7be02..c618665 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,129 @@ -================ mapchete Hub CLI ================ -CLI and Python bindings to mapchete Hub API. +A command line interface for mapchete Hub. + +.. image:: https://img.shields.io/pypi/v/mapchete-hub-cli.svg + :target: https://pypi.org/project/mapchete-hub-cli/ + +.. image:: https://img.shields.io/pypi/l/mapchete-hub-cli.svg + :target: https://github.com/mapchete/mapchete-hub-cli/blob/main/LICENSE + +.. image:: https://img.shields.io/github/actions/workflow/status/mapchete/mapchete-hub-cli/python-package.yml?label=tests + :target: https://github.com/mapchete/mapchete-hub-cli/actions + +.. image:: https://codecov.io/gh/mapchete/mapchete-hub-cli/graph/badge.svg?token=VD1YOF3QA2 + :target: https://codecov.io/gh/mapchete/mapchete-hub-cli + +.. image:: https://img.shields.io/github/repo-size/mapchete/mapchete-hub-cli + :target: https://github.com/mapchete/mapchete-hub-cli + +About mapchete Hub CLI +======================== + +The mapchete Hub CLI provides a user-friendly command line interface (``mhub``) to interact with a running mapchete Hub instance. It simplifies the process of submitting, monitoring, and managing processing jobs without needing to write custom scripts or use tools like `curl`. + +This CLI is the recommended tool for users who want to script workflows or manage mapchete Hub jobs from their terminal. + + +Key Features +============ + +* **💻 Full Job Lifecycle Management**: Intuitive commands to ``execute``, ``job`` (status), ``jobs`` (list), ``cancel``, and ``retry`` processing jobs. +* **⚙️ Inspection**: List available ``processes`` on a remote mapchete Hub and check remote package ``--remote-versions``. +* **🔒 Authentication Support**: Handles basic authentication via command line options (``-u``, ``-p``) or environment variables. +* **🤖 Script-Friendly**: Designed for easy integration into automated scripts and workflows. + + +Installation +============ + +Install the CLI and its dependencies from PyPI: + +.. code-block:: bash + + pip install mapchete-hub-cli + + +Quickstart +========== + +**1. Point to your mapchete Hub** + +All commands require you to specify the mapchete Hub's URL using the ``-h`` or ``--host`` option. If the mapchete Hub is running on the same machine with default settings (``0.0.0.0:5000``), you may not need this option. + +**2. List Available Processes** + +You can see what processes are available on the target mapchete Hub instance: + +.. code-block:: bash + + mhub -h 0.0.0.0:5000 processes -License -------- +**3. Prepare a mapchete File** -MIT License +Create a `mapchete process file `_ using YAML syntax. **Note that all input and output paths must be absolute**, as they are resolved on the remote server. -Copyright (c) 2018 - 2024 `EOX IT Services`_ +For example, `hillshade.mapchete`: + +.. code-block:: yaml + + process: mapchete.processes.examples.hillshade + zoom_levels: + - 10 + pyramid: + grid: geodetic + input: + dem: https://data.example.com/path/to/dem.tif + output: + path: s3://my-bucket/path/to/hillshade_output + format: GTiff + +**4. Execute the Process** + +Use the ``execute`` command to send the job to the mapchete Hub. + +.. code-block:: bash + + mhub -h 0.0.0.0:5000 execute hillshade.mapchete + +The mapchete Hub will return a unique `job_id`: + +.. code-block:: text + + 6227b68b-b7e6-4f40-a396-85763197f481 + +**5. Check Job Status and Progress** + +Use the `job_id` with the ``job`` command to get a detailed status report, or ``progress`` for a quick look at the progress percentage. + +.. code-block:: bash + + # Get detailed status + mhub -h 0.0.0.0:5000 job 6227b68b-b7e6-4f40-a396-85763197f481 + + # Get just the progress + mhub -h 0.0.0.0:5000 progress 6227b68b-b7e6-4f40-a396-85763197f481 + +**6. Manage Jobs** + +You can also list all jobs or cancel a running job: + +.. code-block:: bash + + # List all jobs on the mapchete Hub + mhub -h 0.0.0.0:5000 jobs + + # Cancel a specific job + mhub -h 0.0.0.0:5000 cancel 6227b68b-b7e6-4f40-a396-85763197f481 + + +Contributing +============ + +This is an open-source project and we welcome contributions! Please see the `Contributing Guide `_ in the main ``mapchete`` repository for guidelines on how to get started. + +Acknowledgements +================ -.. _`EOX IT Services`: https://eox.at/ +The initial development of this project was made possible with the resources and support of `EOX IT Services GmbH `_. \ No newline at end of file diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 1cc918d..0000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3.6" - -services: - - mhub_cli_tester: - build: - context: . - args: - BASE_IMAGE: registry.gitlab.eox.at/maps/mapchete_hub/mhub - BASE_IMAGE_TAG: "2024.10.3" - EOX_PYPI_TOKEN: ${EOX_PYPI_TOKEN} - depends_on: - - server - environment: - - MHUB_HOST=http://server:${MHUB_PORT:-5000} - command: /bin/bash -c "echo $MHUB_HOST; sleep 5; pytest -v --cov=mapchete_hub_cli && coverage report" - networks: - - frontend - -networks: - frontend: diff --git a/docker-compose.yml b/docker-compose.yml index 0c82fc0..25b8064 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: command: [ "--bind_ip_all" ] server: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 depends_on: - dask_scheduler - db @@ -41,7 +41,7 @@ services: ] dask_scheduler: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 networks: - backend command: @@ -54,7 +54,7 @@ services: ] dask_worker: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 depends_on: - dask_scheduler networks: diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 65098cc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -click -oyaml -pydantic -requests -tqdm \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 3e5a678..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1 +0,0 @@ -pre_commit \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt deleted file mode 100644 index 001643c..0000000 --- a/requirements_test.txt +++ /dev/null @@ -1,7 +0,0 @@ -httpx -mapchete_hub>=2023.12.0 -pytest -pytest-cov -pytest-env -pytest-mongodb -requests \ No newline at end of file