diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f99c529b..5f6b3df8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,14 +1,14 @@ { - "name": "datavizhub-dev", + "name": "zyra-dev", "dockerComposeFile": [ "../docker-compose.yml" ], - "service": "datavizhub", + "service": "zyra", "workspaceFolder": "/app", "overrideCommand": false, // ✅ Ensures VS Code starts the right services - "runServices": ["datavizhub", "redis"], + "runServices": ["zyra", "redis"], "composeProfiles": ["dev"], "customizations": { diff --git a/.devcontainer/entrypoint.sh b/.devcontainer/entrypoint.sh index ca04b400..1b74b8f4 100644 --- a/.devcontainer/entrypoint.sh +++ b/.devcontainer/entrypoint.sh @@ -19,6 +19,9 @@ if [[ -f .devcontainer/.env ]]; then echo "[entrypoint] Added ${key} to .env from .devcontainer/.env" fi } + # Prefer new ZYRA_* keys; keep DATAVIZHUB_* for compatibility + ensure_kv ZYRA_LLM_PROVIDER + ensure_kv ZYRA_LLM_MODEL ensure_kv DATAVIZHUB_LLM_PROVIDER ensure_kv DATAVIZHUB_LLM_MODEL ensure_kv OLLAMA_BASE_URL @@ -27,7 +30,7 @@ if [[ -f .devcontainer/.env ]]; then fi # Export all variables from .env into the container environment (dev only) -if [[ "${DATAVIZHUB_ENV:-dev}" == "dev" && -f .env ]]; then +if [[ "${ZYRA_ENV:-${DATAVIZHUB_ENV:-dev}}" == "dev" && -f .env ]]; then echo "[entrypoint] Loading environment variables from .env (dev only)" set -a source .env @@ -35,10 +38,27 @@ if [[ "${DATAVIZHUB_ENV:-dev}" == "dev" && -f .env ]]; then fi # ====== SERVICE START SECTION ====== -AUTOSTART_API=${DATAVIZHUB_AUTOSTART_API:-1} -AUTOSTART_RQ=${DATAVIZHUB_AUTOSTART_RQ:-${DATAVIZHUB_USE_REDIS:-0}} -API_HOST=${DATAVIZHUB_API_HOST:-0.0.0.0} -API_PORT=${DATAVIZHUB_API_PORT:-8000} +# Prefer ZYRA_* env names with legacy fallbacks +AUTOSTART_API=${ZYRA_AUTOSTART_API:-${DATAVIZHUB_AUTOSTART_API:-1}} +AUTOSTART_RQ=${ZYRA_AUTOSTART_RQ:-${DATAVIZHUB_AUTOSTART_RQ:-${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}}}} +API_HOST=${ZYRA_API_HOST:-${DATAVIZHUB_API_HOST:-0.0.0.0}} +API_PORT=${ZYRA_API_PORT:-${DATAVIZHUB_API_PORT:-8000}} + +# Map ZYRA_* -> DATAVIZHUB_* for runtime back-compat if legacy vars are unset +map_keys=( + USE_REDIS REDIS_URL AUTOSTART_API AUTOSTART_RQ UPLOAD_DIR MIN_DISK_MB REQUIRE_FFMPEG + API_HOST API_PORT VERBOSITY STRICT_ENV DEFAULT_STDIN + CORS_ALLOW_ALL CORS_ORIGINS API_KEY API_KEY_HEADER + RESULTS_TTL_SECONDS RESULTS_CLEAN_INTERVAL_SECONDS RESULTS_DIR QUEUE + LLM_PROVIDER LLM_MODEL WIZARD_EDITOR_MODE +) +for k in "${map_keys[@]}"; do + zy="ZYRA_${k}" + dv="DATAVIZHUB_${k}" + if [[ -n "${!zy:-}" && -z "${!dv:-}" ]]; then + export "${dv}=${!zy}" + fi +done mkdir -p .cache echo "[entrypoint] ===== $(date) API session =====" >> .cache/api.log @@ -46,10 +66,10 @@ echo "[entrypoint] ===== $(date) RQ worker session =====" >> .cache/rq.log wait_for_redis() { local url host port - url="${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0}" + url="${ZYRA_REDIS_URL:-${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0}}" # Basic validation to avoid accidental command injection or bad values if ! [[ "$url" =~ ^redis://[A-Za-z0-9._-]+(:[0-9]{1,5})?(/[0-9]+)?$ ]]; then - echo "[entrypoint] ERROR: Invalid DATAVIZHUB_REDIS_URL: "$url"" >&2 + echo "[entrypoint] ERROR: Invalid REDIS_URL: "$url"" >&2 return 1 fi host=$(echo "$url" | sed -E 's#^redis://([^:/]+):?([0-9]+)?.*#\1#') @@ -68,24 +88,24 @@ wait_for_redis() { } start_rq_worker() { - if pgrep -f "rq worker datavizhub" >/dev/null 2>&1; then + if pgrep -f "rq worker zyra" >/dev/null 2>&1; then echo "[entrypoint] RQ worker already running" else echo "[entrypoint] Starting RQ worker..." - ( DATAVIZHUB_USE_REDIS=1 poetry run rq worker datavizhub >> .cache/rq.log 2>&1 & ) + ( DATAVIZHUB_USE_REDIS=1 ZYRA_USE_REDIS=1 poetry run rq worker zyra >> .cache/rq.log 2>&1 & ) fi } start_api() { - if pgrep -f "uvicorn datavizhub.api.server:app" >/dev/null 2>&1; then + if pgrep -f "uvicorn zyra.api.server:app" >/dev/null 2>&1; then echo "[entrypoint] API already running" else echo "[entrypoint] Starting API on ${API_HOST}:${API_PORT}" - ( poetry run uvicorn datavizhub.api.server:app --host "${API_HOST}" --port "${API_PORT}" --reload >> .cache/api.log 2>&1 & ) + ( poetry run uvicorn zyra.api.server:app --host "${API_HOST}" --port "${API_PORT}" --reload >> .cache/api.log 2>&1 & ) fi } -if [[ "$AUTOSTART_RQ" == "1" && "$DATAVIZHUB_USE_REDIS" == "1" ]]; then +if [[ "$AUTOSTART_RQ" == "1" && "${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}}" == "1" ]]; then wait_for_redis || exit 1 fi diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 49e1068c..5d86acdd 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -17,10 +17,10 @@ query-filters: - exclude: id: py/path-injection paths: - - src/datavizhub/api/routers/jobs.py - - src/datavizhub/api/workers/executor.py + - src/zyra/api/routers/jobs.py + - src/zyra/api/workers/executor.py - exclude: id: py/uncontrolled-data-in-path-expression paths: - - src/datavizhub/api/routers/jobs.py - - src/datavizhub/api/workers/executor.py + - src/zyra/api/routers/jobs.py + - src/zyra/api/workers/executor.py diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 67698bc1..3a97fb81 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -12,7 +12,7 @@ jobs: environment: name: release - url: https://pypi.org/project/datavizhub/ + url: https://pypi.org/project/zyra/ permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..44b4716f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,48 @@ +name: Release on staging → main merge + +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + release: + if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'staging' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Extract version from pyproject.toml + id: get_version + run: | + VERSION=$(grep -Po '(?<=^version = ")[^"]*' pyproject.toml) + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Ensure version bump + run: | + if git ls-remote --tags origin | grep -q "refs/tags/v${{ steps.get_version.outputs.version }}$"; then + echo "❌ Version v${{ steps.get_version.outputs.version }} already exists. Please bump version in pyproject.toml before merging." + exit 1 + fi + + - name: Create and push Git tag + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git tag v${{ steps.get_version.outputs.version }} + git push origin v${{ steps.get_version.outputs.version }} + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.get_version.outputs.version }} + name: Release v${{ steps.get_version.outputs.version }} + body: | + ## ${{ github.event.pull_request.title }} + ${{ github.event.pull_request.body }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index a6e3f72d..da875131 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ Repository Guidelines Project Structure & Module Organization -- Code lives under `src/datavizhub/`: +- Code lives under `src/zyra/`: - `connectors/`: source/destination integrations and CLI registrars (`connectors.ingest`, `connectors.egress`). Prefer these over legacy `acquisition/` for new work. - `processing/`: data processing (e.g., GRIB/NetCDF/GeoTIFF) exposed via the CLI. - `visualization/`: visualization commands; CLI registration lives in `visualization/cli_register.py`. @@ -49,7 +49,7 @@ Project Structure & Module Organization - `api/` + `api_cli.py`: HTTP API and CLI entry points. - `transform/`: transform helpers (metadata, etc.). - `utils/`: shared helpers/utilities. - - `assets/`: packaged static resources; access with `importlib.resources` (`datavizhub.assets`). + - `assets/`: packaged static resources; access with `importlib.resources` (`zyra.assets`). - `cli.py`, `pipeline_runner.py`: root CLI and pipeline runner. Build, Test, and Development @@ -83,7 +83,7 @@ Security & Configuration - Do not commit secrets. Prefer IAM roles or env vars (e.g., used by S3 connectors). - Avoid hard-coded absolute paths; prefer env-configurable paths (e.g., `DATA_DIR`). -- Use `importlib.resources` for packaged assets under `datavizhub.assets`. +- Use `importlib.resources` for packaged assets under `zyra.assets`. - Pin dependencies when adding new ones; document any system deps (e.g., FFmpeg, PROJ/GEOS). Dependency Management (Poetry) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..9c023b26 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,35 @@ +cff-version: 1.2.0 +title: Zyra +message: ' Modular workflows for reproducible science' +type: software +authors: + - given-names: Eric + family-names: Hackathorn + email: eric.j.hackathorn@noaa.gov + orcid: 'https://orcid.org/0000-0002-9693-2093' + affiliation: NOAA Global Systems Laboratory +identifiers: + - type: doi + value: 10.5281/zenodo.xxxxxxx + description: replace with Zenodo DOI once registered +repository-code: 'https://github.com/NOAA-GSL/zyra' +url: 'https://github.com/NOAA-GSL/zyra/wiki' +repository: 'https://pypi.org/project/zyra/' +abstract: >- + Zyra is an open-source Python framework for creating + reproducible, modular, and visually compelling data + visualizations. It provides a flexible pipeline for data + acquisition, processing, rendering, and dissemination, + making it useful for scientists, educators, and developers + who need to explore, analyze, and communicate complex + scientific data. +keywords: + - Python + - data visualization + - scientific visualization + - open science + - reproducible research + - workflow automation + - data pipeline + - NOAA +license: MIT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ca0addb..1d42fb76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to DataVizHub +# Contributing to Zyra Thanks for your interest in contributing! This project thrives on community contributions, and we welcome improvements of all kinds. @@ -7,7 +7,7 @@ This project thrives on community contributions, and we welcome improvements of ## License and Contributor Terms -- DataVizHub is licensed under the MIT License. See `LICENSE` at the repository root. +- Zyra is licensed under the MIT License. See `LICENSE` at the repository root. - By submitting a pull request, issue suggestion, or any code/documentation/artwork (“Contribution”), you agree to license your Contribution under the MIT License, and you represent that you have the right to do so. diff --git a/Dockerfile b/Dockerfile index 7919ed2f..6194ff89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN poetry install --with dev --all-extras # Healthcheck for non-compose runs HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=5 \ - CMD sh -c "curl -fsS http://localhost:${DATAVIZHUB_API_PORT:-8000}/ready || exit 1" + CMD sh -c "curl -fsS http://localhost:${ZYRA_API_PORT:-${DATAVIZHUB_API_PORT:-8000}}/ready || exit 1" # Automatically load .env variables in interactive shells (dev only) RUN echo 'set -a; [ -f /app/.env ] && source /app/.env; set +a' >> /root/.bashrc diff --git a/README.md b/README.md index f2cecbef..6fca0f3a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ -# DataVizHub +# Zyra (formerly DataVizHub) ## Overview -DataVizHub is a utility library for building data-driven visual products. It provides composable helpers for data transfer (FTP/HTTP/S3/Vimeo), data processing (GRIB/imagery/video), and visualization (matplotlib + basemap overlays). Use these pieces to script your own pipelines; this repo focuses on the reusable building blocks rather than end-user scripts. +Zyra is a utility library for building data-driven visual products. It provides composable helpers for data transfer (FTP/HTTP/S3/Vimeo), data processing (GRIB/imagery/video), and visualization (matplotlib + basemap overlays). Use these pieces to script your own pipelines; this repo focuses on the reusable building blocks rather than end-user scripts. This README documents the library itself and shows how to compose the components. For complete runnable examples, see the examples repos when available, or adapt the snippets below. -[![PyPI version](https://img.shields.io/pypi/v/datavizhub.svg)](https://pypi.org/project/datavizhub/) [![Docs](https://img.shields.io/badge/docs-GitHub_Pages-0A7BBB)](https://noaa-gsl.github.io/datavizhub/) [![Chat with DataVizHub Helper Bot](https://img.shields.io/badge/ChatGPT-DataVizHub_Helper_Bot-00A67E?logo=openai&logoColor=white)](https://chatgpt.com/g/g-6897a3dd5a7481918a55ebe3795f7a26-datavizhub-helper-bot) +[![PyPI version](https://img.shields.io/pypi/v/zyra.svg)](https://pypi.org/project/zyra/) [![Docs](https://img.shields.io/badge/docs-GitHub_Pages-0A7BBB)](https://noaa-gsl.github.io/zyra/) [![Chat with DataVizHub Helper Bot](https://img.shields.io/badge/ChatGPT-DataVizHub_Helper_Bot-00A67E?logo=openai&logoColor=white)](https://chatgpt.com/g/g-6897a3dd5a7481918a55ebe3795f7a26-zyra-helper-bot) + +> Migration notice: the project has been renamed from DataVizHub to Zyra. The new import namespace is `zyra` and new console scripts are `zyra` and `zyra-cli`. For a transition period, the legacy `datavizhub` namespace and CLI remain available and redirect to Zyra under the hood. Please begin migrating your imports and usage examples to `zyra`. ## Table of Contents - [Overview](#overview) @@ -28,10 +30,10 @@ DataVizHub is a utility library for building data-driven visual products. It pro - [Links](#links) ## Features -- [Connectors](#connectors-layer): HTTP/FTP/S3/Vimeo backends (functional API) in `datavizhub.connectors.backends` (legacy `datavizhub.acquisition` still present with deprecations). -- [Processing](#processing-layer): `DataProcessor`, `VideoProcessor`, `GRIBDataProcessor` (in `datavizhub.processing`). -- [Visualization](#visualization-layer): `PlotManager`, `ColormapManager` (with included basemap/overlay assets in `datavizhub.assets`). -- [Utilities](#utilities): `CredentialManager`, `DateManager`, `FileUtils`, `ImageManager`, `JSONFileManager` (in `datavizhub.utils`). +- [Connectors](#connectors-layer): HTTP/FTP/S3/Vimeo backends (functional API) in `zyra.connectors.backends` (legacy `datavizhub.acquisition` still present with deprecations). +- [Processing](#processing-layer): `DataProcessor`, `VideoProcessor`, `GRIBDataProcessor` (in `zyra.processing`). +- [Visualization](#visualization-layer): `PlotManager`, `ColormapManager` (with included basemap/overlay assets in `zyra.assets`). +- [Utilities](#utilities): `CredentialManager`, `DateManager`, `FileUtils`, `ImageManager`, `JSONFileManager` (in `zyra.utils`). ## Project Structure @@ -45,7 +47,7 @@ DataVizHub is a utility library for building data-driven visual products. It pro - `assets/images/`: packaged basemaps and overlays used by plots. Notes -- Legacy `acquisition/` modules remain for back-compat but are deprecated. Prefer `datavizhub.connectors.backends.*` and the CLI groups `acquire`/`decimate`. +- Legacy `acquisition/` modules remain for back-compat but are deprecated. Prefer `zyra.connectors.backends.*` and the CLI groups `acquire`/`decimate`. ## Prerequisites - Python 3.10+ @@ -63,13 +65,13 @@ Notes for development: - Opt into only what you need using `-E ` flags, or use `--all-extras` for a full-featured env. ## Install (pip extras) -- Core only: `pip install datavizhub` -- Connectors deps: `pip install "datavizhub[connectors]"` -- Processing deps: `pip install "datavizhub[processing]"` -- Visualization deps: `pip install "datavizhub[visualization]"` -- Interactive deps: `pip install "datavizhub[interactive]"` -- API service deps: `pip install "datavizhub[api]"` -- Everything: `pip install "datavizhub[all]"` +- Core only: `pip install zyra` +- Connectors deps: `pip install "zyra[connectors]"` +- Processing deps: `pip install "zyra[processing]"` +- Visualization deps: `pip install "zyra[visualization]"` +- Interactive deps: `pip install "zyra[interactive]"` +- API service deps: `pip install "zyra[api]"` +- Everything: `pip install "zyra[all]"` Deprecation note: - `datatransfer` remains available as an alias of `connectors` for backward compatibility and may be removed in a future release. @@ -77,7 +79,7 @@ Deprecation note: Focused installs for GRIB2/NetCDF/GeoTIFF: ``` -pip install "datavizhub[grib2,netcdf,geotiff]" +pip install "zyra[grib2,netcdf,geotiff]" ``` Extras overview: @@ -112,8 +114,8 @@ Install only what you need for a given stage. Each stage can run independently w Examples: - Run the visualization CLI with only the visualization extra installed: - - Heatmap: `python -m datavizhub.cli heatmap --input samples/demo.npy --output heatmap.png` - - Contour: `python -m datavizhub.cli contour --input samples/demo.nc --var T2M --output contour.png --levels 5,10,15 --filled` + - Heatmap: `python -m zyra.cli heatmap --input samples/demo.npy --output heatmap.png` + - Contour: `python -m zyra.cli contour --input samples/demo.nc --var T2M --output contour.png --levels 5,10,15 --filled` Focused extras remain available for targeted installs: - GRIB2 only: `pip install -e .[grib2]` @@ -125,12 +127,12 @@ Note on interactive installs: ## CLI Overview -DataVizHub ships a single `datavizhub` CLI organized into groups that mirror pipeline stages (plus a `transform` helper) and a `run` helper for config-driven pipelines. +DataVizHub ships a single `zyra` CLI organized into groups that mirror pipeline stages (plus a `transform` helper) and a `run` helper for config-driven pipelines. ### CLI Tree ``` -datavizhub +zyra ├─ acquire # Ingest/fetch bytes from sources │ ├─ http # acquire http [--list --pattern REGEX --since ISO --until ISO --date-format FMT] [-o out|-] │ ├─ s3 # acquire s3 --url s3://bucket/key [--unsigned] [--list --pattern REGEX --since ISO --until ISO --date-format FMT] [-o out|-] @@ -164,23 +166,23 @@ Notes ### Quick Usage by Group - Acquire - - HTTP to file: `datavizhub acquire http https://example.com/data.bin -o data.bin` - - HTTP list+filter: `datavizhub acquire http https://example.com/dir/ --list --pattern '\\.png$' --since 2024-01-01 --date-format %Y%m%d` - - S3 to stdout: `datavizhub acquire s3 --url s3://bucket/key -o -` - - S3 list+filter: `datavizhub acquire s3 --url s3://bucket/prefix/ --list --pattern '\\.grib2$' --since 2024-08-01 --date-format %Y%m%d` - - FTP sync directory: `datavizhub acquire ftp ftp://host/path --sync-dir /data/frames --pattern 'image_(\\d{8})\\.png' --since 2024-08-01 --date-format %Y%m%d` + - HTTP to file: `zyra acquire http https://example.com/data.bin -o data.bin` + - HTTP list+filter: `zyra acquire http https://example.com/dir/ --list --pattern '\\.png$' --since 2024-01-01 --date-format %Y%m%d` + - S3 to stdout: `zyra acquire s3 --url s3://bucket/key -o -` + - S3 list+filter: `zyra acquire s3 --url s3://bucket/prefix/ --list --pattern '\\.grib2$' --since 2024-08-01 --date-format %Y%m%d` + - FTP sync directory: `zyra acquire ftp ftp://host/path --sync-dir /data/frames --pattern 'image_(\\d{8})\\.png' --since 2024-08-01 --date-format %Y%m%d` - Process (streaming-friendly) - - Decode GRIB2 to raw bytes via `.idx` subset: `datavizhub process decode-grib2 s3://bucket/file.grib2 --pattern ":TMP:surface:" --raw > subset.grib2` - - Convert stdin to NetCDF: `cat subset.grib2 | datavizhub process convert-format - netcdf --stdout > out.nc` + - Decode GRIB2 to raw bytes via `.idx` subset: `zyra process decode-grib2 s3://bucket/file.grib2 --pattern ":TMP:surface:" --raw > subset.grib2` + - Convert stdin to NetCDF: `cat subset.grib2 | zyra process convert-format - netcdf --stdout > out.nc` - Visualize - - Contour PNG from NetCDF: `datavizhub visualize contour --input out.nc --var TMP --output contour.png --levels 10 --filled` - - Animate frames and compose to MP4: `datavizhub visualize animate --mode heatmap --input cube.npy --output-dir frames && datavizhub visualize compose-video --frames frames -o out.mp4` + - Contour PNG from NetCDF: `zyra visualize contour --input out.nc --var TMP --output contour.png --levels 10 --filled` + - Animate frames and compose to MP4: `zyra visualize animate --mode heatmap --input cube.npy --output-dir frames && zyra visualize compose-video --frames frames -o out.mp4` - Decimate - - Upload to S3 from stdin: `cat out.png | datavizhub decimate s3 -i - --url s3://bucket/products/out.png` -- HTTP POST JSON: `echo '{"ok":true}' | datavizhub decimate post -i - https://example.com/ingest --content-type application/json` + - Upload to S3 from stdin: `cat out.png | zyra decimate s3 -i - --url s3://bucket/products/out.png` +- HTTP POST JSON: `echo '{"ok":true}' | zyra decimate post -i - https://example.com/ingest --content-type application/json` ## Batch Mode @@ -206,7 +208,7 @@ Notes ## Connectors Layer -The `datavizhub.connectors.backends` package provides functional helpers for ingress/egress: +The `zyra.connectors.backends` package provides functional helpers for ingress/egress: - HTTP: `fetch_bytes`, `fetch_text`, `fetch_json`, `post_data`, `list_files`, `get_idx_lines`, `download_byteranges`. - S3: `fetch_bytes`, `upload_bytes`, `list_files`, `exists`, `delete`, `stat`, `get_idx_lines`, `download_byteranges`. @@ -216,7 +218,7 @@ The `datavizhub.connectors.backends` package provides functional helpers for ing Examples: ``` -from datavizhub.connectors.backends import ftp as ftp_backend, s3 as s3_backend +from zyra.connectors.backends import ftp as ftp_backend, s3 as s3_backend # FTP: read a remote file to bytes and write locally data = ftp_backend.fetch_bytes("ftp://ftp.example.com/pub/file.txt") @@ -234,8 +236,8 @@ Optional helpers speed up GRIB workflows and large file transfers. - .idx subsetting (S3 public bucket, unsigned): ```python - from datavizhub.connectors.backends import s3 as s3_backend - from datavizhub.utils.grib import idx_to_byteranges + from zyra.connectors.backends import s3 as s3_backend + from zyra.utils.grib import idx_to_byteranges url = "s3://noaa-hrrr-bdp-pds/hrrr.20230801/conus/hrrr.t00z.wrfsfcf00.grib2" lines = s3_backend.get_idx_lines(url, unsigned=True) @@ -246,12 +248,12 @@ Optional helpers speed up GRIB workflows and large file transfers. - Pattern-based listing (regex) - S3 prefix listing with regex filter: ```python - from datavizhub.connectors.backends import s3 as s3_backend + from zyra.connectors.backends import s3 as s3_backend keys = s3_backend.list_files("s3://bucket/hrrr.20230801/conus/", pattern=r"wrfsfcf\d+\.grib2$") ``` - HTTP directory-style index scraping with regex filter: ```python - from datavizhub.connectors.backends import http as http_backend + from zyra.connectors.backends import http as http_backend urls = http_backend.list_files( "https://nomads.ncep.noaa.gov/pub/data/nccf/com/hrrr/prod/", pattern=r"\.grib2$", @@ -260,8 +262,8 @@ Optional helpers speed up GRIB workflows and large file transfers. - Parallel HTTP range downloads via `.idx`: ```python - from datavizhub.connectors.backends import http as http_backend - from datavizhub.utils.grib import idx_to_byteranges + from zyra.connectors.backends import http as http_backend + from zyra.utils.grib import idx_to_byteranges lines = http_backend.get_idx_lines("https://example.com/path/file.grib2") ranges = idx_to_byteranges(lines, r"GUST") @@ -269,7 +271,7 @@ Optional helpers speed up GRIB workflows and large file transfers. ``` Notes -- Migration: `datavizhub.acquisition.*` is deprecated. Prefer `datavizhub.connectors.backends.*` and `datavizhub.utils.grib`. +- Migration: `datavizhub.acquisition.*` is deprecated. Prefer `zyra.connectors.backends.*` and `zyra.utils.grib`. - Pattern filters use Python regular expressions (`re.search`) applied to full keys/paths/URLs. - `.idx` resolution appends `.idx` to the GRIB path unless a fully qualified `.idx` path is given. - For unsigned public S3 buckets, pass `unsigned=True` as shown above. @@ -292,16 +294,16 @@ YAML templating with environment variables since_period: "P1Y" date_format: "%Y%m%d" ``` - - Run with strict env: `datavizhub run pipeline.yaml --strict-env` + - Run with strict env: `zyra run pipeline.yaml --strict-env` - Provide env: `export FTP_USER=anonymous; export FTP_PASS='test@example.com'` (when embedding directly, URL-encode `@` as `%40`). Transform: Frames Metadata - Compute metadata for a frames directory between acquire and visualize: - - CLI: `datavizhub transform metadata --frames-dir ./frames --datetime-format %Y%m%d --period-seconds 3600 -o frames_meta.json` + - CLI: `zyra transform metadata --frames-dir ./frames --datetime-format %Y%m%d --period-seconds 3600 -o frames_meta.json` ## Processing Layer -The `datavizhub.processing` package standardizes processors under a common `DataProcessor` interface. +The `zyra.processing` package standardizes processors under a common `DataProcessor` interface. - DataProcessor: abstract base with `load(input_source)`, `process(**kwargs)`, `save(output_path=None)`, and optional `validate()`. - Processors: `VideoProcessor` (image sequences → video via FFmpeg), `GRIBDataProcessor` (GRIB files → NumPy arrays + utilities). @@ -311,7 +313,7 @@ Examples: ``` # Video: compile image frames into a video -from datavizhub.processing.video_processor import VideoProcessor +from zyra.processing.video_processor import VideoProcessor vp = VideoProcessor(input_directory="./frames", output_file="./out/movie.mp4") vp.load("./frames") @@ -322,7 +324,7 @@ if vp.validate(): ``` # GRIB: read a GRIB file to arrays and dates -from datavizhub.processing.grib_data_processor import GRIBDataProcessor +from zyra.processing.grib_data_processor import GRIBDataProcessor gp = GRIBDataProcessor() data_list, dates = gp.process(grib_file_path="/path/to/file.grib2", shift_180=True) @@ -333,7 +335,7 @@ data_list, dates = gp.process(grib_file_path="/path/to/file.grib2", shift_180=Tr Decode a GRIB2 subset returned as bytes, extract a variable, and write NetCDF: ``` -from datavizhub.processing import grib_decode, extract_variable, convert_to_format +from zyra.processing import grib_decode, extract_variable, convert_to_format dec = grib_decode(data_bytes, backend="cfgrib") # default backend da = extract_variable(dec, r"^TMP$") # exact/regex match @@ -343,7 +345,7 @@ nc_bytes = convert_to_format(dec, "netcdf", var="TMP") Work with NetCDF directly and subset spatially/temporally: ``` -from datavizhub.processing import load_netcdf, subset_netcdf +from zyra.processing import load_netcdf, subset_netcdf ds = load_netcdf(nc_bytes) sub = subset_netcdf(ds, variables=["TMP"], bbox=(-130,20,-60,55), time_range=("2024-01-01","2024-01-02")) @@ -356,9 +358,9 @@ Notes and fallbacks: - Generic NetCDF→GRIB2 is not supported by `wgrib2`. If `cdo` is installed, `convert_to_grib2()` uses `cdo -f grb2 copy` automatically; otherwise a clear exception is raised. CLI helpers (grouped commands): -- `datavizhub process decode-grib2 [--backend cfgrib|pygrib|wgrib2]` -- `datavizhub process extract-variable [--backend ...]` -- `datavizhub process convert-format -o out.ext [--var NAME] [--backend ...]` +- `zyra process decode-grib2 [--backend cfgrib|pygrib|wgrib2]` +- `zyra process extract-variable [--backend ...]` +- `zyra process convert-format -o out.ext [--var NAME] [--backend ...]` ## Development, Test, Lint @@ -382,17 +384,17 @@ CLI helpers (grouped commands): Sample pipeline configs live under `samples/pipelines/`: - `nc_passthrough.yaml`: read NetCDF bytes from stdin and emit NetCDF to stdout. - - Usage: `cat tests/testdata/demo.nc | datavizhub run samples/pipelines/nc_passthrough.yaml > out.nc` + - Usage: `cat tests/testdata/demo.nc | zyra run samples/pipelines/nc_passthrough.yaml > out.nc` - `nc_to_file.yaml`: read NetCDF from stdin, then write to `out.nc` via `decimate local`. - - Usage: `cat tests/testdata/demo.nc | datavizhub run samples/pipelines/nc_to_file.yaml` + - Usage: `cat tests/testdata/demo.nc | zyra run samples/pipelines/nc_to_file.yaml` - `ftp_to_s3.yaml`: template for FTP → video composition → S3 upload (placeholders, not CI-safe). - - Dry-run mapping: `datavizhub run samples/pipelines/ftp_to_s3.yaml --dry-run` + - Dry-run mapping: `zyra run samples/pipelines/ftp_to_s3.yaml --dry-run` - Notes: Requires network access and credentials when running without `--dry-run`. - `ftp_to_local.yaml`: FTP → transform → video → local file copy (no S3/Vimeo). - - Dry-run mapping: `datavizhub run samples/pipelines/ftp_to_local.yaml --dry-run` + - Dry-run mapping: `zyra run samples/pipelines/ftp_to_local.yaml --dry-run` - Live run: writes `/tmp/frames_meta.json` and `/tmp/video.mp4` locally. Requires FTP network access only. - `extract_variable_to_file.yaml`: extract TMP from stdin, convert to NetCDF, and write to file. - - Usage: `cat tests/testdata/demo.nc | datavizhub run samples/pipelines/extract_variable_to_file.yaml` + - Usage: `cat tests/testdata/demo.nc | zyra run samples/pipelines/extract_variable_to_file.yaml` - `compose_video_to_local.yaml`: compose frames in a directory to MP4 and copy locally. Overrides @@ -404,19 +406,19 @@ Overrides Dry run and argv output - Validate and print the expanded stage argv without executing: - - Text: `datavizhub run pipeline.yaml --dry-run` - - JSON: `datavizhub run pipeline.yaml --dry-run --print-argv-format=json` + - Text: `zyra run pipeline.yaml --dry-run` + - JSON: `zyra run pipeline.yaml --dry-run --print-argv-format=json` - Structure: ```json [ - {"stage": 1, "name": "acquire", "argv": ["datavizhub", "acquire", "http", "https://..."]}, - {"stage": 2, "name": "process", "argv": ["datavizhub", "process", "convert-format", "-", "netcdf"]} + {"stage": 1, "name": "acquire", "argv": ["zyra", "acquire", "http", "https://..."]}, + {"stage": 2, "name": "process", "argv": ["zyra", "process", "convert-format", "-", "netcdf"]} ] ``` Error handling - Stop on first error (default) or continue executing remaining stages: - - `datavizhub run pipeline.yaml --continue-on-error` + - `zyra run pipeline.yaml --continue-on-error` - Lint/format (if enabled in your env): - `poetry run black . && poetry run isort . && poetry run flake8` @@ -427,9 +429,9 @@ The CLI supports streaming binary data through stdout/stdin so you can compose o - `.idx` → extract → convert (one-liner): ```bash - datavizhub process decode-grib2 file.grib2 --pattern "TMP" --raw | \ - datavizhub process extract-variable - "TMP" --stdout --format grib2 | \ - datavizhub process convert-format - geotiff --stdout > tmp.tif + zyra process decode-grib2 file.grib2 --pattern "TMP" --raw | \ + zyra process extract-variable - "TMP" --stdout --format grib2 | \ + zyra process convert-format - geotiff --stdout > tmp.tif ``` - Notes on tools and fallbacks: @@ -441,8 +443,8 @@ The CLI supports streaming binary data through stdout/stdin so you can compose o - `process convert-format` can read from stdin (`-`) and auto-detects GRIB2 vs NetCDF by magic bytes. NetCDF is opened with xarray; GRIB2 uses the configured backend to decode. Bytes-first demos: -- Use `.idx`-aware subsetting directly with URLs: `datavizhub process decode-grib2 https://.../file.grib2 --pattern ":(UGRD|VGRD):10 m above ground:"` -- Pipe small outputs without temp files: `datavizhub process convert-format local.grib2 netcdf --stdout | hexdump -C | head` +- Use `.idx`-aware subsetting directly with URLs: `zyra process decode-grib2 https://.../file.grib2 --pattern ":(UGRD|VGRD):10 m above ground:"` +- Pipe small outputs without temp files: `zyra process convert-format local.grib2 netcdf --stdout | hexdump -C | head` Offline demo assets: - Tiny NetCDF file: `tests/testdata/demo.nc` @@ -456,13 +458,13 @@ Plot a data array with a basemap ``` import numpy as np from importlib.resources import files, as_file -from datavizhub.visualization import PlotManager, ColormapManager +from zyra.visualization import PlotManager, ColormapManager # Example data data = np.random.rand(180, 360) # Locate packaged basemap asset -resource = files("datavizhub.assets").joinpath("images/earth_vegetation.jpg") +resource = files("zyra.assets").joinpath("images/earth_vegetation.jpg") with as_file(resource) as p: basemap_path = str(p) @@ -483,7 +485,7 @@ Tile basemaps (static images) ``` poetry install --with dev -E visualization -poetry run python -m datavizhub.cli heatmap \ +poetry run python -m zyra.cli heatmap \ --input samples/demo.npy \ --output out.png \ --map-type tile \ @@ -493,7 +495,7 @@ poetry run python -m datavizhub.cli heatmap \ - Contour over a named tile source: ``` -poetry run python -m datavizhub.cli contour \ +poetry run python -m zyra.cli contour \ --input samples/demo.npy --output contour.png \ --levels 10 --filled \ --map-type tile --tile-source Stamen.TerrainBackground @@ -502,7 +504,7 @@ poetry run python -m datavizhub.cli contour \ - Vector quiver over tiles: ``` -poetry run python -m datavizhub.cli vector \ +poetry run python -m zyra.cli vector \ --u /path/U.npy --v /path/V.npy \ --output vec.png \ --map-type tile --tile-zoom 2 @@ -529,13 +531,13 @@ plotter.save("/tmp/heatmap_classified.png") Render interactive HTML (Folium or Plotly) via the CLI. Install extras as needed: - Poetry: `poetry install --with dev -E interactive` (or `-E visualization -E interactive`) -- Pip: `pip install "datavizhub[interactive]"` +- Pip: `pip install "zyra[interactive]"` Examples - Folium heatmap from a NumPy array: ``` -python -m datavizhub.cli interactive \ +python -m zyra.cli interactive \ --input samples/demo.npy \ --output out.html \ --engine folium \ @@ -545,7 +547,7 @@ python -m datavizhub.cli interactive \ - Plotly heatmap (standalone HTML): ``` -python -m datavizhub.cli interactive \ +python -m zyra.cli interactive \ --input samples/demo.npy \ --output out_plotly.html \ --engine plotly \ @@ -556,7 +558,7 @@ python -m datavizhub.cli interactive \ - Folium points from CSV: ``` -python -m datavizhub.cli interactive \ +python -m zyra.cli interactive \ --input samples/points.csv \ --output points.html \ --engine folium \ @@ -566,7 +568,7 @@ python -m datavizhub.cli interactive \ - Folium points with a time column (TimeDimension): ``` -python -m datavizhub.cli interactive \ +python -m zyra.cli interactive \ --input samples/points_time.csv \ --output points_time.html \ --engine folium \ @@ -579,7 +581,7 @@ python -m datavizhub.cli interactive \ - Folium vector quiver from U/V arrays: ``` -python -m datavizhub.cli interactive \ +python -m zyra.cli interactive \ --mode vector \ --u /path/U.npy \ --v /path/V.npy \ @@ -598,8 +600,8 @@ CRS notes Compose FTP fetch + video + Vimeo upload ```python -from datavizhub.connectors.backends import ftp as ftp_backend, vimeo as vimeo_backend -from datavizhub.processing import VideoProcessor +from zyra.connectors.backends import ftp as ftp_backend, vimeo as vimeo_backend +from zyra.processing import VideoProcessor # Download a frame from FTP and write it to disk (repeat for remaining frames) frame_bytes = ftp_backend.fetch_bytes("ftp://ftp.example.com/pub/images/img_0001.png") @@ -613,7 +615,7 @@ vimeo_backend.upload_path("/tmp/out.mp4", name="Latest Render") ## Utilities -The `datavizhub.utils` package provides shared helpers for credentials, dates, files, images, and small JSON configs. +The `zyra.utils` package provides shared helpers for credentials, dates, files, images, and small JSON configs. - CredentialManager: read/manage dotenv-style secrets without exporting globally. - DateManager: parse timestamps in filenames, compute date ranges, and reason about frame cadences. @@ -625,7 +627,7 @@ Examples: ``` # Credentials -from datavizhub.utils import CredentialManager +from zyra.utils import CredentialManager with CredentialManager(".env", namespace="MYAPP_") as cm: cm.read_credentials(expected_keys=["API_KEY"]) # expects MYAPP_API_KEY @@ -634,7 +636,7 @@ with CredentialManager(".env", namespace="MYAPP_") as cm: ``` # Dates -from datavizhub.utils import DateManager +from zyra.utils import DateManager dm = DateManager(["%Y%m%d"]) start, end = dm.get_date_range("7D") @@ -644,7 +646,7 @@ print(dm.is_date_in_range("frame_20240102.png", start, end)) Batch fetching (functional connectors): ``` -from datavizhub.connectors.backends import http as http_backend +from zyra.connectors.backends import http as http_backend urls = [ "https://example.com/a.bin", @@ -660,8 +662,8 @@ for u in urls: Minimal pipeline: build video from images and upload to S3 ```python -from datavizhub.processing import VideoProcessor -from datavizhub.connectors.backends import s3 as s3_backend +from zyra.processing import VideoProcessor +from zyra.connectors.backends import s3 as s3_backend vp = VideoProcessor(input_directory="/data/images", output_file="/data/out/movie.mp4") vp.load("/data/images") @@ -685,10 +687,10 @@ with open("/data/out/movie.mp4", "rb") as f: - Project structure, dev workflow, testing, and contribution tips: see [AGENTS.md](AGENTS.md). ## Documentation -- Primary: Project wiki at https://github.com/NOAA-GSL/datavizhub/wiki - - API Routers and Endpoints: https://github.com/NOAA-GSL/datavizhub/wiki/DataVizHub-API-Routers-and-Endpoints - - Security Quickstart: https://github.com/NOAA-GSL/datavizhub/wiki/DataVizHub-API-Security-Quickstart -- API docs (GitHub Pages): https://noaa-gsl.github.io/datavizhub/ +- Primary: Project wiki at https://github.com/NOAA-GSL/zyra/wiki + - API Routers and Endpoints: https://github.com/NOAA-GSL/zyra/wiki/DataVizHub-API-Routers-and-Endpoints + - Security Quickstart: https://github.com/NOAA-GSL/zyra/wiki/DataVizHub-API-Security-Quickstart +- API docs (GitHub Pages): https://noaa-gsl.github.io/zyra/ - CI-synced wiki: A GitHub Action mirrors the wiki into `docs/source/wiki/` so Sphinx can build it with the docs. Sync commits occur only on `main`; PRs/branches use the synced copy for builds without committing changes. - Wizard REPL (experimental): see docs/wizard-cli.md for autocomplete and edit-before-run usage. @@ -704,7 +706,7 @@ CAPABILITIES vs. FEATURES: Examples: ``` -from datavizhub.processing.video_processor import VideoProcessor +from zyra.processing.video_processor import VideoProcessor vp = VideoProcessor("./frames", "./out.mp4") print(vp.features) # {'load','process','save','validate'} @@ -714,13 +716,13 @@ print(vp.features) # {'load','process','save','validate'} Distributed under the MIT License. See [LICENSE](LICENSE). ## Links -- Source: https://github.com/NOAA-GSL/datavizhub -- PyPI: https://pypi.org/project/datavizhub/ +- Source: https://github.com/NOAA-GSL/zyra +- PyPI: https://pypi.org/project/zyra/ ## API Service Expose the 4-stage CLI over HTTP using FastAPI. - Install with API extras: `poetry install --with dev -E api` or `--all-extras`. -- Run locally: `poetry run uvicorn datavizhub.api.server:app --reload --host 0.0.0.0 --port 8000`. +- Run locally: `poetry run uvicorn zyra.api.server:app --reload --host 0.0.0.0 --port 8000`. - Convenience script: `./scripts/start-api.sh`. Quick links: @@ -755,7 +757,7 @@ POST /cli/run Notes: - For convenience, `args.input` is treated as `file_or_url` for processing commands. - This service runs CLI functions in-process; no shelling out is used. - - Optional async backend: set `DATAVIZHUB_USE_REDIS=1` and `DATAVIZHUB_REDIS_URL=redis://host:6379/0`; run an RQ worker: `poetry run rq worker datavizhub`. + - Optional async backend: set `DATAVIZHUB_USE_REDIS=1` and `DATAVIZHUB_REDIS_URL=redis://host:6379/0`; run an RQ worker: `poetry run rq worker zyra`. - WebSocket usage: connect to `ws://localhost:8000/ws/jobs/{job_id}`. - Without Redis, in-memory streaming is enabled by default. - websocat quickstart: @@ -811,7 +813,7 @@ Strict file_id resolution: - Set `DATAVIZHUB_STRICT_FILE_ID=1` to return `404` if any `file_id` cannot be resolved to an uploaded file path at request time. Results, TTL, and cleanup: -- Results are stored under `DATAVIZHUB_RESULTS_DIR` (default `/tmp/datavizhub_results/{job_id}/`). +- Results are stored under `DATAVIZHUB_RESULTS_DIR` (default `/tmp/zyra_results/{job_id}/`). - `/jobs/{job_id}/download` enforces TTL via `DATAVIZHUB_RESULTS_TTL_SECONDS` (default 86400s). - A background cleanup task removes expired files and prunes empty job dirs at `DATAVIZHUB_RESULTS_CLEAN_INTERVAL_SECONDS` (default 3600s). - For heavier-duty cleanup, consider a sidecar (e.g., `tmpreaper`) targeting the results dir. @@ -826,7 +828,7 @@ MIME detection (optional): - Falls back to `mimetypes` when `python-magic` is not installed. WebSocket client extra: -- Install `poetry install --with ws` to enable the `datavizhub-cli --ws` streaming option (bundles `websockets`). +- Install `poetry install --with ws` to enable the `zyra-cli --ws` streaming option (bundles `websockets`). See Batch Mode section for multi-input workflows across acquisition, processing, and visualization. ### Running Tests diff --git a/SECURITY.md b/SECURITY.md index 9ef2cda9..74c5360d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,7 +38,7 @@ contained, validated paths (e.g., `os.open` with `O_NOFOLLOW`, `open(mf)` after ## Operational guidance -- Configure `DATAVIZHUB_RESULTS_DIR` to a dedicated directory owned by the +- Configure `ZYRA_RESULTS_DIR` to a dedicated directory owned by the service. Do not point it at shared or sensitive system paths. - Do not relax allowlists without revisiting containment and tests. - Keep tests for traversal and TTL behavior green. diff --git a/docker-compose.yml b/docker-compose.yml index 085256c7..8eefc9aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ services: - datavizhub: + zyra: build: context: . dockerfile: Dockerfile @@ -22,18 +22,29 @@ services: VIMEO_ACCESS_TOKEN: ${VIMEO_ACCESS_TOKEN} OPENAI_API_KEY: ${OPENAI_API_KEY} - # Redis / API autostart - DATAVIZHUB_USE_REDIS: ${DATAVIZHUB_USE_REDIS:-0} - DATAVIZHUB_REDIS_URL: ${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0} - DATAVIZHUB_AUTOSTART_API: 1 - DATAVIZHUB_AUTOSTART_RQ: ${DATAVIZHUB_USE_REDIS:-0} - DATAVIZHUB_UPLOAD_DIR: /data/uploads - DATAVIZHUB_MIN_DISK_MB: 100 - DATAVIZHUB_REQUIRE_FFMPEG: 0 + # Redis / API autostart (prefer ZYRA_*; fallback to legacy DATAVIZHUB_*) + ZYRA_USE_REDIS: ${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}} + ZYRA_REDIS_URL: ${ZYRA_REDIS_URL:-${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0}} + ZYRA_AUTOSTART_API: ${ZYRA_AUTOSTART_API:-1} + ZYRA_AUTOSTART_RQ: ${ZYRA_AUTOSTART_RQ:-${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}}} + ZYRA_UPLOAD_DIR: ${ZYRA_UPLOAD_DIR:-/data/uploads} + ZYRA_MIN_DISK_MB: ${ZYRA_MIN_DISK_MB:-100} + ZYRA_REQUIRE_FFMPEG: ${ZYRA_REQUIRE_FFMPEG:-0} - # API network settings - DATAVIZHUB_API_HOST: 0.0.0.0 - DATAVIZHUB_API_PORT: 8000 + # API network settings (prefer ZYRA_*) + ZYRA_API_HOST: ${ZYRA_API_HOST:-0.0.0.0} + ZYRA_API_PORT: ${ZYRA_API_PORT:-8000} + + # Back-compat: also set legacy names from ZYRA_* when missing + DATAVIZHUB_USE_REDIS: ${DATAVIZHUB_USE_REDIS:-${ZYRA_USE_REDIS:-0}} + DATAVIZHUB_REDIS_URL: ${DATAVIZHUB_REDIS_URL:-${ZYRA_REDIS_URL:-redis://redis:6379/0}} + DATAVIZHUB_AUTOSTART_API: ${DATAVIZHUB_AUTOSTART_API:-${ZYRA_AUTOSTART_API:-1}} + DATAVIZHUB_AUTOSTART_RQ: ${DATAVIZHUB_AUTOSTART_RQ:-${ZYRA_AUTOSTART_RQ:-0}} + DATAVIZHUB_UPLOAD_DIR: ${DATAVIZHUB_UPLOAD_DIR:-${ZYRA_UPLOAD_DIR:-/data/uploads}} + DATAVIZHUB_MIN_DISK_MB: ${DATAVIZHUB_MIN_DISK_MB:-${ZYRA_MIN_DISK_MB:-100}} + DATAVIZHUB_REQUIRE_FFMPEG: ${DATAVIZHUB_REQUIRE_FFMPEG:-${ZYRA_REQUIRE_FFMPEG:-0}} + DATAVIZHUB_API_HOST: ${DATAVIZHUB_API_HOST:-${ZYRA_API_HOST:-0.0.0.0}} + DATAVIZHUB_API_PORT: ${DATAVIZHUB_API_PORT:-${ZYRA_API_PORT:-8000}} ports: - "8000:8000" diff --git a/docs/source/api/datavizhub.api.routers.cli.rst b/docs/source/api/datavizhub.api.routers.cli.rst index 7a4794dd..07850e5d 100644 --- a/docs/source/api/datavizhub.api.routers.cli.rst +++ b/docs/source/api/datavizhub.api.routers.cli.rst @@ -1,8 +1 @@ -datavizhub.api.routers.cli module -================================= - -.. automodule:: datavizhub.api.routers.cli - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.api.routers.cli.rst diff --git a/docs/source/api/datavizhub.api.rst b/docs/source/api/datavizhub.api.rst index 87d86690..9d309bef 100644 --- a/docs/source/api/datavizhub.api.rst +++ b/docs/source/api/datavizhub.api.rst @@ -1,12 +1 @@ -datavizhub.api package -====================== - -Submodules ----------- - -.. toctree:: - :maxdepth: 2 - - datavizhub.api.server - datavizhub.api.routers.cli - +.. include:: zyra.api.rst diff --git a/docs/source/api/datavizhub.api.server.rst b/docs/source/api/datavizhub.api.server.rst index 41438acc..9fd4b099 100644 --- a/docs/source/api/datavizhub.api.server.rst +++ b/docs/source/api/datavizhub.api.server.rst @@ -1,8 +1 @@ -datavizhub.api.server module -============================ - -.. automodule:: datavizhub.api.server - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.api.server.rst diff --git a/docs/source/api/datavizhub.assets.rst b/docs/source/api/datavizhub.assets.rst index 099ea4ed..6ed6b0aa 100644 --- a/docs/source/api/datavizhub.assets.rst +++ b/docs/source/api/datavizhub.assets.rst @@ -1,16 +1 @@ -datavizhub.assets package -========================= - -.. automodule:: datavizhub.assets - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. automodule:: datavizhub.assets.images - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.assets.rst diff --git a/docs/source/api/datavizhub.cli.rst b/docs/source/api/datavizhub.cli.rst index 5af7b291..f58638ee 100644 --- a/docs/source/api/datavizhub.cli.rst +++ b/docs/source/api/datavizhub.cli.rst @@ -1,8 +1 @@ -datavizhub.cli module -===================== - -.. automodule:: datavizhub.cli - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.cli.rst diff --git a/docs/source/api/datavizhub.connectors.backends.rst b/docs/source/api/datavizhub.connectors.backends.rst index 239621eb..310da730 100644 --- a/docs/source/api/datavizhub.connectors.backends.rst +++ b/docs/source/api/datavizhub.connectors.backends.rst @@ -1,23 +1,22 @@ -datavizhub.connectors.backends package +zyra.connectors.backends package ===================================== -.. automodule:: datavizhub.connectors.backends.http +.. automodule:: zyra.connectors.backends.http :members: :undoc-members: :show-inheritance: -.. automodule:: datavizhub.connectors.backends.ftp +.. automodule:: zyra.connectors.backends.ftp :members: :undoc-members: :show-inheritance: -.. automodule:: datavizhub.connectors.backends.s3 +.. automodule:: zyra.connectors.backends.s3 :members: :undoc-members: :show-inheritance: -.. automodule:: datavizhub.connectors.backends.vimeo +.. automodule:: zyra.connectors.backends.vimeo :members: :undoc-members: :show-inheritance: - diff --git a/docs/source/api/datavizhub.connectors.egress.rst b/docs/source/api/datavizhub.connectors.egress.rst index d8e1e445..4da8e4c3 100644 --- a/docs/source/api/datavizhub.connectors.egress.rst +++ b/docs/source/api/datavizhub.connectors.egress.rst @@ -1,8 +1,7 @@ -datavizhub.connectors.egress module +zyra.connectors.egress module =================================== -.. automodule:: datavizhub.connectors.egress +.. automodule:: zyra.connectors.egress :members: :undoc-members: :show-inheritance: - diff --git a/docs/source/api/datavizhub.connectors.ingest.rst b/docs/source/api/datavizhub.connectors.ingest.rst index 041ae0c9..75815e49 100644 --- a/docs/source/api/datavizhub.connectors.ingest.rst +++ b/docs/source/api/datavizhub.connectors.ingest.rst @@ -1,8 +1,7 @@ -datavizhub.connectors.ingest module +zyra.connectors.ingest module =================================== -.. automodule:: datavizhub.connectors.ingest +.. automodule:: zyra.connectors.ingest :members: :undoc-members: :show-inheritance: - diff --git a/docs/source/api/datavizhub.connectors.rst b/docs/source/api/datavizhub.connectors.rst index 59bed8b1..a0dffeba 100644 --- a/docs/source/api/datavizhub.connectors.rst +++ b/docs/source/api/datavizhub.connectors.rst @@ -1,21 +1 @@ -datavizhub.connectors package -============================= - -Subpackages ------------ - -.. toctree:: - :maxdepth: 2 - - datavizhub.connectors.backends - datavizhub.connectors.ingest - datavizhub.connectors.egress - -Module contents ---------------- - -.. automodule:: datavizhub.connectors - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.connectors.rst diff --git a/docs/source/api/datavizhub.processing.rst b/docs/source/api/datavizhub.processing.rst index 23d5b35f..93472ac1 100644 --- a/docs/source/api/datavizhub.processing.rst +++ b/docs/source/api/datavizhub.processing.rst @@ -1,35 +1 @@ -datavizhub.processing package -============================= - -.. automodule:: datavizhub.processing - :members: - :undoc-members: - :show-inheritance: - -Modules -------- - -.. automodule:: datavizhub.processing.base - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.processing.video_processor - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.processing.grib_data_processor - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.processing.grib_utils - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.processing.netcdf_data_processor - :members: - :undoc-members: - :show-inheritance: +.. include:: zyra.processing.rst diff --git a/docs/source/api/datavizhub.rst b/docs/source/api/datavizhub.rst index 62541c77..254db45a 100644 --- a/docs/source/api/datavizhub.rst +++ b/docs/source/api/datavizhub.rst @@ -1,22 +1 @@ -datavizhub package -================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 2 - datavizhub.connectors - datavizhub.transform - datavizhub.processing - datavizhub.visualization - datavizhub.utils - datavizhub.assets - -Module contents ---------------- - -.. automodule:: datavizhub - :members: - :undoc-members: - :show-inheritance: +.. include:: zyra.rst diff --git a/docs/source/api/datavizhub.transform.rst b/docs/source/api/datavizhub.transform.rst index 53ea40c7..cd68b25e 100644 --- a/docs/source/api/datavizhub.transform.rst +++ b/docs/source/api/datavizhub.transform.rst @@ -1,8 +1 @@ -datavizhub.transform module -=========================== - -.. automodule:: datavizhub.transform - :members: - :undoc-members: - :show-inheritance: - +.. include:: zyra.transform.rst diff --git a/docs/source/api/datavizhub.utils.rst b/docs/source/api/datavizhub.utils.rst index 8129ad82..68ae773c 100644 --- a/docs/source/api/datavizhub.utils.rst +++ b/docs/source/api/datavizhub.utils.rst @@ -1,55 +1 @@ -datavizhub.utils package -======================== - -.. automodule:: datavizhub.utils - :members: - :undoc-members: - :show-inheritance: - -Modules -------- - -.. automodule:: datavizhub.utils.credential_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.date_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.file_utils - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.image_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.json_file_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.cli_helpers - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.io_utils - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.geo_utils - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.utils.grib - :members: - :undoc-members: - :show-inheritance: +.. include:: zyra.utils.rst diff --git a/docs/source/api/datavizhub.visualization.rst b/docs/source/api/datavizhub.visualization.rst index c2ecffe0..8adfecc8 100644 --- a/docs/source/api/datavizhub.visualization.rst +++ b/docs/source/api/datavizhub.visualization.rst @@ -1,121 +1 @@ -datavizhub.visualization package -================================ - -.. automodule:: datavizhub.visualization - :members: - :undoc-members: - :show-inheritance: - -Core modules ------------- - -.. automodule:: datavizhub.visualization.base - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.plot_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.colormap_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.basemap - :members: - :undoc-members: - :show-inheritance: - -Managers --------- - -.. automodule:: datavizhub.visualization.heatmap_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.contour_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.vector_field_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.vector_particles_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.timeseries_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.animate_manager - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.interactive_manager - :members: - :undoc-members: - :show-inheritance: - -Styles and CLI --------------- - -.. automodule:: datavizhub.visualization.styles - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_utils - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_animate - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_compose_video - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_contour - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_heatmap - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_interactive - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_timeseries - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: datavizhub.visualization.cli_vector - :members: - :undoc-members: - :show-inheritance: +.. include:: zyra.visualization.rst diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst index f47235a5..b4c01933 100644 --- a/docs/source/api/modules.rst +++ b/docs/source/api/modules.rst @@ -1,8 +1,7 @@ -DataVizHub API +Zyra API ============== .. toctree:: :maxdepth: 2 - datavizhub - + zyra diff --git a/docs/source/api/zyra.api.routers.cli.rst b/docs/source/api/zyra.api.routers.cli.rst new file mode 100644 index 00000000..17a8ffb6 --- /dev/null +++ b/docs/source/api/zyra.api.routers.cli.rst @@ -0,0 +1,7 @@ +zyra.api.routers.cli module +================================= + +.. automodule:: zyra.api.routers.cli + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.api.rst b/docs/source/api/zyra.api.rst new file mode 100644 index 00000000..1dfb6448 --- /dev/null +++ b/docs/source/api/zyra.api.rst @@ -0,0 +1,11 @@ +zyra.api package +====================== + +Submodules +---------- + +.. toctree:: + :maxdepth: 2 + + zyra.api.server + zyra.api.routers.cli diff --git a/docs/source/api/zyra.api.server.rst b/docs/source/api/zyra.api.server.rst new file mode 100644 index 00000000..d031651d --- /dev/null +++ b/docs/source/api/zyra.api.server.rst @@ -0,0 +1,7 @@ +zyra.api.server module +============================ + +.. automodule:: zyra.api.server + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.assets.rst b/docs/source/api/zyra.assets.rst new file mode 100644 index 00000000..10687c26 --- /dev/null +++ b/docs/source/api/zyra.assets.rst @@ -0,0 +1,15 @@ +zyra.assets package +========================= + +.. automodule:: zyra.assets + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. automodule:: zyra.assets.images + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.cli.rst b/docs/source/api/zyra.cli.rst new file mode 100644 index 00000000..197d9344 --- /dev/null +++ b/docs/source/api/zyra.cli.rst @@ -0,0 +1,7 @@ +zyra.cli module +===================== + +.. automodule:: zyra.cli + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.connectors.rst b/docs/source/api/zyra.connectors.rst new file mode 100644 index 00000000..f75b8733 --- /dev/null +++ b/docs/source/api/zyra.connectors.rst @@ -0,0 +1,20 @@ +zyra.connectors package +============================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 2 + + zyra.connectors.backends + zyra.connectors.ingest + zyra.connectors.egress + +Module contents +--------------- + +.. automodule:: zyra.connectors + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.processing.rst b/docs/source/api/zyra.processing.rst new file mode 100644 index 00000000..638c300c --- /dev/null +++ b/docs/source/api/zyra.processing.rst @@ -0,0 +1,35 @@ +zyra.processing package +============================= + +.. automodule:: zyra.processing + :members: + :undoc-members: + :show-inheritance: + +Modules +------- + +.. automodule:: zyra.processing.base + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.processing.video_processor + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.processing.grib_data_processor + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.processing.grib_utils + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.processing.netcdf_data_processor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.rst b/docs/source/api/zyra.rst new file mode 100644 index 00000000..e26300d0 --- /dev/null +++ b/docs/source/api/zyra.rst @@ -0,0 +1,22 @@ +zyra package +================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 2 + zyra.connectors + zyra.transform + zyra.processing + zyra.visualization + zyra.utils + zyra.assets + +Module contents +--------------- + +.. automodule:: zyra + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.transform.rst b/docs/source/api/zyra.transform.rst new file mode 100644 index 00000000..ee23dfba --- /dev/null +++ b/docs/source/api/zyra.transform.rst @@ -0,0 +1,7 @@ +zyra.transform module +=========================== + +.. automodule:: zyra.transform + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.utils.rst b/docs/source/api/zyra.utils.rst new file mode 100644 index 00000000..bacae639 --- /dev/null +++ b/docs/source/api/zyra.utils.rst @@ -0,0 +1,55 @@ +zyra.utils package +======================== + +.. automodule:: zyra.utils + :members: + :undoc-members: + :show-inheritance: + +Modules +------- + +.. automodule:: zyra.utils.credential_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.date_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.file_utils + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.image_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.json_file_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.cli_helpers + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.io_utils + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.geo_utils + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.utils.grib + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/zyra.visualization.rst b/docs/source/api/zyra.visualization.rst new file mode 100644 index 00000000..62cc1f8d --- /dev/null +++ b/docs/source/api/zyra.visualization.rst @@ -0,0 +1,121 @@ +zyra.visualization package +================================ + +.. automodule:: zyra.visualization + :members: + :undoc-members: + :show-inheritance: + +Core modules +------------ + +.. automodule:: zyra.visualization.base + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.plot_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.colormap_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.basemap + :members: + :undoc-members: + :show-inheritance: + +Managers +-------- + +.. automodule:: zyra.visualization.heatmap_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.contour_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.vector_field_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.vector_particles_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.timeseries_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.animate_manager + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.interactive_manager + :members: + :undoc-members: + :show-inheritance: + +Styles and CLI +-------------- + +.. automodule:: zyra.visualization.styles + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_utils + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_animate + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_compose_video + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_contour + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_heatmap + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_interactive + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_timeseries + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: zyra.visualization.cli_vector + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/conf.py b/docs/source/conf.py index b87458d9..5c0b7a3c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,7 +12,7 @@ # -- Project information ----------------------------------------------------- -project = "DataVizHub" +project = "Zyra" author = "NOAA-GSL" copyright = f"{datetime.now():%Y}, {author}" @@ -103,6 +103,15 @@ "ffmpeg_python", "ffmpeg-python", # Mock API submodules that cause side effects on import during docs build + # Use Zyra namespace; legacy kept for transition + "zyra.api", + "zyra.api.server", + "zyra.api.routers", + "zyra.api.routers.cli", + "zyra.api.routers.files", + "zyra.api.workers", + "zyra.api.workers.executor", + "zyra.api.workers.jobs", "datavizhub.api", "datavizhub.api.server", "datavizhub.api.routers", diff --git a/docs/source/index.rst b/docs/source/index.rst index e55ac22e..794655e9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,7 +1,7 @@ -DataVizHub Documentation +Zyra Documentation ======================== -Welcome to the DataVizHub documentation. +Welcome to the Zyra documentation. Contents -------- @@ -16,14 +16,14 @@ Contents :maxdepth: 2 :caption: Packages - api/datavizhub.visualization - api/datavizhub.processing - api/datavizhub.connectors - api/datavizhub.utils - api/datavizhub.api - api/datavizhub.assets - api/datavizhub.cli - api/datavizhub.transform + api/zyra.visualization + api/zyra.processing + api/zyra.connectors + api/zyra.utils + api/zyra.api + api/zyra.assets + api/zyra.cli + api/zyra.transform .. toctree:: :maxdepth: 2 @@ -42,5 +42,5 @@ External Wiki For human-authored guidance, design notes, and usage guides, see the project wiki: -- Online: https://github.com/NOAA-GSL/datavizhub/wiki +- Online: https://github.com/NOAA-GSL/zyra/wiki - Synced copy is included in this documentation under the "Wiki" section. diff --git a/docs/source/wiki b/docs/source/wiki new file mode 160000 index 00000000..74e8bc22 --- /dev/null +++ b/docs/source/wiki @@ -0,0 +1 @@ +Subproject commit 74e8bc221733c24ff02253e277678d83bad87bd3 diff --git "a/docs/source/wiki/Branch\342\200\220Aware-Wiki-Sync-for-DataVizHub.md" "b/docs/source/wiki/Branch\342\200\220Aware-Wiki-Sync-for-DataVizHub.md" deleted file mode 100644 index d4b55e1b..00000000 --- "a/docs/source/wiki/Branch\342\200\220Aware-Wiki-Sync-for-DataVizHub.md" +++ /dev/null @@ -1,175 +0,0 @@ -## Goals -- Mirror the **GitHub Wiki** into `docs/wiki/`. -- Make wiki pages part of the **Sphinx site**. -- Ensure **contributors in feature branches** can build docs with up-to-date wiki. -- Keep commit history clean: **sync commits only happen in `main`**. -- Remove **Docker-based wiki sync** to simplify builds. - ---- - -## 1. Add Branch-Aware GitHub Action for Wiki Sync - -Create `.github/workflows/sync-wiki.yml`: - -```yaml -name: Sync Wiki to Docs - -on: - push: - branches: - - "**" # Run on all branches - pull_request: # Ensure PR builds include wiki - schedule: - - cron: "0 6 * * *" # Daily sync at 6 AM UTC - workflow_dispatch: # Manual trigger - -jobs: - sync-wiki: - runs-on: ubuntu-latest - - steps: - - name: Checkout main repo - uses: actions/checkout@v4 - with: - path: main - - - name: Checkout wiki repo - uses: actions/checkout@v4 - with: - repository: NOAA-GSL/datavizhub.wiki - path: wiki - - - name: Sync wiki into docs/wiki - run: | - rsync -av --delete wiki/ main/docs/wiki/ - - - name: Commit wiki changes (main only) - run: | - cd main - if [ "$GITHUB_REF_NAME" = "main" ]; then - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add docs/wiki/ - if ! git diff --cached --quiet; then - git commit -m "chore(docs): sync wiki into docs/wiki" - git push - else - echo "No changes to commit" - fi - else - echo "Skipping commit: branch is $GITHUB_REF_NAME" - fi -``` - -On feature branches and PRs: wiki is **synced locally** so docs build works. -On `main`: wiki changes are **committed and pushed**. - ---- - -## 2. Update Sphinx to Support Wiki Pages - -In `docs/source/conf.py`: - -```python -extensions = [ - "myst_parser", # Enables Markdown - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", -] - -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", -} -``` - -Install `myst-parser` via Poetry or pip. - ---- - -## 3. Add Wiki to Docs Site Navigation - -In `docs/source/index.rst`: - -```rst -Welcome to DataVizHub's Documentation -===================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - api/index - wiki/index - contributing/wiki-sync -``` - -Create `docs/wiki/index.md`: - -```markdown -# DataVizHub Wiki - -This section mirrors the official [GitHub Wiki](https://github.com/NOAA-GSL/datavizhub/wiki). - -Contents are synced automatically. -``` - ---- - -## 4. Add Contributor Documentation - -Create `docs/source/contributing/wiki-sync.md`: - -```markdown -# Wiki Sync Workflow - -## Why -- Ensures wiki knowledge is AI-accessible. -- Keeps docs version-controlled with code. -- Removes duplication between Docker and CI. - -## How -- GitHub Action syncs wiki → `docs/wiki/`. -- Sphinx builds wiki pages into the docs site. -- Commits only happen on `main`. - -## Editing Rules -- Edit via the GitHub Wiki → synced to repo on next run. -- Do not edit `docs/wiki/` directly (overwritten on sync). -``` - ---- - -## 5. Remove Docker-Based Wiki Sync -- Delete wiki fetch steps from `Dockerfile`. -- Ensure `docs/wiki/` is not ignored in `.dockerignore`. -- Containers now build purely from repo contents. - ---- - -## 6. Verify End-to-End -1. Run workflow manually (`workflow_dispatch`). -2. Confirm `docs/wiki/` populated in branch builds. -3. Merge to `main` → verify wiki commit appears. -4. Build docs (`make html`) → wiki pages visible. -5. Confirm container builds still include synced docs. - ---- - -## Deliverables -1. `.github/workflows/sync-wiki.yml` (branch-aware). -2. Updated `conf.py` with `myst_parser`. -3. Updated `index.rst` with wiki & contributing guide. -4. `docs/wiki/index.md` entrypoint. -5. `docs/source/contributing/wiki-sync.md` contributor guide. -6. `Dockerfile` cleaned of wiki sync logic. - ---- - -## Benefits -- Always up-to-date wiki in every branch. -- Clean commit history (sync commits only in `main`). -- Docs site integrates wiki automatically. -- Simplified container builds. -- AI assistants can read wiki content directly from repo. \ No newline at end of file diff --git a/docs/source/wiki/DataVizHub-API-Routers-and-Endpoints.md b/docs/source/wiki/DataVizHub-API-Routers-and-Endpoints.md deleted file mode 100644 index 7dc236ea..00000000 --- a/docs/source/wiki/DataVizHub-API-Routers-and-Endpoints.md +++ /dev/null @@ -1,67 +0,0 @@ -This page maps the FastAPI routers to their endpoints, with brief descriptions and auth notes. - -## Routers Overview - -- `datavizhub.api.routers.cli` - - `GET /cli/commands` — Discovery: stages, commands, and argument schemas (with examples) - - `GET /cli/examples` — Curated example request bodies and pipelines - - `POST /cli/run` — Execute a CLI command (sync or async) - - `GET /examples` — Interactive examples page (Upload → Run → Download, WS streaming) - - Auth: Requires API key header when `DATAVIZHUB_API_KEY` is set (except `/docs` and `/redoc`) - -- `datavizhub.api.routers.files` - - `POST /upload` — Multipart file upload; returns `{file_id, path}` - - Auth: Requires API key header when enabled - - Notes: Uploaded files are written under `DATAVIZHUB_UPLOAD_DIR` (default `/tmp/datavizhub_uploads`) - -- `datavizhub.api.routers.jobs` - - `GET /jobs/{job_id}` — Job status (stdout, stderr, exit_code, output_file, resolved_input_paths) - - `DELETE /jobs/{job_id}` — Cancel a queued job - - `GET /jobs/{job_id}/manifest` — JSON list of job artifacts (name, path, size, mtime, media_type) - - `GET /jobs/{job_id}/download` — Download an artifact; supports `?file=NAME` and `?zip=1` - - Auth: Requires API key header when enabled - - Notes: Results under `DATAVIZHUB_RESULTS_DIR/{job_id}`; TTL via `DATAVIZHUB_RESULTS_TTL_SECONDS`; periodic prune via cleanup loop - -- `datavizhub.api.routers.ws` - - `WS /ws/jobs/{job_id}` — Live JSON messages for logs/progress/final payload - - Query params: - - `api_key` (when `DATAVIZHUB_API_KEY` is set) — required - - `stream=stdout,stderr,progress` — filter which keys to stream - - Auth: Fail-fast on bad/missing key (closes with code 1008, no data) - - Modes: Redis pub/sub or in-memory pub/sub parity - -## Auth Recap - -- HTTP endpoints (CLI, Files, Jobs) require the API key header when configured -- WebSocket requires `?api_key=` in the URL when configured -- `/examples` page can be gated with `DATAVIZHUB_REQUIRE_KEY_FOR_EXAMPLES=1` -- OpenAPI docs remain readable without a key - -## Common Workflows - -- Upload → Run → Download - 1. `POST /upload` (store `file_id`) - 2. `POST /cli/run` (use `file_id:` placeholder in args) — async returns `job_id` - 3. Stream: `ws://.../ws/jobs/{job_id}?api_key=&stream=progress` - 4. `GET /jobs/{job_id}` until `status=succeeded`; `GET /jobs/{job_id}/download` to fetch artifact - -- Discovery and Examples - - `GET /cli/commands` for per-command schemas and examples - - `GET /cli/examples` for curated examples; try them in `/examples` - -## Tags in OpenAPI - -- `cli` — CLI discovery and execution -- `files` — Uploads -- `jobs` — Job status/manifest/download -- `ws` — WebSocket streaming -- `system` — `GET /health`, `GET /ready` - -## Environment Variables (selected) - -- Auth: `DATAVIZHUB_API_KEY`, `DATAVIZHUB_API_KEY_HEADER`, `DATAVIZHUB_REQUIRE_KEY_FOR_EXAMPLES` -- CORS: `DATAVIZHUB_CORS_ALLOW_ALL`, `DATAVIZHUB_CORS_ORIGINS` -- Uploads: `DATAVIZHUB_UPLOAD_DIR` -- Results: `DATAVIZHUB_RESULTS_DIR`, `DATAVIZHUB_RESULTS_TTL_SECONDS`, `DATAVIZHUB_RESULTS_CLEAN_INTERVAL_SECONDS` -- Streaming: `DATAVIZHUB_USE_REDIS`, `DATAVIZHUB_REDIS_URL`, `DATAVIZHUB_QUEUE` - diff --git a/docs/source/wiki/DataVizHub-API-Security-Quickstart.md b/docs/source/wiki/DataVizHub-API-Security-Quickstart.md deleted file mode 100644 index 16b6eea4..00000000 --- a/docs/source/wiki/DataVizHub-API-Security-Quickstart.md +++ /dev/null @@ -1,75 +0,0 @@ -This page describes how to enable and use API key authentication, how WebSocket authentication works, and how to configure CORS for browser clients. - -## API Key Authentication - -- Enable with an environment variable: - - `DATAVIZHUB_API_KEY`: the required API key value - - `DATAVIZHUB_API_KEY_HEADER` (optional): header name (default: `X-API-Key`) -- Effect: - - All HTTP routes in the CLI, Files, and Jobs routers require the API key header - - OpenAPI docs (`/docs`, `/redoc`) remain readable without a key - - WebSockets accept the API key as a query string: `?api_key=` - -### Examples - -- HTTP request with header (default header name): - -```bash -curl -H 'X-API-Key: $DATAVIZHUB_API_KEY' \ - http://localhost:8000/cli/commands -``` - -- WebSocket connection (e.g., websocat/wscat) with key and filters: - -```bash -npx wscat -c "ws://localhost:8000/ws/jobs/?api_key=$DATAVIZHUB_API_KEY&stream=progress" -``` - -- TestClient (Python): - -```python -from fastapi.testclient import TestClient -from datavizhub.api.server import app - -client = TestClient(app) -r = client.get('/cli/commands', headers={'X-API-Key': 'your-key'}) -assert r.status_code == 200 -``` - -## Gating `/examples` in Production - -- To block the interactive examples page in production, set: - - `DATAVIZHUB_REQUIRE_KEY_FOR_EXAMPLES=1` - - `DATAVIZHUB_API_KEY=` -- `/examples` then returns 401 unless a valid key is provided via header or `?api_key=` query parameter. -- The examples UI includes an API key field; when present it sends the key in HTTP request headers and as a WS query param. - -## WebSocket Authentication Behavior - -- If `DATAVIZHUB_API_KEY` is defined and the `api_key` query parameter is missing or incorrect: - - The server closes the connection immediately with code 1008 (policy violation) and does not send any payload. -- When the key is valid, messages stream as JSON; you can filter by keys with `?stream=stdout,stderr,progress`. - -## CORS Configuration - -CORS is restrictive by default. Enable explicitly via environment variables: - -- Allow all (dev only): `DATAVIZHUB_CORS_ALLOW_ALL=1` -- Specific origins (recommended for prod): - - `DATAVIZHUB_CORS_ORIGINS="https://app.example.com,https://tools.example.org"` - -The server then enables: -- `allow_credentials=True` -- `allow_methods=*` -- `allow_headers=*` - -## Reverse Proxy and TLS (Overview) - -- Run the API behind Nginx/Caddy for HTTPS termination and WS upgrade support -- Ensure the proxy preserves `Host`, supports WebSocket upgrade headers, and forwards custom headers (e.g., `X-API-Key`) - -## Future: Rate Limiting and Observability - -- Rate limiting (e.g., SlowAPI) can be added later for abuse control -- Structured logs + error reporting (e.g., Sentry) can be enabled with environment-based opt-in - diff --git a/docs/source/wiki/DataVizHub-CLI-Expansion-Plan-with-Pipeline-Configs-and-Connectors-Refactor.md b/docs/source/wiki/DataVizHub-CLI-Expansion-Plan-with-Pipeline-Configs-and-Connectors-Refactor.md deleted file mode 100644 index 0e85ff0f..00000000 --- a/docs/source/wiki/DataVizHub-CLI-Expansion-Plan-with-Pipeline-Configs-and-Connectors-Refactor.md +++ /dev/null @@ -1,188 +0,0 @@ -## Goal -Transform the existing CLI into a complete, modular, four-stage pipeline: -``` -[ Acquisition (Ingest) ] → [ Processing ] → [ Visualization ] → [ Decimation (Egress) ] -``` -- Rename `acquisition/` to `connectors/` to reflect both inbound and outbound data flow. -- Split into `ingest/` and `egress/` submodules, sharing common backend code in `backends/`. -- Each stage has its own CLI subcommands. -- All commands accept stdin/stdout for chaining. -- Add `run` command to execute pipelines from YAML/JSON config files. - ---- - -## 1. Refactor CLI Structure - -### 1.1 Top-Level Groups -- Modify `src/datavizhub/cli.py`: - - Create **four top-level subparsers**: `acquire`, `process`, `visualize`, `decimate`, plus `run`. - - Remove existing flat commands (`decode-grib2`, `convert-format`, etc.) and nest under `process`. - -### 1.2 Module Self-Registration -- In each stage's `__init__.py`, add: -```python -def register_cli(subparsers): - """Register CLI commands for this stage.""" -``` -- `cli.py` calls: -```python -from datavizhub.connectors import ingest, egress -from datavizhub import processing, visualization - -ingest.register_cli(acquire_subparser) -processing.register_cli(process_subparser) -visualization.register_cli(visualize_subparser) -egress.register_cli(decimate_subparser) -``` - ---- - -## 2. Connectors Module Structure - -``` -src/datavizhub/connectors/ - backends/ - s3.py - http.py - ftp.py - vimeo.py - ingest/ - __init__.py - ingest_manager.py - egress/ - __init__.py - egress_manager.py -``` - -### 2.1 Ingest -- `ingest_manager.py` maps CLI commands to inbound fetchers: - - `acquire http ` → `backends/http.py` - - `acquire s3 /` → `backends/s3.py` - - `acquire ftp /` → `backends/ftp.py` - - `acquire vimeo ` → `backends/vimeo.py` -- All commands: - - Accept `--output` (default `-` = stdout). - - Stream binary data directly. - -### 2.2 Egress -- `egress_manager.py` maps CLI commands to outbound writers: - - `decimate local ` - - `decimate s3 /` → `backends/s3.py` - - `decimate ftp /` → `backends/ftp.py` - - `decimate post ` → `backends/http.py` -- All commands: - - Accept stdin (`-`) as input. - - Write binary data directly. - ---- - -## 3. Processing (`src/datavizhub/processing/`) -- Move existing CLI functions into `process` namespace: - - `decode-grib2` - - `extract-variable` - - `convert-format` -- Add missing processors: - - NetCDF subset/extract - - Video conversion -- All commands: - - Accept stdin/stdout. - - Auto-detect formats. - ---- - -## 4. Visualization (`src/datavizhub/visualization/`) -- New commands: - - `visualize plot --type contour|timeseries --var ` - - `visualize colormap --set ` - - `visualize animate --frames --output