Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
"name": "datavizhub-dev",
"name": "zyra-dev",
"dockerComposeFile": [
"../docker-compose.yml"
],
"service": "datavizhub",
"service": "zyra",
"workspaceFolder": "/app",
"overrideCommand": false,

// ✅ Ensures VS Code starts the right services
"runServices": ["datavizhub", "redis"],
"runServices": ["zyra", "redis"],
"composeProfiles": ["dev"],

"customizations": {
Expand Down
44 changes: 32 additions & 12 deletions .devcontainer/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ if [[ -f .devcontainer/.env ]]; then
echo "[entrypoint] Added ${key} to .env from .devcontainer/.env"
fi
}
# Prefer new ZYRA_* keys; keep DATAVIZHUB_* for compatibility
ensure_kv ZYRA_LLM_PROVIDER
ensure_kv ZYRA_LLM_MODEL
ensure_kv DATAVIZHUB_LLM_PROVIDER
ensure_kv DATAVIZHUB_LLM_MODEL
ensure_kv OLLAMA_BASE_URL
Expand All @@ -27,29 +30,46 @@ if [[ -f .devcontainer/.env ]]; then
fi

# Export all variables from .env into the container environment (dev only)
if [[ "${DATAVIZHUB_ENV:-dev}" == "dev" && -f .env ]]; then
if [[ "${ZYRA_ENV:-${DATAVIZHUB_ENV:-dev}}" == "dev" && -f .env ]]; then
echo "[entrypoint] Loading environment variables from .env (dev only)"
set -a
source .env
set +a
fi

# ====== SERVICE START SECTION ======
AUTOSTART_API=${DATAVIZHUB_AUTOSTART_API:-1}
AUTOSTART_RQ=${DATAVIZHUB_AUTOSTART_RQ:-${DATAVIZHUB_USE_REDIS:-0}}
API_HOST=${DATAVIZHUB_API_HOST:-0.0.0.0}
API_PORT=${DATAVIZHUB_API_PORT:-8000}
# Prefer ZYRA_* env names with legacy fallbacks
AUTOSTART_API=${ZYRA_AUTOSTART_API:-${DATAVIZHUB_AUTOSTART_API:-1}}
AUTOSTART_RQ=${ZYRA_AUTOSTART_RQ:-${DATAVIZHUB_AUTOSTART_RQ:-${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}}}}
API_HOST=${ZYRA_API_HOST:-${DATAVIZHUB_API_HOST:-0.0.0.0}}
API_PORT=${ZYRA_API_PORT:-${DATAVIZHUB_API_PORT:-8000}}

# Map ZYRA_* -> DATAVIZHUB_* for runtime back-compat if legacy vars are unset
map_keys=(
USE_REDIS REDIS_URL AUTOSTART_API AUTOSTART_RQ UPLOAD_DIR MIN_DISK_MB REQUIRE_FFMPEG
API_HOST API_PORT VERBOSITY STRICT_ENV DEFAULT_STDIN
CORS_ALLOW_ALL CORS_ORIGINS API_KEY API_KEY_HEADER
RESULTS_TTL_SECONDS RESULTS_CLEAN_INTERVAL_SECONDS RESULTS_DIR QUEUE
LLM_PROVIDER LLM_MODEL WIZARD_EDITOR_MODE
)
for k in "${map_keys[@]}"; do
zy="ZYRA_${k}"
dv="DATAVIZHUB_${k}"
if [[ -n "${!zy:-}" && -z "${!dv:-}" ]]; then
export "${dv}=${!zy}"
fi
done

mkdir -p .cache
echo "[entrypoint] ===== $(date) API session =====" >> .cache/api.log
echo "[entrypoint] ===== $(date) RQ worker session =====" >> .cache/rq.log

wait_for_redis() {
local url host port
url="${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0}"
url="${ZYRA_REDIS_URL:-${DATAVIZHUB_REDIS_URL:-redis://redis:6379/0}}"
# Basic validation to avoid accidental command injection or bad values
if ! [[ "$url" =~ ^redis://[A-Za-z0-9._-]+(:[0-9]{1,5})?(/[0-9]+)?$ ]]; then
echo "[entrypoint] ERROR: Invalid DATAVIZHUB_REDIS_URL: "$url"" >&2
echo "[entrypoint] ERROR: Invalid REDIS_URL: "$url"" >&2
return 1
fi
host=$(echo "$url" | sed -E 's#^redis://([^:/]+):?([0-9]+)?.*#\1#')
Expand All @@ -68,24 +88,24 @@ wait_for_redis() {
}

start_rq_worker() {
if pgrep -f "rq worker datavizhub" >/dev/null 2>&1; then
if pgrep -f "rq worker zyra" >/dev/null 2>&1; then
echo "[entrypoint] RQ worker already running"
else
echo "[entrypoint] Starting RQ worker..."
( DATAVIZHUB_USE_REDIS=1 poetry run rq worker datavizhub >> .cache/rq.log 2>&1 & )
( DATAVIZHUB_USE_REDIS=1 ZYRA_USE_REDIS=1 poetry run rq worker zyra >> .cache/rq.log 2>&1 & )
fi
}

start_api() {
if pgrep -f "uvicorn datavizhub.api.server:app" >/dev/null 2>&1; then
if pgrep -f "uvicorn zyra.api.server:app" >/dev/null 2>&1; then
echo "[entrypoint] API already running"
else
echo "[entrypoint] Starting API on ${API_HOST}:${API_PORT}"
( poetry run uvicorn datavizhub.api.server:app --host "${API_HOST}" --port "${API_PORT}" --reload >> .cache/api.log 2>&1 & )
( poetry run uvicorn zyra.api.server:app --host "${API_HOST}" --port "${API_PORT}" --reload >> .cache/api.log 2>&1 & )
fi
}

if [[ "$AUTOSTART_RQ" == "1" && "$DATAVIZHUB_USE_REDIS" == "1" ]]; then
if [[ "$AUTOSTART_RQ" == "1" && "${ZYRA_USE_REDIS:-${DATAVIZHUB_USE_REDIS:-0}}" == "1" ]]; then
wait_for_redis || exit 1
fi

Expand Down
8 changes: 4 additions & 4 deletions .github/codeql/codeql-config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ query-filters:
- exclude:
id: py/path-injection
paths:
- src/datavizhub/api/routers/jobs.py
- src/datavizhub/api/workers/executor.py
- src/zyra/api/routers/jobs.py
- src/zyra/api/workers/executor.py
- exclude:
id: py/uncontrolled-data-in-path-expression
paths:
- src/datavizhub/api/routers/jobs.py
- src/datavizhub/api/workers/executor.py
- src/zyra/api/routers/jobs.py
- src/zyra/api/workers/executor.py
2 changes: 1 addition & 1 deletion .github/workflows/publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:

environment:
name: release
url: https://pypi.org/project/datavizhub/
url: https://pypi.org/project/zyra/

permissions:
contents: read
Expand Down
48 changes: 48 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Release on staging → main merge

on:
pull_request:
types:
- closed
branches:
- main

jobs:
release:
if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'staging'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Extract version from pyproject.toml
id: get_version
run: |
VERSION=$(grep -Po '(?<=^version = ")[^"]*' pyproject.toml)
echo "version=$VERSION" >> $GITHUB_OUTPUT

- name: Ensure version bump
run: |
if git ls-remote --tags origin | grep -q "refs/tags/v${{ steps.get_version.outputs.version }}$"; then
echo "❌ Version v${{ steps.get_version.outputs.version }} already exists. Please bump version in pyproject.toml before merging."
exit 1
fi

- name: Create and push Git tag
run: |
git config user.name "github-actions"
git config user.email "github-actions@github.com"
git tag v${{ steps.get_version.outputs.version }}
git push origin v${{ steps.get_version.outputs.version }}

- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
tag_name: v${{ steps.get_version.outputs.version }}
name: Release v${{ steps.get_version.outputs.version }}
body: |
## ${{ github.event.pull_request.title }}
${{ github.event.pull_request.body }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
6 changes: 3 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ Repository Guidelines

Project Structure & Module Organization

- Code lives under `src/datavizhub/`:
- Code lives under `src/zyra/`:
- `connectors/`: source/destination integrations and CLI registrars (`connectors.ingest`, `connectors.egress`). Prefer these over legacy `acquisition/` for new work.
- `processing/`: data processing (e.g., GRIB/NetCDF/GeoTIFF) exposed via the CLI.
- `visualization/`: visualization commands; CLI registration lives in `visualization/cli_register.py`.
- `wizard/`: interactive assistant and related utilities.
- `api/` + `api_cli.py`: HTTP API and CLI entry points.
- `transform/`: transform helpers (metadata, etc.).
- `utils/`: shared helpers/utilities.
- `assets/`: packaged static resources; access with `importlib.resources` (`datavizhub.assets`).
- `assets/`: packaged static resources; access with `importlib.resources` (`zyra.assets`).
- `cli.py`, `pipeline_runner.py`: root CLI and pipeline runner.

Build, Test, and Development
Expand Down Expand Up @@ -83,7 +83,7 @@ Security & Configuration

- Do not commit secrets. Prefer IAM roles or env vars (e.g., used by S3 connectors).
- Avoid hard-coded absolute paths; prefer env-configurable paths (e.g., `DATA_DIR`).
- Use `importlib.resources` for packaged assets under `datavizhub.assets`.
- Use `importlib.resources` for packaged assets under `zyra.assets`.
- Pin dependencies when adding new ones; document any system deps (e.g., FFmpeg, PROJ/GEOS).

Dependency Management (Poetry)
Expand Down
35 changes: 35 additions & 0 deletions CITATION.cff
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
cff-version: 1.2.0
title: Zyra
message: ' Modular workflows for reproducible science'
type: software
authors:
- given-names: Eric
family-names: Hackathorn
email: eric.j.hackathorn@noaa.gov
orcid: 'https://orcid.org/0000-0002-9693-2093'
affiliation: NOAA Global Systems Laboratory
identifiers:
- type: doi
value: 10.5281/zenodo.xxxxxxx
description: replace with Zenodo DOI once registered
repository-code: 'https://github.com/NOAA-GSL/zyra'
url: 'https://github.com/NOAA-GSL/zyra/wiki'
repository: 'https://pypi.org/project/zyra/'
abstract: >-
Zyra is an open-source Python framework for creating
reproducible, modular, and visually compelling data
visualizations. It provides a flexible pipeline for data
acquisition, processing, rendering, and dissemination,
making it useful for scientists, educators, and developers
who need to explore, analyze, and communicate complex
scientific data.
keywords:
- Python
- data visualization
- scientific visualization
- open science
- reproducible research
- workflow automation
- data pipeline
- NOAA
license: MIT
4 changes: 2 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Contributing to DataVizHub
# Contributing to Zyra

Thanks for your interest in contributing!
This project thrives on community contributions, and we welcome improvements of all kinds.
Expand All @@ -7,7 +7,7 @@ This project thrives on community contributions, and we welcome improvements of

## License and Contributor Terms

- DataVizHub is licensed under the MIT License. See `LICENSE` at the repository root.
- Zyra is licensed under the MIT License. See `LICENSE` at the repository root.
- By submitting a pull request, issue suggestion, or any code/documentation/artwork (“Contribution”),
you agree to license your Contribution under the MIT License, and you represent that you have the
right to do so.
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ RUN poetry install --with dev --all-extras

# Healthcheck for non-compose runs
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=5 \
CMD sh -c "curl -fsS http://localhost:${DATAVIZHUB_API_PORT:-8000}/ready || exit 1"
CMD sh -c "curl -fsS http://localhost:${ZYRA_API_PORT:-${DATAVIZHUB_API_PORT:-8000}}/ready || exit 1"

# Automatically load .env variables in interactive shells (dev only)
RUN echo 'set -a; [ -f /app/.env ] && source /app/.env; set +a' >> /root/.bashrc
Expand Down
Loading
Loading